Skip to content

Commit

Permalink
use codeclysm/extract and remove extract code
Browse files Browse the repository at this point in the history
  • Loading branch information
@umbynos
umbynos committed Dec 7, 2023
1 parent 3363502 commit 223ef12
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 345 deletions.
290 changes: 14 additions & 276 deletions tools/download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,28 +16,23 @@
package tools

import (
"archive/tar"
"archive/zip"
"bytes"
"compress/bzip2"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"strings"

"github.com/arduino/arduino-create-agent/utilities"
"github.com/arduino/arduino-create-agent/v2/pkgs"
"github.com/blang/semver"
"github.com/codeclysm/extract/v3"
)

// public vars to allow override in the tests
Expand All @@ -46,10 +41,6 @@ var (
Arch = runtime.GOARCH
)

func mimeType(data []byte) (string, error) {
return http.DetectContentType(data[0:512]), nil
}

func pathExists(path string) bool {
_, err := os.Stat(path)
if err == nil {
Expand Down Expand Up @@ -139,23 +130,22 @@ func (t *Tools) Download(pack, name, version, behaviour string) error {
return err
}

srcType, err := mimeType(body)
ctx := context.Background()

reader := bytes.NewReader(body)
err = extract.Archive(ctx, reader, location, func(original string) string {
// Split the original path into components
components := strings.Split(original, string(os.PathSeparator))
// If there's a root directory, remove it
if len(components) > 1 {
return filepath.Join(components[1:]...)
}
return original
})
if err != nil {
return err
}

switch srcType {
case "application/zip":
location, err = extractZip(t.logger, body, location)
case "application/x-bz2":
case "application/octet-stream":
location, err = extractBz2(t.logger, body, location)
case "application/x-gzip":
location, err = extractTarGz(t.logger, body, location)
default:
return errors.New("Unknown extension for file " + correctSystem.URL)
}

if err != nil {
t.logger("Error extracting the archive: " + err.Error())
return err
Expand Down Expand Up @@ -207,258 +197,6 @@ func findTool(pack, name, version string, data pkgs.Index) (pkgs.Tool, pkgs.Syst
return correctTool, correctSystem
}

func commonPrefix(sep byte, paths []string) string {
// Handle special cases.
switch len(paths) {
case 0:
return ""
case 1:
return path.Clean(paths[0])
}

c := []byte(path.Clean(paths[0]))

// We add a trailing sep to handle: common prefix directory is included in the path list
// (e.g. /home/user1, /home/user1/foo, /home/user1/bar).
// path.Clean will have cleaned off trailing / separators with
// the exception of the root directory, "/" making it "//"
// but this will get fixed up to "/" below).
c = append(c, sep)

// Ignore the first path since it's already in c
for _, v := range paths[1:] {
// Clean up each path before testing it
v = path.Clean(v) + string(sep)

// Find the first non-common byte and truncate c
if len(v) < len(c) {
c = c[:len(v)]
}
for i := 0; i < len(c); i++ {
if v[i] != c[i] {
c = c[:i]
break
}
}
}

// Remove trailing non-separator characters and the final separator
for i := len(c) - 1; i >= 0; i-- {
if c[i] == sep {
c = c[:i]
break
}
}

return string(c)
}

func removeStringFromSlice(s []string, r string) []string {
for i, v := range s {
if v == r {
return append(s[:i], s[i+1:]...)
}
}
return s
}

func findBaseDir(dirList []string) string {
if len(dirList) == 1 {
return path.Dir(dirList[0]) + "/"
}

// https://github.com/backdrop-ops/contrib/issues/55#issuecomment-73814500
dontdiff := []string{"pax_global_header"}
for _, v := range dontdiff {
dirList = removeStringFromSlice(dirList, v)
}

commonBaseDir := commonPrefix('/', dirList)
if commonBaseDir != "" {
commonBaseDir = commonBaseDir + "/"
}
return commonBaseDir
}

func extractZip(log func(msg string), body []byte, location string) (string, error) {
path, _ := utilities.SaveFileonTempDir("tooldownloaded.zip", bytes.NewReader(body))
r, err := zip.OpenReader(path)
if err != nil {
return location, err
}

var dirList []string

for _, f := range r.File {
dirList = append(dirList, f.Name)
}

basedir := findBaseDir(dirList)
log(fmt.Sprintf("selected baseDir %s from Zip Archive Content: %v", basedir, dirList))

for _, f := range r.File {
fullname := filepath.Join(location, strings.Replace(f.Name, basedir, "", -1))
log(fmt.Sprintf("generated fullname %s removing %s from %s", fullname, basedir, f.Name))
if f.FileInfo().IsDir() {
os.MkdirAll(fullname, f.FileInfo().Mode().Perm())
} else {
os.MkdirAll(filepath.Dir(fullname), 0755)
perms := f.FileInfo().Mode().Perm()
out, err := os.OpenFile(fullname, os.O_CREATE|os.O_RDWR, perms)
if err != nil {
return location, err
}
rc, err := f.Open()
if err != nil {
return location, err
}
_, err = io.CopyN(out, rc, f.FileInfo().Size())
if err != nil {
return location, err
}
rc.Close()
out.Close()

mtime := f.FileInfo().ModTime()
err = os.Chtimes(fullname, mtime, mtime)
if err != nil {
return location, err
}
}
}
return location, nil
}

func extractTarGz(log func(msg string), body []byte, location string) (string, error) {
bodyCopy := make([]byte, len(body))
copy(bodyCopy, body)
tarFile, _ := gzip.NewReader(bytes.NewReader(body))
tarReader := tar.NewReader(tarFile)

var dirList []string

for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
dirList = append(dirList, header.Name)
}

basedir := findBaseDir(dirList)
log(fmt.Sprintf("selected baseDir %s from TarGz Archive Content: %v", basedir, dirList))

tarFile, _ = gzip.NewReader(bytes.NewReader(bodyCopy))
tarReader = tar.NewReader(tarFile)

for {
header, err := tarReader.Next()
if err == io.EOF {
break
} else if err != nil {
return location, err
}

path := filepath.Join(location, strings.Replace(header.Name, basedir, "", -1))
info := header.FileInfo()

// Create parent folder
dirmode := info.Mode() | os.ModeDir | 0700
if err = os.MkdirAll(filepath.Dir(path), dirmode); err != nil {
return location, err
}

if info.IsDir() {
if err = os.MkdirAll(path, info.Mode()); err != nil {
return location, err
}
continue
}

if header.Typeflag == tar.TypeSymlink {
_ = os.Symlink(header.Linkname, path)
continue
}

file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
if err != nil {
continue
}
_, err = io.Copy(file, tarReader)
if err != nil {
return location, err
}
file.Close()
}
return location, nil
}

func extractBz2(log func(msg string), body []byte, location string) (string, error) {
bodyCopy := make([]byte, len(body))
copy(bodyCopy, body)
tarFile := bzip2.NewReader(bytes.NewReader(body))
tarReader := tar.NewReader(tarFile)

var dirList []string

for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
dirList = append(dirList, header.Name)
}

basedir := findBaseDir(dirList)
log(fmt.Sprintf("selected baseDir %s from Bz2 Archive Content: %v", basedir, dirList))

tarFile = bzip2.NewReader(bytes.NewReader(bodyCopy))
tarReader = tar.NewReader(tarFile)

for {
header, err := tarReader.Next()
if err == io.EOF {
break
} else if err != nil {
continue
//return location, err
}

path := filepath.Join(location, strings.Replace(header.Name, basedir, "", -1))
info := header.FileInfo()

// Create parent folder
dirmode := info.Mode() | os.ModeDir | 0700
if err = os.MkdirAll(filepath.Dir(path), dirmode); err != nil {
return location, err
}

if info.IsDir() {
if err = os.MkdirAll(path, info.Mode()); err != nil {
return location, err
}
continue
}

if header.Typeflag == tar.TypeSymlink {
_ = os.Symlink(header.Linkname, path)
continue
}

file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
if err != nil {
continue
//return location, err
}
_, err = io.Copy(file, tarReader)
if err != nil {
return location, err
}
file.Close()
}
return location, nil
}

func (t *Tools) installDrivers(location string) error {
OkPressed := 6
extension := ".bat"
Expand Down
29 changes: 0 additions & 29 deletions tools/download_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ package tools

import (
"encoding/json"
"fmt"
"testing"
"time"

Expand Down Expand Up @@ -95,34 +94,6 @@ func TestDownloadFallbackPlatform(t *testing.T) {
}
}

func Test_findBaseDir(t *testing.T) {
cases := []struct {
dirList []string
want string
}{
{[]string{"bin/bossac"}, "bin/"},
{[]string{"bin/", "bin/bossac"}, "bin/"},
{[]string{"bin/", "bin/bossac", "example"}, ""},
{[]string{"avrdude/bin/",
"avrdude/bin/avrdude.exe",
"avrdude/bin/remove_giveio.bat",
"avrdude/bin/status_giveio.bat",
"avrdude/bin/giveio.sys",
"avrdude/bin/loaddrv.exe",
"avrdude/bin/libusb0.dll",
"avrdude/bin/install_giveio.bat",
"avrdude/etc/avrdude.conf"}, "avrdude/"},
{[]string{"pax_global_header", "bin/", "bin/bossac"}, "bin/"},
}
for _, tt := range cases {
t.Run(fmt.Sprintln(tt.dirList), func(t *testing.T) {
if got := findBaseDir(tt.dirList); got != tt.want {
t.Errorf("findBaseDir() = got %v, want %v", got, tt.want)
}
})
}
}

func TestDownload(t *testing.T) {
testCases := []struct {
name string
Expand Down
Loading

0 comments on commit 223ef12

Please sign in to comment.