From 223ef12e74a9ce3de7f59a7b2c9f08c78c9c7bec Mon Sep 17 00:00:00 2001 From: Umberto Baldi Date: Thu, 7 Dec 2023 18:53:09 +0100 Subject: [PATCH] use `codeclysm/extract` and remove extract code --- tools/download.go | 290 ++--------------------------------------- tools/download_test.go | 29 ----- utilities/utilities.go | 40 ------ 3 files changed, 14 insertions(+), 345 deletions(-) diff --git a/tools/download.go b/tools/download.go index 76f4056c5..cfd97c503 100644 --- a/tools/download.go +++ b/tools/download.go @@ -16,28 +16,23 @@ package tools import ( - "archive/tar" - "archive/zip" "bytes" - "compress/bzip2" - "compress/gzip" + "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" - "fmt" "io" "net/http" "os" "os/exec" - "path" "path/filepath" "runtime" "strings" - "github.com/arduino/arduino-create-agent/utilities" "github.com/arduino/arduino-create-agent/v2/pkgs" "github.com/blang/semver" + "github.com/codeclysm/extract/v3" ) // public vars to allow override in the tests @@ -46,10 +41,6 @@ var ( Arch = runtime.GOARCH ) -func mimeType(data []byte) (string, error) { - return http.DetectContentType(data[0:512]), nil -} - func pathExists(path string) bool { _, err := os.Stat(path) if err == nil { @@ -139,23 +130,22 @@ func (t *Tools) Download(pack, name, version, behaviour string) error { return err } - srcType, err := mimeType(body) + ctx := context.Background() + + reader := bytes.NewReader(body) + err = extract.Archive(ctx, reader, location, func(original string) string { + // Split the original path into components + components := strings.Split(original, string(os.PathSeparator)) + // If there's a root directory, remove it + if len(components) > 1 { + return filepath.Join(components[1:]...) + } + return original + }) if err != nil { return err } - switch srcType { - case "application/zip": - location, err = extractZip(t.logger, body, location) - case "application/x-bz2": - case "application/octet-stream": - location, err = extractBz2(t.logger, body, location) - case "application/x-gzip": - location, err = extractTarGz(t.logger, body, location) - default: - return errors.New("Unknown extension for file " + correctSystem.URL) - } - if err != nil { t.logger("Error extracting the archive: " + err.Error()) return err @@ -207,258 +197,6 @@ func findTool(pack, name, version string, data pkgs.Index) (pkgs.Tool, pkgs.Syst return correctTool, correctSystem } -func commonPrefix(sep byte, paths []string) string { - // Handle special cases. - switch len(paths) { - case 0: - return "" - case 1: - return path.Clean(paths[0]) - } - - c := []byte(path.Clean(paths[0])) - - // We add a trailing sep to handle: common prefix directory is included in the path list - // (e.g. /home/user1, /home/user1/foo, /home/user1/bar). - // path.Clean will have cleaned off trailing / separators with - // the exception of the root directory, "/" making it "//" - // but this will get fixed up to "/" below). - c = append(c, sep) - - // Ignore the first path since it's already in c - for _, v := range paths[1:] { - // Clean up each path before testing it - v = path.Clean(v) + string(sep) - - // Find the first non-common byte and truncate c - if len(v) < len(c) { - c = c[:len(v)] - } - for i := 0; i < len(c); i++ { - if v[i] != c[i] { - c = c[:i] - break - } - } - } - - // Remove trailing non-separator characters and the final separator - for i := len(c) - 1; i >= 0; i-- { - if c[i] == sep { - c = c[:i] - break - } - } - - return string(c) -} - -func removeStringFromSlice(s []string, r string) []string { - for i, v := range s { - if v == r { - return append(s[:i], s[i+1:]...) - } - } - return s -} - -func findBaseDir(dirList []string) string { - if len(dirList) == 1 { - return path.Dir(dirList[0]) + "/" - } - - // https://github.com/backdrop-ops/contrib/issues/55#issuecomment-73814500 - dontdiff := []string{"pax_global_header"} - for _, v := range dontdiff { - dirList = removeStringFromSlice(dirList, v) - } - - commonBaseDir := commonPrefix('/', dirList) - if commonBaseDir != "" { - commonBaseDir = commonBaseDir + "/" - } - return commonBaseDir -} - -func extractZip(log func(msg string), body []byte, location string) (string, error) { - path, _ := utilities.SaveFileonTempDir("tooldownloaded.zip", bytes.NewReader(body)) - r, err := zip.OpenReader(path) - if err != nil { - return location, err - } - - var dirList []string - - for _, f := range r.File { - dirList = append(dirList, f.Name) - } - - basedir := findBaseDir(dirList) - log(fmt.Sprintf("selected baseDir %s from Zip Archive Content: %v", basedir, dirList)) - - for _, f := range r.File { - fullname := filepath.Join(location, strings.Replace(f.Name, basedir, "", -1)) - log(fmt.Sprintf("generated fullname %s removing %s from %s", fullname, basedir, f.Name)) - if f.FileInfo().IsDir() { - os.MkdirAll(fullname, f.FileInfo().Mode().Perm()) - } else { - os.MkdirAll(filepath.Dir(fullname), 0755) - perms := f.FileInfo().Mode().Perm() - out, err := os.OpenFile(fullname, os.O_CREATE|os.O_RDWR, perms) - if err != nil { - return location, err - } - rc, err := f.Open() - if err != nil { - return location, err - } - _, err = io.CopyN(out, rc, f.FileInfo().Size()) - if err != nil { - return location, err - } - rc.Close() - out.Close() - - mtime := f.FileInfo().ModTime() - err = os.Chtimes(fullname, mtime, mtime) - if err != nil { - return location, err - } - } - } - return location, nil -} - -func extractTarGz(log func(msg string), body []byte, location string) (string, error) { - bodyCopy := make([]byte, len(body)) - copy(bodyCopy, body) - tarFile, _ := gzip.NewReader(bytes.NewReader(body)) - tarReader := tar.NewReader(tarFile) - - var dirList []string - - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } - dirList = append(dirList, header.Name) - } - - basedir := findBaseDir(dirList) - log(fmt.Sprintf("selected baseDir %s from TarGz Archive Content: %v", basedir, dirList)) - - tarFile, _ = gzip.NewReader(bytes.NewReader(bodyCopy)) - tarReader = tar.NewReader(tarFile) - - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } else if err != nil { - return location, err - } - - path := filepath.Join(location, strings.Replace(header.Name, basedir, "", -1)) - info := header.FileInfo() - - // Create parent folder - dirmode := info.Mode() | os.ModeDir | 0700 - if err = os.MkdirAll(filepath.Dir(path), dirmode); err != nil { - return location, err - } - - if info.IsDir() { - if err = os.MkdirAll(path, info.Mode()); err != nil { - return location, err - } - continue - } - - if header.Typeflag == tar.TypeSymlink { - _ = os.Symlink(header.Linkname, path) - continue - } - - file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()) - if err != nil { - continue - } - _, err = io.Copy(file, tarReader) - if err != nil { - return location, err - } - file.Close() - } - return location, nil -} - -func extractBz2(log func(msg string), body []byte, location string) (string, error) { - bodyCopy := make([]byte, len(body)) - copy(bodyCopy, body) - tarFile := bzip2.NewReader(bytes.NewReader(body)) - tarReader := tar.NewReader(tarFile) - - var dirList []string - - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } - dirList = append(dirList, header.Name) - } - - basedir := findBaseDir(dirList) - log(fmt.Sprintf("selected baseDir %s from Bz2 Archive Content: %v", basedir, dirList)) - - tarFile = bzip2.NewReader(bytes.NewReader(bodyCopy)) - tarReader = tar.NewReader(tarFile) - - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } else if err != nil { - continue - //return location, err - } - - path := filepath.Join(location, strings.Replace(header.Name, basedir, "", -1)) - info := header.FileInfo() - - // Create parent folder - dirmode := info.Mode() | os.ModeDir | 0700 - if err = os.MkdirAll(filepath.Dir(path), dirmode); err != nil { - return location, err - } - - if info.IsDir() { - if err = os.MkdirAll(path, info.Mode()); err != nil { - return location, err - } - continue - } - - if header.Typeflag == tar.TypeSymlink { - _ = os.Symlink(header.Linkname, path) - continue - } - - file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()) - if err != nil { - continue - //return location, err - } - _, err = io.Copy(file, tarReader) - if err != nil { - return location, err - } - file.Close() - } - return location, nil -} - func (t *Tools) installDrivers(location string) error { OkPressed := 6 extension := ".bat" diff --git a/tools/download_test.go b/tools/download_test.go index 38181458c..faf38d13d 100644 --- a/tools/download_test.go +++ b/tools/download_test.go @@ -17,7 +17,6 @@ package tools import ( "encoding/json" - "fmt" "testing" "time" @@ -95,34 +94,6 @@ func TestDownloadFallbackPlatform(t *testing.T) { } } -func Test_findBaseDir(t *testing.T) { - cases := []struct { - dirList []string - want string - }{ - {[]string{"bin/bossac"}, "bin/"}, - {[]string{"bin/", "bin/bossac"}, "bin/"}, - {[]string{"bin/", "bin/bossac", "example"}, ""}, - {[]string{"avrdude/bin/", - "avrdude/bin/avrdude.exe", - "avrdude/bin/remove_giveio.bat", - "avrdude/bin/status_giveio.bat", - "avrdude/bin/giveio.sys", - "avrdude/bin/loaddrv.exe", - "avrdude/bin/libusb0.dll", - "avrdude/bin/install_giveio.bat", - "avrdude/etc/avrdude.conf"}, "avrdude/"}, - {[]string{"pax_global_header", "bin/", "bin/bossac"}, "bin/"}, - } - for _, tt := range cases { - t.Run(fmt.Sprintln(tt.dirList), func(t *testing.T) { - if got := findBaseDir(tt.dirList); got != tt.want { - t.Errorf("findBaseDir() = got %v, want %v", got, tt.want) - } - }) - } -} - func TestDownload(t *testing.T) { testCases := []struct { name string diff --git a/utilities/utilities.go b/utilities/utilities.go index 7372ab380..4f40aaf73 100644 --- a/utilities/utilities.go +++ b/utilities/utilities.go @@ -16,7 +16,6 @@ package utilities import ( - "archive/zip" "bytes" "crypto" "crypto/rsa" @@ -29,7 +28,6 @@ import ( "io" "os" "os/exec" - "path" "path/filepath" "strings" @@ -120,44 +118,6 @@ func call(stack []*exec.Cmd, pipes []*io.PipeWriter) (err error) { return stack[0].Wait() } -// Unzip will unzip a file to a destination -func Unzip(zippath string, destination string) (err error) { - r, err := zip.OpenReader(zippath) - if err != nil { - return err - } - for _, f := range r.File { - fullname := path.Join(destination, f.Name) - if f.FileInfo().IsDir() { - os.MkdirAll(fullname, f.FileInfo().Mode().Perm()) - } else { - os.MkdirAll(filepath.Dir(fullname), 0755) - perms := f.FileInfo().Mode().Perm() - out, err := os.OpenFile(fullname, os.O_CREATE|os.O_RDWR, perms) - if err != nil { - return err - } - rc, err := f.Open() - if err != nil { - return err - } - _, err = io.CopyN(out, rc, f.FileInfo().Size()) - if err != nil { - return err - } - rc.Close() - out.Close() - - mtime := f.FileInfo().ModTime() - err = os.Chtimes(fullname, mtime, mtime) - if err != nil { - return err - } - } - } - return -} - // SafeJoin performs a filepath.Join of 'parent' and 'subdir' but returns an error // if the resulting path points outside of 'parent'. func SafeJoin(parent, subdir string) (string, error) {