Skip to content

Commit

Permalink
Dependency Updates and Performance Improvements (#58)
Browse files Browse the repository at this point in the history 
* move deprecated docker types

* push

* Update packages + go from `github.com/mholt/archiver/v4` -> `github.com/mholt/archives`

* add back the check for ignored files when compressing

* Make unpacking .tar.gz, .tar.bz2, .7z, .tar, .rar work

* Also try to handle other formats

* Import the log library

* The error: "invalid argument" was fixt in mholt/archives

* Update imports

* Allow for rar Extraction

* Remove the duplicate conversion
  • Loading branch information
@QuintenQVD0
QuintenQVD0 authored Jan 4, 2025
1 parent faf0156 commit c335010
Show file tree
Hide file tree
Showing 13 changed files with 252 additions and 158 deletions.
2 changes: 1 addition & 1 deletion cmd/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ func rootCmdRun(cmd *cobra.Command, _ []string) {
log.WithField("error", err).Fatal("failed to initialize cron system")
} else {
log.WithField("subsystem", "cron").Info("starting cron processes")
s.StartAsync()
s.Start()
}

go func() {
Expand Down
3 changes: 1 addition & 2 deletions environment/docker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (

"emperror.dev/errors"
"github.com/apex/log"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"

Expand Down Expand Up @@ -39,7 +38,7 @@ func ConfigureDocker(ctx context.Context) error {
}

nw := config.Get().Docker.Network
resource, err := cli.NetworkInspect(ctx, nw.Name, types.NetworkInspectOptions{})
resource, err := cli.NetworkInspect(ctx, nw.Name, network.InspectOptions{})
if err != nil {
if !client.IsErrNotFound(err) {
return err
Expand Down
21 changes: 10 additions & 11 deletions environment/docker/container.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@ import (
"emperror.dev/errors"
"github.com/apex/log"
"github.com/buger/jsonparser"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
dockerImage "github.com/docker/docker/api/types/image"
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"
dockerImage "github.com/docker/docker/api/types/image" // Alias the correct images package

"github.com/pelican-dev/wings/config"
"github.com/pelican-dev/wings/environment"
Expand Down Expand Up @@ -196,17 +196,17 @@ func (e *Environment) Create() error {

networkMode := container.NetworkMode(cfg.Docker.Network.Mode)
if a.ForceOutgoingIP {
enableIPv6 := false // define a bool variable
enableIPv6 := false
e.log().Debug("environment/docker: forcing outgoing IP address")
networkName := "ip-" + strings.ReplaceAll(strings.ReplaceAll(a.DefaultMapping.Ip, ".", "-"), ":", "-")
networkMode = container.NetworkMode(networkName)

if _, err := e.client.NetworkInspect(ctx, networkName, types.NetworkInspectOptions{}); err != nil {
if _, err := e.client.NetworkInspect(ctx, networkName, network.InspectOptions{}); err != nil {
if !client.IsErrNotFound(err) {
return err
}

if _, err := e.client.NetworkCreate(ctx, networkName, types.NetworkCreate{
if _, err := e.client.NetworkCreate(ctx, networkName, network.CreateOptions{
Driver: "bridge",
EnableIPv6: &enableIPv6,
Internal: false,
Expand Down Expand Up @@ -440,16 +440,15 @@ func (e *Environment) ensureImageExists(image string) error {
}

func (e *Environment) convertMounts() []mount.Mount {
var out []mount.Mount

for _, m := range e.Configuration.Mounts() {
out = append(out, mount.Mount{
mounts := e.Configuration.Mounts()
out := make([]mount.Mount, len(mounts))
for i, m := range mounts {
out[i] = mount.Mount{
Type: mount.TypeBind,
Source: m.Source,
Target: m.Target,
ReadOnly: m.ReadOnly,
})
}
}

return out
}
8 changes: 4 additions & 4 deletions environment/docker/stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"time"

"emperror.dev/errors"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/goccy/go-json"

"github.com/pelican-dev/wings/environment"
Expand Down Expand Up @@ -57,7 +57,7 @@ func (e *Environment) pollResources(ctx context.Context) error {
case <-ctx.Done():
return ctx.Err()
default:
var v types.StatsJSON
var v container.StatsResponse
if err := dec.Decode(&v); err != nil {
if err != io.EOF && !errors.Is(err, context.Canceled) {
e.log().WithField("error", err).Warn("error while processing Docker stats output for container")
Expand Down Expand Up @@ -103,7 +103,7 @@ func (e *Environment) pollResources(ctx context.Context) error {
// bothering me about it. It should also reflect a slightly more correct memory value anyways.
//
// @see https://github.com/docker/cli/blob/96e1d1d6/cli/command/container/stats_helpers.go#L227-L249
func calculateDockerMemory(stats types.MemoryStats) uint64 {
func calculateDockerMemory(stats container.MemoryStats) uint64 {
if v, ok := stats.Stats["total_inactive_file"]; ok && v < stats.Usage {
return stats.Usage - v
}
Expand All @@ -119,7 +119,7 @@ func calculateDockerMemory(stats types.MemoryStats) uint64 {
// by the defined CPU limits on the container.
//
// @see https://github.com/docker/cli/blob/aa097cf1aa19099da70930460250797c8920b709/cli/command/container/stats_helpers.go#L166
func calculateDockerAbsoluteCpu(pStats types.CPUStats, stats types.CPUStats) float64 {
func calculateDockerAbsoluteCpu(pStats container.CPUStats, stats container.CPUStats) float64 {
// Calculate the change in CPU usage between the current and previous reading.
cpuDelta := float64(stats.CPUUsage.TotalUsage) - float64(pStats.CPUUsage.TotalUsage)

Expand Down
53 changes: 28 additions & 25 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,49 +14,50 @@ require (
github.com/buger/jsonparser v1.1.1
github.com/cenkalti/backoff/v4 v4.3.0
github.com/creasty/defaults v1.8.0
github.com/docker/docker v27.2.0+incompatible
github.com/docker/docker v27.4.1+incompatible
github.com/docker/go-connections v0.5.0
github.com/fatih/color v1.17.0
github.com/fatih/color v1.18.0
github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf
github.com/gabriel-vasile/mimetype v1.4.5
github.com/gabriel-vasile/mimetype v1.4.7
github.com/gammazero/workerpool v1.1.3
github.com/gbrlsnchs/jwt/v3 v3.0.1
github.com/gin-gonic/gin v1.10.0
github.com/glebarez/sqlite v1.11.0
github.com/go-co-op/gocron v1.37.0
github.com/goccy/go-json v0.10.3
github.com/go-co-op/gocron/v2 v2.14.0
github.com/goccy/go-json v0.10.4
github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.5.3
github.com/iancoleman/strcase v0.3.0
github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0
github.com/juju/ratelimit v1.0.2
github.com/klauspost/compress v1.17.9
github.com/klauspost/compress v1.17.11
github.com/klauspost/pgzip v1.2.6
github.com/magiconair/properties v1.8.7
github.com/magiconair/properties v1.8.9
github.com/mattn/go-colorable v0.1.13
github.com/mholt/archiver/v4 v4.0.0-alpha.8
github.com/mholt/archives v0.0.0-20241226194006-fc8400ac3529
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/sftp v1.13.6
github.com/pkg/sftp v1.13.7
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
github.com/shirou/gopsutil/v3 v3.24.5
github.com/spf13/cobra v1.8.1
github.com/stretchr/testify v1.9.0
golang.org/x/crypto v0.27.0
golang.org/x/sync v0.8.0
golang.org/x/sys v0.25.0
github.com/stretchr/testify v1.10.0
golang.org/x/crypto v0.31.0
golang.org/x/sync v0.10.0
golang.org/x/sys v0.28.0
gopkg.in/ini.v1 v1.67.0
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.1
gorm.io/gorm v1.25.11
gorm.io/gorm v1.25.12
)

require (
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/Microsoft/hcsshim v0.12.2 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/STARRY-S/zip v0.2.1 // indirect
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/bodgit/plumbing v1.3.0 // indirect
github.com/bodgit/sevenzip v1.5.1 // indirect
github.com/bodgit/sevenzip v1.6.0 // indirect
github.com/bodgit/windows v1.0.1 // indirect
github.com/bytedance/sonic v1.11.6 // indirect
github.com/bytedance/sonic/loader v0.1.1 // indirect
Expand All @@ -66,7 +67,7 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/gammazero/deque v0.2.1 // indirect
Expand All @@ -79,17 +80,18 @@ require (
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.20.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/jonboulle/clockwork v0.4.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magefile/mage v1.15.0 // indirect
Expand All @@ -101,7 +103,7 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/ncruces/go-strftime v0.1.9 // indirect
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
Expand All @@ -113,6 +115,7 @@ require (
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/sorairolake/lzip-go v0.3.5 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/therootcompany/xz v1.0.1 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
Expand All @@ -127,16 +130,16 @@ require (
go.opentelemetry.io/otel/metric v1.25.0 // indirect
go.opentelemetry.io/otel/sdk v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.25.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/arch v0.8.0 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/term v0.24.0 // indirect
golang.org/x/text v0.18.0 // indirect
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/net v0.31.0 // indirect
golang.org/x/term v0.27.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
golang.org/x/tools v0.22.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gotest.tools/v3 v3.0.2 // indirect
Expand Down
Loading

0 comments on commit c335010

Please sign in to comment.