add trace level logging to enable sdk debug mode (#430)

CHANGELOG.md

@@ -15,6 +15,7 @@
 - Added `--ignore-glacier-warnings` flag to `cp`, `mv` and `select` commands. ([#346](https://github.com/peak/s5cmd/issues/346))
 - Added `--request-payer` flag to include `x-amz-request-payer` in header while sending GET, POST and HEAD requests. ([#297](https://github.com/peak/s5cmd/issues/297)) [@Kirill888](https://github.com/Kirill888)
 - Added `--use-list-objects-v1` flag to force using S3 ListObjects API instead of ListObjectsV2 API. ([#405](https://github.com/peak/s5cmd/issues/405)) [@greenpau](https://github.com/greenpau)
+- Added trace log level(`--log=trace`) which enables SDK debug logs.([#363](https://github.com/peak/s5cmd/issues/363))
 
 #### Improvements
 - Upgraded minimum required Go version to 1.16.
@@ -25,6 +26,7 @@
 - Print error if the commands file of `run` command is not accessible. ([#410](https://github.com/peak/s5cmd/pull/410))
 - Updated region detection call to use current session's address resolving method ([#314](https://github.com/peak/s5cmd/issues/314))
 - Fixed a bug where lines with large tokens fail in `run` command. `sync` was failing when it finds multiple files to remove. ([#435](https://github.com/peak/s5cmd/issues/435), [#436](https://github.com/peak/s5cmd/issues/436))
+- Print usage error if given log level(`--log`) is not valid. ([#430](https://github.com/peak/s5cmd/pull/430))
 
 ## v1.4.0 - 21 Sep 2021
 

README.md

@@ -509,6 +509,7 @@ ERROR "cp s3://somebucket/file.txt file.txt": object already exists
     "error": "'cp s3://somebucket/file.txt file.txt': object already exists"
 }
 ```
+
 ## Benchmarks
 Some benchmarks regarding the performance of `s5cmd` are introduced below. For more
 details refer to this [post](https://medium.com/@joshua_robinson/s5cmd-for-high-performance-object-storage-7071352cc09d)

command/app.go

@@ -49,10 +49,13 @@ var app = &cli.App{
 			Name:  "no-verify-ssl",
 			Usage: "disable SSL certificate verification",
 		},
-		&cli.StringFlag{
-			Name:  "log",
-			Value: "info",
-			Usage: "log level: (debug, info, error)",
+		&cli.GenericFlag{
+			Name: "log",
+			Value: &EnumValue{
+				Enum:    []string{"trace", "debug", "info", "error"},
+				Default: "info",
+			},
+			Usage: "log level: (trace, debug, info, error)",
 		},
 		&cli.BoolFlag{
 			Name:  "install-completion",
@@ -152,13 +155,13 @@ var app = &cli.App{
 // NewStorageOpts creates storage.Options object from the given context.
 func NewStorageOpts(c *cli.Context) storage.Options {
 	return storage.Options{
-		MaxRetries:       c.Int("retry-count"),
-		Endpoint:         c.String("endpoint-url"),
-		NoVerifySSL:      c.Bool("no-verify-ssl"),
 		DryRun:           c.Bool("dry-run"),
+		Endpoint:         c.String("endpoint-url"),
+		MaxRetries:       c.Int("retry-count"),
 		NoSignRequest:    c.Bool("no-sign-request"),
-		UseListObjectsV1: c.Bool("use-list-objects-v1"),
+		NoVerifySSL:      c.Bool("no-verify-ssl"),
 		RequestPayer:     c.String("request-payer"),
+		UseListObjectsV1: c.Bool("use-list-objects-v1"),
 	}
 }
 

command/flag.go

@@ -0,0 +1,30 @@
+package command
+
+import (
+	"fmt"
+	"strings"
+)
+
+type EnumValue struct {
+	Enum     []string
+	Default  string
+	selected string
+}
+
+func (e *EnumValue) Set(value string) error {
+	for _, enum := range e.Enum {
+		if enum == value {
+			e.selected = value
+			return nil
+		}
+	}
+
+	return fmt.Errorf("allowed values: [%s]", strings.Join(e.Enum, ", "))
+}
+
+func (e EnumValue) String() string {
+	if e.selected == "" {
+		return e.Default
+	}
+	return e.selected
+}

command/select.go

@@ -47,10 +47,13 @@ func NewSelectCommand() *cli.Command {
 				Usage: "input compression format",
 				Value: "NONE",
 			},
-			&cli.StringFlag{
+			&cli.GenericFlag{
 				Name:  "format",
 				Usage: "input data format (only JSON supported for the moment)",
-				Value: "JSON",
+				Value: &EnumValue{
+					Enum:    []string{"JSON"},
+					Default: "JSON",
+				},
 			},
 			&cli.StringSliceFlag{
 				Name:  "exclude",

e2e/app_test.go

@@ -119,3 +119,20 @@ func TestUsageError(t *testing.T) {
 		1: equals("See 's5cmd --help' for usage"),
 	})
 }
+
+func TestInvalidLoglevel(t *testing.T) {
+	t.Parallel()
+
+	_, s5cmd, cleanup := setup(t)
+	defer cleanup()
+
+	cmd := s5cmd("--log", "notexist", "ls")
+	result := icmd.RunCmd(cmd)
+
+	result.Assert(t, icmd.Expected{ExitCode: 1})
+
+	assertLines(t, result.Stderr(), map[int]compareFunc{
+		0: equals(`Incorrect Usage: invalid value "notexist" for flag -log: allowed values: [trace, debug, info, error]`),
+		1: equals("See 's5cmd --help' for usage"),
+	})
+}

log/log.go

@@ -22,6 +22,11 @@ func Init(level string, json bool) {
 	global = New(level, json)
 }
 
+// Trace prints message in trace mode.
+func Trace(msg Message) {
+	global.printf(levelTrace, msg, os.Stdout)
+}
+
 // Debug prints message in debug mode.
 func Debug(msg Message) {
 	global.printf(levelDebug, msg, os.Stdout)
@@ -94,7 +99,8 @@ func (l *Logger) out() {
 type logLevel int
 
 const (
-	levelDebug logLevel = iota
+	levelTrace logLevel = iota
+	levelDebug
 	levelInfo
 	levelError
 )
@@ -108,6 +114,12 @@ func (l logLevel) String() string {
 		return "ERROR "
 	case levelDebug:
 		return "DEBUG "
+	case levelTrace:
+		// levelTrace is used for printing aws sdk logs and
+		// aws-sdk-go already adds "DEBUG" prefix to logs.
+		// So do not add another prefix to log which makes it
+		// look weird.
+		return ""
 	default:
 		return "UNKNOWN "
 	}
@@ -123,6 +135,8 @@ func levelFromString(s string) logLevel {
 		return levelInfo
 	case "error":
 		return levelError
+	case "trace":
+		return levelTrace
 	default:
 		return levelInfo
 	}

log/message.go

@@ -63,6 +63,18 @@ type DebugMessage struct {
 	Err       string `json:"error"`
 }
 
+type TraceMessage struct {
+	Message string `json:"message"`
+}
+
+func (t TraceMessage) String() string {
+	return t.Message
+}
+
+func (t TraceMessage) JSON() string {
+	return strutil.JSON(t)
+}
+
 // String is the string representation of ErrorMessage.
 func (d DebugMessage) String() string {
 	if d.Command == "" {

storage/s3.go

@@ -740,6 +740,15 @@ func (s *S3) RemoveBucket(ctx context.Context, name string) error {
 	return err
 }
 
+type sdkLogger struct{}
+
+func (l sdkLogger) Log(args ...interface{}) {
+	msg := log.TraceMessage{
+		Message: fmt.Sprint(args...),
+	}
+	log.Trace(msg)
+}
+
 // SessionCache holds session.Session according to s3Opts and it synchronizes
 // access/modification.
 type SessionCache struct {
@@ -790,7 +799,9 @@ func (sc *SessionCache) newSession(ctx context.Context, opts Options) (*session.
 		WithEndpoint(endpointURL.String()).
 		WithS3ForcePathStyle(!isVirtualHostStyle).
 		WithS3UseAccelerate(useAccelerate).
-		WithHTTPClient(httpClient)
+		WithHTTPClient(httpClient).
+		WithLogLevel(aws.LogDebug).
+		WithLogger(sdkLogger{})
 
 	awsCfg.Retryer = newCustomRetryer(opts.MaxRetries)