From 3596ce62344d8d0a0f550aa68a5b3d3a707ddf54 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Fri, 25 Oct 2024 01:18:18 +0900 Subject: [PATCH] Add s3 config to list objects version (#6280) Signed-off-by: SungJin1212 --- CHANGELOG.md | 1 + docs/blocks-storage/querier.md | 4 ++ docs/blocks-storage/store-gateway.md | 4 ++ docs/configuration/config-file-reference.md | 16 ++++++ pkg/storage/bucket/s3/bucket_client.go | 7 +-- pkg/storage/bucket/s3/config.go | 30 +++++++---- pkg/storage/bucket/s3/config_test.go | 55 +++++++++++++++++++++ 7 files changed, 105 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af865528c6..30515f6472 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * [FEATURE] Ruler: Minimize chances of missed rule group evaluations that can occur due to OOM kills, bad underlying nodes, or due to an unhealthy ruler that appears in the ring as healthy. This feature is enabled via `-ruler.enable-ha-evaluation` flag. #6129 * [FEATURE] Store Gateway: Add an in-memory chunk cache. #6245 * [FEATURE] Chunk Cache: Support multi level cache and add metrics. #6249 +* [ENHANCEMENT] S3 Bucket Client: Add a list objects version configs to configure list api object version. #6280 * [ENHANCEMENT] Query Frontend: Add new query stats metrics `cortex_query_samples_scanned_total` and `cortex_query_peak_samples` to track scannedSamples and peakSample per user. #6228 * [ENHANCEMENT] Ingester: Add `blocks-storage.tsdb.wal-compression-type` to support zstd wal compression type. #6232 * [ENHANCEMENT] Query Frontend: Add info field to query response. #6207 diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 360775fe7e..6e4b5d78ef 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -312,6 +312,10 @@ blocks_storage: # CLI flag: -blocks-storage.s3.send-content-md5 [send_content_md5: | default = true] + # The list api version. Supported values are: v1, v2, and ''. + # CLI flag: -blocks-storage.s3.list-objects-version + [list_objects_version: | default = ""] + # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: blocks-storage [sse: ] diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 66047f3c52..98eaf3c8f3 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -403,6 +403,10 @@ blocks_storage: # CLI flag: -blocks-storage.s3.send-content-md5 [send_content_md5: | default = true] + # The list api version. Supported values are: v1, v2, and ''. + # CLI flag: -blocks-storage.s3.list-objects-version + [list_objects_version: | default = ""] + # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: blocks-storage [sse: ] diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index a841909e5c..a3e9eb04ab 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -563,6 +563,10 @@ s3: # CLI flag: -alertmanager-storage.s3.send-content-md5 [send_content_md5: | default = true] + # The list api version. Supported values are: v1, v2, and ''. + # CLI flag: -alertmanager-storage.s3.list-objects-version + [list_objects_version: | default = ""] + # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: alertmanager-storage [sse: ] @@ -842,6 +846,10 @@ s3: # CLI flag: -blocks-storage.s3.send-content-md5 [send_content_md5: | default = true] + # The list api version. Supported values are: v1, v2, and ''. + # CLI flag: -blocks-storage.s3.list-objects-version + [list_objects_version: | default = ""] + # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: blocks-storage [sse: ] @@ -4611,6 +4619,10 @@ s3: # CLI flag: -ruler-storage.s3.send-content-md5 [send_content_md5: | default = true] + # The list api version. Supported values are: v1, v2, and ''. + # CLI flag: -ruler-storage.s3.list-objects-version + [list_objects_version: | default = ""] + # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: ruler-storage [sse: ] @@ -4898,6 +4910,10 @@ s3: # CLI flag: -runtime-config.s3.send-content-md5 [send_content_md5: | default = true] + # The list api version. Supported values are: v1, v2, and ''. + # CLI flag: -runtime-config.s3.list-objects-version + [list_objects_version: | default = ""] + # The s3_sse_config configures the S3 server-side encryption. # The CLI flags prefix for this block config is: runtime-config [sse: ] diff --git a/pkg/storage/bucket/s3/bucket_client.go b/pkg/storage/bucket/s3/bucket_client.go index 7a72e65133..ce25b8526b 100644 --- a/pkg/storage/bucket/s3/bucket_client.go +++ b/pkg/storage/bucket/s3/bucket_client.go @@ -102,9 +102,10 @@ func newS3Config(cfg Config) (s3.Config, error) { Transport: cfg.HTTP.Transport, }, // Enforce signature version 2 if CLI flag is set - SignatureV2: cfg.SignatureVersion == SignatureVersionV2, - BucketLookupType: bucketLookupType, - AWSSDKAuth: cfg.AccessKeyID == "", + ListObjectsVersion: cfg.ListObjectsVersion, + SignatureV2: cfg.SignatureVersion == SignatureVersionV2, + BucketLookupType: bucketLookupType, + AWSSDKAuth: cfg.AccessKeyID == "", }, nil } diff --git a/pkg/storage/bucket/s3/config.go b/pkg/storage/bucket/s3/config.go index e138079139..bb7bb9f9f8 100644 --- a/pkg/storage/bucket/s3/config.go +++ b/pkg/storage/bucket/s3/config.go @@ -31,16 +31,21 @@ const ( BucketAutoLookup = "auto" BucketVirtualHostLookup = "virtual-hosted" BucketPathLookup = "path" + + ListObjectsVersionV1 = "v1" + ListObjectsVersionV2 = "v2" ) var ( supportedSignatureVersions = []string{SignatureVersionV4, SignatureVersionV2} supportedSSETypes = []string{SSEKMS, SSES3} supportedBucketLookupTypes = []string{BucketAutoLookup, BucketVirtualHostLookup, BucketPathLookup} + supportedListObjectsVersion = []string{ListObjectsVersionV1, ListObjectsVersionV2} errUnsupportedSignatureVersion = errors.New("unsupported signature version") errUnsupportedSSEType = errors.New("unsupported S3 SSE type") errInvalidSSEContext = errors.New("invalid S3 SSE encryption context") errInvalidBucketLookupType = errors.New("invalid bucket lookup type") + errInvalidListObjectsVersion = errors.New("invalid list object version") ) // HTTPConfig stores the http.Transport configuration for the s3 minio client. @@ -58,15 +63,16 @@ func (cfg *HTTPConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { // Config holds the config options for an S3 backend type Config struct { - Endpoint string `yaml:"endpoint"` - Region string `yaml:"region"` - BucketName string `yaml:"bucket_name"` - SecretAccessKey flagext.Secret `yaml:"secret_access_key"` - AccessKeyID string `yaml:"access_key_id"` - Insecure bool `yaml:"insecure"` - SignatureVersion string `yaml:"signature_version"` - BucketLookupType string `yaml:"bucket_lookup_type"` - SendContentMd5 bool `yaml:"send_content_md5"` + Endpoint string `yaml:"endpoint"` + Region string `yaml:"region"` + BucketName string `yaml:"bucket_name"` + SecretAccessKey flagext.Secret `yaml:"secret_access_key"` + AccessKeyID string `yaml:"access_key_id"` + Insecure bool `yaml:"insecure"` + SignatureVersion string `yaml:"signature_version"` + BucketLookupType string `yaml:"bucket_lookup_type"` + SendContentMd5 bool `yaml:"send_content_md5"` + ListObjectsVersion string `yaml:"list_objects_version"` SSE SSEConfig `yaml:"sse"` HTTP HTTPConfig `yaml:"http"` @@ -88,6 +94,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.StringVar(&cfg.SignatureVersion, prefix+"s3.signature-version", SignatureVersionV4, fmt.Sprintf("The signature version to use for authenticating against S3. Supported values are: %s.", strings.Join(supportedSignatureVersions, ", "))) f.StringVar(&cfg.BucketLookupType, prefix+"s3.bucket-lookup-type", BucketAutoLookup, fmt.Sprintf("The s3 bucket lookup style. Supported values are: %s.", strings.Join(supportedBucketLookupTypes, ", "))) f.BoolVar(&cfg.SendContentMd5, prefix+"s3.send-content-md5", true, "If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum algorithm to verify the provided digest. If false, use CRC32C algorithm instead.") + f.StringVar(&cfg.ListObjectsVersion, prefix+"s3.list-objects-version", "", fmt.Sprintf("The list api version. Supported values are: %s, and ''.", strings.Join(supportedListObjectsVersion, ", "))) cfg.SSE.RegisterFlagsWithPrefix(prefix+"s3.sse.", f) cfg.HTTP.RegisterFlagsWithPrefix(prefix, f) } @@ -100,6 +107,11 @@ func (cfg *Config) Validate() error { if !util.StringsContain(supportedBucketLookupTypes, cfg.BucketLookupType) { return errInvalidBucketLookupType } + if cfg.ListObjectsVersion != "" { + if !util.StringsContain(supportedListObjectsVersion, cfg.ListObjectsVersion) { + return errInvalidListObjectsVersion + } + } if err := cfg.SSE.Validate(); err != nil { return err diff --git a/pkg/storage/bucket/s3/config_test.go b/pkg/storage/bucket/s3/config_test.go index c61bfe9c37..b1f38ce6f4 100644 --- a/pkg/storage/bucket/s3/config_test.go +++ b/pkg/storage/bucket/s3/config_test.go @@ -168,6 +168,61 @@ func TestSSEConfig_Validate(t *testing.T) { } } +func TestS3Config_Validate(t *testing.T) { + tests := map[string]struct { + cfg *Config + expectedErr error + }{ + "should pass with valid config": { + cfg: &Config{ + SignatureVersion: SignatureVersionV4, + BucketLookupType: BucketAutoLookup, + ListObjectsVersion: ListObjectsVersionV2, + }, + expectedErr: nil, + }, + "should fail with invalid signature version": { + cfg: &Config{ + SignatureVersion: "v3", + BucketLookupType: BucketAutoLookup, + ListObjectsVersion: ListObjectsVersionV2, + }, + expectedErr: errUnsupportedSignatureVersion, + }, + "should fail with invalid bucket lookup type": { + cfg: &Config{ + SignatureVersion: SignatureVersionV4, + BucketLookupType: "dummy", + ListObjectsVersion: ListObjectsVersionV2, + }, + expectedErr: errInvalidBucketLookupType, + }, + "should fail with invalid list objects version": { + cfg: &Config{ + SignatureVersion: SignatureVersionV4, + BucketLookupType: BucketAutoLookup, + ListObjectsVersion: "v3", + }, + expectedErr: errInvalidListObjectsVersion, + }, + "should pass with empty list objects version": { + cfg: &Config{ + SignatureVersion: SignatureVersionV4, + BucketLookupType: BucketAutoLookup, + ListObjectsVersion: "", + }, + expectedErr: nil, + }, + } + + for testName, test := range tests { + t.Run(testName, func(t *testing.T) { + err := test.cfg.Validate() + require.Equal(t, test.expectedErr, err) + }) + } +} + func TestSSEConfig_BuildMinioConfig(t *testing.T) { tests := map[string]struct { cfg *SSEConfig