diff --git a/awscrt/s3.py b/awscrt/s3.py index d5cf6fcb..03999b64 100644 --- a/awscrt/s3.py +++ b/awscrt/s3.py @@ -102,6 +102,9 @@ class S3ChecksumAlgorithm(IntEnum): SHA256 = 4 """SHA-256""" + CRC64NVME = 5 + """CRC64NVME""" + class S3ChecksumLocation(IntEnum): """Where to put the checksum.""" diff --git a/crt/aws-c-common b/crt/aws-c-common index 63187b97..fadfef49 160000 --- a/crt/aws-c-common +++ b/crt/aws-c-common @@ -1 +1 @@ -Subproject commit 63187b976a482309e23296c5f967fc19c4131746 +Subproject commit fadfef492042ae53387d4369a6de652c930a5be4 diff --git a/crt/aws-c-s3 b/crt/aws-c-s3 index 5877f40f..337155f6 160000 --- a/crt/aws-c-s3 +++ b/crt/aws-c-s3 @@ -1 +1 @@ -Subproject commit 5877f40f87c77ccf2b278839995a6ee91983080f +Subproject commit 337155f6c07d39e61234e705ed6e58c31d4841eb diff --git a/crt/aws-c-sdkutils b/crt/aws-c-sdkutils index 0818f28e..ce09f797 160000 --- a/crt/aws-c-sdkutils +++ b/crt/aws-c-sdkutils @@ -1 +1 @@ -Subproject commit 0818f28ee436b892f09fbe8e3a6ae37ff40e9436 +Subproject commit ce09f79768653dbdc810fc14cad8685dd90acba1 diff --git a/crt/aws-checksums b/crt/aws-checksums index 0d2f5521..3e4101b9 160000 --- a/crt/aws-checksums +++ b/crt/aws-checksums @@ -1 +1 @@ -Subproject commit 0d2f5521f61215f38f791d106ae304402208112d +Subproject commit 3e4101b9f85a2c090774d27ae2131fca1082f522 diff --git a/crt/aws-lc b/crt/aws-lc index 8ffe277c..59828538 160000 --- a/crt/aws-lc +++ b/crt/aws-lc @@ -1 +1 @@ -Subproject commit 8ffe277c21915ca82dc78a3bdc6a92e10c284b92 +Subproject commit 59828538a790094113eacd5dd23d01be2885b36a diff --git a/crt/s2n b/crt/s2n index ffe0bf42..493b7716 160000 --- a/crt/s2n +++ b/crt/s2n @@ -1 +1 @@ -Subproject commit ffe0bf42da8f139eff8fd2237f47fbde40b478fb +Subproject commit 493b77167dc367c394de23cfe78a029298e2a254 diff --git a/test/test_s3.py b/test/test_s3.py index 2ff06920..8da9b11a 100644 --- a/test/test_s3.py +++ b/test/test_s3.py @@ -253,6 +253,7 @@ def setUp(self): self.num_threads = 0 self.special_path = "put_object_test_10MB@$%.txt" self.non_ascii_file_name = "ÉxÅmple.txt" + self.part_size = 5 * MB self.response_headers = None self.response_status_code = None @@ -363,13 +364,12 @@ def _test_s3_put_get_object( request_type, exception_name=None, enable_s3express=False, - region="us-west-2", mem_limit=None, **kwargs): s3_client = s3_client_new( False, - region, - 5 * MB, + self.region, + self.part_size, enable_s3express=enable_s3express, mem_limit=mem_limit) signing_config = None @@ -392,8 +392,14 @@ def _test_s3_put_get_object( self.assertTrue(shutdown_event.wait(self.timeout)) if exception_name is None: - finished_future.result() - self._validate_successful_response(request_type is S3RequestType.PUT_OBJECT) + try: + finished_future.result() + self._validate_successful_response(request_type is S3RequestType.PUT_OBJECT) + except S3ResponseError as e: + print(e.status_code) + print(e.headers) + print(e.body) + raise e else: e = finished_future.exception() self.assertEqual(e.name, exception_name) @@ -436,14 +442,16 @@ def test_put_object_unknown_content_length_single_part(self): put_body_stream.close() def test_get_object_s3express(self): + self.region = "us-east-1" request = self._get_object_request("/crt-download-10MB", enable_s3express=True) - self._test_s3_put_get_object(request, S3RequestType.GET_OBJECT, enable_s3express=True, region="us-east-1") + self._test_s3_put_get_object(request, S3RequestType.GET_OBJECT, enable_s3express=True) def test_put_object_s3express(self): + self.region = "us-east-1" put_body_stream = open(self.temp_put_obj_file_path, "rb") content_length = os.stat(self.temp_put_obj_file_path).st_size request = self._put_object_request(put_body_stream, content_length, enable_s3express=True) - self._test_s3_put_get_object(request, S3RequestType.PUT_OBJECT, enable_s3express=True, region="us-east-1") + self._test_s3_put_get_object(request, S3RequestType.PUT_OBJECT, enable_s3express=True) put_body_stream.close() def test_put_object_multiple_times(self): @@ -586,28 +594,45 @@ def on_done_remove_file(**kwargs): "the transferred length reported does not match body we sent") self._validate_successful_response(request_type is S3RequestType.PUT_OBJECT) - def test_put_get_with_checksum(self): - put_body = b'hello world' - put_body_stream = BytesIO(put_body) - content_length = len(put_body) - path = '/hello-world.txt' + def _round_trip_with_checksums_helper( + self, + algo=S3ChecksumAlgorithm.CRC32, + mpu=True, + provide_full_object_checksum=False): + if not mpu: + # increase the part size for the client to use single part upload + self.part_size = 20 * MB - # calculate expected CRC32 header value: - # a string containing the url-safe-base64-encoding of a big-endian-32-bit-CRC - crc32_int = zlib.crc32(put_body) - crc32_big_endian = crc32_int.to_bytes(4, 'big') - crc32_base64_bytes = base64.urlsafe_b64encode(crc32_big_endian) - crc32_base64_str = crc32_base64_bytes.decode() + put_body_stream = open(self.temp_put_obj_file_path, "rb") + content_length = os.stat(self.temp_put_obj_file_path).st_size + # construct different path to prevent race condition between tests + path = '/hello-world-' + algo.name + if mpu: + path += "-mpu" + if provide_full_object_checksum: + path += "-full-object" + + if algo == S3ChecksumAlgorithm.CRC32: + checksum_header_name = 'x-amz-checksum-crc32' + checksum_str = 'a9ccsg==' + elif algo == S3ChecksumAlgorithm.CRC64NVME: + checksum_header_name = 'x-amz-checksum-crc64nvme' + checksum_str = 'tPMvgM0jSDQ=' + else: + raise Exception("Checksum algo not supported by test helper") # upload, with client adding checksum upload_request = self._put_object_request(put_body_stream, content_length, path=path) upload_checksum_config = S3ChecksumConfig( - algorithm=S3ChecksumAlgorithm.CRC32, + algorithm=algo, location=S3ChecksumLocation.TRAILER) + if provide_full_object_checksum: + upload_request.headers.add(checksum_header_name, checksum_str) + # checksum will be provided from the header, don't set the checksum configs + upload_checksum_config = None + self._test_s3_put_get_object(upload_request, S3RequestType.PUT_OBJECT, checksum_config=upload_checksum_config) - self.assertEqual(HttpHeaders(self.response_headers).get('x-amz-checksum-crc32'), - crc32_base64_str) # download, with client validating checksum download_request = self._get_object_request(path) @@ -615,9 +640,31 @@ def test_put_get_with_checksum(self): self._test_s3_put_get_object(download_request, S3RequestType.GET_OBJECT, checksum_config=download_checksum_config) self.assertTrue(self.done_did_validate_checksum) - self.assertEqual(self.done_checksum_validation_algorithm, S3ChecksumAlgorithm.CRC32) - self.assertEqual(HttpHeaders(self.response_headers).get('x-amz-checksum-crc32'), - crc32_base64_str) + self.assertEqual(self.done_checksum_validation_algorithm, algo) + self.assertEqual(HttpHeaders(self.response_headers).get(checksum_header_name), + checksum_str) + put_body_stream.close() + + def test_round_trip_with_trailing_checksum(self): + self._round_trip_with_checksums_helper(S3ChecksumAlgorithm.CRC32, mpu=False) + + def test_round_trip_with_full_object_checksum_mpu(self): + self._round_trip_with_checksums_helper( + S3ChecksumAlgorithm.CRC64NVME, + mpu=True, + provide_full_object_checksum=True) + + def test_round_trip_with_full_object_checksum_single_part(self): + self._round_trip_with_checksums_helper( + S3ChecksumAlgorithm.CRC64NVME, + mpu=False, + provide_full_object_checksum=True) + + def test_round_trip_with_full_object_checksum_mpu_crc32(self): + self._round_trip_with_checksums_helper(S3ChecksumAlgorithm.CRC32, mpu=True, provide_full_object_checksum=True) + + def test_round_trip_with_full_object_checksum_single_part_crc32(self): + self._round_trip_with_checksums_helper(S3ChecksumAlgorithm.CRC32, mpu=False, provide_full_object_checksum=True) def _on_progress_cancel_after_first_chunk(self, progress): self.transferred_len += progress