From 70b0c3904ff589afda87ab9c91e9a761010d883a Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Wed, 4 Sep 2024 13:26:48 -0700 Subject: [PATCH] Recv file path binding (#825) --- crt/aws-c-s3 | 2 +- .../amazon/awssdk/crt/s3/S3Client.java | 11 +- .../awssdk/crt/s3/S3MetaRequestOptions.java | 135 ++++++++++++++++++ src/native/s3_client.c | 23 ++- .../amazon/awssdk/crt/test/S3ClientTest.java | 41 +++++- 5 files changed, 207 insertions(+), 5 deletions(-) diff --git a/crt/aws-c-s3 b/crt/aws-c-s3 index 0ab4d58ef..502cd6249 160000 --- a/crt/aws-c-s3 +++ b/crt/aws-c-s3 @@ -1 +1 @@ -Subproject commit 0ab4d58ef0bd97970d43828cb6b57a3de5747343 +Subproject commit 502cd6249c6523583c19b122c65e02cf74301b3e diff --git a/src/main/java/software/amazon/awssdk/crt/s3/S3Client.java b/src/main/java/software/amazon/awssdk/crt/s3/S3Client.java index f7cca31c2..b5946beb9 100644 --- a/src/main/java/software/amazon/awssdk/crt/s3/S3Client.java +++ b/src/main/java/software/amazon/awssdk/crt/s3/S3Client.java @@ -157,6 +157,10 @@ public S3MetaRequest makeMetaRequest(S3MetaRequestOptions options) { if (options.getRequestFilePath() != null) { requestFilePath = options.getRequestFilePath().toString().getBytes(UTF8); } + byte[] responseFilePath = null; + if (options.getResponseFilePath() != null) { + responseFilePath = options.getResponseFilePath().toString().getBytes(UTF8); + } AwsSigningConfig signingConfig = options.getSigningConfig(); boolean didCreateSigningConfig = false; @@ -177,7 +181,9 @@ public S3MetaRequest makeMetaRequest(S3MetaRequestOptions options) { ChecksumAlgorithm.marshallAlgorithmsForJNI(checksumConfig.getValidateChecksumAlgorithmList()), httpRequestBytes, options.getHttpRequest().getBodyStream(), requestFilePath, signingConfig, responseHandlerNativeAdapter, endpoint == null ? null : endpoint.toString().getBytes(UTF8), - options.getResumeToken(), options.getObjectSizeHint()); + options.getResumeToken(), options.getObjectSizeHint(), responseFilePath, + options.getResponseFileOption().getNativeValue(), options.getResponseFilePosition(), + options.getResponseFileDeleteOnFailure()); metaRequest.setMetaRequestNativeHandle(metaRequestNativeHandle); @@ -246,5 +252,6 @@ private static native long s3ClientMakeMetaRequest(long clientId, S3MetaRequest int[] validateAlgorithms, byte[] httpRequestBytes, HttpRequestBodyStream httpRequestBodyStream, byte[] requestFilePath, AwsSigningConfig signingConfig, S3MetaRequestResponseHandlerNativeAdapter responseHandlerNativeAdapter, - byte[] endpoint, ResumeToken resumeToken, Long objectSizeHint); + byte[] endpoint, ResumeToken resumeToken, Long objectSizeHint, byte[] responseFilePath, + int responseFileOption, long responseFilePosition, boolean responseFileDeleteOnFailure); } diff --git a/src/main/java/software/amazon/awssdk/crt/s3/S3MetaRequestOptions.java b/src/main/java/software/amazon/awssdk/crt/s3/S3MetaRequestOptions.java index d3cb7fd84..4a00337e6 100644 --- a/src/main/java/software/amazon/awssdk/crt/s3/S3MetaRequestOptions.java +++ b/src/main/java/software/amazon/awssdk/crt/s3/S3MetaRequestOptions.java @@ -5,6 +5,7 @@ package software.amazon.awssdk.crt.s3; import software.amazon.awssdk.crt.http.HttpRequest; +import software.amazon.awssdk.crt.http.HttpStreamResponseHandler; import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider; import software.amazon.awssdk.crt.auth.signing.AwsSigningConfig; @@ -84,6 +85,10 @@ private static Map buildEnumMapping() { private ChecksumConfig checksumConfig; private HttpRequest httpRequest; private Path requestFilePath; + private Path responseFilePath; + private ResponseFileOption responseFileOption = ResponseFileOption.CREATE_OR_REPLACE; + private long responseFilePosition = 0; + private boolean responseFileDeleteOnFailure = false; private S3MetaRequestResponseHandler responseHandler; private CredentialsProvider credentialsProvider; private AwsSigningConfig signingConfig; @@ -313,4 +318,134 @@ public S3MetaRequestOptions withObjectSizeHint(Long objectSizeHint) { public Long getObjectSizeHint() { return objectSizeHint; } + + public enum ResponseFileOption { + /** + * Create a new file if it doesn't exist, otherwise replace the existing file. + */ + CREATE_OR_REPLACE(0), + + /** + * Always create a new file. If the file already exists, + * AWS_ERROR_S3_RECV_FILE_EXISTS will be raised. + */ + CREATE_NEW(1), + + /** + * Create a new file if it doesn't exist, otherwise append to the existing file. + */ + CREATE_OR_APPEND(2), + + /** + * Write to an existing file at the specified position, defined by the + * {@link withHttpRequest}. + * If the file does not exist, AWS_ERROR_S3_RECV_FILE_NOT_EXISTS will be raised. + * If {@link withHttpRequest} is not configured, start overwriting data at the + * beginning of the file (byte 0). + */ + WRITE_TO_POSITION(3); + + ResponseFileOption(int nativeValue) { + this.nativeValue = nativeValue; + } + + public int getNativeValue() { + return nativeValue; + } + + public static ResponseFileOption getEnumValueFromInteger(int value) { + ResponseFileOption enumValue = enumMapping.get(value); + if (enumValue != null) { + return enumValue; + } + + throw new RuntimeException("Invalid S3 ResponseFileOption"); + } + + private static Map buildEnumMapping() { + Map enumMapping = new HashMap(); + enumMapping.put(CREATE_OR_REPLACE.getNativeValue(), CREATE_OR_REPLACE); + enumMapping.put(CREATE_NEW.getNativeValue(), CREATE_NEW); + enumMapping.put(CREATE_OR_APPEND.getNativeValue(), CREATE_OR_APPEND); + enumMapping.put(WRITE_TO_POSITION.getNativeValue(), WRITE_TO_POSITION); + return enumMapping; + } + + private int nativeValue; + + private static Map enumMapping = buildEnumMapping(); + } + + /** + * If set, this file will be used to write the response body to a file. + * And the {@link HttpStreamResponseHandler#onResponseBody} will not be invoked. + * {@link withResponseFileOption} configures the write behavior. + * + * @param responseFilePath path to file to write response body to. + * @return this + */ + public S3MetaRequestOptions withResponseFilePath(Path responseFilePath) { + this.responseFilePath = responseFilePath; + return this; + } + + public Path getResponseFilePath() { + return responseFilePath; + } + + /** + * Sets the option for how to handle the response file when downloading an + * object from S3. + * This option is only applicable when {@link withResponseFilePath} is set. + * + * By default, the option is set to + * {@link ResponseFileOption#CREATE_OR_REPLACE}. + * + * @param responseFileOption The option for handling the response file. + * @return this + */ + public S3MetaRequestOptions withResponseFileOption(ResponseFileOption responseFileOption) { + this.responseFileOption = responseFileOption; + return this; + } + + public ResponseFileOption getResponseFileOption() { + return responseFileOption; + } + + /** + * Sets the position to start writing to the response file. + * This option is only applicable when {@link withResponseFileOption} is set + * to {@link ResponseFileOption#WRITE_TO_POSITION}. + * + * @param responseFilePosition The position to start writing to the response + * file. + * @return this + */ + public S3MetaRequestOptions withResponseFilePosition(long responseFilePosition) { + this.responseFilePosition = responseFilePosition; + return this; + } + + public long getResponseFilePosition() { + return responseFilePosition; + } + + /** + * Sets whether to delete the response file on failure when downloading an + * object from S3. + * This option is only applicable when a response file path is set. + * + * @param responseFileDeleteOnFailure True to delete the response file on + * failure, + * False to leave it as-is. + * @return this + */ + public S3MetaRequestOptions withResponseFileDeleteOnFailure(boolean responseFileDeleteOnFailure) { + this.responseFileDeleteOnFailure = responseFileDeleteOnFailure; + return this; + } + public boolean getResponseFileDeleteOnFailure() { + return responseFileDeleteOnFailure; + } } diff --git a/src/native/s3_client.c b/src/native/s3_client.c index 6d19db0bf..7754387b3 100644 --- a/src/native/s3_client.c +++ b/src/native/s3_client.c @@ -975,7 +975,11 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_s3_S3Client_s3ClientMake jobject java_response_handler_jobject, jbyteArray jni_endpoint, jobject java_resume_token_jobject, - jobject jni_object_size_hint) { + jobject jni_object_size_hint, + jbyteArray jni_response_filepath, + jint jni_response_file_option, + jlong jni_response_file_position, + jboolean jni_response_file_delete_on_failure) { (void)jni_class; aws_cache_jni_ids(env); @@ -985,6 +989,8 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_s3_S3Client_s3ClientMake AWS_ZERO_STRUCT(operation_name); struct aws_byte_cursor request_filepath; AWS_ZERO_STRUCT(request_filepath); + struct aws_byte_cursor response_filepath; + AWS_ZERO_STRUCT(response_filepath); struct aws_s3_meta_request_resume_token *resume_token = s_native_resume_token_from_java_new(env, java_resume_token_jobject); struct aws_s3_meta_request *meta_request = NULL; @@ -1043,6 +1049,17 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_s3_S3Client_s3ClientMake } } + if (jni_response_filepath) { + response_filepath = aws_jni_byte_cursor_from_jbyteArray_acquire(env, jni_response_filepath); + if (response_filepath.ptr == NULL) { + goto done; + } + if (response_filepath.len == 0) { + aws_jni_throw_illegal_argument_exception(env, "Response file path cannot be empty"); + goto done; + } + } + struct aws_uri endpoint; AWS_ZERO_STRUCT(endpoint); if (jni_endpoint != NULL) { @@ -1097,6 +1114,10 @@ JNIEXPORT jlong JNICALL Java_software_amazon_awssdk_crt_s3_S3Client_s3ClientMake .endpoint = jni_endpoint != NULL ? &endpoint : NULL, .resume_token = resume_token, .object_size_hint = jni_object_size_hint != NULL ? &object_size_hint : NULL, + .recv_filepath = response_filepath, + .recv_file_option = jni_response_file_option, + .recv_file_position = jni_response_file_position, + .recv_file_delete_on_failure = jni_response_file_delete_on_failure, }; meta_request = aws_s3_client_make_meta_request(client, &meta_request_options); diff --git a/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java b/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java index 8fd141f58..ff4e5d4d1 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java @@ -126,7 +126,7 @@ public void testS3ClientCreateDestroy() { } } - + @Test public void testS3ClientCreateDestroyWithTLS() { skipIfAndroid(); @@ -351,6 +351,45 @@ public void onFinished(S3FinishedResponseContext context) { } } + @Test + public void testS3GetWithResponseFilePath() { + skipIfAndroid(); + skipIfNetworkUnavailable(); + Assume.assumeTrue(hasAwsCredentials()); + S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION); + try (S3Client client = createS3Client(clientOptions)) { + CompletableFuture onFinishedFuture = new CompletableFuture<>(); + Path responsePath = Files.createTempFile("testS3GetFilePath", ".txt"); + S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() { + @Override + public void onFinished(S3FinishedResponseContext context) { + Log.log(Log.LogLevel.Info, Log.LogSubject.JavaCrtS3, + "Meta request finished with error code " + context.getErrorCode()); + if (context.getErrorCode() != 0) { + onFinishedFuture.completeExceptionally(makeExceptionFromFinishedResponseContext(context)); + return; + } + onFinishedFuture.complete(Integer.valueOf(context.getErrorCode())); + } + }; + + HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT) }; + HttpRequest httpRequest = new HttpRequest("GET", PRE_EXIST_1MB_PATH, headers, null); + + S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions() + .withMetaRequestType(MetaRequestType.GET_OBJECT).withHttpRequest(httpRequest) + .withResponseFilePath(responsePath) + .withResponseHandler(responseHandler); + + try (S3MetaRequest metaRequest = client.makeMetaRequest(metaRequestOptions)) { + Assert.assertEquals(Integer.valueOf(0), onFinishedFuture.get()); + } + Files.deleteIfExists(responsePath); + } catch (Exception ex) { + Assert.fail(ex.getMessage()); + } + } + @Test public void testS3GetWithSizeHint() { skipIfAndroid();