From bd959127e96010ca83354e711fcd9cfa6c1f2329 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 8 Feb 2019 15:20:53 -0800 Subject: [PATCH 1/2] Do not check existence of file if it is not SST_V2. Note: AmazonServiceException can be for either Amazon unable to process request or for slow down. In either case, it is better to try to upload. --- .../com/netflix/priam/aws/S3FileSystemBase.java | 16 ++++------------ .../netflix/priam/backup/AbstractFileSystem.java | 3 ++- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 0bdb4ee4a..188d42f31 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -13,8 +13,6 @@ */ package com.netflix.priam.aws; -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.BucketLifecycleConfiguration; import com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Rule; @@ -168,18 +166,12 @@ public boolean doesRemoteFileExist(Path remotePath) throws BackupRestoreExceptio boolean exists = false; try { exists = s3Client.doesObjectExist(getShard(), remotePath.toString()); - } catch (AmazonServiceException ase) { - // Amazon S3 rejected request - throw new BackupRestoreException( - "AmazonServiceException while checking existence of object: " - + remotePath - + ". Error: " - + ase.getMessage()); - } catch (AmazonClientException ace) { + } catch (Exception ex) { // No point throwing this exception up. logger.error( - "AmazonClientException: client encountered a serious internal problem while trying to communicate with S3", - ace.getMessage()); + "Exception while checking existence of object: {}. Error: {}", + remotePath, + ex.getMessage()); } return exists; diff --git a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java index c8126daff..3e659837c 100644 --- a/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java +++ b/priam/src/main/java/com/netflix/priam/backup/AbstractFileSystem.java @@ -19,6 +19,7 @@ import com.google.inject.Inject; import com.google.inject.Provider; +import com.netflix.priam.backup.AbstractBackupPath.BackupFileType; import com.netflix.priam.config.IConfiguration; import com.netflix.priam.merics.BackupMetrics; import com.netflix.priam.notification.BackupEvent; @@ -182,7 +183,7 @@ public void uploadFile( long uploadedFileSize; // Upload file if it not present at remote location. - if (!doesRemoteFileExist(remotePath)) { + if (path.getType() != BackupFileType.SST_V2 || !doesRemoteFileExist(remotePath)) { uploadedFileSize = new BoundedExponentialRetryCallable(500, 10000, retry) { @Override From 1f53c5fb9253cd72cf2e0876ead483f2cefa2e98 Mon Sep 17 00:00:00 2001 From: Arun Agrawal Date: Fri, 8 Feb 2019 15:35:24 -0800 Subject: [PATCH 2/2] Change the exception scope --- .../src/main/java/com/netflix/priam/aws/S3FileSystemBase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java index 188d42f31..7287f61ee 100755 --- a/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java +++ b/priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java @@ -13,6 +13,7 @@ */ package com.netflix.priam.aws; +import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.BucketLifecycleConfiguration; import com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Rule; @@ -166,7 +167,7 @@ public boolean doesRemoteFileExist(Path remotePath) throws BackupRestoreExceptio boolean exists = false; try { exists = s3Client.doesObjectExist(getShard(), remotePath.toString()); - } catch (Exception ex) { + } catch (AmazonClientException ex) { // No point throwing this exception up. logger.error( "Exception while checking existence of object: {}. Error: {}",