Merge pull request #732 from arunagrawal84/3.x

Move forgotten files to the `lost+found` directory

priam/src/main/java/com/netflix/priam/backup/SnapshotBackup.java

@@ -38,6 +38,9 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.*;
@@ -126,6 +129,7 @@ public void execute() throws Exception {
 
         } catch (Exception e) {
             logger.error("Exception occurred while taking snapshot: {}. Exception: {}", snapshotName, e.getLocalizedMessage());
+            e.printStackTrace();
             snapshotStatusMgr.failed(backupMetadata);
             throw e;
         } finally {
@@ -200,12 +204,12 @@ protected void processColumnFamily(String keyspace, String columnFamily, File ba
             return;
         }
 
-        findForgottenFiles(snapshotDir);
+        findAndMoveForgottenFiles(snapshotDir);
         // Add files to this dir
         abstractBackupPaths.addAll(upload(snapshotDir, BackupFileType.SNAP, config.enableAsyncSnapshot()));
     }
 
-    private void findForgottenFiles(File snapshotDir) {
+    private void findAndMoveForgottenFiles(File snapshotDir) {
         try {
             Collection<File> snapshotFiles = FileUtils.listFiles(snapshotDir, FileFilterUtils.fileFileFilter(), null);
             File columnfamilyDir = snapshotDir.getParentFile().getParentFile();
@@ -238,14 +242,27 @@ private void findForgottenFiles(File snapshotDir) {
             if (columnfamilyFiles.size() == 0)
                 return;
 
-            columnfamilyFiles.parallelStream().forEach(file -> logger.info("Forgotten file: {} found for CF: {}", file.getAbsolutePath(), columnfamilyDir.getName()));
-
-            //TODO: The eventual plan is to move the forgotten files to a lost+found directory and clean the directory after 'x' amount of time. This behavior should be configurable. 
-            backupMetrics.incrementForgottenFiles(columnfamilyFiles.size());
             logger.warn("# of forgotten files: {} found for CF: {}", columnfamilyFiles.size(), columnfamilyDir.getName());
+            backupMetrics.incrementForgottenFiles(columnfamilyFiles.size());
+
+            //Move the files to lost_found directory if configured.
+            final Path destDir = Paths.get(columnfamilyDir.getAbsolutePath(), "lost+found");
+            for (File file : columnfamilyFiles) {
+                logger.warn("Forgotten file: {} found for CF: {}", file.getAbsolutePath(), columnfamilyDir.getName());
+                if (config.isForgottenFileMoveEnabled()) {
+                    try {
+                        FileUtils.moveFileToDirectory(file, destDir.toFile(), true);
+                    } catch (IOException e) {
+                        logger.error("Exception occurred while trying to move forgottenFile: {}. Ignoring the error and continuing with remaining backup/forgotten files.", file);
+                        e.printStackTrace();
+                    }
+                }
+            }
+
         } catch (Exception e) {
             //Eat the exception, if there, for any reason. This should not stop the snapshot for any reason.
             logger.error("Exception occurred while trying to find forgottenFile. Ignoring the error and continuing with remaining backup", e);
+            e.printStackTrace();
         }
     }
 

priam/src/main/java/com/netflix/priam/config/IConfiguration.java

@@ -918,6 +918,16 @@ default int getForgottenFileGracePeriodDays() {
         return 1;
     }
 
+    /**
+     * If any forgotten file is found in Cassandra, it is usually good practice to move/delete them so when cassandra
+     * restarts, it does not load old data which should be removed else you may run into data resurrection issues.
+     * This behavior is fixed in 3.x.
+     * This configuration will allow Priam to move the forgotten files to a "lost_found" directory for user to review
+     * at later time at the same time ensuring that Cassandra does not resurrect data.
+     * @return true if Priam should move forgotten file to "lost_found" directory of that CF.
+     */
+    default boolean isForgottenFileMoveEnabled() { return false; }
+
     /**
      * A method for allowing access to outside programs to Priam configuration when paired with the Priam configuration
      * HTTP endpoint at /v1/config/structured/all/property

priam/src/main/java/com/netflix/priam/config/PriamConfiguration.java

@@ -258,10 +258,10 @@ public PriamConfiguration(ICredential provider, IConfigSource config, InstanceEn
     public void initialize() {
         try {
             if (this.insEnvIdentity.isClassic()) {
-                this.instanceDataRetriever =  (InstanceDataRetriever) Class.forName("com.netflix.priam.identity.config.AwsClassicInstanceDataRetriever").newInstance();
+                this.instanceDataRetriever = (InstanceDataRetriever) Class.forName("com.netflix.priam.identity.config.AwsClassicInstanceDataRetriever").newInstance();
 
             } else if (this.insEnvIdentity.isNonDefaultVpc()) {
-                this.instanceDataRetriever =  (InstanceDataRetriever) Class.forName("com.netflix.priam.identity.config.AWSVpcInstanceDataRetriever").newInstance();
+                this.instanceDataRetriever = (InstanceDataRetriever) Class.forName("com.netflix.priam.identity.config.AWSVpcInstanceDataRetriever").newInstance();
             } else {
                 throw new IllegalStateException("Unable to determine environemt (vpc, classic) for running instance.");
             }
@@ -284,8 +284,8 @@ public void initialize() {
         SystemUtils.createDirs(getDataFileLocation());
     }
 
-    public InstanceDataRetriever getInstanceDataRetriever()  {
-       return instanceDataRetriever;
+    public InstanceDataRetriever getInstanceDataRetriever() {
+        return instanceDataRetriever;
     }
 
     private void setupEnvVars() {
@@ -369,7 +369,7 @@ private void populateProps() {
         config.set(CONFIG_REGION_NAME, REGION);
     }
 
-    public String getInstanceName(){
+    public String getInstanceName() {
         return INSTANCE_ID;
     }
 
@@ -616,7 +616,7 @@ public String getRestoreSnapshot() {
     }
 
     @Override
-    public boolean isRestoreEncrypted(){
+    public boolean isRestoreEncrypted() {
         return config.get(PRIAM_PRE + ".encrypted.restore.enabled", false);
     }
 
@@ -1036,7 +1036,7 @@ public String getFlushInterval() {
 
     @Override
     public String getBackupStatusFileLoc() {
-        return config.get(CONFIG_BACKUP_STATUS_FILE_LOCATION,  getDataFileLocation() + File.separator + "backup.status");
+        return config.get(CONFIG_BACKUP_STATUS_FILE_LOCATION, getDataFileLocation() + File.separator + "backup.status");
     }
 
     @Override
@@ -1085,8 +1085,7 @@ public int getPostRestoreHookHeartbeatCheckFrequencyInMs() {
     }
 
     @Override
-    public String getProperty(String key, String defaultValue)
-    {
+    public String getProperty(String key, String defaultValue) {
         return config.get(key, defaultValue);
     }
 
@@ -1095,4 +1094,14 @@ public String getMergedConfigurationCronExpression() {
         // Every minute on the top of the minute.
         return config.get(PRIAM_PRE + ".configMerge.cron", "0 * * * * ? *");
     }
+
+    @Override
+    public int getForgottenFileGracePeriodDays() {
+        return config.get(PRIAM_PRE + ".forgottenFileGracePeriodDays", 1);
+    }
+
+    @Override
+    public boolean isForgottenFileMoveEnabled() {
+        return config.get(PRIAM_PRE + ".forgottenFileMoveEnabled", false);
+    }
 }