Merge pull request #771 from arunagrawal84/backup2

Add Cass SNAPSHOT JMX status, snapshot version, last validated timestmp. Changes to Servlet API.

CHANGELOG.md

@@ -1,4 +1,10 @@
 # Changelog
+
+## 2019/01/30 3.1.70
+(#765) Add metrics on CassandraConfig resource calls
+(#768) Support configure/tune complex parameters in cassandra.yaml 
+(#771) Add Cass SNAPSHOT JMX status, snapshot version, last validated timestamp. Changes to Servlet API and new APIs. 
+
 ## 2019/01/10 3.1.69 
 * (#762) Backup Verification for Backup 2.0. 
 * (#762) Restore for Backup 2.0 

build.gradle

@@ -1,5 +1,5 @@
 plugins {
-    id 'nebula.netflixoss' version '5.1.1'
+    id 'nebula.netflixoss' version '7.0.0'
     id 'com.github.sherter.google-java-format' version '0.7.1'
 }
 
@@ -47,14 +47,13 @@ subprojects {
         compile 'org.apache.commons:commons-collections4:4.2'
         compile 'commons-io:commons-io:2.6'
         compile 'commons-cli:commons-cli:1.4'
-        compile 'commons-httpclient:commons-httpclient:3.1'
         compile 'com.sun.jersey.contribs:jersey-multipart:1.19.4'
         compile 'com.sun.jersey:jersey-json:1.19.4'
         compile 'com.sun.jersey:jersey-bundle:1.19.4'
         compile 'com.sun.jersey.contribs:jersey-guice:1.19.4'
         compile 'com.google.guava:guava:21.0'
         compile 'com.google.code.findbugs:jsr305:3.0.2'
-        compile 'com.amazonaws:aws-java-sdk:1.11.475'
+        compile 'com.amazonaws:aws-java-sdk:1.11.488'
         compile 'com.google.inject:guice:4.2.2'
         compile 'com.google.inject.extensions:guice-servlet:4.2.2'
         compile 'org.quartz-scheduler:quartz:2.3.0'
@@ -68,7 +67,7 @@ subprojects {
         compile 'xerces:xercesImpl:2.12.0'
         compile 'net.java.dev.jna:jna:5.2.0'
         compile 'org.apache.httpcomponents:httpclient:4.5.6'
-        compile 'org.apache.httpcomponents:httpcore:4.4.10'
+        compile 'org.apache.httpcomponents:httpcore:4.4.11'
         compile 'com.ning:compress-lzf:1.0.4'
         compile 'com.google.code.gson:gson:2.8.5'
         compile 'org.slf4j:slf4j-api:1.7.25'
@@ -79,13 +78,12 @@ subprojects {
             exclude module: 'guava'
         }
         compile 'com.google.apis:google-api-services-storage:v1-rev141-1.25.0'
-        compile 'com.google.http-client:google-http-client-jackson2:1.22.0'
-        compile 'com.netflix.spectator:spectator-api:0.82.0'
+        compile 'com.google.http-client:google-http-client-jackson2:1.28.0'
+        compile 'com.netflix.spectator:spectator-api:0.83.0'
         compileOnly 'javax.servlet:javax.servlet-api:3.1.0'
         testCompile 'org.jmockit:jmockit:1.31'
         testCompile "org.spockframework:spock-core:1.1-groovy-2.4"
         testCompile 'junit:junit:4.12'
-        //testCompile 'mockit:mockit:0.999'
 
         sourceCompatibility = JavaVersion.VERSION_1_8
         targetCompatibility = JavaVersion.VERSION_1_8

priam/src/main/java/com/netflix/priam/backup/BackupMetadata.java

@@ -29,19 +29,23 @@ public final class BackupMetadata implements Serializable {
     private String token;
     private Date start, completed;
     private Status status;
+    private boolean cassandraSnapshotSuccess;
+    private Date lastValidated;
+    private BackupVersion backupVersion;
     private String snapshotLocation;
 
-    public BackupMetadata(String token, Date start) throws Exception {
+    public BackupMetadata(BackupVersion backupVersion, String token, Date start) throws Exception {
         if (start == null || token == null || StringUtils.isEmpty(token))
             throw new Exception(
                     String.format(
                             "Invalid Input: Token: %s or start date: %s is null or empty.",
                             token, start));
-
+        this.backupVersion = backupVersion;
         this.snapshotDate = DateUtil.formatyyyyMMdd(start);
         this.token = token;
         this.start = start;
         this.status = Status.STARTED;
+        this.cassandraSnapshotSuccess = false;
     }
 
     @Override
@@ -53,14 +57,16 @@ public boolean equals(Object o) {
 
         return this.snapshotDate.equals(that.snapshotDate)
                 && this.token.equals(that.token)
-                && this.start.equals(that.start);
+                && this.start.equals(that.start)
+                && this.backupVersion.equals(that.backupVersion);
     }
 
     @Override
     public int hashCode() {
         int result = this.snapshotDate.hashCode();
         result = 31 * result + this.token.hashCode();
         result = 31 * result + this.start.hashCode();
+        result = 31 * result + this.backupVersion.hashCode();
         return result;
     }
 
@@ -145,6 +151,54 @@ public void setSnapshotLocation(String snapshotLocation) {
         this.snapshotLocation = snapshotLocation;
     }
 
+    /**
+     * Find if cassandra snapshot was successful or not. This is a JMX operation and it is possible
+     * that this operation failed.
+     *
+     * @return cassandra snapshot status.
+     */
+    public boolean isCassandraSnapshotSuccess() {
+        return cassandraSnapshotSuccess;
+    }
+
+    /**
+     * Set the cassandra snapshot status to be either finished successfully or fail.
+     *
+     * @param cassandraSnapshotSuccess is set to success if JMX operation for snapshot is
+     *     successful.
+     */
+    public void setCassandraSnapshotSuccess(boolean cassandraSnapshotSuccess) {
+        this.cassandraSnapshotSuccess = cassandraSnapshotSuccess;
+    }
+
+    /**
+     * Get the backup version for the snapshot.
+     *
+     * @return backup version of the snapshot.
+     */
+    public BackupVersion getBackupVersion() {
+        return backupVersion;
+    }
+
+    /**
+     * Return the last validation timestamp of this backup metadata. Validation of backup implies
+     * finding if all the files are successfully stored in remote file system.
+     *
+     * @return date of last backup validation.
+     */
+    public Date getLastValidated() {
+        return lastValidated;
+    }
+
+    /**
+     * Set the last validation date of backup metadata.
+     *
+     * @param lastValidated date value of backup validation.
+     */
+    public void setLastValidated(Date lastValidated) {
+        this.lastValidated = lastValidated;
+    }
+
     @Override
     public String toString() {
         return GsonJsonSerializer.getGson().toJson(this);

priam/src/main/java/com/netflix/priam/backup/BackupStatusMgr.java

@@ -18,7 +18,10 @@
 import com.netflix.priam.health.InstanceState;
 import com.netflix.priam.utils.DateUtil;
 import com.netflix.priam.utils.MaxSizeHashMap;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.*;
+import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -111,7 +114,11 @@ public void finish(BackupMetadata backupMetadata) {
                     Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime());
 
         instanceState.setBackupStatus(backupMetadata);
+        update(backupMetadata);
+    }
 
+    @Override
+    public void update(BackupMetadata backupMetadata) {
         // Retrieve the snapshot metadata and then update the finish date/status.
         retrieveAndUpdate(backupMetadata);
 
@@ -123,19 +130,29 @@ private void retrieveAndUpdate(final BackupMetadata backupMetadata) {
         // Retrieve the snapshot metadata and then update the date/status.
         LinkedList<BackupMetadata> metadataLinkedList = locate(backupMetadata.getSnapshotDate());
 
-        if (metadataLinkedList == null || metadataLinkedList.isEmpty()) {
+        if (metadataLinkedList == null) {
             logger.error(
                     "No previous backupMetaData found. This should not happen. Creating new to ensure app keeps running.");
             metadataLinkedList = new LinkedList<>();
-            metadataLinkedList.addFirst(backupMetadata);
+            backupMetadataMap.put(backupMetadata.getSnapshotDate(), metadataLinkedList);
         }
 
-        metadataLinkedList.forEach(
+        Optional<BackupMetadata> searchedData =
+                metadataLinkedList
+                        .stream()
+                        .filter(backupMetadata1 -> backupMetadata.equals(backupMetadata1))
+                        .findFirst();
+        if (!searchedData.isPresent()) {
+            metadataLinkedList.addFirst(backupMetadata);
+        }
+        searchedData.ifPresent(
                 backupMetadata1 -> {
-                    if (backupMetadata1.equals(backupMetadata)) {
-                        backupMetadata1.setCompleted(backupMetadata.getCompleted());
-                        backupMetadata1.setStatus(backupMetadata.getStatus());
-                    }
+                    backupMetadata1.setCompleted(backupMetadata.getCompleted());
+                    backupMetadata1.setStatus(backupMetadata.getStatus());
+                    backupMetadata1.setCassandraSnapshotSuccess(
+                            backupMetadata.isCassandraSnapshotSuccess());
+                    backupMetadata1.setSnapshotLocation(backupMetadata.getSnapshotLocation());
+                    backupMetadata1.setLastValidated(backupMetadata.getLastValidated());
                 });
     }
 
@@ -150,12 +167,7 @@ public void failed(BackupMetadata backupMetadata) {
         if (backupMetadata.getStatus() != Status.FAILED) backupMetadata.setStatus(Status.FAILED);
 
         instanceState.setBackupStatus(backupMetadata);
-
-        // Retrieve the snapshot metadata and then update the failure date/status.
-        retrieveAndUpdate(backupMetadata);
-
-        // Save the backupMetaDataMap
-        save(backupMetadata);
+        update(backupMetadata);
     }
 
     /**
@@ -174,6 +186,47 @@ public void failed(BackupMetadata backupMetadata) {
      */
     protected abstract LinkedList<BackupMetadata> fetch(String snapshotDate);
 
+    public List<BackupMetadata> getLatestBackupMetadata(
+            BackupVersion backupVersion, DateUtil.DateRange dateRange) {
+        Instant startDay = dateRange.getStartTime().truncatedTo(ChronoUnit.DAYS);
+        Instant endDay = dateRange.getEndTime().truncatedTo(ChronoUnit.DAYS);
+
+        List<BackupMetadata> allBackups = new ArrayList<>();
+        Instant previousDay = endDay;
+        do {
+            // We need to find the latest backupmetadata in this date range.
+            logger.info(
+                    "Will try to find snapshot for : {}",
+                    DateUtil.formatInstant(DateUtil.yyyyMMddHHmm, previousDay));
+            List<BackupMetadata> backupsForDate = locate(new Date(previousDay.toEpochMilli()));
+            if (backupsForDate != null) allBackups.addAll(backupsForDate);
+            previousDay = previousDay.minus(1, ChronoUnit.DAYS);
+        } while (!previousDay.isBefore(startDay));
+
+        // Return all the backups which are FINISHED and were "started" in the dateRange provided.
+        // Do not compare the end time of snapshot as it may take random amount of time to finish
+        // the snapshot.
+        return allBackups
+                .stream()
+                .filter(Objects::nonNull)
+                .filter(backupMetadata -> backupMetadata.getStatus() == Status.FINISHED)
+                .filter(backupMetadata -> backupMetadata.getBackupVersion().equals(backupVersion))
+                .filter(
+                        backupMetadata ->
+                                backupMetadata
+                                                        .getStart()
+                                                        .toInstant()
+                                                        .compareTo(dateRange.getStartTime())
+                                                >= 0
+                                        && backupMetadata
+                                                        .getStart()
+                                                        .toInstant()
+                                                        .compareTo(dateRange.getEndTime())
+                                                <= 0)
+                .sorted(Comparator.comparing(BackupMetadata::getStart).reversed())
+                .collect(Collectors.toList());
+    }
+
     @Override
     public String toString() {
         return "BackupStatusMgr{"

priam/src/main/java/com/netflix/priam/backup/BackupVerification.java

@@ -18,6 +18,9 @@
 import com.google.inject.Singleton;
 import com.google.inject.name.Named;
 import com.netflix.priam.backupv2.IMetaProxy;
+import com.netflix.priam.scheduler.UnsupportedTypeException;
+import com.netflix.priam.utils.DateUtil;
+import com.netflix.priam.utils.DateUtil.DateRange;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.*;
@@ -33,38 +36,84 @@
 public class BackupVerification {
 
     private static final Logger logger = LoggerFactory.getLogger(BackupVerification.class);
-    private final IMetaProxy metaProxy;
+    private final IMetaProxy metaV1Proxy;
+    private final IMetaProxy metaV2Proxy;
+    private final IBackupStatusMgr backupStatusMgr;
     private final Provider<AbstractBackupPath> abstractBackupPathProvider;
 
     @Inject
     BackupVerification(
-            @Named("v1") IMetaProxy metaProxy,
+            @Named("v1") IMetaProxy metaV1Proxy,
+            @Named("v2") IMetaProxy metaV2Proxy,
+            IBackupStatusMgr backupStatusMgr,
             Provider<AbstractBackupPath> abstractBackupPathProvider) {
-        this.metaProxy = metaProxy;
+        this.metaV1Proxy = metaV1Proxy;
+        this.metaV2Proxy = metaV2Proxy;
+        this.backupStatusMgr = backupStatusMgr;
         this.abstractBackupPathProvider = abstractBackupPathProvider;
     }
 
-    public Optional<BackupMetadata> getLatestBackupMetaData(List<BackupMetadata> metadata) {
-        return metadata.stream()
-                .filter(backupMetadata -> backupMetadata.getStatus() == Status.FINISHED)
-                .sorted(Comparator.comparing(BackupMetadata::getStart).reversed())
-                .findFirst();
+    private IMetaProxy getMetaProxy(BackupVersion backupVersion) {
+        switch (backupVersion) {
+            case SNAPSHOT_BACKUP:
+                return metaV1Proxy;
+            case SNAPSHOT_META_SERVICE:
+                return metaV2Proxy;
+        }
+
+        return null;
     }
 
-    public Optional<BackupVerificationResult> verifyBackup(List<BackupMetadata> metadata) {
-        if (metadata == null || metadata.isEmpty()) return Optional.empty();
+    public Optional<BackupVerificationResult> verifyBackup(
+            BackupVersion backupVersion, boolean force, DateRange dateRange)
+            throws UnsupportedTypeException, IllegalArgumentException {
+        IMetaProxy metaProxy = getMetaProxy(backupVersion);
+        if (metaProxy == null) {
+            throw new UnsupportedTypeException(
+                    "BackupVersion type: " + backupVersion + " is not supported");
+        }
 
-        Optional<BackupMetadata> latestBackupMetaData = getLatestBackupMetaData(metadata);
+        if (dateRange == null) {
+            throw new IllegalArgumentException("dateRange provided is null");
+        }
 
-        if (!latestBackupMetaData.isPresent()) {
-            logger.error("No backup found which finished during the time provided.");
-            return Optional.empty();
+        List<BackupMetadata> metadata =
+                backupStatusMgr.getLatestBackupMetadata(backupVersion, dateRange);
+        if (metadata == null || metadata.isEmpty()) return Optional.empty();
+        for (BackupMetadata backupMetadata : metadata) {
+            if (backupMetadata.getLastValidated() != null && !force) {
+                // Backup is already validated. Nothing to do.
+                BackupVerificationResult result = new BackupVerificationResult();
+                result.valid = true;
+                result.manifestAvailable = true;
+                result.snapshotInstant = backupMetadata.getStart().toInstant();
+                Path snapshotLocation = Paths.get(backupMetadata.getSnapshotLocation());
+                result.remotePath =
+                        snapshotLocation.subpath(1, snapshotLocation.getNameCount()).toString();
+                return Optional.of(result);
+            }
+            BackupVerificationResult backupVerificationResult =
+                    verifyBackup(metaProxy, backupMetadata);
+            if (logger.isDebugEnabled())
+                logger.debug(
+                        "BackupVerification: metadata: {}, result: {}",
+                        backupMetadata,
+                        backupVerificationResult);
+            if (backupVerificationResult.valid) {
+                backupMetadata.setLastValidated(new Date(DateUtil.getInstant().toEpochMilli()));
+                backupStatusMgr.update(backupMetadata);
+                return Optional.of(backupVerificationResult);
+            }
         }
+        return Optional.empty();
+    }
 
-        Path metadataLocation = Paths.get(latestBackupMetaData.get().getSnapshotLocation());
+    private BackupVerificationResult verifyBackup(
+            IMetaProxy metaProxy, BackupMetadata latestBackupMetaData) {
+        Path metadataLocation = Paths.get(latestBackupMetaData.getSnapshotLocation());
         metadataLocation = metadataLocation.subpath(1, metadataLocation.getNameCount());
         AbstractBackupPath abstractBackupPath = abstractBackupPathProvider.get();
         abstractBackupPath.parseRemote(metadataLocation.toString());
-        return Optional.of((metaProxy.isMetaFileValid(abstractBackupPath)));
+        return metaProxy.isMetaFileValid(abstractBackupPath);
     }
 }