Skip to content

Commit

Permalink
Merge pull request #898 from Netflix/feature/CASS-1856-3.x
Browse files Browse the repository at this point in the history 
Cease requiring the backup to be fully in S3.
  • Loading branch information
@mattl-netflix
mattl-netflix authored Jul 15, 2020
2 parents 3827a40 + 01150b9 commit cad9d06
Show file tree
Hide file tree
Showing 3 changed files with 92 additions and 129 deletions.
4 changes: 3 additions & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,9 @@ subprojects {
compileOnly 'javax.servlet:javax.servlet-api:3.1.0'
testCompile 'org.jmockit:jmockit:1.31'
testCompile "org.spockframework:spock-core:1.1-groovy-2.4"
testCompile 'junit:junit:4.12'
testCompile "com.google.truth:truth:1.0.1"
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.3.1'
testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.3.1'

sourceCompatibility = JavaVersion.VERSION_1_8
targetCompatibility = JavaVersion.VERSION_1_8
Expand Down
47 changes: 18 additions & 29 deletions priam/src/main/java/com/netflix/priam/backupv2/BackupVerificationTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import com.netflix.priam.scheduler.TaskTimer;
import com.netflix.priam.utils.DateUtil;
import com.netflix.priam.utils.DateUtil.DateRange;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;
import org.slf4j.Logger;
Expand Down Expand Up @@ -66,45 +67,37 @@ public BackupVerificationTask(
@Override
public void execute() throws Exception {
// Ensure that backup version 2.0 is actually enabled.
if (backupRestoreConfig.getSnapshotMetaServiceCronExpression().equalsIgnoreCase("-1")) {
logger.info(
"Not executing the Verification Service for backups as V2 backups are not enabled.");
if (backupRestoreConfig.getSnapshotMetaServiceCronExpression().equals("-1")) {
logger.info("Skipping backup verification. V2 backups are not enabled.");
return;
}

if (instanceState.getRestoreStatus() != null
&& instanceState.getRestoreStatus().getStatus() != null
&& instanceState.getRestoreStatus().getStatus() == Status.STARTED) {
logger.info(
"Not executing the Verification Service for backups as Priam is in restore mode.");
logger.info("Skipping backup verification. Priam is in restore mode.");
return;
}

// Validate the backup done in last x hours.
DateRange dateRange =
new DateRange(
DateUtil.getInstant()
.minus(
backupRestoreConfig.getBackupVerificationSLOInHours(),
ChronoUnit.HOURS),
DateUtil.getInstant());
Instant now = DateUtil.getInstant();
Instant slo =
now.minus(backupRestoreConfig.getBackupVerificationSLOInHours(), ChronoUnit.HOURS);
DateRange dateRange = new DateRange(slo, now);
List<BackupVerificationResult> verificationResults =
backupVerification.verifyAllBackups(BackupVersion.SNAPSHOT_META_SERVICE, dateRange);

verificationResults
.stream()
.forEach(
backupVerificationResult -> {
logger.info(
"Sending {} message for backup: {}",
AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED,
backupVerificationResult.snapshotInstant);
backupNotificationMgr.notify(backupVerificationResult);
});
verificationResults.forEach(
result -> {
logger.info(
"Sending {} message for backup: {}",
AbstractBackupPath.BackupFileType.SNAPSHOT_VERIFIED,
result.snapshotInstant);
backupNotificationMgr.notify(result);
});

if (!BackupRestoreUtil.getLatestValidMetaPath(
backupVerification.getMetaProxy(BackupVersion.SNAPSHOT_META_SERVICE),
dateRange)
if (!backupVerification
.verifyBackup(BackupVersion.SNAPSHOT_META_SERVICE, false /* force */, dateRange)
.isPresent()) {
logger.error(
"Not able to find any snapshot which is valid in our SLO window: {} hours",
Expand All @@ -113,10 +106,6 @@ public void execute() throws Exception {
}
}

public BackupMetrics getBackupMetrics() {
return backupMetrics;
}

/**
* Interval between trying to verify data manifest file on Remote file system.
*
Expand Down
170 changes: 71 additions & 99 deletions priam/src/test/java/com/netflix/priam/backupv2/TestBackupVerificationTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,179 +17,151 @@

package com.netflix.priam.backupv2;

import com.google.common.collect.ImmutableList;
import com.google.common.truth.Truth;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.netflix.priam.backup.*;
import com.netflix.priam.config.IConfiguration;
import com.netflix.priam.health.InstanceState;
import com.netflix.priam.merics.BackupMetrics;
import com.netflix.priam.merics.Metrics;
import com.netflix.priam.notification.BackupNotificationMgr;
import com.netflix.priam.scheduler.UnsupportedTypeException;
import com.netflix.priam.utils.DateUtil.DateRange;
import com.netflix.spectator.api.Counter;
import com.netflix.spectator.api.Registry;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import javax.inject.Inject;
import mockit.*;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.Assertions;

/** Created by aagrawal on 2/1/19. */
public class TestBackupVerificationTask {
private static BackupVerificationTask backupVerificationService;
private static IConfiguration configuration;
private static BackupVerification backupVerification;
private static BackupNotificationMgr backupNotificationMgr;
@Inject private BackupVerificationTask backupVerificationService;
private Counter badVerifications;
@Mocked private BackupVerification backupVerification;
@Mocked private BackupNotificationMgr backupNotificationMgr;

public TestBackupVerificationTask() {
@Before
public void setUp() {
new MockBackupVerification();
new MockBackupNotificationMgr();
Injector injector = Guice.createInjector(new BRTestModule());
if (configuration == null) configuration = injector.getInstance(IConfiguration.class);
if (backupVerificationService == null)
backupVerificationService = injector.getInstance(BackupVerificationTask.class);
injector.injectMembers(this);
badVerifications =
injector.getInstance(Registry.class)
.counter(Metrics.METRIC_PREFIX + "backup.verification.failure");
}

static class MockBackupVerification extends MockUp<BackupVerification> {
public static boolean emptyBackupVerificationList = false;
public static boolean throwError = false;
public static boolean validBackupVerificationResult = true;
private static final class MockBackupVerification extends MockUp<BackupVerification> {
private static boolean throwError;
private static ImmutableList<BackupVerificationResult> results;

public static void setResults(BackupVerificationResult... newResults) {
results = ImmutableList.copyOf(newResults);
}

public static void shouldThrow(boolean newThrowError) {
throwError = newThrowError;
}

@Mock
public List<BackupVerificationResult> verifyAllBackups(
BackupVersion backupVersion, DateRange dateRange)
throws UnsupportedTypeException, IllegalArgumentException {
if (throwError) throw new IllegalArgumentException("DummyError");

if (emptyBackupVerificationList) {
return new ArrayList<>();
}
List<BackupVerificationResult> result = new ArrayList<>();
if (validBackupVerificationResult) {
result.add(getValidBackupVerificationResult());
} else {
result.add(getInvalidBackupVerificationResult());
}
return result;
return results;
}
}

static class MockBackupNotificationMgr extends MockUp<BackupNotificationMgr> {
@Mock
public void notify(BackupVerificationResult backupVerificationResult) {
// do nothing just return
return;
public Optional<BackupVerificationResult> verifyBackup(
BackupVersion backupVersion, boolean force, DateRange dateRange)
throws UnsupportedTypeException, IllegalArgumentException {
if (throwError) throw new IllegalArgumentException("DummyError");
return results.isEmpty() ? Optional.empty() : Optional.of(results.get(0));
}
}

private static final class MockBackupNotificationMgr extends MockUp<BackupNotificationMgr> {}

@Test
public void throwError() throws Exception {
MockBackupVerification.throwError = true;
MockBackupVerification.emptyBackupVerificationList = false;
try {
backupVerificationService.execute();
Assert.assertTrue(false);
} catch (IllegalArgumentException e) {
if (!e.getMessage().equalsIgnoreCase("DummyError")) Assert.assertTrue(false);
}
public void throwError() {
MockBackupVerification.shouldThrow(true);
Assertions.assertThrows(
IllegalArgumentException.class, () -> backupVerificationService.execute());
}

@Test
public void normalOperation() throws Exception {
MockBackupVerification.throwError = false;
MockBackupVerification.emptyBackupVerificationList = false;
new Expectations() {
{
backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure();
maxTimes = 0;
}
};
public void validBackups() throws Exception {
MockBackupVerification.shouldThrow(false);
MockBackupVerification.setResults(getValidBackupVerificationResult());
backupVerificationService.execute();
Truth.assertThat(badVerifications.count()).isEqualTo(0);
new Verifications() {
{
backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure();
maxTimes = 0;
backupNotificationMgr.notify((BackupVerificationResult) any);
times = 1;
}
};
}

@Test
public void normalOperationPriorVerifiedBackups(
@Mocked BackupRestoreUtil backupRestoreUtil,
@Mocked AbstractBackupPath remoteBackupPath)
throws Exception {
MockBackupVerification.throwError = false;
MockBackupVerification.emptyBackupVerificationList = true;
new Expectations() {
{
backupRestoreUtil.getLatestValidMetaPath((IMetaProxy) any, (DateRange) any);
result = Optional.of(remoteBackupPath);
maxTimes = 1;
}
};
public void invalidBackups() throws Exception {
MockBackupVerification.shouldThrow(false);
MockBackupVerification.setResults(getInvalidBackupVerificationResult());
backupVerificationService.execute();
Truth.assertThat(badVerifications.count()).isEqualTo(0);
new Verifications() {
{
backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure();
maxTimes = 1;
backupNotificationMgr.notify((BackupVerificationResult) any);
times = 1;
}
};
}

@Test
public void normalOperationInvalidBackups() throws Exception {
MockBackupVerification.throwError = false;
MockBackupVerification.emptyBackupVerificationList = false;
MockBackupVerification.validBackupVerificationResult = false;
new Expectations() {
{
backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure();
maxTimes = 0;
}
};
public void noBackups() throws Exception {
MockBackupVerification.shouldThrow(false);
MockBackupVerification.setResults();
backupVerificationService.execute();
Truth.assertThat(badVerifications.count()).isEqualTo(1);
new Verifications() {
{
backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure();
backupNotificationMgr.notify((BackupVerificationResult) any);
maxTimes = 0;
}
};
}

@Test
public void failCalls() throws Exception {
MockBackupVerification.throwError = false;
MockBackupVerification.emptyBackupVerificationList = true;
public void testRestoreMode(@Mocked InstanceState state) throws Exception {
new Expectations() {
{
backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure();
maxTimes = 1;
state.getRestoreStatus().getStatus();
result = Status.STARTED;
}
};
backupVerificationService.execute();
Truth.assertThat(badVerifications.count()).isEqualTo(0);
new Verifications() {
{
backupVerificationService.getBackupMetrics().incrementBackupVerificationFailure();
maxTimes = 1;
backupVerification.verifyBackup((BackupVersion) any, anyBoolean, (DateRange) any);
maxTimes = 0;
}
};
}

@Test
public void testRestoreMode(@Mocked InstanceState state) throws Exception {
new Expectations() {
{
state.getRestoreStatus().getStatus();
result = Status.STARTED;
backupVerification.verifyAllBackups((BackupVersion) any, (DateRange) any);
maxTimes = 0;
}
};
backupVerificationService.execute();
}

@Test
public void testGetBackupMetrics() {
BackupMetrics backupMetrics = backupVerificationService.getBackupMetrics();
Assert.assertTrue(backupMetrics != null);
{
backupNotificationMgr.notify((BackupVerificationResult) any);
maxTimes = 0;
}
};
}

private static BackupVerificationResult getInvalidBackupVerificationResult() {
Expand Down

0 comments on commit cad9d06

Please sign in to comment.