Skip to content

Commit

Permalink
Merge pull request #882 from mattl-netflix/feature/CASS-1608
Browse files Browse the repository at this point in the history 
Preliminary refactoring in advance of backporting PropertiesFileTuner…
  • Loading branch information
@mattl-netflix
mattl-netflix authored Jun 8, 2020
2 parents d926738 + 8782aa1 commit 8e1ee55
Show file tree
Hide file tree
Showing 6 changed files with 39 additions and 90 deletions.
1 change: 1 addition & 0 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ allprojects {

dependencies {
compile 'org.apache.commons:commons-lang3:3.5'
compile 'org.apache.commons:commons-text:1.8'
compile 'commons-logging:commons-logging:1.2'
compile 'org.apache.commons:commons-collections4:4.1'
compile 'commons-io:commons-io:2.6'
Expand Down
18 changes: 4 additions & 14 deletions priam/src/main/java/com/netflix/priam/config/IConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
*/
package com.netflix.priam.config;

import com.google.common.collect.ImmutableSet;
import com.google.inject.ImplementedBy;
import com.netflix.priam.tuner.JVMOption;
import com.netflix.priam.config.PriamConfiguration;
Expand Down Expand Up @@ -45,16 +46,6 @@ public interface IConfiguration {
*/
String getCassHome();

/**
* Returns the path to the Cassandra configuration directory, this varies by distribution but the default
* of finding the yaml file and going up one directory is a reasonable default. If you have a custom Yaml
* Location that doesn't have all the other configuration files you may want to tune this.
* @return
*/
default String getCassConfigurationDirectory() {
return new File(getYamlLocation()).getParentFile().getPath();
}

String getYamlLocation();

/**
Expand Down Expand Up @@ -805,11 +796,10 @@ default int getPostRestoreHookHeartbeatCheckFrequencyInMs() {
}

/**
* Return a comma delimited list of property files that should be tuned in the configuration
* directory by Priam. These files live relative to the configuration directory.
* @return A comma delimited list of relative file paths to the configuration directory
* Return a list of property file paths from the configuration directory by Priam that should be tuned.
* @return the files paths
*/
default String getTunablePropertyFiles() { return ""; }
default ImmutableSet<String> getTunablePropertyFiles() { return ImmutableSet.of(); }

/**
* Escape hatch for getting any arbitrary property by key
Expand Down
78 changes: 23 additions & 55 deletions priam/src/main/java/com/netflix/priam/tuner/PropertiesFileTuner.java
View file
Original file line number Diff line number Diff line change
@@ -1,87 +1,55 @@
package com.netflix.priam.tuner;

import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;


import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Splitter;
import com.google.inject.Inject;
import com.netflix.priam.config.IConfiguration;
import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.commons.lang3.text.StrSubstitutor;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.text.StringSubstitutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.inject.Inject;
import com.netflix.priam.config.IConfiguration;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Map;

/**
* Support tuning standard .properties files
* <p>
*/
public class PropertiesFileTuner
{
private static final Logger logger = LoggerFactory.getLogger(JVMOptionsTuner.class);
private static final Logger logger = LoggerFactory.getLogger(PropertiesFileTuner.class);
protected final IConfiguration config;
protected final String propertyPrefix;

@Inject
public PropertiesFileTuner(IConfiguration config, String prefix) {
public PropertiesFileTuner(IConfiguration config)
{
this.config = config;
this.propertyPrefix = prefix;
}

@SuppressWarnings("unchecked")
public void updateAndSaveProperties(String configPath) throws IOException, ConfigurationException
public void updateAndSaveProperties(String propertyFile) throws IOException, ConfigurationException
{
File propertiesFile = new File(configPath);
try {
if (propertiesFile.exists() && !propertiesFile.canWrite()) {
throw new IOException("Can't write and therefore cannot tune" + configPath);
}

PropertiesConfiguration properties = new PropertiesConfiguration();
properties.getLayout().load(properties, new FileReader(propertiesFile.getPath()));

Set<String> keys = new HashSet<>();
properties.getKeys().forEachRemaining(keys::add);

Map<String, String> overridenProperties = new HashMap<>();
String propertyOverrides = config.getProperty("propertyOverrides." + propertyPrefix, null);

if (propertyOverrides != null) {
properties.getLayout().load(properties, new FileReader(propertyFile));
String overrides = config.getProperty("propertyOverrides." + FilenameUtils.getBaseName(propertyFile), null);
if (overrides != null) {
// Allow use of the IConfiguration object as template strings
ObjectMapper objectMapper = new ObjectMapper();
Map<String, Object> map = objectMapper.convertValue(config, Map.class);
StrSubstitutor sub = new StrSubstitutor(map);
String resolvedPropertyOverrides = sub.replace(propertyOverrides);

String[] pairs = resolvedPropertyOverrides.split(",");
for (String kv : pairs) {
String[] entry = kv.split("=");
if (entry.length != 2)
continue;
keys.add(entry[0]);
overridenProperties.put(entry[0], entry[1]);
}
Map<String, Object> map = new ObjectMapper().convertValue(config, Map.class);
String resolvedOverrides = new StringSubstitutor(map).replace(overrides);
Splitter.on(",").withKeyValueSeparator("=").split(resolvedOverrides).forEach(properties::setProperty);
}

for (String key: keys) {
if (overridenProperties.containsKey(key))
properties.setProperty(key, overridenProperties.get(key));
}

properties.getLayout().save(properties, new FileWriter(propertiesFile.getPath()));
properties.getLayout().save(properties, new FileWriter(propertyFile));
}
catch (IOException | ConfigurationException e) {
logger.error("Could not tune " + configPath, e);
logger.error("Could not tune " + propertyFile + ". Does it exist? Is it writable?", e);
throw e;
}
}

}
16 changes: 3 additions & 13 deletions priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,19 +132,9 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed
// TODO: port commit log backups to the PropertiesFileTuner implementation
configureCommitLogBackups();

File configurationDirectory = new File(config.getCassConfigurationDirectory());
if (configurationDirectory.exists() && configurationDirectory.isDirectory()) {
String[] tunablePropertyFiles = config.getTunablePropertyFiles().split(",");
for (String file: tunablePropertyFiles) {
// e.g. cassandra-rackdc.properties
String[] propertiesFile = file.split("\\.");
if (propertiesFile.length > 1 && !propertiesFile[0].isEmpty())
{
String prefix = propertiesFile[0];
PropertiesFileTuner propertyTuner = new PropertiesFileTuner(config, prefix);
propertyTuner.updateAndSaveProperties(Paths.get(configurationDirectory.getPath(), file).toString());
}
}
PropertiesFileTuner propertyTuner = new PropertiesFileTuner(config);
for (String propertyFile : config.getTunablePropertyFiles()) {
propertyTuner.updateAndSaveProperties(propertyFile);
}
}

Expand Down
11 changes: 5 additions & 6 deletions priam/src/test/java/com/netflix/priam/config/FakeConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package com.netflix.priam.config;

import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.inject.Singleton;
import com.netflix.priam.tuner.JVMOption;
Expand All @@ -27,10 +28,7 @@
import com.netflix.priam.scheduler.UnsupportedTypeException;

import java.io.File;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;

@Singleton
public class FakeConfiguration implements IConfiguration
Expand Down Expand Up @@ -874,8 +872,9 @@ public int getPostRestoreHookTimeOutInDays() {
}

@Override
public String getTunablePropertyFiles() {
return "cassandra-rackdc.properties";
public ImmutableSet<String> getTunablePropertyFiles() {
String path = new File(getYamlLocation()).getParentFile().getPath();
return ImmutableSet.of(path + "/cassandra-rackdc.properties");
}

@Override
Expand Down
5 changes: 3 additions & 2 deletions priam/src/test/java/com/netflix/priam/tuner/StandardTunerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public void setup()

config = new FakeConfiguration();
tuner = new StandardTuner(config);
File targetDir = new File(config.getCassConfigurationDirectory());
File targetDir = new File(config.getYamlLocation()).getParentFile();
if(!targetDir.exists())
targetDir.mkdirs();
}
Expand Down Expand Up @@ -109,7 +109,8 @@ public void testPropertiesFiles() throws Exception
Files.copy(new File("src/main/resources/incr-restore-cassandra.yaml"), new File(target));

File testRackDcFile = new File("src/test/resources/conf/cassandra-rackdc.properties");
File rackDcFile = new File(Paths.get(config.getCassConfigurationDirectory(), "cassandra-rackdc.properties").normalize().toString());
String propertiesPath = new File(config.getYamlLocation()).getParentFile().getPath();
File rackDcFile = new File(Paths.get(propertiesPath, "cassandra-rackdc.properties").normalize().toString());
Files.copy(testRackDcFile, rackDcFile);

try {
Expand Down

0 comments on commit 8e1ee55

Please sign in to comment.