diff --git a/continuity.api/continuity.api.gradle b/continuity.api/continuity.api.gradle index e76e5304..abb3eeec 100644 --- a/continuity.api/continuity.api.gradle +++ b/continuity.api/continuity.api.gradle @@ -8,6 +8,7 @@ dependencyManagement { dependencies { compile project(':continuity.idpa') + compile project(':continuity.dsl') compile("org.springframework.boot:spring-boot-starter-web") compile("org.springframework.boot:spring-boot-starter-amqp") diff --git a/continuity.api/src/main/java/org/continuity/api/amqp/AmqpApi.java b/continuity.api/src/main/java/org/continuity/api/amqp/AmqpApi.java index cc095a89..6b7fbdfd 100644 --- a/continuity.api/src/main/java/org/continuity/api/amqp/AmqpApi.java +++ b/continuity.api/src/main/java/org/continuity/api/amqp/AmqpApi.java @@ -78,7 +78,7 @@ private SessionLogs() { /** * AMQP API of the workload model services, e.g., wessbas. * - * @author Henning Schulz + * @author Henning Schulz, Alper Hidiroglu * */ public static class WorkloadModel { @@ -86,6 +86,8 @@ public static class WorkloadModel { private static final String SCOPE = "workloadmodel"; public static final ExchangeDefinition TASK_CREATE = ExchangeDefinition.task(SCOPE, "create").nonDurable().autoDelete().withRoutingKey(WorkloadType.INSTANCE); + + public static final ExchangeDefinition MIX_CREATE = ExchangeDefinition.task(SCOPE, "createmix").nonDurable().autoDelete().withRoutingKey(WorkloadType.INSTANCE); public static final ExchangeDefinition EVENT_CREATED = ExchangeDefinition.event(SCOPE, "created").nonDurable().autoDelete().withRoutingKey(WorkloadType.INSTANCE); @@ -146,5 +148,22 @@ private IdpaApplication() { } } + + /** + * AMQP API of the forecast service. + * + * @author Alper Hidiroglu + * + */ + public static class Forecast { + + private static final String SCOPE = "forecast"; + + public static final ExchangeDefinition TASK_CREATE = ExchangeDefinition.task(SCOPE, "create").nonDurable().autoDelete().withRoutingKey(ServiceName.INSTANCE); + + private Forecast() { + } + + } } diff --git a/continuity.api/src/main/java/org/continuity/api/entities/artifact/ForecastBundle.java b/continuity.api/src/main/java/org/continuity/api/entities/artifact/ForecastBundle.java new file mode 100644 index 00000000..aa484541 --- /dev/null +++ b/continuity.api/src/main/java/org/continuity/api/entities/artifact/ForecastBundle.java @@ -0,0 +1,56 @@ +package org.continuity.api.entities.artifact; + +import java.util.Date; +import java.util.LinkedList; + +import com.fasterxml.jackson.annotation.JsonFormat; + +/** + * + * @author Alper Hidiroglu + * + */ +public class ForecastBundle { + + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH-mm-ss-SSSX") + private Date timestamp; + + private int workloadIntensity; + + private LinkedList probabilities; + + public ForecastBundle(Date timestamp, Integer workloadIntensity, LinkedList probabilities) { + this.timestamp = timestamp; + this.workloadIntensity = workloadIntensity; + this.probabilities = probabilities; + } + + public ForecastBundle() { + + } + + public Date getTimestamp() { + return timestamp; + } + + public void setTimestamp(Date timestamp) { + this.timestamp = timestamp; + } + + public Integer getWorkloadIntensity() { + return workloadIntensity; + } + + public void setWorkloadIntensity(Integer workloadIntensity) { + this.workloadIntensity = workloadIntensity; + } + + public LinkedList getProbabilities() { + return probabilities; + } + + public void setProbabilities(LinkedList probabilities) { + this.probabilities = probabilities; + } + +} diff --git a/continuity.api/src/main/java/org/continuity/api/entities/artifact/SessionsBundle.java b/continuity.api/src/main/java/org/continuity/api/entities/artifact/SessionsBundle.java new file mode 100644 index 00000000..32e4d43c --- /dev/null +++ b/continuity.api/src/main/java/org/continuity/api/entities/artifact/SessionsBundle.java @@ -0,0 +1,36 @@ +package org.continuity.api.entities.artifact; + +import java.util.List; + +/** + * + * @author Alper Hidiroglu + * + */ +public class SessionsBundle { + + private int behaviorId; + private List sessions; + + public SessionsBundle(int behaviorId, List sessions) { + this.behaviorId = behaviorId; + this.sessions = sessions; + } + + public SessionsBundle() { + + } + + public int getBehaviorId() { + return behaviorId; + } + public void setBehaviorId(int behaviorId) { + this.behaviorId = behaviorId; + } + public List getSessions() { + return sessions; + } + public void setSessions(List sessions) { + this.sessions = sessions; + } +} diff --git a/continuity.api/src/main/java/org/continuity/api/entities/artifact/SessionsBundlePack.java b/continuity.api/src/main/java/org/continuity/api/entities/artifact/SessionsBundlePack.java new file mode 100644 index 00000000..b8fb380f --- /dev/null +++ b/continuity.api/src/main/java/org/continuity/api/entities/artifact/SessionsBundlePack.java @@ -0,0 +1,41 @@ +package org.continuity.api.entities.artifact; + +import java.util.Date; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonFormat; + +public class SessionsBundlePack { + + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH-mm-ss-SSSX") + private Date timestamp; + + private List sessionsBundles; + + public SessionsBundlePack(Date timestamp, List sessionsBundles) { + super(); + this.timestamp = timestamp; + this.sessionsBundles = sessionsBundles; + } + + public SessionsBundlePack() { + + } + + public Date getTimestamp() { + return timestamp; + } + + public void setTimestamp(Date timestamp) { + this.timestamp = timestamp; + } + + public List getSessionsBundles() { + return sessionsBundles; + } + + public void setSessionsBundles(List sessionsBundles) { + this.sessionsBundles = sessionsBundles; + } + +} diff --git a/continuity.api/src/main/java/org/continuity/api/entities/artifact/SimplifiedSession.java b/continuity.api/src/main/java/org/continuity/api/entities/artifact/SimplifiedSession.java new file mode 100644 index 00000000..57ab459e --- /dev/null +++ b/continuity.api/src/main/java/org/continuity/api/entities/artifact/SimplifiedSession.java @@ -0,0 +1,48 @@ +package org.continuity.api.entities.artifact; + +/** + * Represents a simplified session. + * @author Alper Hidiroglu + * + */ +public class SimplifiedSession { + + private String id; + private long startTime; + private long endTime; + + /** + * Constructor. + * @param id + * @param startTime + * @param endTime + */ + public SimplifiedSession(String id, long startTime, long endTime) { + this.id = id; + this.startTime = startTime; + this.endTime = endTime; + } + + public SimplifiedSession() { + + } + + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } + public long getStartTime() { + return startTime; + } + public void setStartTime(long startTime) { + this.startTime = startTime; + } + public long getEndTime() { + return endTime; + } + public void setEndTime(long endTime) { + this.endTime = endTime; + } +} diff --git a/continuity.api/src/main/java/org/continuity/api/entities/config/Order.java b/continuity.api/src/main/java/org/continuity/api/entities/config/Order.java index 4f1ef19f..3c73065c 100644 --- a/continuity.api/src/main/java/org/continuity/api/entities/config/Order.java +++ b/continuity.api/src/main/java/org/continuity/api/entities/config/Order.java @@ -3,13 +3,14 @@ import java.util.Set; import org.continuity.api.entities.links.LinkExchangeModel; +import org.continuity.dsl.description.ForecastInput; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; -@JsonPropertyOrder({ "goal", "mode", "tag", "testing-context", "options", "source" }) +@JsonPropertyOrder({ "goal", "mode", "tag", "testing-context", "options", "source", "forecast-input" }) public class Order { private String tag; @@ -28,6 +29,10 @@ public class Order { @JsonInclude(Include.NON_NULL) private OrderOptions options; + + @JsonInclude(Include.NON_NULL) + @JsonProperty("forecast-input") + private ForecastInput forecastInput; @JsonProperty("modularization") @JsonInclude(Include.NON_NULL) @@ -88,5 +93,13 @@ public ModularizationOptions getModularizationOptions() { public void setModularizationOptions(ModularizationOptions modularizationOptions) { this.modularizationOptions = modularizationOptions; } + + public ForecastInput getForecastInput() { + return forecastInput; + } + + public void setForecastInput(ForecastInput forecastInput) { + this.forecastInput = forecastInput; + } } diff --git a/continuity.api/src/main/java/org/continuity/api/entities/config/OrderGoal.java b/continuity.api/src/main/java/org/continuity/api/entities/config/OrderGoal.java index b72abfa2..dff568d4 100644 --- a/continuity.api/src/main/java/org/continuity/api/entities/config/OrderGoal.java +++ b/continuity.api/src/main/java/org/continuity/api/entities/config/OrderGoal.java @@ -8,7 +8,7 @@ public enum OrderGoal { - CREATE_SESSION_LOGS, CREATE_WORKLOAD_MODEL, CREATE_LOAD_TEST, EXECUTE_LOAD_TEST; + CREATE_SESSION_LOGS, CREATE_BEHAVIOR_MIX, CREATE_FORECAST, CREATE_WORKLOAD_MODEL, CREATE_LOAD_TEST, EXECUTE_LOAD_TEST; private static final Map prettyStringToGoal = new HashMap<>(); diff --git a/continuity.api/src/main/java/org/continuity/api/entities/config/OrderMode.java b/continuity.api/src/main/java/org/continuity/api/entities/config/OrderMode.java index 7aa83e59..71a00151 100644 --- a/continuity.api/src/main/java/org/continuity/api/entities/config/OrderMode.java +++ b/continuity.api/src/main/java/org/continuity/api/entities/config/OrderMode.java @@ -8,7 +8,7 @@ public enum OrderMode { - PAST_SESSIONS, PAST_REQUESTS; + PAST_SESSIONS, PAST_REQUESTS, FORECASTED_WORKLOAD; private static final Map prettyStringToMode = new HashMap<>(); diff --git a/continuity.api/src/main/java/org/continuity/api/entities/config/TaskDescription.java b/continuity.api/src/main/java/org/continuity/api/entities/config/TaskDescription.java index 14828a45..5576ad2f 100644 --- a/continuity.api/src/main/java/org/continuity/api/entities/config/TaskDescription.java +++ b/continuity.api/src/main/java/org/continuity/api/entities/config/TaskDescription.java @@ -1,6 +1,7 @@ package org.continuity.api.entities.config; import org.continuity.api.entities.links.LinkExchangeModel; +import org.continuity.dsl.description.ForecastInput; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; @@ -25,6 +26,8 @@ public class TaskDescription { @JsonProperty("modularization-options") private ModularizationOptions modularizationOptions; + + private ForecastInput forecastInput; public String getTaskId() { return taskId; @@ -73,5 +76,13 @@ public ModularizationOptions getModularizationOptions() { public void setModularizationOptions(ModularizationOptions modularizationOptions) { this.modularizationOptions = modularizationOptions; } + + public ForecastInput getForecastInput() { + return forecastInput; + } + + public void setForecastInput(ForecastInput forecastInput) { + this.forecastInput = forecastInput; + } } diff --git a/continuity.api/src/main/java/org/continuity/api/entities/links/ForecastLinks.java b/continuity.api/src/main/java/org/continuity/api/entities/links/ForecastLinks.java new file mode 100644 index 00000000..41c8312b --- /dev/null +++ b/continuity.api/src/main/java/org/continuity/api/entities/links/ForecastLinks.java @@ -0,0 +1,56 @@ +package org.continuity.api.entities.links; + +import java.lang.reflect.Field; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class ForecastLinks extends AbstractLinks { + + @JsonProperty(value = "link", required = false) + @JsonInclude(Include.NON_NULL) + private String link; + + public ForecastLinks(LinkExchangeModel parent) { + super(parent); + } + + public ForecastLinks() { + this(null); + } + + public String getLink() { + return link; + } + + public ForecastLinks setLink(String forecastLink) { + this.link = forecastLink; + return this; + } + + @Override + public boolean isEmpty() { + for (Field field : ForecastLinks.class.getDeclaredFields()) { + try { + if ((field.getName() != "parent") && (field.get(this) != null)) { + return false; + } + } catch (IllegalArgumentException | IllegalAccessException e) { + e.printStackTrace(); + } + } + + return true; + } + + @Override + public void merge(ForecastLinks other) throws IllegalArgumentException, IllegalAccessException { + for (Field field : ForecastLinks.class.getDeclaredFields()) { + if ((field.getName() != "parent") && (field.get(this) == null)) { + field.set(this, field.get(other)); + } + } + } + +} diff --git a/continuity.api/src/main/java/org/continuity/api/entities/links/LinkExchangeModel.java b/continuity.api/src/main/java/org/continuity/api/entities/links/LinkExchangeModel.java index 686620cd..e31947f4 100644 --- a/continuity.api/src/main/java/org/continuity/api/entities/links/LinkExchangeModel.java +++ b/continuity.api/src/main/java/org/continuity/api/entities/links/LinkExchangeModel.java @@ -31,6 +31,11 @@ public class LinkExchangeModel { @JsonInclude(value = Include.CUSTOM, valueFilter = AbstractLinks.ValueFilter.class) @JsonManagedReference private final SessionLogsLinks sessionLogsLinks = new SessionLogsLinks(this); + + @JsonProperty(value = "forecast", required = false) + @JsonInclude(value = Include.CUSTOM, valueFilter = AbstractLinks.ValueFilter.class) + @JsonManagedReference + private final ForecastLinks forecastLinks = new ForecastLinks(this); @JsonProperty(value = "workload-model", required = false) @JsonInclude(value = Include.CUSTOM, valueFilter = AbstractLinks.ValueFilter.class) @@ -41,6 +46,11 @@ public class LinkExchangeModel { @JsonInclude(value = Include.CUSTOM, valueFilter = AbstractLinks.ValueFilter.class) @JsonManagedReference private final LoadTestLinks loadTestLinks = new LoadTestLinks(this); + + @JsonProperty(value = "sessions-bundles", required = false) + @JsonInclude(value = Include.CUSTOM, valueFilter = AbstractLinks.ValueFilter.class) + @JsonManagedReference + private final SessionsBundlesLinks sessionsBundlesLinks = new SessionsBundlesLinks(this); public String getTag() { return tag; @@ -62,6 +72,10 @@ public MeasurementDataLinks getMeasurementDataLinks() { public SessionLogsLinks getSessionLogsLinks() { return sessionLogsLinks; } + + public ForecastLinks getForecastLinks() { + return forecastLinks; + } public WorkloadModelLinks getWorkloadModelLinks() { return workloadModelLinks; @@ -70,6 +84,10 @@ public WorkloadModelLinks getWorkloadModelLinks() { public LoadTestLinks getLoadTestLinks() { return loadTestLinks; } + + public SessionsBundlesLinks getSessionsBundlesLinks() { + return sessionsBundlesLinks; + } public void merge(LinkExchangeModel other) { if (this.getTag() == null) { diff --git a/continuity.api/src/main/java/org/continuity/api/entities/links/SessionsBundlesLinks.java b/continuity.api/src/main/java/org/continuity/api/entities/links/SessionsBundlesLinks.java new file mode 100644 index 00000000..6cdb0c88 --- /dev/null +++ b/continuity.api/src/main/java/org/continuity/api/entities/links/SessionsBundlesLinks.java @@ -0,0 +1,68 @@ +package org.continuity.api.entities.links; + +import java.lang.reflect.Field; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class SessionsBundlesLinks extends AbstractLinks { + + @JsonProperty(value = "link", required = false) + @JsonInclude(Include.NON_NULL) + private String link; + + @JsonProperty(value = "status", required = false) + @JsonInclude(Include.NON_NULL) + private SessionsStatus status = SessionsStatus.CHANGED; + + public SessionsBundlesLinks(LinkExchangeModel parent) { + super(parent); + } + + public SessionsBundlesLinks() { + this(null); + } + + public String getLink() { + return link; + } + + public SessionsBundlesLinks setLink(String sessionsBundlesLink) { + this.link = sessionsBundlesLink; + return this; + } + + public SessionsStatus getStatus() { + return status; + } + + public void setStatus(SessionsStatus status) { + this.status = status; + } + + @Override + public boolean isEmpty() { + for (Field field : SessionsBundlesLinks.class.getDeclaredFields()) { + try { + if ((field.getName() != "parent") && (field.get(this) != null)) { + return false; + } + } catch (IllegalArgumentException | IllegalAccessException e) { + e.printStackTrace(); + } + } + + return true; + } + + @Override + public void merge(SessionsBundlesLinks other) throws IllegalArgumentException, IllegalAccessException { + for (Field field : SessionsBundlesLinks.class.getDeclaredFields()) { + if ((field.getName() != "parent") && (field.get(this) == null)) { + field.set(this, field.get(other)); + } + } + } + +} diff --git a/continuity.api/src/main/java/org/continuity/api/entities/links/SessionsStatus.java b/continuity.api/src/main/java/org/continuity/api/entities/links/SessionsStatus.java new file mode 100644 index 00000000..28fceee2 --- /dev/null +++ b/continuity.api/src/main/java/org/continuity/api/entities/links/SessionsStatus.java @@ -0,0 +1,30 @@ +package org.continuity.api.entities.links; + +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +public enum SessionsStatus { + + CHANGED, NOT_CHANGED; + + private static final Map prettyStringToApproach = new HashMap<>(); + + static { + for (SessionsStatus status : values()) { + prettyStringToApproach.put(status.toPrettyString(), status); + } + } + + @JsonCreator + public static SessionsStatus fromPrettyString(String key) { + return prettyStringToApproach.get(key); + } + + @JsonValue + public String toPrettyString() { + return toString().replace("_", "-").toLowerCase(); + } +} diff --git a/continuity.api/src/main/java/org/continuity/api/rest/RestApi.java b/continuity.api/src/main/java/org/continuity/api/rest/RestApi.java index 2ce28b66..16491357 100644 --- a/continuity.api/src/main/java/org/continuity/api/rest/RestApi.java +++ b/continuity.api/src/main/java/org/continuity/api/rest/RestApi.java @@ -666,6 +666,49 @@ private Paths() { } } + + /** + * REST API of the Forecast service. + * + * @author Alper Hidiroglu + * + */ + public static class Forecast { + + public static final String SERVICE_NAME = "forecast"; + + private Forecast() { + } + + public static class ForecastResult { + public static final String ROOT = "/forecastbundle"; + + /** {@value #ROOT}/{id} */ + public static final RestEndpoint GET = RestEndpoint.of(SERVICE_NAME, ROOT, Paths.GET, RequestMethod.GET); + + public static class Paths { + + public static final String GET = "/{id}"; + + private Paths() { + } + } + } + public static class Context { + public static final String ROOT = "/context"; + + /** {@value #ROOT}/submit */ + public static final RestEndpoint SUBMIT = RestEndpoint.of(SERVICE_NAME, ROOT, Paths.SUBMIT, RequestMethod.GET); + + public static class Paths { + + public static final String SUBMIT = "/submit"; + + private Paths() { + } + } + } + } /** * REST API of the WESSBAS service. @@ -772,9 +815,32 @@ public static class Paths { private Paths() { } } - } + + /** + * Sessions Bundles API of the WESSBAS service. + * + * @author Alper Hidiroglu + * + */ + public static class SessionsBundles { + + public static final String ROOT = "/sessionsbundles"; + + /** {@value #ROOT}/{id} */ + public static final RestEndpoint GET = RestEndpoint.of(SERVICE_NAME, ROOT, Paths.GET, RequestMethod.GET); + + private SessionsBundles() { + } + public static class Paths { + + public static final String GET = "/{id}"; + + private Paths() { + } + } + } } /** diff --git a/continuity.cli/src/main/java/org/continuity/cli/commands/OrderCommands.java b/continuity.cli/src/main/java/org/continuity/cli/commands/OrderCommands.java index e2327f33..6c65864d 100644 --- a/continuity.cli/src/main/java/org/continuity/cli/commands/OrderCommands.java +++ b/continuity.cli/src/main/java/org/continuity/cli/commands/OrderCommands.java @@ -9,6 +9,8 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; import org.continuity.api.entities.config.LoadTestType; import org.continuity.api.entities.config.ModularizationApproach; @@ -20,12 +22,17 @@ import org.continuity.api.entities.config.WorkloadModelType; import org.continuity.api.entities.links.MeasurementDataLinkType; import org.continuity.api.entities.links.LinkExchangeModel; +import org.continuity.api.entities.links.SessionsStatus; import org.continuity.api.entities.report.OrderReport; import org.continuity.api.entities.report.OrderResponse; import org.continuity.api.rest.RestApi; import org.continuity.cli.config.PropertiesProvider; import org.continuity.cli.storage.OrderStorage; import org.continuity.commons.utils.WebUtils; +import org.continuity.dsl.description.ForecastInput; +import org.continuity.dsl.description.ContextParameter; +import org.continuity.dsl.description.ForecastOptions; +import org.continuity.dsl.description.Measurement; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -177,6 +184,13 @@ private Order initializeOrder() { options.setLoadTestType(LoadTestType.JMETER); options.setWorkloadModelType(WorkloadModelType.WESSBAS); order.setOptions(options); + + Measurement measurement = new Measurement("Name of measurement containing contextual data"); + List covariates = new LinkedList(); + covariates.add(measurement); + ForecastOptions forecastOpt = new ForecastOptions("2019/01/01 00:00:00", "daily, hourly, minutely or secondly", "telescope or prophet", "http://localhost:8086"); + ForecastInput forecastInput = new ForecastInput(covariates, forecastOpt); + order.setForecastInput(forecastInput); ModularizationOptions modularizationOptions = new ModularizationOptions(); HashMap services = new HashMap(); @@ -188,6 +202,8 @@ private Order initializeOrder() { LinkExchangeModel links = new LinkExchangeModel(); links.getMeasurementDataLinks().setLink("LINK_TO_DATA").setTimestamp(new Date(0)).setLinkType(MeasurementDataLinkType.OPEN_XTRACE); links.getSessionLogsLinks().setLink("SESSION_LOGS_LINK"); + links.getSessionsBundlesLinks().setLink("SESSIONS_BUNDLES_LINKS").setStatus(SessionsStatus.NOT_CHANGED); + links.getForecastLinks().setLink("FORECAST_LINKS"); links.getWorkloadModelLinks().setType(WorkloadModelType.WESSBAS).setLink("WORKLOAD_MODEL_LINK"); links.getLoadTestLinks().setType(LoadTestType.JMETER).setLink("LOADTEST_LINK"); order.setSource(links); diff --git a/continuity.dsl/.gitignore b/continuity.dsl/.gitignore new file mode 100644 index 00000000..4a95481e --- /dev/null +++ b/continuity.dsl/.gitignore @@ -0,0 +1,2 @@ +/bin/ +/build/ diff --git a/continuity.dsl/continuity.dsl.gradle b/continuity.dsl/continuity.dsl.gradle new file mode 100644 index 00000000..a063d47c --- /dev/null +++ b/continuity.dsl/continuity.dsl.gradle @@ -0,0 +1,7 @@ +dependencies { + compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.9.1' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.9.1' + compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.9.1' + + compile group: 'org.apache.commons', name: 'commons-collections4', version: '4.1' +} \ No newline at end of file diff --git a/continuity.dsl/src/main/java/org/continuity/dsl/Deserializer.java b/continuity.dsl/src/main/java/org/continuity/dsl/Deserializer.java new file mode 100644 index 00000000..8a92b049 --- /dev/null +++ b/continuity.dsl/src/main/java/org/continuity/dsl/Deserializer.java @@ -0,0 +1,33 @@ +package org.continuity.dsl; + +import java.io.File; + +import org.continuity.dsl.description.ForecastInput; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +/** + * Utility class. + * + * @author Alper Hidiroglu + * + */ +public class Deserializer { + + public ForecastInput deserialize() { + ForecastInput descr = null; + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + try { + + descr = mapper.readValue(new File("C:/Users/ahi/Desktop/ContextDescriptions/context.yaml"), + ForecastInput.class); + + } catch (Exception e) { + + // TODO Auto-generated catch block + e.printStackTrace(); + } + return descr; + } +} diff --git a/continuity.dsl/src/main/java/org/continuity/dsl/description/ContextParameter.java b/continuity.dsl/src/main/java/org/continuity/dsl/description/ContextParameter.java new file mode 100644 index 00000000..2c2b305c --- /dev/null +++ b/continuity.dsl/src/main/java/org/continuity/dsl/description/ContextParameter.java @@ -0,0 +1,16 @@ +package org.continuity.dsl.description; + +import org.continuity.dsl.deserializer.CovariateDeserializer; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +/** + * Context parameter. + * + * @author Alper Hidiroglu + * + */ +@JsonDeserialize(using = CovariateDeserializer.class) +public interface ContextParameter { + +} diff --git a/continuity.dsl/src/main/java/org/continuity/dsl/description/ForecastDateConverter.java b/continuity.dsl/src/main/java/org/continuity/dsl/description/ForecastDateConverter.java new file mode 100644 index 00000000..1541fbeb --- /dev/null +++ b/continuity.dsl/src/main/java/org/continuity/dsl/description/ForecastDateConverter.java @@ -0,0 +1,28 @@ +package org.continuity.dsl.description; + +import java.text.SimpleDateFormat; +import java.util.Date; + +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.type.TypeFactory; +import com.fasterxml.jackson.databind.util.Converter; + +public class ForecastDateConverter implements Converter { + + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); + + @Override + public String convert(Date date) { + return dateFormat.format(date); + } + + @Override + public JavaType getInputType(TypeFactory typeFactory) { + return typeFactory.constructType(Date.class); + } + + @Override + public JavaType getOutputType(TypeFactory typeFactory) { + return typeFactory.constructType(String.class); + } +} diff --git a/continuity.dsl/src/main/java/org/continuity/dsl/description/ForecastInput.java b/continuity.dsl/src/main/java/org/continuity/dsl/description/ForecastInput.java new file mode 100644 index 00000000..1bb4f435 --- /dev/null +++ b/continuity.dsl/src/main/java/org/continuity/dsl/description/ForecastInput.java @@ -0,0 +1,72 @@ +package org.continuity.dsl.description; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +/** + * Represents a context input. + * + * @author Alper Hidiroglu + * + */ +@JsonPropertyOrder({ "context", "forecast-options" }) +public class ForecastInput { + + private List context; + + @JsonProperty("forecast-options") + private ForecastOptions forecastOptions; + + @JsonCreator + public ForecastInput(@JsonProperty(value = "context", required = false) List context, + @JsonProperty(value = "forecast-options", required = true) ForecastOptions forecastOptions) { + this.context = context; + this.forecastOptions = forecastOptions; + } + + /** + * Returns context for the workload forecasting. + * + * @return The context covariates. + */ + public List getContext() { + return context; + } + + /** + * Sets the context for the workload forecasting. + * + * @param context The context covariates. + * + */ + public void setContext(List context) { + this.context = context; + } + + /** + * Gets further information for the workload forecasting. + * + * @return The forecasting information. + */ + public ForecastOptions getForecastOptions() { + return forecastOptions; + } + + /** + * Sets further information for the workload forecasting. + * + * @param forecast The forecasting information. + */ + public void setForecastOptions(ForecastOptions forecast) { + this.forecastOptions = forecast; + } + + @Override + public String toString() { + return "Forecast-Input [context=" + context + ", forecastOptions=" + forecastOptions + "]"; + } + +} diff --git a/continuity.dsl/src/main/java/org/continuity/dsl/description/ForecastOptions.java b/continuity.dsl/src/main/java/org/continuity/dsl/description/ForecastOptions.java new file mode 100644 index 00000000..17f1e9a7 --- /dev/null +++ b/continuity.dsl/src/main/java/org/continuity/dsl/description/ForecastOptions.java @@ -0,0 +1,103 @@ +package org.continuity.dsl.description; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +/** + * Further information for the workload forecasting. + * + * @author Alper Hidiroglu + * + */ +@JsonPropertyOrder({ "forecast-date", "interval", "forecaster", "influx-link" }) +public class ForecastOptions { + + @JsonProperty("forecast-date") + @JsonSerialize(converter=ForecastDateConverter.class) + private Date forecastDate; + + private String interval; + + private String forecaster; + + @JsonProperty("influx-link") + private String influxLink; + + @JsonIgnore + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); + + @JsonCreator + public ForecastOptions(@JsonProperty(value = "forecast-date", required = true) String forecastPeriod, @JsonProperty(value = "interval", required = true) String interval + ,@JsonProperty(value = "forecaster", required = true) String forecaster, @JsonProperty(value = "influx-link", required = true) String influxLink) { + this.forecastDate = null; + try { + forecastDate = dateFormat.parse(forecastPeriod); + } catch (ParseException e) { + e.printStackTrace(); + } + this.forecaster = forecaster; + this.interval = interval; + this.influxLink = influxLink; + } + + public String getInterval() { + return interval; + } + + public void setInterval(String interval) { + this.interval = interval; + } + + /** + * Gets the period of the workload forecast. + * + * @return The period of the workload forecast. + */ + public Date getForecastDate() { + return forecastDate; + } + + /** + * Sets the period of the workload forecasting. + * + * @param forecastPeriod The period of the workload forecast. + */ + public void setForecastDate(Date forecastDate) { + this.forecastDate = forecastDate; + } + + public String getForecaster() { + return forecaster; + } + + public void setForecaster(String forecaster) { + this.forecaster = forecaster; + } + + @JsonIgnore + public long getDateAsTimestamp() { + // 13 digits + return this.forecastDate.getTime(); + } + + + public String getInfluxLink() { + return influxLink; + } + + public void setInfluxLink(String influxLink) { + this.influxLink = influxLink; + } + + @Override + public String toString() { + return "Forecast [forecast-date=" + forecastDate + ", interval=" + interval + ", forecaster=" + forecaster + "]"; + } +} diff --git a/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureEvent.java b/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureEvent.java new file mode 100644 index 00000000..32f4a665 --- /dev/null +++ b/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureEvent.java @@ -0,0 +1,46 @@ +package org.continuity.dsl.description; + +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +public class FutureEvent { + + private String value; + + @JsonProperty("time") + @JsonSerialize(converter=FutureOccurrencesConverter.class) + private FutureOccurrences time; + + @JsonCreator + public FutureEvent(@JsonProperty(value = "value", required = true) String value, @JsonProperty(value = "time", required = true) List time) { + this.value = value; + this.time = new FutureOccurrences(time); + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + @JsonIgnore + public FutureOccurrences getTime() { + return time; + } + + @JsonIgnore + public void setTime(ArrayList time) { + this.time = new FutureOccurrences(time); + } + @Override + public String toString() { + return "FutureEvent [value=" + value + ", time=" + time + "]"; + } +} diff --git a/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureEvents.java b/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureEvents.java new file mode 100644 index 00000000..e1c3ca89 --- /dev/null +++ b/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureEvents.java @@ -0,0 +1,38 @@ +package org.continuity.dsl.description; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +/** + * Represents the event context type. + * + * @author Alper Hidiroglu + * + */ +@JsonDeserialize(as = FutureEvents.class) +public class FutureEvents extends Measurement implements ContextParameter { + + private List future; + + @JsonCreator + public FutureEvents(@JsonProperty(value = "measurement", required = true) String measurement, @JsonProperty(value = "future", required = true) List future) { + super(measurement); + this.future = future; + } + + public List getFuture() { + return future; + } + + public void setFuture(List future) { + this.future = future; + } + + @Override + public String toString() { + return "FutureEvents [measurement=" + super.getMeasurement() + ", future=" + future + "]"; + } +} diff --git a/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureNumber.java b/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureNumber.java new file mode 100644 index 00000000..1e6449ad --- /dev/null +++ b/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureNumber.java @@ -0,0 +1,46 @@ +package org.continuity.dsl.description; + +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +public class FutureNumber { + + private double value; + + @JsonProperty("time") + @JsonSerialize(converter=FutureOccurrencesConverter.class) + private FutureOccurrences time; + + @JsonCreator + public FutureNumber(@JsonProperty(value = "value", required = true) double value, @JsonProperty(value = "time", required = true) List time) { + this.value = value; + this.time = new FutureOccurrences(time); + } + + public double getValue() { + return value; + } + + public void setValue(double value) { + this.value = value; + } + + @JsonIgnore + public FutureOccurrences getTime() { + return time; + } + + @JsonIgnore + public void setTime(ArrayList time) { + this.time = new FutureOccurrences(time); + } + @Override + public String toString() { + return "FutureNumber [value=" + value + ", time=" + time + "]"; + } +} diff --git a/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureNumbers.java b/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureNumbers.java new file mode 100644 index 00000000..1236297b --- /dev/null +++ b/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureNumbers.java @@ -0,0 +1,38 @@ +package org.continuity.dsl.description; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +/** + * Represents the continuous data context type. + * @author Alper Hidiroglu + * + */ +@JsonDeserialize(as = FutureNumbers.class) +public class FutureNumbers extends Measurement implements ContextParameter { + + private List future; + + @JsonCreator + public FutureNumbers(@JsonProperty(value = "measurement", required = true) String measurement, @JsonProperty(value = "future", required = true) List future) { + super(measurement); + this.future = future; + } + + public List getFuture() { + return future; + } + + public void setFuture(List future) { + this.future = future; + } + + @Override + public String toString() { + return "FutureNumbers [measurement=" + super.getMeasurement() + ", future=" + future + "]"; + } + +} diff --git a/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureOccurrences.java b/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureOccurrences.java new file mode 100644 index 00000000..047083c8 --- /dev/null +++ b/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureOccurrences.java @@ -0,0 +1,100 @@ +package org.continuity.dsl.description; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.apache.commons.math3.util.Pair; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +/** + * Future occurrences of a value. + * @author Alper Hidiroglu + */ +public class FutureOccurrences { + + private List singleDates; + private List> rangeDates; + + @JsonIgnore + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); + + public FutureOccurrences(List futureDates) { + singleDates = new ArrayList(); + rangeDates = new ArrayList>(); + String delims = "to"; + + for(String date: futureDates) { + String[] tokens = date.split(delims); + if(tokens.length == 1) { + Date d = null; + try { + d = dateFormat.parse(tokens[0]); + // 13 digits + } catch (ParseException e) { + e.printStackTrace(); + } + singleDates.add(d); + } else if (tokens.length == 2) { + Date dFrom = null; + Date dTo = null; + try { + dFrom = dateFormat.parse(tokens[0]); + dTo = dateFormat.parse(tokens[1]); + } catch (ParseException e) { + e.printStackTrace(); + } + Pair rangeTimestamp = new Pair<>(dFrom, dTo); + rangeDates.add(rangeTimestamp); + } else { + System.out.println("Invalid context input!"); + } + } + } + + public FutureOccurrences() { + + } + + public List getSingleDates() { + return singleDates; + } + + public void setSingleDates(List singleDates) { + this.singleDates = singleDates; + } + + public List> getRangeDates() { + return rangeDates; + } + + public void setRangeDates(List> rangeTimestamps) { + this.rangeDates = rangeTimestamps; + } + + @JsonIgnore + public List getFutureDatesAsTimestamps(long interval) { + List timestamps = new ArrayList(); + for(Date date: this.singleDates) { + long timestamp = date.getTime(); + timestamps.add(timestamp); + } + for(Pair range: this.rangeDates) { + long timestampFrom = range.getKey().getTime(); + long timestampTo = range.getValue().getTime(); + while(timestampFrom <= timestampTo) { + timestamps.add(timestampFrom); + timestampFrom += interval; + } + } + return timestamps; + } + + @Override + public String toString() { + return "FutureOccurrences [single-dates=" + singleDates + ", range-dates=" + rangeDates + "]"; + } +} diff --git a/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureOccurrencesConverter.java b/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureOccurrencesConverter.java new file mode 100644 index 00000000..7d0479e9 --- /dev/null +++ b/continuity.dsl/src/main/java/org/continuity/dsl/description/FutureOccurrencesConverter.java @@ -0,0 +1,40 @@ +package org.continuity.dsl.description; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.apache.commons.math3.util.Pair; + +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.type.TypeFactory; +import com.fasterxml.jackson.databind.util.Converter; + +public class FutureOccurrencesConverter implements Converter> { + + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); + + @Override + public List convert(FutureOccurrences futOcc) { + List dates = new ArrayList(); + for(Date date: futOcc.getSingleDates()) { + dates.add(dateFormat.format(date)); + } + for(Pair datePair: futOcc.getRangeDates()) { + String rangeDates = dateFormat.format(datePair.getKey()) + " " + "to" + " " + dateFormat.format(datePair.getValue()); + dates.add(rangeDates); + } + return dates; + } + + @Override + public JavaType getInputType(TypeFactory typeFactory) { + return typeFactory.constructType(FutureOccurrences.class); + } + + @Override + public JavaType getOutputType(TypeFactory typeFactory) { + return typeFactory.constructType(List.class); + } +} diff --git a/continuity.dsl/src/main/java/org/continuity/dsl/description/Measurement.java b/continuity.dsl/src/main/java/org/continuity/dsl/description/Measurement.java new file mode 100644 index 00000000..36a6294e --- /dev/null +++ b/continuity.dsl/src/main/java/org/continuity/dsl/description/Measurement.java @@ -0,0 +1,30 @@ +package org.continuity.dsl.description; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +@JsonDeserialize(as = Measurement.class) +public class Measurement implements ContextParameter { + + @JsonProperty("measurement") + private String measurement; + + @JsonCreator + public Measurement(@JsonProperty(value = "measurement", required = true) String measurement) { + this.measurement = measurement; + } + + public String getMeasurement() { + return measurement; + } + + public void setMeasurement(String measurement) { + this.measurement = measurement; + } + + @Override + public String toString() { + return "Measurement [measurement=" + measurement + "]"; + } +} diff --git a/continuity.dsl/src/main/java/org/continuity/dsl/deserializer/CovariateDeserializer.java b/continuity.dsl/src/main/java/org/continuity/dsl/deserializer/CovariateDeserializer.java new file mode 100644 index 00000000..42876c1f --- /dev/null +++ b/continuity.dsl/src/main/java/org/continuity/dsl/deserializer/CovariateDeserializer.java @@ -0,0 +1,45 @@ +package org.continuity.dsl.deserializer; + +import java.io.IOException; + +import org.continuity.dsl.description.FutureNumbers; +import org.continuity.dsl.description.ContextParameter; +import org.continuity.dsl.description.FutureEvents; +import org.continuity.dsl.description.Measurement; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * Custom deserializer to differentiate between different types of covariate. + * values. + * + * @author Alper Hidiroglu + * + */ +public class CovariateDeserializer extends JsonDeserializer { + + @Override + public ContextParameter deserialize(JsonParser p, DeserializationContext cntxt) + throws IOException, JsonProcessingException { + ObjectMapper mapper = (ObjectMapper) p.getCodec(); + ObjectNode root = mapper.readTree(p); + + ContextParameter covariate = null; + + if (!root.has("future")) { + covariate = mapper.readValue(root.toString(), Measurement.class); + } else if (root.get("future").findValue("value").isTextual()) { + covariate = mapper.readValue(root.toString(), FutureEvents.class); + } else if (root.get("future").findValue("value").isNumber()) { + covariate = mapper.readValue(root.toString(), FutureNumbers.class); + } else { + throw new IOException("Invalid context input!"); + } + return covariate; + } +} \ No newline at end of file diff --git a/continuity.dsl/src/main/java/org/continuity/dsl/main/Main.java b/continuity.dsl/src/main/java/org/continuity/dsl/main/Main.java new file mode 100644 index 00000000..a1711709 --- /dev/null +++ b/continuity.dsl/src/main/java/org/continuity/dsl/main/Main.java @@ -0,0 +1,52 @@ +package org.continuity.dsl.main; + +import java.io.File; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; + +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; +import org.continuity.dsl.description.ForecastInput; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +/** + * Testing purposes. + * + * @author Alper Hidiroglu + * + */ +public class Main { + + /** + * Test of YAML to object mapping. + * + * @param args + */ + public static void main(String[] args) { + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + + try { + + ForecastInput descr = mapper.readValue(new File("C:/Users/ahi/Desktop/ContextDescriptions/context.yaml"), + ForecastInput.class); + + System.out.println(ReflectionToStringBuilder.toString(descr, + ToStringStyle.MULTI_LINE_STYLE)); + + DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); + String date = Instant.ofEpochMilli(1335391200000L).atOffset(ZoneOffset.UTC).format(dtf).toString(); + System.out.println(date.toString()); + + // StringCovariate covar = (StringCovariate) descr.getCovariates().get(0); + + } catch (Exception e) { + + // TODO Auto-generated catch block + e.printStackTrace(); + + } + } +} diff --git a/continuity.forecast/.gitignore b/continuity.forecast/.gitignore new file mode 100644 index 00000000..4a95481e --- /dev/null +++ b/continuity.forecast/.gitignore @@ -0,0 +1,2 @@ +/bin/ +/build/ diff --git a/continuity.forecast/Dockerfile b/continuity.forecast/Dockerfile new file mode 100644 index 00000000..b4d035ad --- /dev/null +++ b/continuity.forecast/Dockerfile @@ -0,0 +1,22 @@ +FROM rocker/r-base:latest +VOLUME /tmp +VOLUME /storage + +## Install Java +RUN apt-get update \ + && apt-get install -y openjdk-8-jdk r-cran-rjava libcurl4-openssl-dev\ + && R CMD javareconf + +ENV LD_LIBRARY_PATH /usr/lib/R/site-library/rJava/jri/ +ENV R_HOME=/usr/lib/R + +## Install R packages +RUN mkdir /r-library +RUN R -e "install.packages(c('xgboost', 'cluster', 'forecast', 'e1071', 'prophet'), repos='https://ftp.fau.de/cran/')" + +COPY telescope-multi/ telescope-multi/ +COPY prophet/ prophet/ +ARG JAR_FILE +ADD ${JAR_FILE} app.jar + +ENTRYPOINT ["java","-jar","/app.jar", "--port=80", "--spring.rabbitmq.host=rabbitmq", "--eureka.uri=http://eureka:8761/eureka", "--persist.path=/persisted"] \ No newline at end of file diff --git a/continuity.forecast/continuity.forecast.gradle b/continuity.forecast/continuity.forecast.gradle new file mode 100644 index 00000000..eeaffe97 --- /dev/null +++ b/continuity.forecast/continuity.forecast.gradle @@ -0,0 +1,53 @@ + +apply plugin: 'org.springframework.boot' +apply plugin: 'io.spring.dependency-management' + +dependencyManagement { + imports { + mavenBom 'org.springframework.cloud:spring-cloud-starter-parent:Finchley.M4' + } +} + +dependencies { + compile project(':continuity.api') + compile project(':continuity.commons') + compile project(':continuity.dsl') + + compile files('lib/JRI.jar') + compile files('lib/JRIEngine.jar') + compile files('lib/REngine.jar') + + // Spring Boot + + compile("org.springframework.boot:spring-boot-starter-web") + compile("org.springframework.boot:spring-boot-starter-amqp") + compile("org.springframework.cloud:spring-cloud-starter-netflix-eureka-client") + + testCompile("org.springframework.boot:spring-boot-starter-test") + + // Swagger + + compile("io.springfox:springfox-swagger2:2.7.0") + compile("io.springfox:springfox-swagger-ui:2.7.0") + + // Required for Eclipse + compile("org.codehaus.groovy:groovy-all:2.4.7") + + // InfluxDB + compile("org.influxdb:influxdb-java:2.10") + +} + +group = 'continuityproject' + +apply plugin: 'com.palantir.docker' + +docker { + name "${project.group}/forecast" + copySpec.from('.') { + include 'prophet/**' + include 'telescope-multi/**' + } + files jar.archivePath + buildArgs(['JAR_FILE': "${jar.archiveName}"]) +} \ No newline at end of file diff --git a/continuity.forecast/lib/JRI.jar b/continuity.forecast/lib/JRI.jar new file mode 100644 index 00000000..6132e6e0 Binary files /dev/null and b/continuity.forecast/lib/JRI.jar differ diff --git a/continuity.forecast/lib/JRIEngine.jar b/continuity.forecast/lib/JRIEngine.jar new file mode 100644 index 00000000..1b337853 Binary files /dev/null and b/continuity.forecast/lib/JRIEngine.jar differ diff --git a/continuity.forecast/lib/REngine.jar b/continuity.forecast/lib/REngine.jar new file mode 100644 index 00000000..e30f70c1 Binary files /dev/null and b/continuity.forecast/lib/REngine.jar differ diff --git a/continuity.forecast/prophet/AddRegressors.R b/continuity.forecast/prophet/AddRegressors.R new file mode 100644 index 00000000..8758f327 --- /dev/null +++ b/continuity.forecast/prophet/AddRegressors.R @@ -0,0 +1,2 @@ +df[[covarname]] <- values +m <- add_regressor(m, covarname) \ No newline at end of file diff --git a/continuity.forecast/prophet/ExtendFutureDataframe.R b/continuity.forecast/prophet/ExtendFutureDataframe.R new file mode 100644 index 00000000..d7365175 --- /dev/null +++ b/continuity.forecast/prophet/ExtendFutureDataframe.R @@ -0,0 +1 @@ +future[[covarname]] <- values \ No newline at end of file diff --git a/continuity.forecast/prophet/FitModelAndCreateFutureDataframe.R b/continuity.forecast/prophet/FitModelAndCreateFutureDataframe.R new file mode 100644 index 00000000..8fb60161 --- /dev/null +++ b/continuity.forecast/prophet/FitModelAndCreateFutureDataframe.R @@ -0,0 +1,3 @@ +m <- fit.prophet(m, df) +forecast.period <- as.numeric(period) +future <- make_future_dataframe(m, periods = forecast.period, freq = 60 * 60) diff --git a/continuity.forecast/prophet/ForecastProphet.R b/continuity.forecast/prophet/ForecastProphet.R new file mode 100644 index 00000000..679ccd07 --- /dev/null +++ b/continuity.forecast/prophet/ForecastProphet.R @@ -0,0 +1,2 @@ +forecast <- predict(m, future) +forecastValues <- tail(forecast[['yhat']], forecast.period) diff --git a/continuity.forecast/prophet/InitializeVariables.R b/continuity.forecast/prophet/InitializeVariables.R new file mode 100644 index 00000000..f5f2807a --- /dev/null +++ b/continuity.forecast/prophet/InitializeVariables.R @@ -0,0 +1,2 @@ +df <- data.frame(ds=dates, y=intensities) +m <- prophet() \ No newline at end of file diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/ForecastApplication.java b/continuity.forecast/src/main/java/org/continuity/forecast/ForecastApplication.java new file mode 100644 index 00000000..5d311561 --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/ForecastApplication.java @@ -0,0 +1,21 @@ +package org.continuity.forecast; + +import java.io.IOException; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.netflix.eureka.EnableEurekaClient; + +/** + * @author Alper Hidiroglu + * + */ +@SpringBootApplication +@EnableEurekaClient +public class ForecastApplication { + + public static void main(String[] args) throws IOException { + SpringApplication.run(ForecastApplication.class, args); + } +} + diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/amqp/ForecastAmqpHandler.java b/continuity.forecast/src/main/java/org/continuity/forecast/amqp/ForecastAmqpHandler.java new file mode 100644 index 00000000..60c2643c --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/amqp/ForecastAmqpHandler.java @@ -0,0 +1,113 @@ +package org.continuity.forecast.amqp; + +import java.util.Date; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.math3.util.Pair; +import org.continuity.api.amqp.AmqpApi; +import org.continuity.api.entities.artifact.ForecastBundle; +import org.continuity.api.entities.config.TaskDescription; +import org.continuity.api.entities.links.LinkExchangeModel; +import org.continuity.api.entities.links.SessionsStatus; +import org.continuity.api.entities.report.TaskError; +import org.continuity.api.entities.report.TaskReport; +import org.continuity.api.rest.RestApi; +import org.continuity.commons.storage.MixedStorage; +import org.continuity.forecast.config.RabbitMqConfig; +import org.continuity.forecast.controllers.ForecastController; +import org.continuity.forecast.managers.ForecastPipelineManager; +import org.continuity.forecast.managers.IntensitiesPipelineManager; +import org.influxdb.InfluxDB; +import org.influxdb.InfluxDBFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.amqp.core.AmqpTemplate; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +/** + * Handles received monitoring data in order to create Behavior Mix and workload intensity. + * + * @author Alper Hidiroglu + * + */ +@Component +public class ForecastAmqpHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(ForecastAmqpHandler.class); + + @Autowired + private AmqpTemplate amqpTemplate; + + @Autowired + private RestTemplate restTemplate; + + @Autowired + private MixedStorage storage; + + @Autowired + private ConcurrentHashMap> dateAndAmountOfUsersStorage; + + @Value("${spring.application.name}") + private String applicationName; + + /** + * Listener to the RabbitMQ {@link RabbitMqConfig#TASK_CREATE_QUEUE_NAME}. Creates a forecast bundle based on sessions. + * + * @param task + * The description of the task to be done. + * @return The id that can be used to retrieve the forecast bundle later on. + * @see ForecastController + */ + @RabbitListener(queues = RabbitMqConfig.TASK_CREATE_QUEUE_NAME) + public void onMonitoringDataAvailable(TaskDescription task) { + LOGGER.info("Task {}: Received new task to be processed for tag '{}'", task.getTaskId(), task.getTag()); + + String linkToSessions = task.getSource().getSessionsBundlesLinks().getLink(); + + List pathParams = RestApi.Wessbas.SessionsBundles.GET.parsePathParameters(linkToSessions); + + TaskReport report; + + if (linkToSessions == null) { + LOGGER.error("Task {}: Link to sessions is missing for tag {}!", task.getTaskId(), task.getTag()); + report = TaskReport.error(task.getTaskId(), TaskError.MISSING_SOURCE); + } else { + InfluxDB influxDb = InfluxDBFactory.connect(task.getForecastInput().getForecastOptions().getInfluxLink(), "admin", "admin"); + + boolean statusChanged = task.getSource().getSessionsBundlesLinks().getStatus().equals(SessionsStatus.CHANGED) ? true : false; + + // calculate new intensities + if(statusChanged) { + IntensitiesPipelineManager intensitiesPipelineManager = new IntensitiesPipelineManager(restTemplate, influxDb, task.getTag(), task.getForecastInput()); + intensitiesPipelineManager.runPipeline(linkToSessions); + Pair dateAndAmountOfUserGroups = intensitiesPipelineManager.getDateAndAmountOfUserGroups(); + dateAndAmountOfUsersStorage.put(pathParams.get(0), dateAndAmountOfUserGroups); + } + + ForecastPipelineManager pipelineManager = new ForecastPipelineManager(influxDb, task.getTag(), task.getForecastInput()); + ForecastBundle forecastBundle = pipelineManager.runPipeline(dateAndAmountOfUsersStorage.get(pathParams.get(0))); + influxDb.close(); + + if (forecastBundle == null) { + LOGGER.info("Task {}: Could not create forecast for tag '{}'.", task.getTaskId(), task.getTag()); + + report = TaskReport.error(task.getTaskId(), TaskError.INTERNAL_ERROR); + } else { + String storageId = storage.put(forecastBundle, task.getTag(), task.isLongTermUse()); + String forecastLink = RestApi.Forecast.ForecastResult.GET.requestUrl(storageId).withoutProtocol().get(); + + LOGGER.info("Task {}: Created a new forecast with id '{}'.", task.getTaskId(), storageId); + + report = TaskReport.successful(task.getTaskId(), new LinkExchangeModel().getForecastLinks().setLink(forecastLink).parent()); + } + } + + amqpTemplate.convertAndSend(AmqpApi.Global.EVENT_FINISHED.name(), AmqpApi.Global.EVENT_FINISHED.formatRoutingKey().of(RabbitMqConfig.SERVICE_NAME), report); + } + +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/config/RabbitMqConfig.java b/continuity.forecast/src/main/java/org/continuity/forecast/config/RabbitMqConfig.java new file mode 100644 index 00000000..2fd3ac50 --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/config/RabbitMqConfig.java @@ -0,0 +1,106 @@ +package org.continuity.forecast.config; + +import org.continuity.api.amqp.AmqpApi; +import org.springframework.amqp.core.AmqpTemplate; +import org.springframework.amqp.core.Binding; +import org.springframework.amqp.core.BindingBuilder; +import org.springframework.amqp.core.MessagePostProcessor; +import org.springframework.amqp.core.Queue; +import org.springframework.amqp.core.QueueBuilder; +import org.springframework.amqp.core.TopicExchange; +import org.springframework.amqp.rabbit.config.SimpleRabbitListenerContainerFactory; +import org.springframework.amqp.rabbit.connection.ConnectionFactory; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter; +import org.springframework.amqp.support.converter.MessageConverter; +import org.springframework.boot.autoconfigure.amqp.SimpleRabbitListenerContainerFactoryConfigurer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author Alper Hidiroglu + * + */ +@Configuration +public class RabbitMqConfig { + + public static final String SERVICE_NAME = "forecast"; + + public static final String TASK_CREATE_QUEUE_NAME = "continuity.forecast.task.forecast.create"; + + public static final String TASK_CREATE_ROUTING_KEY = "#"; + + public static final String DEAD_LETTER_QUEUE_NAME = AmqpApi.DEAD_LETTER_EXCHANGE.deriveQueueName(SERVICE_NAME); + + // General + + @Bean + MessageConverter jsonMessageConverter() { + return new Jackson2JsonMessageConverter(); + } + + @Bean + MessagePostProcessor typeRemovingProcessor() { + return m -> { + m.getMessageProperties().getHeaders().remove("__TypeId__"); + return m; + }; + } + + @Bean + AmqpTemplate rabbitTemplate(ConnectionFactory connectionFactory) { + final RabbitTemplate rabbitTemplate = new RabbitTemplate(connectionFactory); + rabbitTemplate.setMessageConverter(jsonMessageConverter()); + rabbitTemplate.setBeforePublishPostProcessors(typeRemovingProcessor()); + + return rabbitTemplate; + } + + @Bean + SimpleRabbitListenerContainerFactory containerFactory(ConnectionFactory connectionFactory, SimpleRabbitListenerContainerFactoryConfigurer configurer) { + SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory(); + configurer.configure(factory, connectionFactory); + factory.setMessageConverter(jsonMessageConverter()); + factory.setAfterReceivePostProcessors(typeRemovingProcessor()); + return factory; + } + + @Bean + TopicExchange taskCreateExchange() { + return AmqpApi.Forecast.TASK_CREATE.create(); + } + + @Bean + Queue taskCreateQueue() { + return QueueBuilder.nonDurable(TASK_CREATE_QUEUE_NAME).withArgument(AmqpApi.DEAD_LETTER_EXCHANGE_KEY, AmqpApi.DEAD_LETTER_EXCHANGE.name()) + .withArgument(AmqpApi.DEAD_LETTER_ROUTING_KEY_KEY, SERVICE_NAME).build(); + } + + @Bean + Binding taskCreateBinding() { + return BindingBuilder.bind(taskCreateQueue()).to(taskCreateExchange()).with(TASK_CREATE_ROUTING_KEY); + } + + @Bean + TopicExchange taskFinishedExchange() { + return AmqpApi.Global.EVENT_FINISHED.create(); + } + + // Dead letter exchange and queue + + @Bean + TopicExchange deadLetterExchange() { + return AmqpApi.DEAD_LETTER_EXCHANGE.create(); + } + + @Bean + Queue deadLetterQueue() { + return new Queue(DEAD_LETTER_QUEUE_NAME, true); + } + + @Bean + Binding deadLetterBinding() { + return BindingBuilder.bind(deadLetterQueue()).to(deadLetterExchange()).with(SERVICE_NAME); + } + +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/config/RestConfig.java b/continuity.forecast/src/main/java/org/continuity/forecast/config/RestConfig.java new file mode 100644 index 00000000..680e0ad6 --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/config/RestConfig.java @@ -0,0 +1,24 @@ +package org.continuity.forecast.config; + +import org.springframework.cloud.client.loadbalancer.LoadBalanced; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.web.client.RestTemplate; + +@Configuration +public class RestConfig { + + @LoadBalanced + @Bean + @Primary + RestTemplate eurekaRestTemplate() { + return new RestTemplate(); + } + + @Bean + RestTemplate plainRestTemplate() { + return new RestTemplate(); + } + +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/config/StorageConfig.java b/continuity.forecast/src/main/java/org/continuity/forecast/config/StorageConfig.java new file mode 100644 index 00000000..7d700b6b --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/config/StorageConfig.java @@ -0,0 +1,27 @@ +package org.continuity.forecast.config; + +import java.nio.file.Paths; +import java.util.Date; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.math3.util.Pair; +import org.continuity.api.entities.artifact.ForecastBundle; +import org.continuity.commons.storage.MixedStorage; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class StorageConfig { + + @Bean + public MixedStorage forecastStorage(@Value("${storage.path:storage}") String storagePath) { + return new MixedStorage<>(Paths.get(storagePath), new ForecastBundle()); + } + + @Bean + public ConcurrentHashMap> dateStorage() { + return new ConcurrentHashMap>(); + } + +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/context/BooleanCovariateValue.java b/continuity.forecast/src/main/java/org/continuity/forecast/context/BooleanCovariateValue.java new file mode 100644 index 00000000..0aef6f36 --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/context/BooleanCovariateValue.java @@ -0,0 +1,23 @@ +package org.continuity.forecast.context; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +@JsonDeserialize(as = BooleanCovariateValue.class) +public class BooleanCovariateValue extends GeneralCovariateValue implements CovariateValue { + + private boolean value; + + public boolean isValue() { + return value; + } + + public void setValue(boolean value) { + this.value = value; + } + + @Override + public String toString() { + return "BooleanCovariateValue [value=" + value + "]"; + } + +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/context/CovariateData.java b/continuity.forecast/src/main/java/org/continuity/forecast/context/CovariateData.java new file mode 100644 index 00000000..3aabc67f --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/context/CovariateData.java @@ -0,0 +1,50 @@ +package org.continuity.forecast.context; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +/** + * @author Alper Hidiroglu + */ +@JsonPropertyOrder({ "tag", "covar-name", "values" }) +public class CovariateData { + + private String tag; + + @JsonProperty("covar-name") + private String covarName; + + private List values; + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } + + public String getCovarName() { + return covarName; + } + + public void setCovarName(String covarName) { + this.covarName = covarName; + } + + public List getValues() { + return values; + } + + public void setValues(List values) { + this.values = values; + } + + @Override + public String toString() { + return "CovariateData [tag=" + tag + ", covarName=" + covarName + ", values=" + values + "]"; + } + +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/context/CovariateValue.java b/continuity.forecast/src/main/java/org/continuity/forecast/context/CovariateValue.java new file mode 100644 index 00000000..ac482af5 --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/context/CovariateValue.java @@ -0,0 +1,10 @@ +package org.continuity.forecast.context; + +import org.continuity.forecast.context.deserializer.CovariateDeserializer; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +@JsonDeserialize(using = CovariateDeserializer.class) +public interface CovariateValue { + +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/context/GeneralCovariateValue.java b/continuity.forecast/src/main/java/org/continuity/forecast/context/GeneralCovariateValue.java new file mode 100644 index 00000000..06ae0c62 --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/context/GeneralCovariateValue.java @@ -0,0 +1,15 @@ +package org.continuity.forecast.context; + +public class GeneralCovariateValue { + + private long timestamp; + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/context/NumericalCovariateValue.java b/continuity.forecast/src/main/java/org/continuity/forecast/context/NumericalCovariateValue.java new file mode 100644 index 00000000..dfb04531 --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/context/NumericalCovariateValue.java @@ -0,0 +1,23 @@ +package org.continuity.forecast.context; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +@JsonDeserialize(as = NumericalCovariateValue.class) +public class NumericalCovariateValue extends GeneralCovariateValue implements CovariateValue { + + private double value; + + public double getValue() { + return value; + } + + public void setValue(double value) { + this.value = value; + } + + @Override + public String toString() { + return "NumericalCovariateValue [value=" + value + "]"; + } + +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/context/StringCovariateValue.java b/continuity.forecast/src/main/java/org/continuity/forecast/context/StringCovariateValue.java new file mode 100644 index 00000000..6a8f49f8 --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/context/StringCovariateValue.java @@ -0,0 +1,23 @@ +package org.continuity.forecast.context; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +@JsonDeserialize(as = StringCovariateValue.class) +public class StringCovariateValue extends GeneralCovariateValue implements CovariateValue { + + private String value; + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + @Override + public String toString() { + return "StringCovariateValue [value=" + value + "]"; + } + +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/context/deserializer/CovariateDeserializer.java b/continuity.forecast/src/main/java/org/continuity/forecast/context/deserializer/CovariateDeserializer.java new file mode 100644 index 00000000..2c088297 --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/context/deserializer/CovariateDeserializer.java @@ -0,0 +1,45 @@ +package org.continuity.forecast.context.deserializer; + +import java.io.IOException; + +import org.continuity.forecast.context.BooleanCovariateValue; +import org.continuity.forecast.context.CovariateValue; +import org.continuity.forecast.context.NumericalCovariateValue; +import org.continuity.forecast.context.StringCovariateValue; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * Custom deserializer to differentiate between different types of covariate + * values. + * + * @author Alper Hidiroglu + * + */ +public class CovariateDeserializer extends JsonDeserializer { + + @Override + public CovariateValue deserialize(JsonParser p, DeserializationContext cntxt) + throws IOException, JsonProcessingException { + ObjectMapper mapper = (ObjectMapper) p.getCodec(); + ObjectNode root = mapper.readTree(p); + + CovariateValue covarVal = null; + + if (root.has("value")) { + if (root.get("value").isTextual()) { + covarVal = mapper.readValue(root.toString(), StringCovariateValue.class); + } else if (root.get("value").isBoolean()) { + covarVal = mapper.readValue(root.toString(), BooleanCovariateValue.class); + } else if (root.get("value").isNumber()) { + covarVal = mapper.readValue(root.toString(), NumericalCovariateValue.class); + } + } + return covarVal; + } +} \ No newline at end of file diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/controllers/ContextController.java b/continuity.forecast/src/main/java/org/continuity/forecast/controllers/ContextController.java new file mode 100644 index 00000000..1bc58fe1 --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/controllers/ContextController.java @@ -0,0 +1,39 @@ +package org.continuity.forecast.controllers; + +import static org.continuity.api.rest.RestApi.Forecast.Context.ROOT; +import static org.continuity.api.rest.RestApi.Forecast.Context.Paths.SUBMIT; + +import org.continuity.forecast.context.CovariateData; +import org.continuity.forecast.managers.CovariateDataManager; +import org.influxdb.InfluxDB; +import org.influxdb.InfluxDBFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +/** + * Example json input: {"tag":"appl-name","covar-name":"covar-name", "values":[{"timestamp": 123456789, "value": "some value"}]} + * @author Alper Hidiroglu + * + */ +@RestController() +@RequestMapping(ROOT) +public class ContextController { + + private static final Logger LOGGER = LoggerFactory.getLogger(ContextController.class); + + @RequestMapping(value = SUBMIT, method = RequestMethod.POST) + public ResponseEntity getData(@RequestBody CovariateData covarData) { + LOGGER.info("Received new order to process"); + InfluxDB influxDb = InfluxDBFactory.connect("http://127.0.0.1:8086", "admin", "admin"); + CovariateDataManager manager = new CovariateDataManager(influxDb); + manager.handleOrder(covarData); + influxDb.close(); + return new ResponseEntity(covarData.toString(), HttpStatus.OK); + } +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/controllers/ForecastController.java b/continuity.forecast/src/main/java/org/continuity/forecast/controllers/ForecastController.java new file mode 100644 index 00000000..50c987c5 --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/controllers/ForecastController.java @@ -0,0 +1,44 @@ +package org.continuity.forecast.controllers; + +import static org.continuity.api.rest.RestApi.Forecast.ForecastResult.ROOT; +import static org.continuity.api.rest.RestApi.Forecast.ForecastResult.Paths.GET; + +import org.continuity.api.entities.artifact.ForecastBundle; +import org.continuity.commons.storage.MixedStorage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +/** + * + * @author Alper Hi + * @author Henning Schulz + * + */ +@RestController() +@RequestMapping(ROOT) +public class ForecastController { + + private static final Logger LOGGER = LoggerFactory.getLogger(ForecastController.class); + + @Autowired + private MixedStorage storage; + + @RequestMapping(value = GET, method = RequestMethod.GET) + public ResponseEntity getForecastBundleFromLink(@PathVariable String id) { + ForecastBundle bundle = storage.get(id); + + if (bundle == null) { + LOGGER.warn("Could not find forecast for id {}!", id); + return ResponseEntity.notFound().build(); + } else { + LOGGER.info("Returned forecast for id {}!", id); + return ResponseEntity.ok(bundle); + } + } +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/managers/CovariateDataManager.java b/continuity.forecast/src/main/java/org/continuity/forecast/managers/CovariateDataManager.java new file mode 100644 index 00000000..292646b5 --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/managers/CovariateDataManager.java @@ -0,0 +1,77 @@ +package org.continuity.forecast.managers; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.continuity.forecast.context.BooleanCovariateValue; +import org.continuity.forecast.context.CovariateData; +import org.continuity.forecast.context.CovariateValue; +import org.continuity.forecast.context.NumericalCovariateValue; +import org.continuity.forecast.context.StringCovariateValue; +import org.influxdb.BatchOptions; +import org.influxdb.InfluxDB; +import org.influxdb.dto.Point; + +/** + * Manages covariate data. + * @author Alper Hidiroglu + * + */ +public class CovariateDataManager { + + private InfluxDB influxDb; + + /** + * @param influxDb + */ + public CovariateDataManager(InfluxDB influxDb) { + this.influxDb = influxDb; + } + + /** + * @param covarData + */ + @SuppressWarnings("deprecation") + public void handleOrder(CovariateData covarData) { + String dbName = covarData.getTag(); + if (!influxDb.describeDatabases().contains(dbName)) { + influxDb.createDatabase(dbName); + } + influxDb.setDatabase(dbName); + influxDb.setRetentionPolicy("autogen"); + + writeDataPoints(influxDb, dbName, covarData.getCovarName(), covarData.getValues()); + } + + /** + * @param influxDb + * @param dbName + * @param covarName + * @param values + */ + public void writeDataPoints(InfluxDB influxDb, String dbName, String covarName, List values) { + influxDb.enableBatch(BatchOptions.DEFAULTS); + for(CovariateValue value: values) { + Point point = null; + if(value instanceof StringCovariateValue) { + point = Point.measurement(covarName) + .time(((StringCovariateValue) value).getTimestamp(), TimeUnit.MILLISECONDS) + .addField("value", ((StringCovariateValue) value).getValue()) + .build(); + } else if(value instanceof BooleanCovariateValue) { + point = Point.measurement(covarName) + .time(((BooleanCovariateValue) value).getTimestamp(), TimeUnit.MILLISECONDS) + .addField("value", ((BooleanCovariateValue) value).isValue()) + .build(); + } else if(value instanceof NumericalCovariateValue) { + point = Point.measurement(covarName) + .time(((NumericalCovariateValue) value).getTimestamp(), TimeUnit.MILLISECONDS) + .addField("value", ((NumericalCovariateValue) value).getValue()) + .build(); + } + influxDb.write(point); + + } + influxDb.disableBatch(); + } +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/managers/ForecastPipelineManager.java b/continuity.forecast/src/main/java/org/continuity/forecast/managers/ForecastPipelineManager.java new file mode 100644 index 00000000..8604fcd6 --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/managers/ForecastPipelineManager.java @@ -0,0 +1,797 @@ +package org.continuity.forecast.managers; + +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; + +import org.apache.commons.math3.util.Pair; +import org.continuity.api.entities.artifact.ForecastBundle; +import org.continuity.dsl.description.ContextParameter; +import org.continuity.dsl.description.ForecastInput; +import org.continuity.dsl.description.FutureEvent; +import org.continuity.dsl.description.FutureEvents; +import org.continuity.dsl.description.FutureNumber; +import org.continuity.dsl.description.FutureNumbers; +import org.continuity.dsl.description.FutureOccurrences; +import org.continuity.dsl.description.Measurement; +import org.influxdb.InfluxDB; +import org.influxdb.dto.Query; +import org.influxdb.dto.QueryResult; +import org.influxdb.dto.QueryResult.Result; +import org.influxdb.dto.QueryResult.Series; +import org.rosuda.JRI.Rengine; + +/** + * Manager for the workload forecasting. + * + * @author Alper Hidiroglu + * + */ +public class ForecastPipelineManager { + + // private static final Logger LOGGER = LoggerFactory.getLogger(ForecastPipelineManager.class); + + private InfluxDB influxDb; + + private String tag; + + private ForecastInput forecastInput; + + private int workloadIntensity; + + public int getWorkloadIntensity() { + return workloadIntensity; + } + + public void setWorkloadIntensity(int workloadIntensity) { + this.workloadIntensity = workloadIntensity; + } + + /** + * Constructor. + */ + public ForecastPipelineManager(InfluxDB influxDb, String tag, ForecastInput forecastInput) { + this.influxDb = influxDb; + this.tag = tag; + this.forecastInput = forecastInput; + } + + public void setupDatabase() { + String dbName = this.tag; + influxDb.setDatabase(dbName); + influxDb.setRetentionPolicy("autogen"); + } + + /** + * Runs the pipeline. + * + * @return The generated forecast bundle. + */ + public ForecastBundle runPipeline(Pair dateAndAmountOfUsers) { + setupDatabase(); + ForecastBundle forecastBundle = generateForecastBundle(dateAndAmountOfUsers); + + return forecastBundle; + } + + /** + * Generates the forecast bundle. + * + * @param logs + * @return + * @throws IOException + * @throws ExtractionException + * @throws ParseException + */ + private ForecastBundle generateForecastBundle(Pair dateAndAmountOfUsers) { + // initialize intensity + this.workloadIntensity = 1; + // updates also the workload intensity + LinkedList probabilities = forecastWorkload(dateAndAmountOfUsers.getValue()); + // forecast result + return new ForecastBundle(dateAndAmountOfUsers.getKey(), this.workloadIntensity, probabilities); + } + + /** + * Returns aggregated workload intensity and adapted behavior mix probabilities. + * + * @param bundleList + * @return + */ + private LinkedList forecastWorkload(int amountOfUserGroups) { + Rengine re = initializeRengine(); + + LinkedList probabilities = new LinkedList(); + int sumOfIntensities = 0; + List forecastedIntensities = new LinkedList(); + + if (forecastInput.getForecastOptions().getForecaster().equalsIgnoreCase("Telescope")) { + initializeTelescope(re); + + for (int i = 0; i < amountOfUserGroups; i++) { + int intensity = forecastIntensityForUserGroupTelescope(i, re); + forecastedIntensities.add(intensity); + sumOfIntensities += intensity; + } + } else if (forecastInput.getForecastOptions().getForecaster().equalsIgnoreCase("Prophet")) { + initializeProphet(re); + + for (int i = 0; i < amountOfUserGroups; i++) { + int intensity = forecastIntensityForUserGroupProphet(i, re); + forecastedIntensities.add(intensity); + sumOfIntensities += intensity; + } + } + re.end(); + // updates the workload intensity + setWorkloadIntensity(sumOfIntensities); + + for (int intensity : forecastedIntensities) { + double probability = (double) intensity / (double) sumOfIntensities; + probabilities.add(probability); + } + return probabilities; + } + + /** + * Forecasting the intensities for a user group using Prophet. + * + * @param i + * @param re + * @return + */ + private int forecastIntensityForUserGroupProphet(int i, Rengine re) { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + sdf.setTimeZone(TimeZone.getDefault()); + + Pair, ArrayList> timestampsAndIntensities = getIntensitiesOfUserGroupFromDatabase(i); + ArrayList timestampsOfIntensities = timestampsAndIntensities.getKey(); + ArrayList intensities = timestampsAndIntensities.getValue(); + + Pair>>, ArrayList>>> covariates = calculateCovariates(timestampsOfIntensities, intensities); + + ArrayList datesOfIntensities = new ArrayList(); + for (long timestamp : timestampsOfIntensities) { + Date date = new Date(); + date.setTime((long) timestamp); + String resultDateFromTimestamp = sdf.format(date); + datesOfIntensities.add(resultDateFromTimestamp); + } + int size = calculateSizeOfForecast(timestampsOfIntensities); + int intensity = forecastWithProphet(datesOfIntensities, intensities, covariates.getKey(), covariates.getValue(), size, re); + return intensity; + } + + /** + * Counts amount of forecasted values subway. + * + * @param timestampsOfIntensities + * timestamps + * @return the amount + */ + private int calculateSizeOfForecast(ArrayList timestampsOfIntensities) { + long endTimeIntensities = timestampsOfIntensities.get(timestampsOfIntensities.size() - 1); + long endTimeForecast = this.forecastInput.getForecastOptions().getDateAsTimestamp(); + long interval = calculateInterval(this.forecastInput.getForecastOptions().getInterval()); + + // Calculates considered future timestamps + long startTimeForecast = endTimeIntensities + interval; + int size = 0; + while (startTimeForecast <= endTimeForecast) { + size++; + startTimeForecast += interval; + } + return size; + } + + /** + * Forecasting the intensities for a user group using Telescope. + * + * @param i + * @param re + * @return + */ + private int forecastIntensityForUserGroupTelescope(int i, Rengine re) { + Pair, ArrayList> timestampsAndIntensities = getIntensitiesOfUserGroupFromDatabase(i); + ArrayList timestampsOfIntensities = timestampsAndIntensities.getKey(); + ArrayList intensities = timestampsAndIntensities.getValue(); + + Pair>>, ArrayList>>> covariates = calculateCovariates(timestampsOfIntensities, intensities); + int size = calculateSizeOfForecast(timestampsOfIntensities); + int intensity = forecastWithTelescope(intensities, covariates.getKey(), covariates.getValue(), size, re); + return intensity; + } + + /** + * Calculates historical and future covariates. + * + * @param timestampsOfIntensities + * @param intensities + * @return + */ + private Pair>>, ArrayList>>> calculateCovariates(ArrayList timestampsOfIntensities, ArrayList intensities) { + + ArrayList>> histCovariates = new ArrayList>>(); + ArrayList>> futureCovariates = new ArrayList>>(); + + if (forecastInput.getContext() != null) { + + long startTimeOfIntensities = timestampsOfIntensities.get(0); + long endTimeIntensities = timestampsOfIntensities.get(timestampsOfIntensities.size() - 1); + + long endTimeForecast = this.forecastInput.getForecastOptions().getDateAsTimestamp(); + + long interval = calculateInterval(this.forecastInput.getForecastOptions().getInterval()); + + // Calculates considered future timestamps + long startTimeForecast = endTimeIntensities + interval; + long forecastTimestamp = startTimeForecast; + ArrayList futureTimestamps = new ArrayList(); + while (forecastTimestamp <= endTimeForecast) { + futureTimestamps.add(forecastTimestamp); + forecastTimestamp += interval; + } + + for (ContextParameter covar : forecastInput.getContext()) { + if (covar instanceof FutureNumbers) { + FutureNumbers numCovar = (FutureNumbers) covar; + + ArrayList historicalOccurrences = calculateHistoricalOccurrencesNumerical(numCovar, timestampsOfIntensities, startTimeOfIntensities, endTimeIntensities); + ArrayList futureOccurrences = calculateFutureOccurrencesNumerical(numCovar, futureTimestamps, startTimeForecast, endTimeForecast); + + List numericInstances = numCovar.getFuture(); + + for (FutureNumber numInstance : numericInstances) { + double value = numInstance.getValue(); + FutureOccurrences futureTimes = numInstance.getTime(); + List futureTimestampsOfValue = futureTimes.getFutureDatesAsTimestamps(interval); + + for (long futureTimestampOfValue : futureTimestampsOfValue) { + int index = futureTimestamps.indexOf(futureTimestampOfValue); + futureOccurrences.set(index, value); + } + } + Pair> covarPast = new Pair<>(numCovar.getMeasurement(), historicalOccurrences); + Pair> covarFuture = new Pair<>(numCovar.getMeasurement(), futureOccurrences); + histCovariates.add(covarPast); + futureCovariates.add(covarFuture); + + } else if (covar instanceof FutureEvents) { + FutureEvents stringCovar = (FutureEvents) covar; + List stringInstances = stringCovar.getFuture(); + + HashMap> eventMap = calculateEventMap(stringCovar, startTimeOfIntensities, endTimeIntensities, startTimeForecast, endTimeForecast); + + for (Map.Entry> entry : eventMap.entrySet()) { + String value = entry.getKey(); + ArrayList timestamps = entry.getValue(); + + ArrayList historicalOccurrences = new ArrayList(Collections.nCopies(timestampsOfIntensities.size(), 0.0)); + ArrayList futureOccurrences = new ArrayList(Collections.nCopies(futureTimestamps.size(), 0.0)); + + calculateOccurrencesString(futureOccurrences, historicalOccurrences, timestamps, timestampsOfIntensities, futureTimestamps, endTimeIntensities); + + for (FutureEvent stringInstance : stringInstances) { + String valueOfInstance = stringInstance.getValue(); + if (valueOfInstance.equals(value)) { + FutureOccurrences futureTimes = stringInstance.getTime(); + List futureTimestampsOfValue = futureTimes.getFutureDatesAsTimestamps(interval); + + for (long futureTimestampOfValue : futureTimestampsOfValue) { + int index = futureTimestamps.indexOf(futureTimestampOfValue); + futureOccurrences.set(index, 1.0); + } + } + } + Pair> covarPast = new Pair<>(value, historicalOccurrences); + Pair> covarFuture = new Pair<>(value, futureOccurrences); + histCovariates.add(covarPast); + futureCovariates.add(covarFuture); + } + + } else if (covar instanceof Measurement) { + Measurement mCovar = (Measurement) covar; + boolean isNumeric = identifyIfNumericValues(mCovar); + if (isNumeric) { + + ArrayList historicalOccurrences = calculateHistoricalOccurrencesNumerical(mCovar, timestampsOfIntensities, startTimeOfIntensities, endTimeIntensities); + ArrayList futureOccurrences = calculateFutureOccurrencesNumerical(mCovar, futureTimestamps, startTimeForecast, endTimeForecast); + + Pair> covarPast = new Pair<>(mCovar.getMeasurement(), historicalOccurrences); + Pair> covarFuture = new Pair<>(mCovar.getMeasurement(), futureOccurrences); + histCovariates.add(covarPast); + futureCovariates.add(covarFuture); + + } else { + + HashMap> eventMap = calculateEventMap(mCovar, startTimeOfIntensities, endTimeIntensities, startTimeForecast, endTimeForecast); + + for (Map.Entry> entry : eventMap.entrySet()) { + ArrayList timestamps = entry.getValue(); + + ArrayList historicalOccurrences = new ArrayList(Collections.nCopies(timestampsOfIntensities.size(), 0.0)); + ArrayList futureOccurrences = new ArrayList(Collections.nCopies(futureTimestamps.size(), 0.0)); + + calculateOccurrencesString(futureOccurrences, historicalOccurrences, timestamps, timestampsOfIntensities, futureTimestamps, endTimeIntensities); + + Pair> covarPast = new Pair<>(entry.getKey(), historicalOccurrences); + Pair> covarFuture = new Pair<>(entry.getKey(), futureOccurrences); + histCovariates.add(covarPast); + futureCovariates.add(covarFuture); + } + } + } + } + } + Pair>>, ArrayList>>> covariates = new Pair<>(histCovariates, futureCovariates); + return covariates; + } + + /** + * Calculates occurrences for Strings. + * + * @param futureOccurrences + * @param historicalOccurrences + * @param timestamps + * @param timestampsOfIntensities + * @param futureTimestamps + * @param endTimeIntensities + */ + private void calculateOccurrencesString(ArrayList futureOccurrences, ArrayList historicalOccurrences, ArrayList timestamps, ArrayList timestampsOfIntensities, + ArrayList futureTimestamps, long endTimeIntensities) { + + for (long timestamp : timestamps) { + if (timestamp <= endTimeIntensities) { + int index = timestampsOfIntensities.indexOf(timestamp); + historicalOccurrences.set(index, 1.0); + } else { + int index = futureTimestamps.indexOf(timestamp); + futureOccurrences.set(index, 1.0); + } + } + } + + /** + * Calculates a HashMap containing all event (String) covariates. + * + * @param mCovar + * @param startTime + * @param endTimeIntensities + * @param startTimeForecast + * @param endTimeForecast + * @return + */ + private HashMap> calculateEventMap(Measurement mCovar, long startTime, long endTimeIntensities, long startTimeForecast, long endTimeForecast) { + HashMap> eventMap = new HashMap>(); + + ArrayList> histValues = getStringValues(mCovar, convertTimestampToUtcDate(startTime), convertTimestampToUtcDate(endTimeIntensities)); + ArrayList> futValues = getStringValues(mCovar, convertTimestampToUtcDate(startTimeForecast), convertTimestampToUtcDate(endTimeForecast)); + + for (Pair histValue : histValues) { + String value = histValue.getValue(); + long timestamp = histValue.getKey(); + if (eventMap.containsKey(value)) { + eventMap.get(value).add(timestamp); + } else { + ArrayList timestamps = new ArrayList(); + timestamps.add(timestamp); + eventMap.put(value, timestamps); + } + } + + // Only future values where corresponding past values exist + for (Pair futValue : futValues) { + String value = futValue.getValue(); + if (eventMap.containsKey(value)) { + eventMap.get(value).add(futValue.getKey()); + } + } + return eventMap; + } + + /** + * Calculates historical occurrences for doubles. + * + * @param mCovar + * @param timestampsOfIntensities + * @param startTime + * @param endTimeIntensities + * @return + */ + private ArrayList calculateHistoricalOccurrencesNumerical(Measurement mCovar, ArrayList timestampsOfIntensities, long startTime, long endTimeIntensities) { + ArrayList historicalOccurrences = new ArrayList(Collections.nCopies(timestampsOfIntensities.size(), 0.0)); + ArrayList> histValues = getNumericValues(mCovar, convertTimestampToUtcDate(startTime), convertTimestampToUtcDate(endTimeIntensities)); + + for (Pair histValue : histValues) { + int index = timestampsOfIntensities.indexOf(histValue.getKey()); + historicalOccurrences.set(index, histValue.getValue()); + } + + return historicalOccurrences; + } + + /** + * Calculates future occurrences for doubles. + * + * @param mCovar + * @param futureTimestamps + * @param startTimeForecast + * @param endTimeForecast + * @return + */ + private ArrayList calculateFutureOccurrencesNumerical(Measurement mCovar, ArrayList futureTimestamps, long startTimeForecast, long endTimeForecast) { + ArrayList futureOccurrences = new ArrayList(Collections.nCopies(futureTimestamps.size(), 0.0)); + ArrayList> futValues = getNumericValues(mCovar, convertTimestampToUtcDate(startTimeForecast), convertTimestampToUtcDate(endTimeForecast)); + + for (Pair futValue : futValues) { + int index = futureTimestamps.indexOf(futValue.getKey()); + futureOccurrences.set(index, futValue.getValue()); + } + return futureOccurrences; + } + + /** + * Passes intensities dataset and covariates to Prophet which does the forecasting. Aggregates + * the resulting intensities to one intensity value. + * + * @param datesOfIntensities + * @param intensitiesOfUserGroup + * @param size + * @param re + * @return + */ + private int forecastWithProphet(ArrayList datesOfIntensities, ArrayList intensitiesOfUserGroup, ArrayList>> covariates, + ArrayList>> futureCovariates, int size, Rengine re) { + + double[] intensities = intensitiesOfUserGroup.stream().mapToDouble(i -> i).toArray(); + String[] dates = new String[datesOfIntensities.size()]; + dates = datesOfIntensities.toArray(dates); + + re.assign("dates", dates); + re.assign("intensities", intensities); + + re.eval("source(\"prophet/InitializeVariables.R\")"); + + if (!covariates.isEmpty()) { + for (Pair> covariate : covariates) { + double[] values = covariate.getValue().stream().mapToDouble(i -> i).toArray(); + re.assign("values", values); + re.assign("covarname", covariate.getKey()); + re.eval("source(\"prophet/AddRegressors.R\")"); + } + } + + String period = Integer.toString(size); + re.assign("period", period); + + re.eval("source(\"prophet/FitModelAndCreateFutureDataframe.R\")"); + + if (!covariates.isEmpty()) { + int x = 0; + for (Pair> covariate : covariates) { + covariate.getValue().addAll(futureCovariates.get(x).getValue()); + double[] values = covariate.getValue().stream().mapToDouble(i -> i).toArray(); + re.assign("values", values); + re.assign("covarname", covariate.getKey()); + re.eval("source(\"prophet/ExtendFutureDataframe.R\")"); + x++; + } + } + + re.eval("source(\"prophet/ForecastProphet.R\")"); + + double[] forecastedIntensities = re.eval("forecastValues").asDoubleArray(); + + return aggregateWorkload(forecastedIntensities); + } + + /** + * Passes intensities dataset and covariates to Telescope which does the forecasting. Aggregates + * the resulting intensities to one intensity value. + * + * @param intensitiesOfUserGroup + * @return + */ + private int forecastWithTelescope(ArrayList intensitiesOfUserGroup, ArrayList>> covariates, ArrayList>> futureCovariates, + int size, Rengine re) { + if (!covariates.isEmpty()) { + // hist.covar + String matrixString = calculateMatrix("hist", covariates, re); + re.assign("hist.covar.matrix", re.eval(matrixString)); + + // future.covar + String futureMatrixString = calculateMatrix("future", futureCovariates, re); + re.assign("future.covar.matrix", re.eval(futureMatrixString)); + } + + double[] intensities = intensitiesOfUserGroup.stream().mapToDouble(i -> i).toArray(); + + String period = Integer.toString(size); + + re.assign("intensities", intensities); + re.assign("period", period); + + if (!covariates.isEmpty()) { + re.eval("source(\"telescope-multi/ForecastTelescopeWithCovariates.R\")"); + } else { + re.eval("source(\"telescope-multi/ForecastTelescope.R\")"); + } + re.eval("dev.off()"); + + double[] forecastedIntensities = re.eval("forecastValues").asDoubleArray(); + + return aggregateWorkload(forecastedIntensities); + } + + /** + * Aggregates the forecasted workload. TODO: Check other possibilities for workload aggregation. + * Information should be passed by user. + * + * @param forecastedIntensities + * @return + */ + private int aggregateWorkload(double[] forecastedIntensities) { + double maxIntensity = 0; + for (int i = 0; i < forecastedIntensities.length; i++) { + if (forecastedIntensities[i] > maxIntensity) { + maxIntensity = forecastedIntensities[i]; + } + } + return (int) Math.round(maxIntensity); + } + + /** + * Calculates a covariate matrix. Such a matrix is an input to Telescope. + * + * @param string + * @param covariates + * @param re + * @return + */ + private String calculateMatrix(String string, ArrayList>> covariates, Rengine re) { + ArrayList nameOfCovars = new ArrayList(); + for (Pair> covariateValues : covariates) { + String name = covariateValues.getKey() + "." + string; + double[] occurrences = covariateValues.getValue().stream().mapToDouble(i -> i).toArray(); + re.assign(name, occurrences); + nameOfCovars.add(name); + } + + String matrixString = "cbind("; + + boolean isFirst = true; + for (String name : nameOfCovars) { + if (isFirst) { + matrixString += name; + isFirst = false; + } else { + matrixString += "," + name; + } + } + matrixString += ")"; + return matrixString; + } + + /** + * Returns interval in numerical representation. + * + * @param interval + * @return + */ + private long calculateInterval(String interval) { + long numericInterval = 0; + switch (interval) { + case "secondly": + numericInterval = 1000L; + break; + case "minutely": + numericInterval = 60000L; + break; + case "hourly": + numericInterval = 3600000L; + break; + default: + numericInterval = 1000L; + } + return numericInterval; + } + + /** + * Converts a milliseconds timestamp to UTC date as string. + * + * @param timestamp + * @return + */ + private String convertTimestampToUtcDate(long timestamp) { + DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); + String date = Instant.ofEpochMilli(timestamp).atOffset(ZoneOffset.UTC).format(dtf).toString(); + return date; + } + + /** + * Tests if values in a measurement are numerical. + * + * @param mCovar + * @return + */ + private boolean identifyIfNumericValues(Measurement mCovar) { + String measurementName = mCovar.getMeasurement(); + String queryString = "SELECT time, value FROM " + measurementName; + Query query = new Query(queryString, tag); + QueryResult queryResult = influxDb.query(query); + Result result = queryResult.getResults().get(0); + if (result.getSeries() != null) { + Series serie = result.getSeries().get(0); + if (serie.getValues().get(0).get(1).getClass().toString().equalsIgnoreCase("class java.lang.Double")) { + return true; + } + } + return false; + } + + /** + * Gets numeric values from database. + * + * @param covariateValues + * @param covar + * @param startTime + * @param endTime + * @return + */ + private ArrayList> getNumericValues(Measurement covar, String startTime, String endTime) { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + ArrayList> measurements = new ArrayList>(); + String measurementName = covar.getMeasurement(); + String queryString = "SELECT time, value FROM " + measurementName + " WHERE time >= '" + startTime + "' AND time <= '" + endTime + "'"; + Query query = new Query(queryString, tag); + QueryResult queryResult = influxDb.query(query); + for (Result result : queryResult.getResults()) { + if (result.getSeries() != null) { + for (Series serie : result.getSeries()) { + for (List listTuples : serie.getValues()) { + long time = 0; + try { + time = sdf.parse((String) listTuples.get(0)).getTime(); + } catch (ParseException e) { + + } + double measurement = (double) listTuples.get(1); + Pair timeAndValue = new Pair<>(time, measurement); + measurements.add(timeAndValue); + } + } + } + } + return measurements; + } + + /** + * Gets String values from database. + * + * @param covar + * @param startTime + * @param endTime + */ + private ArrayList> getStringValues(Measurement covar, String startTime, String endTime) { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + ArrayList> events = new ArrayList>(); + String measurementName = covar.getMeasurement(); + String queryString = "SELECT time, value FROM " + measurementName + " WHERE time >= '" + startTime + "' AND time <= '" + endTime + "'"; + Query query = new Query(queryString, tag); + QueryResult queryResult = influxDb.query(query); + for (Result result : queryResult.getResults()) { + if (result.getSeries() != null) { + for (Series serie : result.getSeries()) { + for (List listTuples : serie.getValues()) { + long time = 0; + try { + time = sdf.parse((String) listTuples.get(0)).getTime(); + } catch (ParseException e) { + + } + String event = (String) listTuples.get(1); + Pair timeAndValue = new Pair<>(time, event); + events.add(timeAndValue); + } + } + } + } + return events; + } + + /** + * Gets the intensities of a user group from database. + * + * @param userGroupId + * @return + */ + public Pair, ArrayList> getIntensitiesOfUserGroupFromDatabase(int userGroupId) { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + Pair, ArrayList> timestampsAndIntensities = null; + ArrayList timestamps = new ArrayList(); + ArrayList intensities = new ArrayList(); + String measurementName = "userGroup" + userGroupId; + Query query = new Query("SELECT time, value FROM " + measurementName, tag); + QueryResult queryResult = influxDb.query(query); + for (Result result : queryResult.getResults()) { + if (result.getSeries() != null) { + for (Series serie : result.getSeries()) { + for (List listTuples : serie.getValues()) { + long time = 0; + try { + time = sdf.parse((String) listTuples.get(0)).getTime(); + } catch (ParseException e) { + + } + double intensity = (double) listTuples.get(1); + timestamps.add(time); + intensities.add(intensity); + } + } + } + } + timestampsAndIntensities = new Pair<>(timestamps, intensities); + return timestampsAndIntensities; + } + + /** + * Initializes Telescope. + * + * @param re + */ + private void initializeTelescope(Rengine re) { + re.eval("source(\"telescope-multi/R/telescope.R\")"); + re.eval("source(\"telescope-multi/R/cluster_periods.R\")"); + re.eval("source(\"telescope-multi/R/detect_anoms.R\")"); + re.eval("source(\"telescope-multi/R/fitting_models.R\")"); + re.eval("source(\"telescope-multi/R/frequency.R\")"); + re.eval("source(\"telescope-multi/R/outlier.R\")"); + re.eval("source(\"telescope-multi/R/telescope_Utils.R\")"); + re.eval("source(\"telescope-multi/R/vec_anom_detection.R\")"); + re.eval("source(\"telescope-multi/R/xgb.R\")"); + + re.eval("library(xgboost)"); + re.eval("library(cluster)"); + re.eval("library(forecast)"); + re.eval("library(e1071)"); + } + + /** + * Initializes Prophet. + * + * @param re + */ + private void initializeProphet(Rengine re) { + re.eval("library(prophet)"); + } + + /** + * Initializes Rengine. + * + * @return + */ + private Rengine initializeRengine() { + String newargs1[] = { "--no-save" }; + + Rengine re = Rengine.getMainEngine(); + if (re == null) { + re = new Rengine(newargs1, false, null); + } + re.eval(".libPaths('/usr/local/lib/R/site-library/')"); + return re; + } +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/managers/IntensitiesPipelineManager.java b/continuity.forecast/src/main/java/org/continuity/forecast/managers/IntensitiesPipelineManager.java new file mode 100644 index 00000000..f2a18828 --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/managers/IntensitiesPipelineManager.java @@ -0,0 +1,323 @@ +package org.continuity.forecast.managers; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang3.Range; +import org.apache.commons.math3.util.Pair; +import org.continuity.api.entities.artifact.SessionsBundle; +import org.continuity.api.entities.artifact.SessionsBundlePack; +import org.continuity.api.entities.artifact.SimplifiedSession; +import org.continuity.commons.utils.WebUtils; +import org.continuity.dsl.description.ForecastInput; +import org.influxdb.BatchOptions; +import org.influxdb.InfluxDB; +import org.influxdb.dto.Point; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; + + +/** + * Manager for calculating intensities and saving them into database. + * @author Alper Hidiroglu + * + */ +public class IntensitiesPipelineManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(ForecastPipelineManager.class); + + private RestTemplate restTemplate; + + private InfluxDB influxDb; + + private String tag; + + private ForecastInput forecastInput; + + private Pair dateAndAmountOfUserGroups; + + public Pair getDateAndAmountOfUserGroups() { + return dateAndAmountOfUserGroups; + } + + public void setDateAndAmountOfUserGroups(Pair dateAndAmountOfUsers) { + this.dateAndAmountOfUserGroups = dateAndAmountOfUsers; + } + + + private int workloadIntensity; + + public int getWorkloadIntensity() { + return workloadIntensity; + } + + public void setWorkloadIntensity(int workloadIntensity) { + this.workloadIntensity = workloadIntensity; + } + + /** + * Constructor. + */ + public IntensitiesPipelineManager(RestTemplate restTemplate, InfluxDB influxDb, String tag, ForecastInput context) { + this.restTemplate = restTemplate; + this.influxDb = influxDb; + this.tag = tag; + this.forecastInput = context; + } + + public IntensitiesPipelineManager() { + + } + + @SuppressWarnings("deprecation") + public void setupDatabase(){ + String dbName = this.tag; + if (!influxDb.describeDatabases().contains(dbName)) { + influxDb.createDatabase(dbName); + } + influxDb.setDatabase(dbName); + influxDb.setRetentionPolicy("autogen"); + } + + /** + * Runs the pipeline. + */ + public void runPipeline(String linkToSessions) { + setupDatabase(); + SessionsBundlePack sessionsBundles = null; + try { + sessionsBundles = restTemplate.getForObject(WebUtils.addProtocolIfMissing(linkToSessions), SessionsBundlePack.class); + } catch (RestClientException e) { + LOGGER.error("Error when retrieving sessions!", e); + } + + Date date = sessionsBundles.getTimestamp(); + int amountOfUsers = sessionsBundles.getSessionsBundles().size(); + Pair pairDateUserGroupAmount = new Pair<>(date, amountOfUsers); + setDateAndAmountOfUserGroups(pairDateUserGroupAmount); + + influxDb.enableBatch(BatchOptions.DEFAULTS); + + calculateIntensities(sessionsBundles.getSessionsBundles()); + + influxDb.disableBatch(); + } + + /** + * @param bundleList + */ + private void calculateIntensities(List bundleList) { + for (SessionsBundle sessBundle : bundleList) { + List sessions = sessBundle.getSessions(); + int behaviorId = sessBundle.getBehaviorId(); + calculateIntensitiesForUserGroup(sessions, behaviorId); + } + } + + /** + * Calculates the intensities for one user group. Saves the intensities into database. + * Timestamps are in nanoseconds. + * @param sessions + * @return + */ + private void calculateIntensitiesForUserGroup(List sessions, int behaviorId) { + sortSessions(sessions); + long startTime = sessions.get(0).getStartTime(); + + // The time range for which an intensity will be calculated + long rangeLength = calculateInterval(forecastInput.getForecastOptions().getInterval()); + + // rounds start time down + long roundedStartTime = startTime - startTime % rangeLength; + + long highestEndTime = 0; + + for(SimplifiedSession session: sessions) { + if(session.getEndTime() > highestEndTime) { + highestEndTime = session.getEndTime(); + } + } + // rounds highest end time up + long roundedHighestEndTime = highestEndTime; + if (highestEndTime % rangeLength != 0) { + roundedHighestEndTime = (highestEndTime - highestEndTime % rangeLength) + rangeLength; + } + + long completePeriod = roundedHighestEndTime - roundedStartTime; + long amountOfRanges = completePeriod / rangeLength; + + ArrayList> listOfRanges = calculateRanges(roundedStartTime, amountOfRanges, rangeLength); + + // Remove first and last range from list if necessary + if(listOfRanges.get(0).getMinimum() != startTime) { + listOfRanges.remove(0); + } + + if(listOfRanges.get(listOfRanges.size() - 1).getMaximum() != highestEndTime) { + listOfRanges.remove(listOfRanges.size() - 1); + } + + // This map is used to hold necessary information which will be saved into DB + HashMap intensities = new HashMap(); + + for(Range range: listOfRanges) { + ArrayList sessionsInRange = new ArrayList(); + for(SimplifiedSession session: sessions) { + Range sessionRange = Range.between(session.getStartTime(), session.getEndTime()); + if(sessionRange.containsRange(range) || range.contains(session.getStartTime()) + || range.contains(session.getEndTime())) { + sessionsInRange.add(session); + } + } + int intensityOfRange = (int) calculateIntensityForRange(range, sessionsInRange, rangeLength); + intensities.put(range.getMinimum(), intensityOfRange); + } + + saveIntensitiesOfUserGroupIntoDb(intensities, behaviorId); + } + + /** + * Saves intensities into InfluxDB + * @param intensities + */ + @SuppressWarnings("rawtypes") + private void saveIntensitiesOfUserGroupIntoDb(HashMap intensities, int behaviorId) { + String measurementName = "userGroup" + behaviorId; + Iterator iterator = intensities.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry pair = (Map.Entry)iterator.next(); + Point point = Point.measurement(measurementName) + .time((long) pair.getKey(), TimeUnit.NANOSECONDS) + .addField("value", (int) pair.getValue()) + .build(); + + influxDb.write(point); + iterator.remove(); + } + } + + protected long calculateInterval(String interval) { + long numericInterval = 0; + switch(interval) { + case "secondly": + numericInterval = 1000000000L; + break; + case "minutely": + numericInterval = 60000000000L; + break; + case "hourly": + numericInterval = 3600000000000L; + break; + default: + numericInterval = 1000000000L; + } + return numericInterval; + } + + /** + * Calculates the time ranges. + * @param startTime + * @param amountOfRanges + * @param rangeLength + * @return + */ + private ArrayList> calculateRanges(long startTime, long amountOfRanges, long rangeLength ) { + ArrayList> listOfRanges = new ArrayList>(); + for(int i = 0; i < amountOfRanges; i++) { + Range range = Range.between(startTime, startTime + rangeLength); + listOfRanges.add(range); + startTime += rangeLength; + } + return listOfRanges; + } + + /** + * Calculates the workload intensity for a time range. Calculates average, min and max. + * @param range + * @param sessionsInRange + * @param rangeLength + * @return + */ + private long calculateIntensityForRange(Range range, ArrayList sessionsInRange, long rangeLength) { + int counter = 0; + long sumOfTime = 0; + boolean inTimeRange = true; + long endOfRange = range.getMaximum(); + // smallest found timestamp + long lastOccurredEvent = range.getMinimum(); + + // initialize the counter with amount of sessions at the beginning of the range + for(SimplifiedSession session: sessionsInRange) { + Range sessionRange = Range.between(session.getStartTime(), session.getEndTime()); + if(sessionRange.contains(lastOccurredEvent)) { + counter++; + } + } + // min value of range + int minCounter = counter; + // max value of range + int maxCounter = counter; + + while(inTimeRange) { + long minValue = Long.MAX_VALUE; + int currentCounter = counter; + for(SimplifiedSession session: sessionsInRange) { + long startTimeOfSession = session.getStartTime(); + long endTimeOfSession = session.getEndTime(); + if(startTimeOfSession > lastOccurredEvent) { + if(startTimeOfSession == minValue) { + currentCounter ++; + } else if (startTimeOfSession < minValue){ + currentCounter = counter + 1; + minValue = startTimeOfSession; + } + } else if(endTimeOfSession > lastOccurredEvent) { + if(endTimeOfSession == minValue) { + currentCounter --; + } else if (endTimeOfSession < minValue) { + currentCounter = counter - 1; + minValue = endTimeOfSession; + } + } + } + if(minValue > endOfRange) { + minValue = endOfRange; + inTimeRange = false; + } + sumOfTime += counter * (minValue - lastOccurredEvent); + lastOccurredEvent = minValue; + + counter = currentCounter; + + if(counter < minCounter) { + minCounter = counter; + } + if(counter > maxCounter) { + maxCounter = counter; + } + } + return sumOfTime / rangeLength; + } + + + /** + * Sorts sessions. + * @param sessions + */ + private void sortSessions(List sessions) { + sessions.sort((SimplifiedSession sess1, SimplifiedSession sess2) -> { + if (sess1.getStartTime() > sess2.getStartTime()) + return 1; + if (sess1.getStartTime() < sess2.getStartTime()) + return -1; + return 0; + }); + } +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/storage/ForecastFileStorage.java b/continuity.forecast/src/main/java/org/continuity/forecast/storage/ForecastFileStorage.java new file mode 100644 index 00000000..754064f3 --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/storage/ForecastFileStorage.java @@ -0,0 +1,11 @@ +package org.continuity.forecast.storage; + +import java.nio.file.Path; + +public class ForecastFileStorage { + + public ForecastFileStorage(Path path) { + // TODO Auto-generated constructor stub + } + +} diff --git a/continuity.forecast/src/main/java/org/continuity/forecast/test/Main.java b/continuity.forecast/src/main/java/org/continuity/forecast/test/Main.java new file mode 100644 index 00000000..ec9be2ec --- /dev/null +++ b/continuity.forecast/src/main/java/org/continuity/forecast/test/Main.java @@ -0,0 +1,19 @@ +package org.continuity.forecast.test; + +import org.influxdb.InfluxDB; +import org.influxdb.InfluxDBFactory; +import org.influxdb.dto.Pong; + +public class Main { + + public static void main(String[] args) { + + InfluxDB influxDB = InfluxDBFactory.connect("http://127.0.0.1:8086", "admin", "admin"); + + Pong response = influxDB.ping(); + if (response.getVersion().equalsIgnoreCase("unknown")) { + System.out.println("Error pinging server."); + return; + } + } +} diff --git a/continuity.forecast/src/main/resources/application.yml b/continuity.forecast/src/main/resources/application.yml new file mode 100644 index 00000000..c96abbf2 --- /dev/null +++ b/continuity.forecast/src/main/resources/application.yml @@ -0,0 +1,17 @@ +server: + port: ${port:0} +spring: + application: + name: forecast + rabbitmq: + host: localhost + listener: + simple: + default-requeue-rejected: false +eureka: + client: + serviceUrl: + defaultZone: ${eureka.uri:http://localhost:8761/eureka} + instance: + statusPageUrlPath: /swagger-ui.html + preferIpAddress: true \ No newline at end of file diff --git a/continuity.forecast/src/main/resources/banner.txt b/continuity.forecast/src/main/resources/banner.txt new file mode 100644 index 00000000..9441dd7c --- /dev/null +++ b/continuity.forecast/src/main/resources/banner.txt @@ -0,0 +1,13 @@ + + ..ooo.. ..ooo.. + .oOOOOOOOOOOo. .oOOOOOOOOOOo. + .oOOOO° °OOOOo oOOOO° °OOOOo. _____ _ _ _____ _______ + .OOOO °OOOOo. .oOOOO° OOOO. / ____| | | (_) |_ _|__ __| + oOOO °OOOOOOOOOO° OOOo | | ___ _ __ | |_ _ _ __ _ _ | | | |_ _ + OOOO -OOOO- OOOO | | / _ \| '_ \| __| | '_ \| | | | | | | | | | | + °OOO .oOOOOOOOOOOo. OOO° | |___| (_) | | | | |_| | | | | |_| |_| |_ | | |_| | + °OOOO .oOOOO° °OOOOo. OOOO° \_____\___/|_| |_|\__|_|_| |_|\__,_|_____| |_|\__, | + °OOOOOOOOOOOOOO OOOOOOOOOOOOOO° ============================================ __/ |=== + °°OOOOOO°° °°OOOOOO°° Forecast Service |___/ + + :: Spring Boot :: v${spring-boot.version} :: diff --git a/continuity.forecast/telescope-multi/ForecastTelescope.R b/continuity.forecast/telescope-multi/ForecastTelescope.R new file mode 100644 index 00000000..7cdb0329 --- /dev/null +++ b/continuity.forecast/telescope-multi/ForecastTelescope.R @@ -0,0 +1,3 @@ +forecast.period <- as.numeric(period) +foo <- telescope.forecast(tvp = intensities, horizon = forecast.period) +forecastValues <- as.numeric(foo$mean) \ No newline at end of file diff --git a/continuity.forecast/telescope-multi/ForecastTelescopeWithCovariates.R b/continuity.forecast/telescope-multi/ForecastTelescopeWithCovariates.R new file mode 100644 index 00000000..ca91fd99 --- /dev/null +++ b/continuity.forecast/telescope-multi/ForecastTelescopeWithCovariates.R @@ -0,0 +1,3 @@ +forecast.period <- as.numeric(period) +foo <- telescope.forecast(tvp = intensities, horizon = forecast.period, hist.covar = hist.covar.matrix, future.covar = future.covar.matrix) +forecastValues <- as.numeric(foo$mean) \ No newline at end of file diff --git a/continuity.forecast/telescope-multi/R/cluster_periods.R b/continuity.forecast/telescope-multi/R/cluster_periods.R new file mode 100644 index 00000000..e85350af --- /dev/null +++ b/continuity.forecast/telescope-multi/R/cluster_periods.R @@ -0,0 +1,128 @@ +#' @author Marwin Zuefle, Andre Bauer + +#' @description Calculates the clusters for the periods of the time series regarding mean, variance and range +#' +#' @title Calculates the Clusters for Periods +#' @param timeseries The time series to split +#' @param frequency The determined frequency of the time series +#' @param doAnomDet Boolean whether anomaly detection shall be used for clustering +#' @param replace.zeros If TRUE, all zeros will be replaced by the mean of the non-zero neighbors +#' @param debug Optional parameter: If TRUE, debugging information will be displayed. FALSE by default +#' @return The cluster label for each period +calcClustersForPeriods <- function(timeseries,frequency,doAnomDet,replace.zeros,debug=FALSE) { + + min.sil <- 0.66 + + if(doAnomDet) { + timeseries <- removeAnomalies(as.vector(timeseries),frequency,replace.zeros = replace.zeros) + } + + set.seed(750) + + # Calculates the number of full periods + lenSeq <- length(timeseries) + frequency - 1 + times <- seq(frequency,lenSeq)/frequency + amountFullPeriods <- as.integer(length(timeseries)/frequency) + + means <- c() + vars <- c() + ranges <- c() + + # Calculates mean, variance and range over each period + i <- 0 + while(i0) { + clusters <- rep(1,length(clk$cluster)) + } else { + clusters <- clk$cluster + } + + ret <- cbind(means,vars,ranges,clusters) + + return(ret) +} + +#' @description Forecasts the clusters for the forecast periods +#' +#' @title Forecast the Clusters +#' @param clusters The clusters found for the periods +#' @param freuqency The determined frequency +#' @param timeseries The time series +#' @param reps The amount of repeats for ANN +#' @param debug Optional parameter: If TRUE, debugging information will be displayed. FALSE by default +#' @return All cluster labels (history and forecast) +forecastClusters <- function(clusters,frequency,timeseries,reps,horizon,debug=FALSE) { + + # split time series in test and training + clusterTrain <- ts(clusters) + total.length <- length(timeseries) + horizon + + fullper <- as.integer(total.length/frequency) + + # Required number of new clusters + reqCluster <- fullper - length(clusters) + 1 + + # perform ANN forecast + fAnn <- doANN(clusterTrain,reps) + fcAnn <- forecast(fAnn, h = reqCluster)$mean + + fcCluster <- round(fcAnn) + + # Builds vector of all cluster labels + forecastCluster <- c(clusters,fcCluster) + if(debug){ + print(forecastCluster) + } + + # Labels each datapoint with associated cluster + clusterLabel <- c() + for(i in 1:(length(forecastCluster)-1)) { + clusterLabel <- c(clusterLabel,rep(forecastCluster[i],frequency)) + } + + rest <- total.length-(fullper*frequency) + clusterLabel <- c(clusterLabel,rep(forecastCluster[length(forecastCluster)],rest)) + + return(clusterLabel) +} + + diff --git a/continuity.forecast/telescope-multi/R/detect_anoms.R b/continuity.forecast/telescope-multi/R/detect_anoms.R new file mode 100644 index 00000000..82c25e45 --- /dev/null +++ b/continuity.forecast/telescope-multi/R/detect_anoms.R @@ -0,0 +1,125 @@ +#https://github.com/twitter/AnomalyDetection + +detect_anoms <- function(data, k = 0.49, alpha = 0.05, num_obs_per_period = NULL, + use_decomp = TRUE, use_esd = FALSE, one_tail = TRUE, + upper_tail = TRUE, verbose = FALSE) { + # Detects anomalies in a time series using S-H-ESD. + # + # Args: + # data: Time series to perform anomaly detection on. + # k: Maximum number of anomalies that S-H-ESD will detect as a percentage of the data. + # alpha: The level of statistical significance with which to accept or reject anomalies. + # num_obs_per_period: Defines the number of observations in a single period, and used during seasonal decomposition. + # use_decomp: Use seasonal decomposition during anomaly detection. + # use_esd: Uses regular ESD instead of hybrid-ESD. Note hybrid-ESD is more statistically robust. + # one_tail: If TRUE only positive or negative going anomalies are detected depending on if upper_tail is TRUE or FALSE. + # upper_tail: If TRUE and one_tail is also TRUE, detect only positive going (right-tailed) anomalies. If FALSE and one_tail is TRUE, only detect negative (left-tailed) anomalies. + # verbose: Additionally printing for debugging. + # Returns: + # A list containing the anomalies (anoms) and decomposition components (stl). + + if(is.null(num_obs_per_period)){ + stop("must supply period length for time series decomposition") + } + + num_obs <- nrow(data) + + # Check to make sure we have at least two periods worth of data for anomaly context + if(num_obs < num_obs_per_period * 2){ + stop("Anom detection needs at least 2 periods worth of data") + } + + # Check if our timestamps are posix + posix_timestamp <- if (class(data[[1L]])[1L] == "POSIXlt") TRUE else FALSE + + # Handle NAs + if (length(rle(is.na(c(NA,data[[2L]],NA)))$values)>3){ + stop("Data contains non-leading NAs. We suggest replacing NAs with interpolated values (see na.approx in Zoo package).") + } else { + data <- na.omit(data) + } + + # -- Step 1: Decompose data. This returns a univarite remainder which will be used for anomaly detection. Optionally, we might NOT decompose. + data_decomp <- stl(ts(data[[2L]], frequency = num_obs_per_period), + s.window = "periodic", robust = TRUE) + + # Remove the seasonal component, and the median of the data to create the univariate remainder + data <- data.frame(timestamp = data[[1L]], count = (data[[2L]]-data_decomp$time.series[,"seasonal"]-median(data[[2L]]))) + + # Store the smoothed seasonal component, plus the trend component for use in determining the "expected values" option + data_decomp <- data.frame(timestamp=data[[1L]], count=(as.numeric(trunc(data_decomp$time.series[,"trend"]+data_decomp$time.series[,"seasonal"])))) + + if(posix_timestamp){ + data_decomp <- format_timestamp(data_decomp) + } + # Maximum number of outliers that S-H-ESD can detect (e.g. 49% of data) + max_outliers <- trunc(num_obs*k) + + if(max_outliers == 0){ + stop(paste0("With longterm=TRUE, AnomalyDetection splits the data into 2 week periods by default. You have ", num_obs, " observations in a period, which is too few. Set a higher piecewise_median_period_weeks.")) + } + + func_ma <- match.fun(median) + func_sigma <- match.fun(mad) + + ## Define values and vectors. + n <- length(data[[2L]]) + if (posix_timestamp){ + R_idx <- as.POSIXlt(data[[1L]][1L:max_outliers], tz = "UTC") + } else { + R_idx <- 1L:max_outliers + } + + num_anoms <- 0L + + # Compute test statistic until r=max_outliers values have been + # removed from the sample. + for (i in 1L:max_outliers){ + if(verbose) message(paste(i,"/", max_outliers,"completed")) + + if(one_tail){ + if(upper_tail){ + ares <- data[[2L]] - func_ma(data[[2L]]) + } else { + ares <- func_ma(data[[2L]]) - data[[2L]] + } + } else { + ares = abs(data[[2L]] - func_ma(data[[2L]])) + } + + # protect against constant time series + data_sigma <- func_sigma(data[[2L]]) + if(data_sigma == 0) + break + + ares <- ares/data_sigma + R <- max(ares) + + temp_max_idx <- which(ares == R)[1L] + + R_idx[i] <- data[[1L]][temp_max_idx] + + data <- data[-which(data[[1L]] == R_idx[i]), ] + + ## Compute critical value. + if(one_tail){ + p <- 1 - alpha/(n-i+1) + } else { + p <- 1 - alpha/(2*(n-i+1)) + } + + t <- qt(p,(n-i-1L)) + lam <- t*(n-i) / sqrt((n-i-1+t**2)*(n-i+1)) + + if(R > lam) + num_anoms <- i + } + + if(num_anoms > 0) { + R_idx <- R_idx[1L:num_anoms] + } else { + R_idx = NULL + } + + return(list(anoms = R_idx, stl = data_decomp)) +} diff --git a/continuity.forecast/telescope-multi/R/fitting_models.R b/continuity.forecast/telescope-multi/R/fitting_models.R new file mode 100644 index 00000000..a8eecd05 --- /dev/null +++ b/continuity.forecast/telescope-multi/R/fitting_models.R @@ -0,0 +1,107 @@ +#' @author Marwin Zuefle, Andre Bauer + +#' @description Checks the model of the trend: either linear or exponential +#' +#' @title Fitting the Model of the Trend +#' @param tvp The time-value pair +#' @param frequency The frequency of the time-value pair +#' @param difFactor Optional parameter: The factor how much exp needs to be better than linear. 1.5 by default +#' @param debug Optional parameter: If TRUE, debugging information will be displayed. FALSE by default +#' @return The model, either linear or exp and if the estimation is riskys +fittingModels <- function(stl, frequency, difFactor = 1.5, debug = FALSE) { + + stlTrend <- stl$time.series[,2] + # Moves trend to greater than 1 due to log + if(min(stlTrend) < 1) { + stlTrend <- stlTrend + (1 - min(stlTrend)) + } + + times <- seq(frequency,(length(stlTrend)+frequency-1))/frequency + + # Fiting a linear model + lin <- lm(stlTrend~times) + len <- length(stlTrend) + # y axis intercept + cut <- coef(lin)[1] + # slope + m <- coef(lin)[2] + + # Shows some debugging information + if(debug) { + plot(lin) + lines(c(1,(len+frequency-1/frequency)),c(m*frequency+cut,(len+frequency-1)*m+cut),col="red") + } + + # Fitting an exponential model + stlTrendLOG <- log(stlTrend) + lmEXP <- lm(stlTrendLOG~times) + # y axis intercept + cutEXP <- coef(lmEXP)[1] + # slope + mEXP <- coef(lmEXP)[2] + if(debug) { + plot(stlTrendLOG) + lines(c(1,(len+frequency-1)/frequency),c(mEXP*frequency+cutEXP,(len+frequency-1)*mEXP+cutEXP),col="red") + } + + # Creates both models + Counts.exponential2 <- exp(predict(lmEXP,list(Time=stl$time.series[,0]))) + Counts.linear <- predict(lin,list(Time=stl$time.series[,0])) + + # plot all types of models + if(debug){ + plot(exp(stlTrendLOG)) + expModel <- matrix(nrow=len,ncol = 2) + for(i in frequency:(frequency+len-1)) { + expModel[(i-frequency+1),1] <- i/frequency + expModel[(i-frequency+1),2] <- exp(i*(mEXP)+cutEXP) + } + lines(expModel[,1],expModel[,2],col="blue") + lines(times, Counts.exponential2,lwd=2, col = "green", xlab = "Time (s)", ylab = "Counts") + lines(times, Counts.linear,lwd=2, col = "purple", xlab = "Time (s)", ylab = "Counts") + } + + # Calculates RSME between the models and the trend + RMSElog <- mean(sqrt((stlTrend-Counts.exponential2)^2)) + RMSE <- mean(sqrt((stlTrend-Counts.linear)^2)) + + print(paste("RMSE exp fit:",RMSElog)) + print(paste("RMSE lin fit:",RMSE)) + if (RMSE <= difFactor*RMSElog) { + # if the difference is small, the trend model estimation is risky + if(RMSE <= 0.8*difFactor*RMSElog) { + risky_trend_model <- FALSE + } else { + risky_trend_model <- TRUE + } + ret <- "linear" + } else { + # if the difference is small, the trend model estimation is risky + if(RMSE > 1.2*difFactor*RMSElog) { + risky_trend_model <- FALSE + } else { + risky_trend_model <- TRUE + } + ret <- "exp" + } + return( + list( + "trendmodel" = ret, + "risky_trend_model" = risky_trend_model + ) + ) +} + +#' @description Checks if the time series has a significant trend +#' +#' @title Test the Signifcant of Trend +#' @param tvp The time-value pair (not in log) +#' @param frequency The frequency of the time-value pair +#' @return If time series has a significant trend +testTrend <- function(tvp, frequency) { + ts <- ts(tvp,frequency = frequency) + + stl <- stl(ts, s.window = "periodic", t.window = length(tvp)/2) + return(estimateBooster(stl)) + +} diff --git a/continuity.forecast/telescope-multi/R/frequency.R b/continuity.forecast/telescope-multi/R/frequency.R new file mode 100644 index 00000000..79e03a74 --- /dev/null +++ b/continuity.forecast/telescope-multi/R/frequency.R @@ -0,0 +1,193 @@ +#' @author Marwin Zuefle, Andre Bauer + +#' @description Estimates the frequency of the time series based on a periodogram +#' +#' @title Guess the Frequency of the Time Series +#' @param timeValuePair Timestamps with raw values (not as time series object) +#' @param asInteger Optional parameter: Boolean indicating whether frequerncy needs to be an integer +#' @param difFactor Optional parameter: A factor to determine whether a possible frequency is accepted +#' @param ithBest Optional parameter: i-th most likely frequency is tested. 1 by default. +#' @param spans Optional parameter: vector of odd integers giving the widths of modified Daniell smoothers to be used to smooth the periodogram. Null by default +#' @param debug Optional parameter: If TRUE, debugging information will be displayed. FALSE by default +#' @param PGramTvp Optional parameter: An already created periodogram. Null by default +#' @return The found frequency and the created periodogram +guessFrequencyPeriodogram <- function(timeValuePair, asInteger = TRUE, difFactor = 0.5, ithBest = 1, spans = NULL, debug = FALSE, PGramTvp = NULL) { + + num <- ithBest-1 + + # Is there already a periodogram? + if(num==0 || is.null(PGramTvp)) { + PGramTvp <- spec.pgram(x = ts(timeValuePair),plot=FALSE,spans = spans) + } + # Sort the spectrum + SpecSortPgramTvp <- sort(PGramTvp$spec) + # Calculates the frequencies + ProbFreq <- 1/PGramTvp$freq[which(PGramTvp$spec==SpecSortPgramTvp[length(SpecSortPgramTvp)-num])] + + if(length(ProbFreq) == 0 || ProbFreq >= length(timeValuePair)) { + freqPeriodo <- -1 + print("Guessed Frequency longer than time series length!") + } else if(ProbFreq >= length(timeValuePair)/2) { + freqPeriodo <- -1 + print("Guessed Frequency too long: we need more than 2 periods worth of data!") + } else { + + if(asInteger) { + IntProbFreq <- round(ProbFreq) + DiffTvp <- diff(timeValuePair,IntProbFreq) + } else { + DiffTvp <- diff(timeValuePair,ProbFreq) + } + + # Create the periodogram for the difference of the original time series using the estimated frequency as lag + PGramTvpDiff <- spec.pgram(ts(DiffTvp),plot=FALSE,spans = spans) + + # Find the closest frequency in the periodogram of diffs compared to the estimated frequency of the original observations + if((min(abs(PGramTvpDiff$freq-(1/ProbFreq)))+1/ProbFreq) %in% PGramTvpDiff$freq) { + ProbFreqDiff <- 1/(min(abs(PGramTvpDiff$freq-(1/ProbFreq)))+1/ProbFreq) + } else if((-min(abs(PGramTvpDiff$freq-(1/ProbFreq)))+1/ProbFreq) %in% PGramTvpDiff$freq) { + ProbFreqDiff <- 1/(-min(abs(PGramTvpDiff$freq-(1/ProbFreq)))+1/ProbFreq) + } else { + stop("Error determining frequency! Could not find equivalent frequency in diff.") + } + + # Determine the spectrum of the estimated frequency for the original observations as well as the maximum spectrum + lowerBound <- 1/(ProbFreq+0.005) + upperBound <- 1/(ProbFreq-0.005) + maxSpec <- PGramTvp$spec[which(PGramTvp$freq>lowerBound & PGramTvp$freqlowerBoundDiff & PGramTvpDiff$freq=4)] + + + frequency <- -1 + numIters <- 1 + while(frequency==-1) { + # If there is no suitable frequency found during the maximum amount of iterations, the most dominant frequency, the one + # determined using ithBest = 1, is returned + if(numIters>maxIters) { + print("Periodogram could not find dominant frequency") + freq <- guessFrequencyPeriodogram(timeValuePair,asInteger,difFactor,ithBest = 1,PGramTvp = PGramTvp) + PGramTvp <- freq$pgram + break() + } + + # Add a span if there is no suitable frequency found during the first two iterations + if(numIters<=2) { + freq <- guessFrequencyPeriodogram(timeValuePair,asInteger,difFactor,ithBest = ithBest,PGramTvp = PGramTvp) + } else { + freq <- guessFrequencyPeriodogram(timeValuePair,asInteger,difFactor,ithBest = ithBest,spans = 5,PGramTvp = PGramTvp) + } + + print(paste("Iteration:",numIters, "testing frequency:",freq$frequency)) + PGramTvp <- freq$pgram + a <- freq$frequency + + # Create a tolerance area for matching estimated frequencies to listed frequencies + a.p <- c((1-tolerance)*a,(1+tolerance)*a) + if(tolerance*a<1) { + a.p <- c(a-1,a+1) + } + + # Find all suitable frequencies which are between the bounds of a (a.p) + a.p <- round(a.p) + possible.freqs <- allComb[which(findInterval(allComb,a.p)==1)] + # Calculate the differences of the frequencies found in the interval and the estimated frequency + deltas <- abs(possible.freqs-a) + + # If there are any frequencies found in the interval, select the one with the smallest difference + # Otherwise, there is no suitable frequency found in this iteration + if(length(deltas)>0) { + frequency <- possible.freqs[deltas==min(deltas)] + } else { + frequency <- c() + } + if(length(frequency)==0) { + frequency <- -1 + } + lastIterFreq1 <- numIters + numIters <- numIters + 1 + ithBest <- ithBest + 1 + } + + # If there is no "good" frequency found, set frequency to 2 as STL requries at least this value + if(frequency == -1) { + frequency = 2 + print("No frequency found. Set frequency to: 2") + } else { + print(paste("Accepting frequency:",frequency)) + } + + return(list("frequency" = frequency, "pgram" = freq$pgram, "lastIterfreq" = lastIterFreq1)) +} diff --git a/continuity.forecast/telescope-multi/R/outlier.R b/continuity.forecast/telescope-multi/R/outlier.R new file mode 100644 index 00000000..b767e767 --- /dev/null +++ b/continuity.forecast/telescope-multi/R/outlier.R @@ -0,0 +1,61 @@ +#' @author Marwin Zuefle, Andre Bauer + +#' @description Removes the anomalies of the time series +#' +#' @title Remove the Anomalies +#' @param rawValues The raw values as vector of the time series without timestamp +#' @param frequency The frequency of the time series +#' @param replace.zeros If TRUE, all zeros will be replaced by the mean of the non-zero neighbors +#' @return The vector of observations without anomalies (anomalies replaced by mean of normal values around) +removeAnomalies <- function(rawValues,frequency,replace.zeros) { + + vals <- rawValues + # Gets the anomalies + anom <- AnomalyDetectionVec(x = vals, period = frequency, direction = "pos",plot = FALSE) + anomPos <- anom$anoms$index + anomVals <- anom$anoms$anoms + allPos <- c(1:length(vals)) + if(!is.null(anomPos)) { + normPos <- allPos[-anomPos] + + for(i in 1:length(anomPos)) { + # find the next lower and upper non-anomaly neighbor of each anomaly + if(length(which(normPos>anomPos[i]))==0) { + lb <- max(normPos[which(normPosanomPos[i])]) + ub <- min(normPos[which(normPos>anomPos[i])]) + } else { + lb <- max(normPos[which(normPosanomPos[i])]) + } + # Interpolates the values to replace the anomaly + vals[anomPos[i]] <- (vals[lb]+vals[ub])/2 + } + } + + if(replace.zeros) { + anomPos.null <- which(vals==0) + if(length(anomPos.null)>0) { + normPos.null <- allPos[-anomPos.null] + # find the next lower and upper non-zero neighbor for each zero value + for(i in 1:length(anomPos.null)) { + if(length(which(normPos.null>anomPos.null[i]))==0) { + lb <- max(normPos.null[which(normPos.nullanomPos.null[i])]) + ub <- min(normPos.null[which(normPos.null>anomPos.null[i])]) + } else { + lb <- max(normPos.null[which(normPos.nullanomPos.null[i])]) + } + # Interpolates the values to replace the zeros + vals[anomPos.null[i]] <- (vals[lb]+vals[ub])/2 + } + } + } + + return(vals) +} diff --git a/continuity.forecast/telescope-multi/R/telescope.R b/continuity.forecast/telescope-multi/R/telescope.R new file mode 100644 index 00000000..5b19aecd --- /dev/null +++ b/continuity.forecast/telescope-multi/R/telescope.R @@ -0,0 +1,281 @@ +#' @author Marwin Zuefle, Andre Bauer + + +#' @description Forecasts a given univariate time series in a hybrid manner and based on time series decomposition +#' +#' @title Perform the Forecast +#' @param tvp The time value pair: either vector of raw values or n-by-2 matrix (raw values in second column), or time series +#' @param horizon The number of values that should be forecast +#' @param hist.covar The covariates of the history used to build the model (matrix, nrows has to equal length (nrow) of tvp) +#' @param future.covar The covariates of the future used for prediction (matrix, same columns as hist.covar, nrow has to equal horizon) +#' @param repsANN Optional parameter: The amount of repeats for ANN. 20 by default. +#' @param doAnomDet Optional parameter: Boolean whether anomaly detection shall be used for clustering. TRUE by default +#' @param replace.zeros Optional parameter: If TRUE, all zeros will be replaced by the mean of the non-zero neighbors. TRUE by default +#' @param use.indicators Optional parameter: If TRUE, additional information (e.g. a flag wheter there is a high remainder) will be returned. TRUE by default +#' @param save_fc Optional parameter: Boolean wheter the forecast shall be saved as csv. FALSE by default +#' @param csv.path Optional parameter: The path for the saved csv-file. The current workspace by default. +#' @param csv.name Optional parameter: The name of the saved csvfile. Telescope by default. +#' @param debug Optional parameter: If TRUE, debugging information will be displayed. FALSE by default +#' @return The forecast +#' @examples +#' telescope.forecast(AirPassengers, horizon=10) +#' @export +telescope.forecast <- function(tvp, horizon, + hist.covar, + future.covar, + repsANN = 20,doAnomDet = TRUE, replace.zeros = TRUE, use.indicators = TRUE, save_fc = FALSE, csv.path = '', csv.name = "Telescope", debug = FALSE) { + + use.second.freq <- TRUE + sig.dif.factor <- 0.5 + plot <- TRUE + + startTime <- Sys.time() + + # Convert timeseries and extract information + tvp <- extract.info(tvp, use.second.freq) + + # Remove all Anomalies on the raw time series first + if(tvp$frequency>10) { + tvp$values <- removeAnomalies(tvp$values,frequency = tvp$frequency, replace.zeros = replace.zeros) + } + + hist.length <- length(tvp$values) + + # get the minimum value to shift all observations to real positive values + minValue <- min(tvp$values) + if(minValue<=0) { + tvp$values <- tvp$values + abs(minValue) + 1 + } + + # use log if there is a significant trend in the forecast (STL with log delivers a multiplicative decomposition) + use.log <- testTrend(tvp = tvp$values, frequency = tvp$frequency) + + if(use.log) { + tvp$values <- log(tvp$values) + print("using log for stl and forecast") + } + + tsTrain <- ts(tvp$values,frequency=tvp$frequency) + + stlTrain <- stl(tsTrain,s.window = "periodic",t.window=length(tsTrain)/2) + + stlTraintrend <- stlTrain$time.series[,2] + stlTrainremainder <- stlTrain$time.series[,3] + + # check if the time series has a high remainder + high_remainder <- has.highRemainder(tvp$values,stlTrainremainder,use.log, sig.dif.factor) + if(high_remainder){ + print("-------------- ATTENTION: High remainder in STL --------------") + } + + # Search for the next best frequency + if(tvp$use.second.freq) { + print("estimating second freq") + freq2 <- calcFrequencyPeriodogram(timeValuePair = tvp$values, asInteger = TRUE, difFactor = 0.5,maxIters = 10,ithBest = tvp$lastIterfreq + 1, PGramTvp = tvp$pgram)$frequency + + # second stl decomposition + if(freq2 < (length(stlTrainremainder)/2)) { + stlRemainder <- stl(ts(stlTrainremainder,frequency=freq2),s.window="periodic",t.window=length(stlTrainremainder)/2) + stlRemainderSeason <- stlRemainder$time.series[,1] + stlRemainderTrend <- stlRemainder$time.series[,2] + print("finished estimating second freq positive") + } else { + print("second freq too long") + tvp$use.second.freq <- FALSE + } + } + + # Forecast season according to stl decomposition + fcSeason <- forecast.season(tvp, stlTrain, horizon) + + + total.length <- hist.length+horizon + # Add second period to fist period + if(tvp$use.second.freq) { + fullper2 <- as.integer(total.length/freq2) + rest2 <- total.length-(fullper2*freq2) + fcSeason2 <- rep(stlRemainderSeason[1:freq2],fullper2) + if(rest2>0){ + fcSeason2 <- c(fcSeason2,stlRemainderSeason[1:rest2]) + } + + fcSeason <- fcSeason + fcSeason2 + } + + # Forecast trend with ARIMA (without seasonal models) + tsTrainTrend <- ts(stlTraintrend,frequency = tvp$frequency) + + + # Check for trend model: linear or exp + model <- fittingModels(stlTrain,frequency = tvp$frequency,difFactor = 1.5, debug = debug) + + if(model$risky_trend_model) { + print("-------------- ATTENTION: risky trend estimation --------------") + } + + + # Forecast trend according to the underlying trend model + fcTrend <- forecast.trend(model$trendmodel,tsTrainTrend, tvp$frequency, horizon) + + + # Creation of categorical information + # Time series used without trend to use mean as feature + tvpDetrend <- tvp$values-stlTraintrend + tsDetrend <- ts(tvpDetrend,frequency=tvp$frequency) + + # Get clusters + clusters <- calcClustersForPeriods(timeseries = tsDetrend, frequency = tvp$frequency, doAnomDet = doAnomDet, replace.zeros = replace.zeros, debug = debug) + + if(debug){ + xend <- (length(stlTraintrend)+length(fcTrend))/tvp$frequency+1 + print(paste("frequency:",tvp$frequency)) + plot(stlTrain) + par(mfrow=c(1,1)) + plot(ts(fcSeason,frequency=tvp$frequency)) + par(mfrow=c(2,1)) + plot(stlTraintrend,xlim=c(0,xend),ylim=c(min(c(stlTraintrend,fcTrend)),max(c(fcTrend,stlTraintrend)))) + lines(fcTrend,col="red") + plot(stl(ts(tvp$values,frequency=tvp$frequency),s.window = "periodic",t.window=length(tvp$values)/2)$time.series[,2],xlim=c(0,xend)) + print(clusters[,ncol(clusters)]) + } + + # Forecast cluster labels + clusterLabels <- forecastClusters(clusters = clusters[,ncol(clusters)], frequency = tvp$frequency, timeseries = tsTrain, reps = repsANN, horizon = horizon, debug = debug) + + + + # Build the covariates matrix + if(use.log) { + if(tvp$use.second.freq) { + xgbcov <- rbind(as.vector(stlTrain$time.series[,1]) + as.vector(stlRemainderSeason),clusterLabels[1:hist.length]) + } else { + xgbcov <- rbind(stlTrain$time.series[,1],clusterLabels[1:hist.length]) + } + } else { + if(tvp$use.second.freq) { + xgbcov <- rbind(stlTraintrend,as.vector(stlTrain$time.series[,1]) + as.vector(stlRemainderSeason),clusterLabels[1:hist.length]) + } else { + xgbcov <- rbind(stlTraintrend,stlTrain$time.series[,1],clusterLabels[1:hist.length]) + } + } + xgbcov <- as.matrix(t(xgbcov)) + + if(!missing(hist.covar)) { + xgbcov <- cbind(xgbcov, hist.covar) + } + + # Building the training labels + if(use.log) { + xgblabel <- tvp$values-stlTraintrend + } else { + xgblabel <- tvp$values + } + + # Learning the XGBoost model + # xglinear for time series with trend patten in the forecast, gbtree for only seasonal pattern in forecast + if(estimateBooster(stlTrain)) { + booster <- "gblinear" + } else { + booster <- "gbtree" + } + print(booster) + # Train XGBoost + fXGB <- doXGB.train(myts = xgblabel, cov = xgbcov, booster = booster, verbose = 0) + + # Build the covariates matrix for the future + if(use.log) { + testcov <- rbind(fcSeason[(hist.length+1):total.length],clusterLabels[(hist.length+1):total.length]) + } else { + testcov <- rbind(fcTrend,fcSeason[(hist.length+1):total.length],clusterLabels[(hist.length+1):total.length]) + } + testcov <- as.matrix(t(testcov)) + + if(!missing(future.covar)) { + testcov <- cbind(testcov, future.covar) + } + + # Unify names + fXGB$feature_names <- colnames(testcov) + colnames(xgbcov) <- colnames(testcov) + + # Perform forecast using the covariates + predXGB <- predict(fXGB,testcov) + if(use.log) { + predXGB <- exp(predXGB)*exp(fcTrend) + tvp$values <- exp(tvp$values) + } + + # Undo adjustment to positive values + if(minValue<=0) { + predXGB <- predXGB - abs(minValue) - 1 + } + + if(save_fc) { + save.csv(values = predXGB, name = csv.name, path = csv.path) + } + + endTime <- Sys.time() + print(paste("Time elapsed for the whole forecast:",difftime(endTime,startTime,units = "secs"))) + + par(mfrow=c(2,1)) + + # Get model of the history + xgb.model <- predict(fXGB,xgbcov) + if(use.log) { + xgb.model <- exp(xgb.model)*exp(stlTraintrend) + } + # Undo adjustment to positive values + if(minValue<=0) { + xgb.model <- xgb.model - abs(minValue) - 1 + tvp$values <- tvp$values - abs(minValue) - 1 + } + + # Calculates the accuracies of the trained model + accuracyXGB <- accuracy(xgb.model,tvp$values) + # inner MASE value (fitting of the model) + tvpTrain <- tvp$values + MASE <- computeMASE(xgb.model[-1], train = tvpTrain[1], test = tvpTrain[-1], !plot) + MASE_Multistep <- computeMASEsameValue(xgb.model[-1], train = tvpTrain[1], test = tvpTrain[-1], !plot) + inner.accuracy <- cbind(accuracyXGB, MASE, MASE_Multistep) + print(inner.accuracy) + + # Build the time series with history and forecast + fcOnly <- ts(predXGB,frequency=tvp$frequency) + fcAll <- c(tvp$values,predXGB) + fcAll <- ts(fcAll,frequency=tvp$frequency) + + # Plot the model and the time series + y.min <- min(min(tvpTrain[-1]),min(xgb.model[-1])) + y.max <- max(max(tvpTrain[-1]),max(xgb.model[-1])) + plot(1:length(tvpTrain[-1]), tvpTrain[-1],type="l",col="black", main = 'History (black) and Model (red)', xlab = 'Index', ylab = 'Observation', xlim = c(0, total.length), ylim = c(y.min, y.max)) + lines(1:length(xgb.model[-1]), xgb.model[-1], type = "l", col="red") + + # Plot the forecasted time series and the original time series + y.min <- min(min(fcAll),min(tvp$values)) + y.max <- max(max(fcAll),max(tvp$values)) + plot(1:total.length, as.vector(fcAll),type = 'l',col="red",xlab = 'Index', ylab = 'Observation', main = 'History (black) and Forecast (red)', xlim = c(0, total.length), ylim = c(y.min, y.max)) + lines(1:length(tvp$values), tvp$values) + + # Collect information for output + output.mean <- fcOnly + output.x <- tvp$values + output.residuals <- output.x - ts(xgb.model,frequency=tvp$frequency) + output.method <- "Telescope" + output.accuracy <- inner.accuracy + output.fitted <- xgb.model + + + if(use.indicators) { + output.risky.trend.model <- model$risky_trend_model + output.high.stl.remainder <- high_remainder + output <- list(mean=output.mean, x=output.x, residuals=output.residuals, method=output.method, + fitted=output.fitted, riskytrend=output.risky.trend.model, highresiduals=output.high.stl.remainder) + } else { + output <- list(mean=output.mean, x=output.x, residuals=output.residuals, method=output.method, + fitted=output.fitted) + } + + return(structure(output, class = 'forecast')) + +} + diff --git a/continuity.forecast/telescope-multi/R/telescope_Utils.R b/continuity.forecast/telescope-multi/R/telescope_Utils.R new file mode 100644 index 00000000..9981cfa7 --- /dev/null +++ b/continuity.forecast/telescope-multi/R/telescope_Utils.R @@ -0,0 +1,265 @@ +#' @author Marwin Zuefle, Andre Bauer + +#' @description Saves the vales to a csv-file +#' +#' @title Save as CSV +#' @param values The values to be stored in the csv file +#' @param name The name of the csv file to be created +#' @param csv.path Optional parameter: The path for the saved csv-file. The current workspace by default. +save.csv <- function(values, name, path = '') { + save_here <- paste(path, Sys.time(), "_", name, ".csv", sep = "") + save_here <- gsub(" ", "_", save_here) + save_here <- gsub(":", "-", save_here) + save_here <- paste("C:", save_here, sep = "") + write.csv(values, file = save_here) +} + + +#' @description Extract the requird information (e.g., frequency, values, etc.) of the time series +#' +#' @title Extract time series information +#' @param tvp The time value pair: either vector of raw values or n-by-2 matrix (raw values in second column), or time series +#' @param use.sec.freq Determines if a second frequency shall be used +#' @return the time value pair as vector, the frequency of the data, use.sec.freq and if the data was no time series, the last iteration and the periodigram of the frequncy estimation is also returned +extract.info <- function(tvp, use.sec.freq) { + + # If the time value pair is a time series, get the values and the frequency + if (is.ts(tvp)) { + frequency <- frequency(tvp) + use.second.freq <- FALSE + tvp <- tvp[1:length(tvp)] + + return( + list( + "values" = as.vector(tvp), + "frequency" = frequency, + "use.second.freq" = use.second.freq + ) + ) + + } else { + # If the time value pair is not a time series, estimate frequency and extract values + tvp <- as.matrix(tvp) + if (ncol(tvp) == 2) { + tvp <- tvp[, 2] + } else if (ncol(tvp) > 2) { + stop( + "Input time series has to many columns. Either single column with only raw values or two columns with raw values in second column!" + ) + } + freq <- + calcFrequencyPeriodogram( + timeValuePair = tvp, + asInteger = TRUE, + difFactor = 0.5 + ) + + frequency <- freq$frequency[1] + use.second.freq <- use.sec.freq + + return( + list( + "values" = as.vector(tvp), + "frequency" = frequency, + "use.second.freq" = use.second.freq, + "lastIterfreq" = freq$lastIterfreq, + "pgram" = freq$pgram + ) + ) + } + + + +} + +#' @description Forecasts the season part of the time series +#' +#' @title Forecasting Season +#' @param tvp The time value pair as vector +#' @param stlTrain The decomposition of tvp +#' @param horizon The forecast horizon +#' @return The seasonal pattern for the forecast horizon +forecast.season <- function(tvp, stlTrain, horizon) { + # Total length = history + forecast + total.length <- length(tvp$values) + horizon + # As the season is per defintion recurring, repeat season over the horizon + fullper <- as.integer(total.length / tvp$frequency) + rest <- total.length - (fullper * tvp$frequency) + fcSeason <- rep(stlTrain$time.series[1:tvp$frequency, 1], fullper) + if (rest > 0) { + fcSeason <- c(fcSeason, stlTrain$time.series[1:rest, 1]) + } + return(fcSeason) +} + +#' @description Forecats the trend part of the time series +#' +#' @title Forecasting Trend +#' @param model The model of the trend, either exponential or linear +#' @param tsTrainTrend The trend component of the time series +#' @param frequency The frequency of the time series +#' @param horizon The forecast horizon +#' @return The trend pattern for the forecast horizon +forecast.trend <- function(model, tsTrainTrend, frequency, horizon) { + # If trend is exponential, log time series + if (model == "exp") { + tsTrainLOG <- ts(log(tsTrainTrend), frequency = frequency) + fArimaLOG <- doArima(tsTrainLOG, FALSE) + fcArimaLOG <- forecast(fArimaLOG, h = horizon)$mean + print("exponential Trend detected!") + fcArima <- exp(fcArimaLOG) + } else { + fArima <- doArima(tsTrainTrend, FALSE) + fcArima <- forecast(fArima, h = horizon)$mean + } + return(fcArima) +} + +#' @description Checks if the time series has a significant high remainder +#' +#' @title Checking Remainder +#' @param tvp tvp The time value pair as vector +#' @param stlRemainder The remainder part of the deccomposition of the tvp +#' @param use.log A flag if log was used for tvp +#' @param sig.dif.factor The threshold that is to exceed for having a high remainder +#' @return If the the time series has a high remainder compared to the threshold +has.highRemainder <- function(tvp, stlRemainder, use.log, sig.dif.factor) { + # Calculates the IQR of the remainder and original time series + if (use.log) { + remainder.quantiles.log <- quantile(stlRemainder) + # range from 25% to 75% quantile + remainder.quantiles.range <- exp(remainder.quantiles.log[4] - remainder.quantiles.log[2]) + tvp.quantiles <- quantile(tvp) + tvp.quantiles.range <- exp(tvp.quantiles[4] - tvp.quantiles[2]) + } else { + remainder.quantiles <- quantile(stlRemainder) + # range from 25% to 75% quantile + remainder.quantiles.range <- + remainder.quantiles[4] - remainder.quantiles[2] + tvp.quantiles <- quantile(tvp) + tvp.quantiles.range <- tvp.quantiles[4] - tvp.quantiles[2] + } + # If the remainder IQR has a higher propotion than the threshold + if (sig.dif.factor * tvp.quantiles.range < remainder.quantiles.range) { + return(TRUE) + } else { + return(FALSE) + } +} + +#' @description Performs the ARIMA forecast of the timeseries. +#' +#' @title Apply Arima +#' @param ts The timeseries. +#' @param season If false, only non-seasonal models will be fitted +#' @return The Arima Forecast of \code{ts}. +doArima <- function(ts, season = TRUE){ + bool <- is.ts(ts) + if (is.ts(ts)){ + fc <- auto.arima(ts, stepwise = TRUE, seasonal = season) + return(fc) + } + return(NULL) +} + + +#' @description Performs the ANN forecast of the timeseries. +#' +#' @title Apply ANN +#' @param ts The timeseries. +#' @param rep The amount of repeats +#' @return The ANN Forecast +doANN <- function(myts,rep = 20){ + result <- tryCatch({ + nnetar(myts, repeats = rep) + }, error = function(e){ + print(paste("Some error doing ANN, probably stl: ", e, sep="" )) + tryCatch({ + nnetar(myts, repeats = rep, p=1) + }, error = function(e){ + print(paste("Other unknown error: ", e, sep = "")) + snaive(x = ts, h = length(ts)/4) + }) + }) + return(result) +} + +#' @description Creates a tag containing the MASE of the forecasting methods for the timeseries. +#' +#' @title Compute MASE +#' @param forecast The forecasted values. +#' @param train The 'historical' data used for forecasting. +#' @param test The 'future' data used for finding MASE. +#' @param plot Boolean indicating whether the forecast should be plotted. +#' @return MASE between forecast and real data +computeMASE <- function(forecast, train, test, plot){ + if(plot){ + plot(1:length(test), test,type="l",col="black", main = 'History (black) and Model (red)', xlab = 'Index', ylab = 'Observation') + lines(1:length(forecast), forecast, type = "l", col="red") + } + + forecast <- as.vector(forecast) + train <- as.vector(train) + test <- as.vector(test) + + # calculate scaling factor + test <- append(test, train[length(train)], after = 0) + n <- length(test) + if(n == 1){ + stop('Computing MASE: Test vector of length 0 is invalid.') + } + scalingFactor <- sum(abs(test[2:n] - test[1:(n-1)])) / (n-1) + + # Avoding to divide by zero + if(scalingFactor==0) { + scalingFactor<-0.00001 + } + + # calculate MASE + et <- abs(test[2:length(test)]-forecast) + qt <- et/scalingFactor + meanMASE <- mean(abs(qt)) + + return(meanMASE) +} + +#' @description MASE for a naive forecast taking the last observation for the whole forecast +#' +#' @title Compute MASE for same value +#' @param forecast The forecasted values. +#' @param train The 'historical' data used for forecasting. +#' @param test The 'future' data used for finding MASE. +#' @param plot Boolean indicating whether the forecast should be plotted. +#' @return MASE between forecast and real data +computeMASEsameValue <- function(forecast, train, test, plot){ + if(plot){ + plot(1:length(test), test,type="l",col="black", main = deparse(substitute(forecast))) + lines(1:length(forecast), forecast, type = "l", col="red") + } + + forecast <- as.vector(forecast) + train <- as.vector(train) + test <- as.vector(test) + + # calculate scaling factor + n <- length(test) + if(n == 1){ + stop('Computing MASE: Test vector of length 0 is invalid.') + } + lastObservation <- tail(train,1) + maseForecast <- rep(lastObservation,n) + scalingFactor <- sum(abs(maseForecast - test[1:n])) / n + + # Avoding to divide by zero + if(scalingFactor==0) { + scalingFactor<-0.00001 + print("scaling factor is 0") + } + + # calculate MASE + et <- abs(test-forecast) + qt <- et/scalingFactor + meanMASE <- mean(abs(qt)) + + return(meanMASE) +} diff --git a/continuity.forecast/telescope-multi/R/vec_anom_detection.R b/continuity.forecast/telescope-multi/R/vec_anom_detection.R new file mode 100644 index 00000000..3e354033 --- /dev/null +++ b/continuity.forecast/telescope-multi/R/vec_anom_detection.R @@ -0,0 +1,294 @@ +#https://github.com/twitter/AnomalyDetection + +#' Anomaly Detection Using Seasonal Hybrid ESD Test +#' +#' A technique for detecting anomalies in seasonal univariate time series where the input is a +#' series of observations. +#' @name AnomalyDetectionVec +#' @param x Time series as a column data frame, list, or vector, where the column consists of +#' the observations. +#' @param max_anoms Maximum number of anomalies that S-H-ESD will detect as a percentage of the +#' data. +#' @param direction Directionality of the anomalies to be detected. Options are: +#' \code{'pos' | 'neg' | 'both'}. +#' @param alpha The level of statistical significance with which to accept or reject anomalies. +#' @param period Defines the number of observations in a single period, and used during seasonal +#' decomposition. +#' @param only_last Find and report anomalies only within the last period in the time series. +#' @param threshold Only report positive going anoms above the threshold specified. Options are: +#' \code{'None' | 'med_max' | 'p95' | 'p99'}. +#' @param e_value Add an additional column to the anoms output containing the expected value. +#' @param longterm_period Defines the number of observations for which the trend can be considered +#' flat. The value should be an integer multiple of the number of observations in a single period. +#' This increases anom detection efficacy for time series that are greater than a month. +#' @param plot A flag indicating if a plot with both the time series and the estimated anoms, +#' indicated by circles, should also be returned. +#' @param y_log Apply log scaling to the y-axis. This helps with viewing plots that have extremely +#' large positive anomalies relative to the rest of the data. +#' @param xlabel X-axis label to be added to the output plot. +#' @param ylabel Y-axis label to be added to the output plot. +#' @details +#' \code{longterm_period} This option should be set when the input time series is longer than a month. +#' The option enables the approach described in Vallis, Hochenbaum, and Kejariwal (2014).\cr\cr +#' \code{threshold} Filter all negative anomalies and those anomalies whose magnitude is smaller +#' than one of the specified thresholds which include: the median +#' of the daily max values (med_max), the 95th percentile of the daily max values (p95), and the +#' 99th percentile of the daily max values (p99). +#' @param title Title for the output plot. +#' @param verbose Enable debug messages +#' @return The returned value is a list with the following components. +#' @return \item{anoms}{Data frame containing index, values, and optionally expected values.} +#' @return \item{plot}{A graphical object if plotting was requested by the user. The plot contains +#' the estimated anomalies annotated on the input time series.} +#' @return One can save \code{anoms} to a file in the following fashion: +#' \code{write.csv([["anoms"]], file=)} +#' @return One can save \code{plot} to a file in the following fashion: +#' \code{ggsave(, plot=[["plot"]])} +#' @references Vallis, O., Hochenbaum, J. and Kejariwal, A., (2014) "A Novel Technique for +#' Long-Term Anomaly Detection in the Cloud", 6th USENIX, Philadelphia, PA. +#' @references Rosner, B., (May 1983), "Percentage Points for a Generalized ESD Many-Outlier Procedure" +#' , Technometrics, 25(2), pp. 165-172. +#' +#' @docType data +#' @keywords datasets +#' @name raw_data +#' @examples +#' data(raw_data) +#' AnomalyDetectionVec(raw_data[,2], max_anoms=0.02, period=1440, direction='both', plot=TRUE) +#' # To detect only the anomalies in the last period, run the following: +#' AnomalyDetectionVec(raw_data[,2], max_anoms=0.02, period=1440, direction='both', +#' only_last=TRUE, plot=TRUE) +#' @seealso \code{\link{AnomalyDetectionTs}} +#' @export +AnomalyDetectionVec = function(x, max_anoms=0.10, direction='pos', + alpha=0.05, period=NULL, only_last=F, + threshold='None', e_value=F, longterm_period=NULL, + plot=F, y_log=F, xlabel='', ylabel='count', + title=NULL, verbose=FALSE){ + + # Check for supported inputs types and add timestamps + if(is.data.frame(x) && ncol(x) == 1 && is.numeric(x[[1]])){ + x <- data.frame(timestamp=c(1:length(x[[1]])), count=x[[1]]) + } else if(is.vector(x) || is.list(x) && is.numeric(x)) { + x <- data.frame(timestamp=c(1:length(x)), count=x) + } else { + stop("data must be a single data frame, list, or vector that holds numeric values.") + } + + # Sanity check all input parameterss + if(max_anoms > .49){ + stop(paste("max_anoms must be less than 50% of the data points (max_anoms =", round(max_anoms*length(x[[2]]), 0), " data_points =", length(x[[2]]),").")) + } + if(!direction %in% c('pos', 'neg', 'both')){ + stop("direction options are: pos | neg | both.") + } + if(!(0.01 <= alpha || alpha <= 0.1)){ + if(verbose) message("Warning: alpha is the statistical signifigance, and is usually between 0.01 and 0.1") + } + if(is.null(period)){ + stop("Period must be set to the number of data points in a single period") + } + if(!is.logical(only_last)){ + stop("only_last must be either TRUE (T) or FALSE (F)") + } + if(!threshold %in% c('None', 'med_max', 'p95', 'p99')){ + stop("threshold options are: None | med_max | p95 | p99.") + } + if(!is.logical(e_value)){ + stop("e_value must be either TRUE (T) or FALSE (F)") + } + if(!is.logical(plot)){ + stop("plot must be either TRUE (T) or FALSE (F)") + } + if(!is.logical(y_log)){ + stop("y_log must be either TRUE (T) or FALSE (F)") + } + if(!is.character(xlabel)){ + stop("xlabel must be a string") + } + if(!is.character(ylabel)){ + stop("ylabel must be a string") + } + if(!is.character(title) && !is.null(title)){ + stop("title must be a string") + } + if(is.null(title)){ + title <- "" + } else { + title <- paste(title, " : ", sep="") + } + + # -- Main analysis: Perform S-H-ESD + + num_obs <- length(x[[2]]) + + if(max_anoms < 1/num_obs){ + max_anoms <- 1/num_obs + } + + # -- Setup for longterm time series + + # If longterm is enabled, break the data into subset data frames and store in all_data, + if(!is.null(longterm_period)){ + all_data <- vector(mode="list", length=ceiling(length(x[[1]])/(longterm_period))) + # Subset x into two week chunks + for(j in seq(1,length(x[[1]]), by=longterm_period)){ + start_index <- x[[1]][j] + end_index <- min((start_index + longterm_period - 1), num_obs) + # if there is at least longterm_period left, subset it, otherwise subset last_index - longterm_period + if((end_index - start_index + 1) == longterm_period){ + all_data[[ceiling(j/(longterm_period))]] <- subset(x, x[[1]] >= start_index & x[[1]] <= end_index) + }else{ + all_data[[ceiling(j/(longterm_period))]] <- subset(x, x[[1]] > (num_obs-longterm_period) & x[[1]] <= num_obs) + } + } + }else{ + # If longterm is not enabled, then just overwrite all_data list with x as the only item + all_data <- list(x) + } + + # Create empty data frames to store all anoms and seasonal+trend component from decomposition + all_anoms <- data.frame(timestamp=numeric(0), count=numeric(0)) + seasonal_plus_trend <- data.frame(timestamp=numeric(0), count=numeric(0)) + + # Detect anomalies on all data (either entire data in one-pass, or in 2 week blocks if longterm=TRUE) + for(i in 1:length(all_data)) { + + anomaly_direction = switch(direction, + "pos" = data.frame(one_tail=TRUE, upper_tail=TRUE), # upper-tail only (positive going anomalies) + "neg" = data.frame(one_tail=TRUE, upper_tail=FALSE), # lower-tail only (negative going anomalies) + "both" = data.frame(one_tail=FALSE, upper_tail=TRUE)) # Both tails. Tail direction is not actually used. + + # detect_anoms actually performs the anomaly detection and returns the results in a list containing the anomalies + # as well as the decomposed components of the time series for further analysis. + s_h_esd_timestamps <- detect_anoms(all_data[[i]], k=max_anoms, alpha=alpha, num_obs_per_period=period, use_decomp=TRUE, use_esd=FALSE, + one_tail=anomaly_direction$one_tail, upper_tail=anomaly_direction$upper_tail, verbose=verbose) + + # store decomposed components in local variable and overwrite s_h_esd_timestamps to contain only the anom timestamps + data_decomp <- s_h_esd_timestamps$stl + s_h_esd_timestamps <- s_h_esd_timestamps$anoms + + # -- Step 3: Use detected anomaly timestamps to extract the actual anomalies (timestamp and value) from the data + if(!is.null(s_h_esd_timestamps)){ + anoms <- subset(all_data[[i]], (all_data[[i]][[1]] %in% s_h_esd_timestamps)) + } else { + anoms <- data.frame(timestamp=numeric(0), count=numeric(0)) + } + + # Filter the anomalies using one of the thresholding functions if applicable + if(threshold != "None"){ + # Calculate daily max values + if(!is.null(longterm_period)){ + periodic_maxs <- tapply(all_data[[i]][[2]], c(0:(longterm_period-1))%/%period, FUN=max) + }else{ + periodic_maxs <- tapply(all_data[[i]][[2]], c(0:(num_obs-1))%/%period, FUN=max) + } + + # Calculate the threshold set by the user + if(threshold == 'med_max'){ + thresh <- median(periodic_maxs) + }else if (threshold == 'p95'){ + thresh <- quantile(periodic_maxs, .95) + }else if (threshold == 'p99'){ + thresh <- quantile(periodic_maxs, .99) + } + # Remove any anoms below the threshold + anoms <- subset(anoms, anoms[[2]] >= thresh) + } + all_anoms <- rbind(all_anoms, anoms) + seasonal_plus_trend <- rbind(seasonal_plus_trend, data_decomp) + } + + # Cleanup potential duplicates + all_anoms <- all_anoms[!duplicated(all_anoms[[1]]), ] + seasonal_plus_trend <- seasonal_plus_trend[!duplicated(seasonal_plus_trend[[1]]), ] + + # -- If only_last was set by the user, create subset of the data that represent the most recent period + if(only_last){ + x_subset_single_period <- data.frame(timestamp=x[[1]][(num_obs-period+1):num_obs], count=x[[2]][(num_obs-period+1):num_obs]) + # Let's try and show 7 periods prior + past_obs <- period*7 + # If we don't have that much data, then show what we have - the last period + if(num_obs < past_obs){ + past_obs <- num_obs-period + } + + # When plotting anoms for the last period only we only show the previous 7 periods of data + x_subset_previous <- data.frame(timestamp=x[[1]][(num_obs-past_obs+1):(num_obs-period+1)], count=x[[2]][(num_obs-past_obs+1):(num_obs-period+1)]) + + all_anoms <- subset(all_anoms, all_anoms[[1]] >= x_subset_single_period[[1]][1]) + num_obs <- length(x_subset_single_period[[2]]) + } + + # Calculate number of anomalies as a percentage + anom_pct <- (length(all_anoms[[2]]) / num_obs) * 100 + + # If there are no anoms, then let's exit + if(anom_pct == 0){ + if(verbose) message("No anomalies detected.") + return (list("anoms"=data.frame(), "plot"=plot.new())) + } + + if(plot){ + # -- Build title for plots utilizing parameters set by user + plot_title <- paste(title, round(anom_pct, digits=2), "% Anomalies (alpha=", alpha, ", direction=", direction,")", sep="") + if(!is.null(longterm_period)){ + plot_title <- paste(plot_title, ", longterm=T", sep="") + } + + # -- Plot raw time series data + color_name <- paste("\"", title, "\"", sep="") + alpha <- 0.8 + if(only_last){ + all_data <- rbind(x_subset_previous, x_subset_single_period) + lines_at <- seq(1, length(all_data[[2]]), period)+min(all_data[[1]]) + xgraph <- ggplot2::ggplot(all_data, ggplot2::aes_string(x="timestamp", y="count")) + ggplot2::theme_bw() + ggplot2::theme(panel.grid.major = ggplot2::element_blank(), panel.grid.minor = ggplot2::element_blank(), text=ggplot2::element_text(size = 14)) + xgraph <- xgraph + ggplot2::geom_line(data=x_subset_previous, ggplot2::aes_string(colour=color_name), alpha=alpha*.33) + ggplot2::geom_line(data=x_subset_single_period, ggplot2::aes_string(color=color_name), alpha=alpha) + yrange <- get_range(all_data, index=2, y_log=y_log) + xgraph <- xgraph + ggplot2::scale_x_continuous(breaks=lines_at, expand=c(0,0)) + xgraph <- xgraph + ggplot2::geom_vline(xintercept=lines_at, color="gray60") + xgraph <- xgraph + ggplot2::labs(x=xlabel, y=ylabel, title=plot_title) + }else{ + num_periods <- num_obs/period + lines_at <- seq(1, num_obs, period) + + # check to see that we don't have too many breaks + inc <- 2 + while(num_periods > 14){ + num_periods <- num_obs/(period*inc) + lines_at <- seq(1, num_obs, period*inc) + inc <- inc + 1 + } + xgraph <- ggplot2::ggplot(x, ggplot2::aes_string(x="timestamp", y="count")) + ggplot2::theme_bw() + ggplot2::theme(panel.grid.major = ggplot2::element_blank(), panel.grid.minor = ggplot2::element_blank(), text=ggplot2::element_text(size = 14)) + xgraph <- xgraph + ggplot2::geom_line(data=x, ggplot2::aes_string(colour=color_name), alpha=alpha) + yrange <- get_range(x, index=2, y_log=y_log) + xgraph <- xgraph + ggplot2::scale_x_continuous(breaks=lines_at, expand=c(0,0)) + xgraph <- xgraph + ggplot2::geom_vline(xintercept=lines_at, color="gray60") + xgraph <- xgraph + ggplot2::labs(x=xlabel, y=ylabel, title=plot_title) + } + + # Add anoms to the plot as circles. + # We add zzz_ to the start of the name to ensure that the anoms are listed after the data sets. + xgraph <- xgraph + ggplot2::geom_point(data=all_anoms, ggplot2::aes_string(color=paste("\"zzz_",title,"\"",sep="")), size = 3, shape = 1) + + # Hide legend and timestamps + xgraph <- xgraph + ggplot2::theme(axis.text.x=ggplot2::element_blank()) + ggplot2::theme(legend.position="none") + + # Use log scaling if set by user + xgraph <- xgraph + add_formatted_y(yrange, y_log=y_log) + } + + # Store expected values if set by user + if(e_value) { + anoms <- data.frame(index=all_anoms[[1]], anoms=all_anoms[[2]], expected_value=subset(seasonal_plus_trend[[2]], seasonal_plus_trend[[1]] %in% all_anoms[[1]])) + } else { + anoms <- data.frame(index=all_anoms[[1]], anoms=all_anoms[[2]]) + } + + # Lastly, return anoms and optionally the plot if requested by the user + if(plot){ + return (list(anoms = anoms, plot = xgraph)) + } else { + return (list(anoms = anoms, plot = plot.new())) + } +} diff --git a/continuity.forecast/telescope-multi/R/xgb.R b/continuity.forecast/telescope-multi/R/xgb.R new file mode 100644 index 00000000..54bb08c5 --- /dev/null +++ b/continuity.forecast/telescope-multi/R/xgb.R @@ -0,0 +1,93 @@ +#' @author Marwin Zuefle, Andre Bauer + +#' @description Trains XGBoost regarding the time series and its covariates +#' +#' @title XGBoost Model Training +#' @param myts The training part of the time series +#' @param cov The covariates for the training +#' @param booster Select a boosting method: gblinear or gbtree +#' @param verbose Set to 1 to print information of xgboost performance; 0 for no prints +#' @return model The XGBoost model +doXGB.train <- function(myts, cov, booster, verbose) { + combined <- cbind(myts,cov) + feature.columns <- c(2:ncol(combined)) + + set.seed(200) + + nsample <- as.integer(0.2 * nrow(combined)) + + # Create a vector of nsample sample indices between 1 and the amount of rows of the matrix combined + h <- sample(nrow(combined),nsample) + + # In case of only one feature, convert the resulting vector to a matrix + # Create the special kind of matrices for xgboost (training and validation) + if(length(feature.columns)==1) { + dtrain <- xgb.DMatrix(data = as.matrix(combined[h,feature.columns]), label = myts[h]) + dtest <- xgb.DMatrix(data = as.matrix(combined[-h,feature.columns]), label = myts[-h]) + } else { + dtrain <- xgb.DMatrix(data = combined[h,feature.columns], label = myts[h]) + dtest <- xgb.DMatrix(data = combined[-h,feature.columns], label = myts[-h]) + } + + # Create a watchlist using a validation and a training set to prevent xgboost from overfitting + watchlist <- list(val=dtest, train=dtrain) + + param <- list( objective = "reg:linear", + booster = booster, + eta = 0.1, + max_depth = 5, + subsample = 0.8, + min_child_weight = 1, + num_parallel_tree = 2 + ) + + # Builds model + model <- xgb.train(params = param, data = dtrain, nthread = 2, nrounds = 500, + early_stop_rounds = 50, verbose = verbose, watchlist = watchlist, + maximize = FALSE + ) + return(model) +} + +#' @description Estimates the booster for XGBoost based on if the time series has a significant trend +#' +#' @title Estimating the boosting method +#' @param stl.decomp The STL decomposition of a time series +#' @param lower.percentile Optional parameter: The lower percentile for the IPR. 0.05 by default +#' @param upper.percentile Optional parameter: The upper percentile for the IPR. 0.95 by default +#' @param threshold Optional parameter: Threshold what the propotion of trend component has to exceed to be trendy. 0.33 by default +#' @return True if the time series has a significant trend +estimateBooster <- function(stl.decomp, lower.percentile = 0.05, upper.percentile = 0.95, threshold = 0.33) { + + # Calculates the IPR of the noise as noise may have outliers + range.noise <- IPR(stl.decomp$time.series[,3], lower.percentile, upper.percentile) + # Calculates the range + range.trend <- range(stl.decomp$time.series[,2])[2] - range(stl.decomp$time.series[,2])[1] + range.season <- range(stl.decomp$time.series[,1])[2] - range(stl.decomp$time.series[,1])[1] + + # Calculates the propotion of each component + range.vec <- c(range.noise, range.trend, range.season) + range.aggr <- sum(range.vec) + range.ratios <- range.vec / range.aggr + + # If trend component has a higher propotion than the threshold + if(range.ratios[2] >= threshold) { + return(TRUE) + } else { + return(FALSE) + } + +} + +#' @description computes the interpercentile range of vector x from percentile lower.percentile to percentile upper.percentile +#' +#' @title Inter Percentile Range +#' @param x The vector to compute interpercentile range of +#' @param lower.percentile The lower percentile +#' @param upper.percentile The upper percentile +#' @return The range between the lower and upper percentile +IPR <- function(x, lower.percentile, upper.percentile) { + q <- quantile(x, c(lower.percentile, upper.percentile)) + y <- q[2] - q[1] + return(as.double(y)) +} diff --git a/continuity.orchestrator/src/main/java/org/continuity/orchestrator/controllers/OrchestrationController.java b/continuity.orchestrator/src/main/java/org/continuity/orchestrator/controllers/OrchestrationController.java index 8bc8ed7a..ea75c65e 100644 --- a/continuity.orchestrator/src/main/java/org/continuity/orchestrator/controllers/OrchestrationController.java +++ b/continuity.orchestrator/src/main/java/org/continuity/orchestrator/controllers/OrchestrationController.java @@ -27,15 +27,18 @@ import org.continuity.api.entities.config.OrderMode; import org.continuity.api.entities.config.OrderOptions; import org.continuity.api.entities.config.WorkloadModelType; +import org.continuity.api.entities.links.ForecastLinks; import org.continuity.api.entities.links.LinkExchangeModel; import org.continuity.api.entities.links.LoadTestLinks; import org.continuity.api.entities.links.SessionLogsLinks; +import org.continuity.api.entities.links.SessionsBundlesLinks; import org.continuity.api.entities.links.WorkloadModelLinks; import org.continuity.api.entities.report.OrderReport; import org.continuity.api.entities.report.OrderResponse; import org.continuity.api.rest.RestApi; import org.continuity.commons.storage.MemoryStorage; import org.continuity.commons.utils.WebUtils; +import org.continuity.dsl.description.ForecastInput; import org.continuity.orchestrator.entities.CreationStep; import org.continuity.orchestrator.entities.DummyStep; import org.continuity.orchestrator.entities.OrderReportCounter; @@ -125,14 +128,14 @@ public ResponseEntity submitOrder(@RequestBody Order order, HttpServletR for (Map.Entry, Set> entry : sources.entrySet()) { for (LinkExchangeModel source : entry.getValue()) { - createAndSubmitRecipe(orderId, order.getTag(), order.getGoal(), order.getMode(), order.getOptions(), entry.getKey(), source, order.getModularizationOptions()); + createAndSubmitRecipe(orderId, order.getTag(), order.getGoal(), order.getMode(), order.getOptions(), order.getForecastInput(), entry.getKey(), source, order.getModularizationOptions()); } } } else { declareResponseQueue(orderId); orderCounterStorage.putToReserved(orderId, new OrderReportCounter(orderId, 1)); - createAndSubmitRecipe(orderId, order.getTag(), order.getGoal(), order.getMode(), order.getOptions(), order.getTestingContext(), order.getSource(), order.getModularizationOptions()); + createAndSubmitRecipe(orderId, order.getTag(), order.getGoal(), order.getMode(), order.getOptions(), order.getForecastInput(), order.getTestingContext(), order.getSource(), order.getModularizationOptions()); } OrderResponse response = new OrderResponse(); @@ -144,7 +147,7 @@ public ResponseEntity submitOrder(@RequestBody Order order, HttpServletR return ResponseEntity.accepted().body(response); } - private void createAndSubmitRecipe(String orderId, String tag, OrderGoal goal, OrderMode mode, OrderOptions options, Set testingContext, LinkExchangeModel source, ModularizationOptions modularizationOptions) { + private void createAndSubmitRecipe(String orderId, String tag, OrderGoal goal, OrderMode mode, OrderOptions options, ForecastInput forecastInput, Set testingContext, LinkExchangeModel source, ModularizationOptions modularizationOptions) { boolean useTestingContext = ((testingContext != null) && !testingContext.isEmpty()); if (useTestingContext) { @@ -175,7 +178,7 @@ private void createAndSubmitRecipe(String orderId, String tag, OrderGoal goal, O String recipeId = recipeStorage.reserve(tag); LOGGER.info("Processing new recipe {} for order {} with goal {}...", recipeId, orderId, goal); - Recipe recipe = new Recipe(orderId, recipeId, tag, recipeSteps, source, useTestingContext, testingContext, options, modularizationOptions); + Recipe recipe = new Recipe(orderId, recipeId, tag, recipeSteps, source, useTestingContext, testingContext, options, modularizationOptions, forecastInput); if (recipe.hasNext()) { recipeStorage.putToReserved(recipeId, recipe); @@ -253,7 +256,7 @@ private RecipeStep createRecipeStep(String tag, OrderGoal goal, OrderOptions opt step = new CreationStep(stepName, amqpTemplate, AmqpApi.SessionLogs.TASK_CREATE, AmqpApi.SessionLogs.TASK_CREATE.formatRoutingKey().of(tag), isPresent(LinkExchangeModel::getSessionLogsLinks, SessionLogsLinks::getLink)); break; - case CREATE_WORKLOAD_MODEL: + case CREATE_BEHAVIOR_MIX: WorkloadModelType workloadType; if ((options == null) || (options.getWorkloadModelType() == null)) { workloadType = WorkloadModelType.WESSBAS; @@ -261,6 +264,20 @@ private RecipeStep createRecipeStep(String tag, OrderGoal goal, OrderOptions opt workloadType = options.getWorkloadModelType(); } + step = new CreationStep(stepName, amqpTemplate, AmqpApi.WorkloadModel.MIX_CREATE, AmqpApi.WorkloadModel.MIX_CREATE.formatRoutingKey().of(workloadType.toPrettyString()), + isPresent(LinkExchangeModel::getSessionsBundlesLinks, SessionsBundlesLinks::getLink)); + break; + case CREATE_FORECAST: + step = new CreationStep(stepName, amqpTemplate, AmqpApi.Forecast.TASK_CREATE, AmqpApi.Forecast.TASK_CREATE.formatRoutingKey().of("forecast"), + isPresent(LinkExchangeModel::getForecastLinks, ForecastLinks::getLink)); + break; + case CREATE_WORKLOAD_MODEL: + if ((options == null) || (options.getWorkloadModelType() == null)) { + workloadType = WorkloadModelType.WESSBAS; + } else { + workloadType = options.getWorkloadModelType(); + } + Function check = all(isPresent(LinkExchangeModel::getWorkloadModelLinks, WorkloadModelLinks::getLink), isEqual(LinkExchangeModel::getWorkloadModelLinks, WorkloadModelLinks::getType, workloadType)); diff --git a/continuity.orchestrator/src/main/java/org/continuity/orchestrator/entities/Recipe.java b/continuity.orchestrator/src/main/java/org/continuity/orchestrator/entities/Recipe.java index 2e8fb845..1bb9feb4 100644 --- a/continuity.orchestrator/src/main/java/org/continuity/orchestrator/entities/Recipe.java +++ b/continuity.orchestrator/src/main/java/org/continuity/orchestrator/entities/Recipe.java @@ -10,6 +10,7 @@ import org.continuity.api.entities.config.TaskDescription; import org.continuity.api.entities.links.LinkExchangeModel; import org.continuity.api.entities.report.TaskReport; +import org.continuity.dsl.description.ForecastInput; public class Recipe { @@ -26,6 +27,8 @@ public class Recipe { private LinkExchangeModel source; private PropertySpecification properties; + + private ForecastInput forecastInput; private final boolean longTermUse; @@ -34,7 +37,7 @@ public class Recipe { private ModularizationOptions modularizationOptions; public Recipe(String orderId, String recipeId, String tag, List steps, LinkExchangeModel source, boolean longTermUse, Set testingContext, OrderOptions options, - ModularizationOptions modularizationOptions) { + ModularizationOptions modularizationOptions, ForecastInput forecastInput) { this.orderId = orderId; this.recipeId = recipeId; this.iterator = steps.listIterator(steps.size()); @@ -43,6 +46,7 @@ public Recipe(String orderId, String recipeId, String tag, List step this.longTermUse = longTermUse; this.testingContext = testingContext; this.modularizationOptions = modularizationOptions; + this.setForecastInput(forecastInput); initIterator(source); if (options != null) { @@ -83,6 +87,7 @@ public RecipeStep next() { task.setTag(tag); task.setSource(source); task.setProperties(properties); + task.setForecastInput(forecastInput); task.setLongTermUse(longTermUse); task.setModularizationOptions(modularizationOptions); @@ -108,5 +113,13 @@ private void initIterator(LinkExchangeModel source) { } } } + + public ForecastInput getForecastInput() { + return forecastInput; + } + + public void setForecastInput(ForecastInput forecastInput) { + this.forecastInput = forecastInput; + } } diff --git a/continuity.orchestrator/src/main/java/org/continuity/orchestrator/orders/OrderCycleManager.java b/continuity.orchestrator/src/main/java/org/continuity/orchestrator/orders/OrderCycleManager.java index 74d17793..7d1e95e2 100644 --- a/continuity.orchestrator/src/main/java/org/continuity/orchestrator/orders/OrderCycleManager.java +++ b/continuity.orchestrator/src/main/java/org/continuity/orchestrator/orders/OrderCycleManager.java @@ -29,6 +29,7 @@ public class OrderCycleManager { public OrderCycleManager() { cycle(OrderMode.PAST_SESSIONS, OrderGoal.CREATE_SESSION_LOGS, OrderGoal.CREATE_WORKLOAD_MODEL, OrderGoal.CREATE_LOAD_TEST, OrderGoal.EXECUTE_LOAD_TEST); cycle(OrderMode.PAST_REQUESTS, OrderGoal.CREATE_WORKLOAD_MODEL, OrderGoal.CREATE_LOAD_TEST, OrderGoal.EXECUTE_LOAD_TEST); + cycle(OrderMode.FORECASTED_WORKLOAD, OrderGoal.CREATE_SESSION_LOGS, OrderGoal.CREATE_BEHAVIOR_MIX, OrderGoal.CREATE_FORECAST, OrderGoal.CREATE_WORKLOAD_MODEL, OrderGoal.CREATE_LOAD_TEST, OrderGoal.EXECUTE_LOAD_TEST); } /** diff --git a/continuity.session.logs/Dockerfile b/continuity.session.logs/Dockerfile index 1f47678d..f104e4d1 100644 --- a/continuity.session.logs/Dockerfile +++ b/continuity.session.logs/Dockerfile @@ -3,4 +3,4 @@ VOLUME /tmp VOLUME /storage ARG JAR_FILE ADD ${JAR_FILE} app.jar -ENTRYPOINT ["java","-jar","/app.jar", "--port=80", "--eureka.uri=http://eureka:8761/eureka", "--storage.path=/storage"] \ No newline at end of file +ENTRYPOINT ["java","-jar","/app.jar", "--port=80", "--spring.rabbitmq.host=rabbitmq", "--eureka.uri=http://eureka:8761/eureka", "--storage.path=/storage"] \ No newline at end of file diff --git a/continuity.session.logs/continuity.session.logs.gradle b/continuity.session.logs/continuity.session.logs.gradle index ff469200..4accfa17 100644 --- a/continuity.session.logs/continuity.session.logs.gradle +++ b/continuity.session.logs/continuity.session.logs.gradle @@ -45,6 +45,9 @@ dependencies { compile("org.spec.research:open.xtrace.default.impl:0.2.2") compile("rocks.inspectit:shared-all:1.9.1") + + // CSV mapper + compile("com.univocity:univocity-parsers:2.7.5") } group = 'continuityproject' diff --git a/continuity.session.logs/src/main/java/org/continuity/session/logs/amqp/SessionLogsAmqpHandler.java b/continuity.session.logs/src/main/java/org/continuity/session/logs/amqp/SessionLogsAmqpHandler.java index 67ea384d..9c3d6775 100644 --- a/continuity.session.logs/src/main/java/org/continuity/session/logs/amqp/SessionLogsAmqpHandler.java +++ b/continuity.session.logs/src/main/java/org/continuity/session/logs/amqp/SessionLogsAmqpHandler.java @@ -48,7 +48,6 @@ public void createSessionLogs(TaskDescription task) { TaskReport report; String tag = task.getTag(); String link = task.getSource().getMeasurementDataLinks().getLink(); - boolean useOpenXtrace = task.getSource().getMeasurementDataLinks().getLinkType().equals(MeasurementDataLinkType.OPEN_XTRACE) ? true : false; boolean applyModularization = false; if (null != task.getModularizationOptions()) { @@ -73,10 +72,10 @@ public void createSessionLogs(TaskDescription task) { if (applyModularization) { LOGGER.info("Task {}: Creating modularized session logs for tags {} from data {} ...", task.getTaskId(), task.getModularizationOptions().getServices().keySet(), link); - sessionLog = manager.runPipeline(useOpenXtrace, task.getModularizationOptions().getServices()); + sessionLog = manager.runPipeline(task.getSource().getMeasurementDataLinks().getLinkType(), task.getModularizationOptions().getServices()); } else { LOGGER.info("Task {}: Creating session logs for tag {} from data {} ...", task.getTaskId(), tag, link); - sessionLog = manager.runPipeline(useOpenXtrace); + sessionLog = manager.runPipeline(task.getSource().getMeasurementDataLinks().getLinkType()); } String id = storage.put(new SessionLogs(task.getSource().getMeasurementDataLinks().getTimestamp(), sessionLog), tag); String sessionLink = RestApi.SessionLogs.GET.requestUrl(id).withoutProtocol().get(); diff --git a/continuity.session.logs/src/main/java/org/continuity/session/logs/converter/SessionConverterCSVData.java b/continuity.session.logs/src/main/java/org/continuity/session/logs/converter/SessionConverterCSVData.java new file mode 100644 index 00000000..963f1053 --- /dev/null +++ b/continuity.session.logs/src/main/java/org/continuity/session/logs/converter/SessionConverterCSVData.java @@ -0,0 +1,334 @@ +package org.continuity.session.logs.converter; + +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; + +import org.continuity.session.logs.entities.RowObject; + + +/** + * Class for converting a list of RowObjects into session logs. + * + * @author Alper Hidiroglu + * + */ +public class SessionConverterCSVData { + + /** + * Creates session logs from CSV data. + * @param dataList + * @return + */ + public String createSessionLogsFromCSV(ArrayList dataList) { + LinkedHashMap> map = processSessions(dataList); + boolean first = true; + String sessionLogs = ""; + for (Entry> entry : map.entrySet()) { + boolean empty = true; + StringBuffer buffer = new StringBuffer(); + buffer.append(entry.getKey()).append(";"); + for (RowObject rowObject : entry.getValue()) { + appendRowObjectInfo(buffer, rowObject); + empty = false; + } + + if (!empty) { + if (first) { + first = false; + } else { + sessionLogs += "\n"; + } + sessionLogs += buffer.toString(); + } + } + return sessionLogs; + } + + /** + * Creates sessions from the available data in RowObjects. + * + * @return + */ + public LinkedHashMap> processSessions(ArrayList dataList) { + LinkedHashMap> sessions = null; + if(!(dataList.get(0).getSessionID() == null)) { + sessions = extractSessions(dataList); + } else { + if(!(dataList.get(0).getUserName() == null)) { + sessions = calculateSessionsWithUserNames(dataList); + } else { + sessions = calculateSessions(dataList); + } + } + return sessions; + } + + /** + * If dataset contains session identifiers, extract sessions. + * Sorts requests for each session. + * @param dataList + * @return + */ + private LinkedHashMap> extractSessions(ArrayList dataList) { + LinkedHashMap> sessions = new LinkedHashMap>(); + + for(int i = 0; i < dataList.size(); i++) { + String sessionID = dataList.get(i).getSessionID(); + if (sessions.containsKey(sessionID)) { + LinkedList existingList = sessions.get(sessionID); + existingList.add(dataList.get(i)); + sessions.put(sessionID, existingList); + } else { + LinkedList newList = new LinkedList(); + newList.add(dataList.get(i)); + sessions.put(sessionID, newList); + } + } + for (Entry> entry : sessions.entrySet()) { + String sessionID = entry.getKey(); + LinkedList rowObjectList = entry.getValue(); + sortRowObjects(rowObjectList); + sessions.put(sessionID, rowObjectList); + } + return sessions; + } + + /** + * Sorts RowObjects in list. + * @param rowObjects + */ + private void sortRowObjects(LinkedList rowObjects) { + rowObjects.sort((RowObject ro1, RowObject ro2) -> { + long startTimeRo1 = Long.parseLong(ro1.getRequestStartTime()); + long startTimeRo2 = Long.parseLong(ro2.getRequestStartTime()); + int startTimeComparison = Long.compare(startTimeRo1, startTimeRo2); + if (startTimeComparison != 0) { + return startTimeComparison; + } else { + long durationRo1 = Long.parseLong(ro1.getRequestEndTime()) - startTimeRo1; + long durationRo2 = Long.parseLong(ro2.getRequestEndTime()) - startTimeRo2; + return Double.compare(durationRo1, durationRo2); + } + }); + } + + /** + * If dataset has no session identifiers, calculate them with the help of user names and request start times. + * @param dataList + * @return + */ + private LinkedHashMap> calculateSessionsWithUserNames(ArrayList dataList) { + LinkedHashMap> sessions = new LinkedHashMap>(); + + DateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy HH:mm"); + + String currentUserName = ""; + + String currentSession = ""; + + long startOfRandomNumber = 1000000000000000L; + + ArrayList listOfRandomNumbers = new ArrayList(); + for(long i = 0; i < dataList.size(); i++) { + listOfRandomNumbers.add(startOfRandomNumber); + startOfRandomNumber++; + } + Collections.shuffle(listOfRandomNumbers); + + for (int i = 0; i < dataList.size(); i++) { + String userName = dataList.get(i).getUserName(); + if (userName.equals(currentUserName)) { + Date date1 = null; + Date date2 = null; + try { + date1 = dateFormat.parse(dataList.get(i).getRequestStartTime()); + date2 = dateFormat.parse(dataList.get(i - 1).getRequestStartTime()); + } catch (ParseException e) { + e.printStackTrace(); + } + long time1 = date1.getTime(); + long time2 = date2.getTime(); + long difference = time1 - time2; + long minutes = TimeUnit.MILLISECONDS.toMinutes(difference); + if (minutes > 30) { + String newSession = Long.toString(listOfRandomNumbers.get(i)); + currentSession = newSession; + LinkedList newList = new LinkedList(); + newList.add(dataList.get(i)); + sessions.put(newSession, newList); + } else { + LinkedList currentList = sessions.get(currentSession); + currentList.add(dataList.get(i)); + sessions.put(currentSession, currentList); + } + } else { + currentUserName = userName; + String newSession = Long.toString(listOfRandomNumbers.get(i)); + currentSession = newSession; + LinkedList newList = new LinkedList(); + newList.add(dataList.get(i)); + sessions.put(newSession, newList); + } + } + return sessions; + + } + + /** + * If dataset has no session identifiers, calculate them with the help of request start times. + * @param dataList + * @return + */ + private LinkedHashMap> calculateSessions(ArrayList dataList) { + LinkedHashMap> sessions = new LinkedHashMap>(); + + DateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy HH:mm"); + + String currentSession = ""; + + long startOfRandomNumber = 1000000000000000L; + + ArrayList listOfRandomNumbers = new ArrayList(); + for(long i = 0; i < dataList.size(); i++) { + listOfRandomNumbers.add(startOfRandomNumber); + startOfRandomNumber++; + } + Collections.shuffle(listOfRandomNumbers); + + for (int i = 0; i < dataList.size(); i++) { + if(i == 0) { + String newSession = Long.toString(listOfRandomNumbers.get(i)); + currentSession = newSession; + LinkedList newList = new LinkedList(); + newList.add(dataList.get(i)); + sessions.put(newSession, newList); + } else { + Date date1 = null; + Date date2 = null; + try { + date1 = dateFormat.parse(dataList.get(i).getRequestStartTime()); + date2 = dateFormat.parse(dataList.get(i - 1).getRequestStartTime()); + } catch (ParseException e) { + e.printStackTrace(); + } + long time1 = date1.getTime(); + long time2 = date2.getTime(); + long difference = time1 - time2; + long minutes = TimeUnit.MILLISECONDS.toMinutes(difference); + if (minutes > 30) { + String newSession = Long.toString(listOfRandomNumbers.get(i)); + currentSession = newSession; + LinkedList newList = new LinkedList(); + newList.add(dataList.get(i)); + sessions.put(newSession, newList); + } else { + LinkedList currentList = sessions.get(currentSession); + currentList.add(dataList.get(i)); + sessions.put(currentSession, currentList); + } + } + } + return sessions; + } + + /** + * Appends RowObject infos to the String buffer. + * + * @param buffer + * @param rowObject + */ + private void appendRowObjectInfo(StringBuffer buffer, RowObject rowObject) { + DateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy HH:mm"); + + Date startDate = null; + Date endDate = null; + long endTime = 0L; + try { + startDate = dateFormat.parse(rowObject.getRequestStartTime()); + if (!(rowObject.getRequestEndTime() == null)) { + endDate = dateFormat.parse(rowObject.getRequestEndTime()); + endTime = endDate.getTime(); + } + } catch (ParseException e) { + e.printStackTrace(); + } + long startTime = startDate.getTime(); + long startMicros = TimeUnit.MILLISECONDS.toNanos(startTime); + long endMicros = TimeUnit.MILLISECONDS.toNanos(endTime); + + if (!(rowObject.getBusinessTransaction() == null)) { + buffer.append("\"").append(rowObject.getBusinessTransaction()).append("\":"); + } else { + buffer.append("\"").append(rowObject.getRequestURL()).append("\":"); + } + buffer.append(Long.toString(startMicros)).append(":"); + if (!(rowObject.getRequestEndTime() == null)) { + buffer.append(Long.toString(endMicros)).append(":"); + } else { + buffer.append(Long.toString(startMicros)).append(":"); + } + + appendHTTPInfo(buffer, rowObject); + } + + /** + * Appends HTTP infos to the String buffer. + * + * Sets dummy values when required information is not available in the data + * point. + * + * @param buffer + * @param rowObject + */ + private void appendHTTPInfo(StringBuffer buffer, RowObject rowObject) { + if (!(rowObject.getRequestURL() == null)) { + buffer.append(rowObject.getRequestURL()).append(":"); + } else { + buffer.append("/").append(rowObject.getBusinessTransaction()).append(":"); + } + + if (!(rowObject.getPort() == null)) { + buffer.append(rowObject.getPort()).append(":"); + } else { + buffer.append("8080").append(":"); + } + + if (!(rowObject.getHostIP() == null)) { + buffer.append(rowObject.getHostIP()).append(":"); + } else { + buffer.append("127.0.0.1").append(":"); + } + + if (!(rowObject.getProtocol() == null)) { + buffer.append(rowObject.getProtocol()).append(":"); + } else { + buffer.append("HTTP/1.1").append(":"); + } + + if (!(rowObject.getMethod() == null)) { + buffer.append(rowObject.getMethod()).append(":"); + } else { + buffer.append("GET").append(":"); + } + + if (!(rowObject.getParameter() == null)) { + buffer.append(rowObject.getParameter()).append(":"); + } else { + buffer.append("").append(":"); + } + + if (!(rowObject.getEncoding() == null)) { + buffer.append(rowObject.getEncoding()).append(";"); + } else { + buffer.append("UTF-8").append(";"); + } + } +} diff --git a/continuity.session.logs/src/main/java/org/continuity/session/logs/csv/ReadCSV.java b/continuity.session.logs/src/main/java/org/continuity/session/logs/csv/ReadCSV.java new file mode 100644 index 00000000..b3b2a53e --- /dev/null +++ b/continuity.session.logs/src/main/java/org/continuity/session/logs/csv/ReadCSV.java @@ -0,0 +1,46 @@ +package org.continuity.session.logs.csv; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.util.ArrayList; +import java.util.List; + +import org.continuity.session.logs.entities.RowObject; + +import com.univocity.parsers.common.processor.BeanListProcessor; +import com.univocity.parsers.csv.CsvParser; +import com.univocity.parsers.csv.CsvParserSettings; + +/** + * + * @author Alper Hidiroglu + * + */ +public class ReadCSV { + /** + * Reads data from CSV and saves the data into list of RowObjects. + * + * @return + */ + @SuppressWarnings("deprecation") + public ArrayList readDataFromCSV(String link) { + BeanListProcessor rowProcessor = new BeanListProcessor(RowObject.class); + + CsvParserSettings parserSettings = new CsvParserSettings(); + parserSettings.setRowProcessor(rowProcessor); + parserSettings.setHeaderExtractionEnabled(true); + parserSettings.setDelimiterDetectionEnabled(true, ';'); + + CsvParser parser = new CsvParser(parserSettings); + try { + parser.parse(new FileReader(new File(link))); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + + List beans = rowProcessor.getBeans(); + + return (ArrayList) beans; + } +} diff --git a/continuity.session.logs/src/main/java/org/continuity/session/logs/entities/RowObject.java b/continuity.session.logs/src/main/java/org/continuity/session/logs/entities/RowObject.java new file mode 100644 index 00000000..f80eb803 --- /dev/null +++ b/continuity.session.logs/src/main/java/org/continuity/session/logs/entities/RowObject.java @@ -0,0 +1,146 @@ +package org.continuity.session.logs.entities; + +import com.univocity.parsers.annotations.Parsed; + +public class RowObject { + + @Parsed(field = "session-id") + private String sessionID; + + @Parsed(field = "user-name") + private String userName; + + @Parsed(field = "business-transaction") + private String businessTransaction; + + @Parsed(field = "request-start-time") + private String requestStartTime; + + @Parsed(field = "request-end-time") + private String requestEndTime; + + @Parsed(field = "request-url") + private String requestURL; + + @Parsed(field = "port") + private String port; + + @Parsed(field = "host-ip") + private String hostIP; + + @Parsed(field = "protocol") + private String protocol; + + @Parsed(field = "method") + private String method; + + @Parsed(field = "parameter") + private String parameter; + + @Parsed(field = "encoding") + private String encoding; + + public String getSessionID() { + return sessionID; + } + + public void setSessionID(String sessionID) { + this.sessionID = sessionID; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getBusinessTransaction() { + return businessTransaction; + } + + public void setBusinessTransaction(String businessTransaction) { + this.businessTransaction = businessTransaction; + } + + public String getRequestStartTime() { + return requestStartTime; + } + + public void setRequestStartTime(String requestStartTime) { + this.requestStartTime = requestStartTime; + } + + public String getRequestEndTime() { + return requestEndTime; + } + + public void setRequestEndTime(String requestEndTime) { + this.requestEndTime = requestEndTime; + } + + public String getRequestURL() { + return requestURL; + } + + public void setRequestURL(String requestURL) { + this.requestURL = requestURL; + } + + public String getPort() { + return port; + } + + public void setPort(String port) { + this.port = port; + } + + public String getHostIP() { + return hostIP; + } + + public void setHostIP(String hostIP) { + this.hostIP = hostIP; + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + } + + public String getMethod() { + return method; + } + + public void setMethod(String method) { + this.method = method; + } + + public String getParameter() { + return parameter; + } + + public void setParameter(String parameter) { + this.parameter = parameter; + } + + public String getEncoding() { + return encoding; + } + + public void setEncoding(String encoding) { + this.encoding = encoding; + } + + @Override + public String toString() { + return "RowObject [sessionID=" + sessionID + ", userName=" + userName + ", businessTransaction=" + + businessTransaction + ", requestStartTime=" + requestStartTime + ", requestEndTime=" + requestEndTime + + ", requestURL=" + requestURL + ", port=" + port + ", hostIP=" + hostIP + ", protocol=" + protocol + + ", method=" + method + ", parameter=" + parameter + ", encoding=" + encoding + "]"; + } +} diff --git a/continuity.session.logs/src/main/java/org/continuity/session/logs/managers/SessionLogsPipelineManager.java b/continuity.session.logs/src/main/java/org/continuity/session/logs/managers/SessionLogsPipelineManager.java index 10f0d3e8..06d56307 100644 --- a/continuity.session.logs/src/main/java/org/continuity/session/logs/managers/SessionLogsPipelineManager.java +++ b/continuity.session.logs/src/main/java/org/continuity/session/logs/managers/SessionLogsPipelineManager.java @@ -5,7 +5,11 @@ import java.util.List; import java.util.Map; +import org.continuity.api.entities.links.MeasurementDataLinkType; import org.continuity.rest.InspectITRestClient; +import org.continuity.session.logs.converter.SessionConverterCSVData; +import org.continuity.session.logs.csv.ReadCSV; +import org.continuity.session.logs.entities.RowObject; import org.continuity.session.logs.extractor.InspectITSessionLogsExtractor; import org.continuity.session.logs.extractor.ModularizedOPENxtraceSessionLogsExtractor; import org.continuity.session.logs.extractor.OPENxtraceSessionLogsExtractor; @@ -15,6 +19,7 @@ import org.spec.research.open.xtrace.dflt.impl.serialization.OPENxtraceSerializationFactory; import org.spec.research.open.xtrace.dflt.impl.serialization.OPENxtraceSerializationFormat; import org.springframework.util.MultiValueMap; +import org.springframework.util.ResourceUtils; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponents; import org.springframework.web.util.UriComponentsBuilder; @@ -39,8 +44,10 @@ public class SessionLogsPipelineManager { public SessionLogsPipelineManager(String link, String tag, RestTemplate plainRestTemplate, RestTemplate eurekaRestTemplate) { this.link = link; this.tag = tag; - UriComponents uri = UriComponentsBuilder.fromHttpUrl(link).build(); - cmrConfig = uri.getHost() + ":" + uri.getPort(); + if(ResourceUtils.isUrl(link)) { + UriComponents uri = UriComponentsBuilder.fromHttpUrl(link).build(); + cmrConfig = uri.getHost() + ":" + uri.getPort(); + } this.eurekaRestTemplate = eurekaRestTemplate; this.plainRestTemplate = plainRestTemplate; } @@ -50,13 +57,31 @@ public SessionLogsPipelineManager(String link, String tag, RestTemplate plainRes * * @return */ - public String runPipeline(boolean useOpenXtrace) { - if (useOpenXtrace) { - return new OPENxtraceSessionLogsExtractor(tag, eurekaRestTemplate).getSessionLogs(getOPENxtraces()); - } else { + public String runPipeline(MeasurementDataLinkType measurementType) { + switch(measurementType) { + case INSPECTIT: return new InspectITSessionLogsExtractor(tag, eurekaRestTemplate, cmrConfig).getSessionLogs(getInvocationSequences()); + case OPEN_XTRACE: + return new OPENxtraceSessionLogsExtractor(tag, eurekaRestTemplate).getSessionLogs(getOPENxtraces()); + case CSV: + return getSessionLogsFromCSV(this.link); + default: + throw new RuntimeException("The given measurement cannot be resolved!"); } } + + /** + * Generates session logs from CSV file. + * @param link + * @return + */ + private String getSessionLogsFromCSV(String link) { + ReadCSV rcsv = new ReadCSV(); + ArrayList dataList = rcsv.readDataFromCSV(link); + SessionConverterCSVData csvsc = new SessionConverterCSVData(); + String sessionLogs = csvsc.createSessionLogsFromCSV(dataList); + return sessionLogs; + } /** * Runs the pipeline using the session logs modularization. Based on the environment variable, @@ -65,8 +90,8 @@ public String runPipeline(boolean useOpenXtrace) { * * @return */ - public String runPipeline(boolean useOpenXtrace, Map hostnames) { - if (useOpenXtrace) { + public String runPipeline(MeasurementDataLinkType measurementType, Map hostnames) { + if (measurementType.equals(MeasurementDataLinkType.OPEN_XTRACE)) { return new ModularizedOPENxtraceSessionLogsExtractor(tag, eurekaRestTemplate, hostnames).getSessionLogs(getOPENxtraces()); } else { throw new UnsupportedOperationException("Modularization of the session logs is currently only supported with open.XTRACE as source"); diff --git a/continuity.wessbas/continuity.wessbas.gradle b/continuity.wessbas/continuity.wessbas.gradle index 96ad0ad7..72efa4aa 100644 --- a/continuity.wessbas/continuity.wessbas.gradle +++ b/continuity.wessbas/continuity.wessbas.gradle @@ -14,7 +14,7 @@ dependencies { compile group: 'net.sf.markov4jmeter', name: 'm4jdsl', version: '1.0.0' compile group: 'net.sf.markov4jmeter', name: 'commons', version: '1.0.0' - compile group: 'net.sf.markov4jmeter', name: 'behaviormodelextractor', version: '1.0.0' + compile group: 'net.sf.markov4jmeter', name: 'behaviormodelextractor', version: '1.0.1' compile group: 'net.sf.markov4jmeter', name: 'modelgenerator', version: '1.0.0' compile group: 'net.sf.markov4jmeter', name: 'testplangenerator', version: '1.0.0' compile group: 'net.voorn', name: 'markov4jmeter', version: '1.0.20140405' diff --git a/continuity.wessbas/src/main/java/org/continuity/wessbas/amqp/BehaviorMixCreationAmqpHandler.java b/continuity.wessbas/src/main/java/org/continuity/wessbas/amqp/BehaviorMixCreationAmqpHandler.java new file mode 100644 index 00000000..966916a1 --- /dev/null +++ b/continuity.wessbas/src/main/java/org/continuity/wessbas/amqp/BehaviorMixCreationAmqpHandler.java @@ -0,0 +1,91 @@ +package org.continuity.wessbas.amqp; + +import org.continuity.api.amqp.AmqpApi; +import org.continuity.api.entities.artifact.SessionsBundlePack; +import org.continuity.api.entities.config.TaskDescription; +import org.continuity.api.entities.links.LinkExchangeModel; +import org.continuity.api.entities.report.TaskError; +import org.continuity.api.entities.report.TaskReport; +import org.continuity.api.rest.RestApi; +import org.continuity.commons.storage.MixedStorage; +import org.continuity.wessbas.config.RabbitMqConfig; +import org.continuity.wessbas.controllers.WessbasModelController; +import org.continuity.wessbas.entities.BehaviorModelPack; +import org.continuity.wessbas.managers.BehaviorMixManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.amqp.core.AmqpTemplate; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +/** + * Handles received monitoring data in order to create the Behavior mix. + * + * @author Alper Hidiroglu + * + */ +@Component +public class BehaviorMixCreationAmqpHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(BehaviorMixCreationAmqpHandler.class); + + @Autowired + private AmqpTemplate amqpTemplate; + + @Autowired + private RestTemplate restTemplate; + + @Autowired + private MixedStorage storage; + +// @Autowired +// private ConcurrentHashMap pathStorage; + + @Value("${spring.application.name}") + private String applicationName; + + /** + * Listener to the RabbitMQ {@link RabbitMqConfig#TASK_CREATE_QUEUE_NAME}. Creates a new Behavior + * mix based on the specified monitoring data. + * + * @param task + * The description of the task to be done. + * @return The id that can be used to retrieve the mix later on. + * @see WessbasModelController + */ + @RabbitListener(queues = RabbitMqConfig.MIX_CREATE_QUEUE_NAME) + public void onMonitoringDataAvailable(TaskDescription task) { + LOGGER.info("Task {}: Received new task to be processed for tag '{}'", task.getTaskId(), task.getTag()); + + TaskReport report; + + if (task.getSource().getSessionLogsLinks().getLink() == null) { + LOGGER.error("Task {}: Session logs link is missing for tag {}!", task.getTaskId(), task.getTag()); + report = TaskReport.error(task.getTaskId(), TaskError.MISSING_SOURCE); + } else { + BehaviorMixManager behaviorManager = new BehaviorMixManager(restTemplate); + SessionsBundlePack sessionsBundles = behaviorManager.runPipeline(task.getSource().getSessionLogsLinks().getLink()); + BehaviorModelPack behaviorModelPack = new BehaviorModelPack(sessionsBundles, behaviorManager.getWorkingDir()); + + if (sessionsBundles == null) { + LOGGER.info("Task {}: Could not create a new behavior mix for tag '{}'.", task.getTaskId(), task.getTag()); + + report = TaskReport.error(task.getTaskId(), TaskError.INTERNAL_ERROR); + } else { + + String storageId = storage.put(behaviorModelPack, task.getTag(), task.isLongTermUse()); + String behaviorModelPackLink = RestApi.Wessbas.SessionsBundles.GET.requestUrl(storageId).withoutProtocol().get(); + + report = TaskReport.successful(task.getTaskId(), new LinkExchangeModel().getSessionsBundlesLinks().setLink(behaviorModelPackLink).parent()); + + LOGGER.info("Task {}: Created a new sessions-bundle-pack with id '{}'.", task.getTaskId(), storageId); + } + } + + amqpTemplate.convertAndSend(AmqpApi.Global.EVENT_FINISHED.name(), AmqpApi.Global.EVENT_FINISHED.formatRoutingKey().of(RabbitMqConfig.SERVICE_NAME), report); + } + +} diff --git a/continuity.wessbas/src/main/java/org/continuity/wessbas/amqp/WessbasAmqpHandler.java b/continuity.wessbas/src/main/java/org/continuity/wessbas/amqp/WessbasAmqpHandler.java index a595c281..e18440b1 100644 --- a/continuity.wessbas/src/main/java/org/continuity/wessbas/amqp/WessbasAmqpHandler.java +++ b/continuity.wessbas/src/main/java/org/continuity/wessbas/amqp/WessbasAmqpHandler.java @@ -1,15 +1,21 @@ package org.continuity.wessbas.amqp; +import java.nio.file.Path; +import java.util.List; + import org.continuity.api.amqp.AmqpApi; import org.continuity.api.entities.config.TaskDescription; import org.continuity.api.entities.report.TaskError; import org.continuity.api.entities.report.TaskReport; +import org.continuity.api.rest.RestApi; import org.continuity.commons.storage.MixedStorage; import org.continuity.wessbas.config.RabbitMqConfig; import org.continuity.wessbas.controllers.WessbasModelController; +import org.continuity.wessbas.entities.BehaviorModelPack; import org.continuity.wessbas.entities.WessbasBundle; import org.continuity.wessbas.entities.WorkloadModelPack; import org.continuity.wessbas.managers.WessbasPipelineManager; +import org.continuity.wessbas.managers.WorkloadModelManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.amqp.core.AmqpTemplate; @@ -38,6 +44,9 @@ public class WessbasAmqpHandler { @Autowired private MixedStorage storage; + + @Autowired + private MixedStorage storageBehav; @Value("${spring.application.name}") private String applicationName; @@ -57,13 +66,23 @@ public void onMonitoringDataAvailable(TaskDescription task) { TaskReport report; - if (task.getSource().getSessionLogsLinks().getLink() == null) { - LOGGER.error("Task {}: Session logs link is missing for tag {}!", task.getTaskId(), task.getTag()); + if (task.getSource().getSessionLogsLinks().getLink() == null && task.getSource().getForecastLinks().getLink() == null) { + LOGGER.error("Task {}: Session logs link and forecast link is missing for tag {}!", task.getTaskId(), task.getTag()); report = TaskReport.error(task.getTaskId(), TaskError.MISSING_SOURCE); } else { - WessbasPipelineManager pipelineManager = new WessbasPipelineManager(restTemplate); - WessbasBundle workloadModel = pipelineManager.runPipeline(task.getSource().getSessionLogsLinks().getLink()); - + WessbasBundle workloadModel = null; + if(task.getSource().getForecastLinks().getLink() != null) { + WorkloadModelManager modelManager = new WorkloadModelManager(restTemplate); + + List pathParams = RestApi.Wessbas.SessionsBundles.GET.parsePathParameters(task.getSource().getSessionsBundlesLinks().getLink()); + BehaviorModelPack behaviorModelPack = storageBehav.get(pathParams.get(0)); + Path pathToBehaviorFiles = behaviorModelPack.getPathToBehaviorModelFiles(); + + workloadModel = modelManager.runPipeline(task.getSource().getForecastLinks().getLink(), pathToBehaviorFiles); + } else { + WessbasPipelineManager pipelineManager = new WessbasPipelineManager(restTemplate); + workloadModel = pipelineManager.runPipeline(task.getSource().getSessionLogsLinks().getLink()); + } if (workloadModel == null) { LOGGER.info("Task {}: Could not create a new workload model for tag '{}'.", task.getTaskId(), task.getTag()); diff --git a/continuity.wessbas/src/main/java/org/continuity/wessbas/config/RabbitMqConfig.java b/continuity.wessbas/src/main/java/org/continuity/wessbas/config/RabbitMqConfig.java index 5681215f..ec1cc8ce 100644 --- a/continuity.wessbas/src/main/java/org/continuity/wessbas/config/RabbitMqConfig.java +++ b/continuity.wessbas/src/main/java/org/continuity/wessbas/config/RabbitMqConfig.java @@ -18,7 +18,7 @@ import org.springframework.context.annotation.Configuration; /** - * @author Henning Schulz + * @author Henning Schulz, Alper Hidiroglu * */ @Configuration @@ -27,8 +27,12 @@ public class RabbitMqConfig { public static final String SERVICE_NAME = "wessbas"; public static final String TASK_CREATE_QUEUE_NAME = "continuity.wessbas.task.workloadmodel.create"; + + public static final String MIX_CREATE_QUEUE_NAME = "continuity.wessbas.task.behaviormix.createmix"; public static final String TASK_CREATE_ROUTING_KEY = AmqpApi.WorkloadModel.TASK_CREATE.formatRoutingKey().of(SERVICE_NAME); + + public static final String MIX_CREATE_ROUTING_KEY = AmqpApi.WorkloadModel.MIX_CREATE.formatRoutingKey().of(SERVICE_NAME); public static final String DEAD_LETTER_QUEUE_NAME = AmqpApi.DEAD_LETTER_EXCHANGE.deriveQueueName(SERVICE_NAME); @@ -69,17 +73,33 @@ SimpleRabbitListenerContainerFactory containerFactory(ConnectionFactory connecti TopicExchange taskCreateExchange() { return AmqpApi.WorkloadModel.TASK_CREATE.create(); } + + @Bean + TopicExchange mixCreateExchange() { + return AmqpApi.WorkloadModel.MIX_CREATE.create(); + } @Bean Queue taskCreateQueue() { return QueueBuilder.nonDurable(TASK_CREATE_QUEUE_NAME).withArgument(AmqpApi.DEAD_LETTER_EXCHANGE_KEY, AmqpApi.DEAD_LETTER_EXCHANGE.name()) .withArgument(AmqpApi.DEAD_LETTER_ROUTING_KEY_KEY, SERVICE_NAME).build(); } + + @Bean + Queue mixCreateQueue() { + return QueueBuilder.nonDurable(MIX_CREATE_QUEUE_NAME).withArgument(AmqpApi.DEAD_LETTER_EXCHANGE_KEY, AmqpApi.DEAD_LETTER_EXCHANGE.name()) + .withArgument(AmqpApi.DEAD_LETTER_ROUTING_KEY_KEY, SERVICE_NAME).build(); + } @Bean Binding taskCreateBinding() { return BindingBuilder.bind(taskCreateQueue()).to(taskCreateExchange()).with(TASK_CREATE_ROUTING_KEY); } + + @Bean + Binding mixCreateBinding() { + return BindingBuilder.bind(mixCreateQueue()).to(mixCreateExchange()).with(MIX_CREATE_ROUTING_KEY); + } @Bean TopicExchange eventCreatedExchange() { diff --git a/continuity.wessbas/src/main/java/org/continuity/wessbas/config/StorageConfig.java b/continuity.wessbas/src/main/java/org/continuity/wessbas/config/StorageConfig.java index 4783c95a..c5cd2d65 100644 --- a/continuity.wessbas/src/main/java/org/continuity/wessbas/config/StorageConfig.java +++ b/continuity.wessbas/src/main/java/org/continuity/wessbas/config/StorageConfig.java @@ -3,6 +3,7 @@ import java.nio.file.Paths; import org.continuity.commons.storage.MixedStorage; +import org.continuity.wessbas.entities.BehaviorModelPack; import org.continuity.wessbas.entities.WessbasBundle; import org.continuity.wessbas.storage.WessbasFileStorage; import org.springframework.beans.factory.annotation.Value; @@ -16,5 +17,10 @@ public class StorageConfig { public MixedStorage wessbasStorage(@Value("${storage.path:storage}") String storagePath) { return new MixedStorage<>(WessbasBundle.class, new WessbasFileStorage(Paths.get(storagePath))); } + + @Bean + public MixedStorage behaviorModelStorage(@Value("${storage.path:storage}") String storagePath) { + return new MixedStorage<>(Paths.get(storagePath), new BehaviorModelPack()); + } } diff --git a/continuity.wessbas/src/main/java/org/continuity/wessbas/controllers/SessionsBundlePackController.java b/continuity.wessbas/src/main/java/org/continuity/wessbas/controllers/SessionsBundlePackController.java new file mode 100644 index 00000000..7ac1fed5 --- /dev/null +++ b/continuity.wessbas/src/main/java/org/continuity/wessbas/controllers/SessionsBundlePackController.java @@ -0,0 +1,45 @@ +package org.continuity.wessbas.controllers; + +import static org.continuity.api.rest.RestApi.Wessbas.SessionsBundles.Paths.GET; + +import org.continuity.api.entities.artifact.SessionsBundlePack; +import org.continuity.api.rest.RestApi; +import org.continuity.commons.storage.MixedStorage; +import org.continuity.wessbas.entities.BehaviorModelPack; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +/** + * + * @author Alper Hidiroglu + * + */ +@RestController() +@RequestMapping(RestApi.Wessbas.SessionsBundles.ROOT) +public class SessionsBundlePackController { + + private static final Logger LOGGER = LoggerFactory.getLogger(SessionsBundlePackController.class); + + @Autowired + private MixedStorage storage; + + @RequestMapping(value = GET, method = RequestMethod.GET) + public ResponseEntity getSessionsBundlePackFromLink(@PathVariable String id) { + BehaviorModelPack behaviorModelPack = storage.get(id); + SessionsBundlePack sessionsBundles = behaviorModelPack.getSessionsBundlePack(); + + if (sessionsBundles == null) { + LOGGER.warn("Could not find sessions-bundle-pack for id {}!", id); + return ResponseEntity.notFound().build(); + } else { + LOGGER.info("Returned sessions-bundle-pack for id {}!", id); + return ResponseEntity.ok(sessionsBundles); + } + } +} diff --git a/continuity.wessbas/src/main/java/org/continuity/wessbas/entities/BehaviorModelPack.java b/continuity.wessbas/src/main/java/org/continuity/wessbas/entities/BehaviorModelPack.java new file mode 100644 index 00000000..7cdf2a5b --- /dev/null +++ b/continuity.wessbas/src/main/java/org/continuity/wessbas/entities/BehaviorModelPack.java @@ -0,0 +1,49 @@ +package org.continuity.wessbas.entities; + +import java.nio.file.Path; + +import org.continuity.api.entities.artifact.SessionsBundlePack; + +/** + * Pack that holds sessions for each user group and the path to the created behavior model files. + * @author Alper Hidiroglu + * + */ +public class BehaviorModelPack { + + /** + * Sessions for each user group. + */ + private SessionsBundlePack sessionsBundlePack; + + /** + * Path to behavior model files. + */ + private Path pathToBehaviorModelFiles; + + public BehaviorModelPack(SessionsBundlePack sessionsBundlePack, Path pathToBehaviorModelFiles) { + this.sessionsBundlePack = sessionsBundlePack; + this.pathToBehaviorModelFiles = pathToBehaviorModelFiles; + } + + public BehaviorModelPack() { + + } + + public SessionsBundlePack getSessionsBundlePack() { + return sessionsBundlePack; + } + + public void setSessionsBundlePack(SessionsBundlePack sessionsBundlePack) { + this.sessionsBundlePack = sessionsBundlePack; + } + + public Path getPathToBehaviorModelFiles() { + return pathToBehaviorModelFiles; + } + + public void setPathToBehaviorModelFiles(Path pathToBehaviorModelFiles) { + this.pathToBehaviorModelFiles = pathToBehaviorModelFiles; + } + +} diff --git a/continuity.wessbas/src/main/java/org/continuity/wessbas/managers/BehaviorMixManager.java b/continuity.wessbas/src/main/java/org/continuity/wessbas/managers/BehaviorMixManager.java new file mode 100644 index 00000000..806d8b4e --- /dev/null +++ b/continuity.wessbas/src/main/java/org/continuity/wessbas/managers/BehaviorMixManager.java @@ -0,0 +1,167 @@ +package org.continuity.wessbas.managers; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; + +import org.continuity.api.entities.artifact.SessionLogs; +import org.continuity.api.entities.artifact.SessionsBundle; +import org.continuity.api.entities.artifact.SessionsBundlePack; +import org.continuity.api.entities.artifact.SimplifiedSession; +import org.continuity.commons.utils.WebUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; + +import net.sf.markov4jmeter.behavior.BehaviorMix; +import net.sf.markov4jmeter.behavior.Session; +import net.sf.markov4jmeter.behaviormodelextractor.BehaviorModelExtractor; +import net.sf.markov4jmeter.behaviormodelextractor.extraction.ExtractionException; +import net.sf.markov4jmeter.m4jdslmodelgenerator.GeneratorException; +import wessbas.commons.parser.ParseException; + +/** + * + * @author Alper Hidiroglu + * + */ +public class BehaviorMixManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(BehaviorMixManager.class); + + private RestTemplate restTemplate; + + private final Path workingDir; + + public Path getWorkingDir() { + return workingDir; + } + + /** + * Constructor. + */ + public BehaviorMixManager(RestTemplate restTemplate) { + this.restTemplate = restTemplate; + + Path tmpDir; + try { + tmpDir = Files.createTempDirectory("wessbas"); + } catch (IOException e) { + LOGGER.error("Could not create a temp directory!"); + e.printStackTrace(); + tmpDir = Paths.get("wessbas"); + } + + workingDir = tmpDir; + + LOGGER.info("Set working directory to {}", workingDir); + } + + /** + * Runs the pipeline and returns a SessionsBundlePack that holds a list of SessionBundles. + * + * + * @param task + * Input monitoring data to be transformed into a WESSBAS DSL instance. + * + * @return The generated workload model. + */ + public SessionsBundlePack runPipeline(String sessionLogsLink) { + + SessionLogs sessionLog; + try { + sessionLog = restTemplate.getForObject(WebUtils.addProtocolIfMissing(sessionLogsLink), SessionLogs.class); + } catch (RestClientException e) { + LOGGER.error("Error when retrieving the session logs!", e); + return null; + } + BehaviorMix mix; + SessionsBundlePack sessionsBundles; + + try { + mix = convertSessionLogIntoBehaviorMix(sessionLog.getLogs()); + sessionsBundles = extractSessions(sessionLog.getDataTimestamp(), mix); + + } catch (Exception e) { + LOGGER.error("Could not create the Behavior Mix!", e); + mix = null; + sessionsBundles = null; + } + + return sessionsBundles; + } + + /** + * This method extracts the Behavior Mix from a session log. + * + * @param sessionLog + * @throws IOException + * @throws GeneratorException + * @throws SecurityException + */ + private BehaviorMix convertSessionLogIntoBehaviorMix(String sessionLog) throws IOException, SecurityException, GeneratorException, ExtractionException, ParseException { + Path sessionLogsPath = writeSessionLogIntoFile(sessionLog); + BehaviorMix mix = createBehaviorMix(sessionLogsPath); + return mix; + } + + /** + * + * @param sessionLog + * @return + * @throws IOException + */ + private Path writeSessionLogIntoFile(String sessionLog) throws IOException { + Path sessionLogsPath = workingDir.resolve("sessions.dat"); + Files.write(sessionLogsPath, Collections.singletonList(sessionLog), StandardOpenOption.CREATE); + return sessionLogsPath; + } + + /** + * Creates the Behavior Mix and writes the corresponding files. + * @param sessionLogsPath + * @return + * @throws IOException + * @throws ParseException + * @throws ExtractionException + */ + private BehaviorMix createBehaviorMix(Path sessionLogsPath) throws IOException, ParseException, ExtractionException { + Path outputDir = workingDir.resolve("behaviormodelextractor"); + outputDir.toFile().mkdir(); + + BehaviorModelExtractor extractor = new BehaviorModelExtractor(); + extractor.init(null, null, 0); + BehaviorMix mix = extractor.extractBehaviorMix(sessionLogsPath.toString(), outputDir.toString()); + + extractor.writeIntoFiles(mix, outputDir.toString()); + + return mix; + } + + /** + * + * @param mix + * @return + */ + private SessionsBundlePack extractSessions(Date timestamp, BehaviorMix mix) { + SessionsBundlePack sessionsBundles = new SessionsBundlePack(timestamp, new LinkedList()); + for(int i = 0; i < mix.getEntries().size(); i++) { + List simplifiedSessions = new LinkedList(); + for(Session session: mix.getEntries().get(i).getSessions()) { + SimplifiedSession simpleSession = new SimplifiedSession(session.getId(), session.getStartTime(), session.getEndTime()); + simplifiedSessions.add(simpleSession); + } + SessionsBundle sessBundle = new SessionsBundle(i, simplifiedSessions); + sessionsBundles.getSessionsBundles().add(sessBundle); + } + return sessionsBundles; + } + +} diff --git a/continuity.wessbas/src/main/java/org/continuity/wessbas/managers/WorkloadModelManager.java b/continuity.wessbas/src/main/java/org/continuity/wessbas/managers/WorkloadModelManager.java new file mode 100644 index 00000000..b7c1bef8 --- /dev/null +++ b/continuity.wessbas/src/main/java/org/continuity/wessbas/managers/WorkloadModelManager.java @@ -0,0 +1,118 @@ +package org.continuity.wessbas.managers; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Properties; + +import org.continuity.api.entities.artifact.ForecastBundle; +import org.continuity.commons.utils.WebUtils; +import org.continuity.wessbas.entities.WessbasBundle; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.client.RestTemplate; + +import m4jdsl.WorkloadModel; +import net.sf.markov4jmeter.m4jdslmodelgenerator.GeneratorException; +import net.sf.markov4jmeter.m4jdslmodelgenerator.M4jdslModelGenerator; +import wessbas.commons.util.XmiEcoreHandler; + +/** + * Manages the workload model pipeline from the input data to the output WESSBAS DSL + * instance. + * + * @author Alper Hidiroglu + * + */ +public class WorkloadModelManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(WessbasPipelineManager.class); + + private RestTemplate restTemplate; + + private Path workingDir; + + /** + * Constructor. + */ + public WorkloadModelManager(RestTemplate restTemplate) { + this.restTemplate = restTemplate; + } + + /** + * Runs the pipeline and calls the callback when the model was created. + * + * + * @param task + * Input monitoring data to be transformed into a WESSBAS DSL instance. + * + * @return The generated workload model. + */ + public WessbasBundle runPipeline(String forecastLink, Path pathToFiles) { + + ForecastBundle forecastBundle = restTemplate.getForObject(WebUtils.addProtocolIfMissing(forecastLink), ForecastBundle.class); + + this.workingDir = pathToFiles; + + LOGGER.info("Working directory is {}", workingDir); + + WorkloadModel workloadModel; + + try { + workloadModel = generateWessbasDSLInstance(forecastBundle); + } catch (Exception e) { + LOGGER.error("Could not create a WESSBAS workload model!", e); + workloadModel = null; + } + + return new WessbasBundle(forecastBundle.getTimestamp(), workloadModel); + } + + /** + * This method generates a Wessbas DSL instance. + * + * @param sessionLog + * @throws IOException + * @throws GeneratorException + * @throws SecurityException + */ + private WorkloadModel generateWessbasDSLInstance(ForecastBundle forecastBundle) throws IOException, SecurityException, GeneratorException { + // set 1 as default and configure actual number on demand + Properties intensityProps = createWorkloadIntensity(forecastBundle.getWorkloadIntensity()); + Properties behaviorProps = loadBehaviorMix(forecastBundle); + WorkloadModel workloadModel = generateWessbasModel(intensityProps, behaviorProps); + // update the behavior mix + for(int i = 0; i < forecastBundle.getProbabilities().size(); i++) { + workloadModel.getBehaviorMix().getRelativeFrequencies().get(i).setValue(forecastBundle.getProbabilities().get(i)); + } + + final String xmiOutputFilePath = "workloadmodel/workloadmodel.xmi"; + XmiEcoreHandler.getInstance().ecoreToXMI(workloadModel, xmiOutputFilePath); + return workloadModel; + } + + private Properties loadBehaviorMix(ForecastBundle forecastBundle) throws IOException { + Properties behaviorProperties = new Properties(); + behaviorProperties.load(Files.newInputStream(workingDir.resolve("behaviormodelextractor").resolve("behaviormix.txt"))); + return behaviorProperties; + } + + private Properties createWorkloadIntensity(int numberOfUsers) throws IOException { + Properties properties = new Properties(); + properties.put("workloadIntensity.type", "constant"); + properties.put("wl.type.value", Integer.toString(numberOfUsers)); + + properties.store(Files.newOutputStream(workingDir.resolve("workloadIntensity.properties"), StandardOpenOption.CREATE), null); + return properties; + } + + private WorkloadModel generateWessbasModel(Properties workloadIntensityProperties, Properties behaviorModelsProperties) throws FileNotFoundException, SecurityException, GeneratorException { + M4jdslModelGenerator generator = new M4jdslModelGenerator(); + final String sessionDatFilePath = workingDir.resolve("sessions.dat").toString(); + + return generator.generateWorkloadModel(workloadIntensityProperties, behaviorModelsProperties, null, sessionDatFilePath, false); + } + +} \ No newline at end of file diff --git a/docker-compose-debug.yml b/docker-compose-debug.yml index 1483463d..85a3391e 100644 --- a/docker-compose-debug.yml +++ b/docker-compose-debug.yml @@ -190,8 +190,35 @@ services: - -jar - /app.jar - --port=80 + - --spring.rabbitmq.host=rabbitmq + - --eureka.uri=http://eureka:8761/eureka + + forecast: + image: continuityproject/forecast + hostname: forecast + networks: + - continuity-network + ports: + - '8087:80' + - '5007:5000' + + depends_on: + - rabbitmq + - eureka + + entrypoint: + - java + - -Xdebug + - -Xnoagent + - -Djava.compiler=NONE + - -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5000 + - -jar + - /app.jar + - --port=80 + - --spring.rabbitmq.host=rabbitmq - --eureka.uri=http://eureka:8761/eureka + networks: continuity-network: driver: bridge diff --git a/docker-compose.yml b/docker-compose.yml index b83f21f1..7bf6e584 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -85,9 +85,16 @@ services: depends_on: - rabbitmq - eureka - environment: - - "JAVA_OPTS=-DUSE_OPEN_XTRACE=false" - + forecast: + image: continuityproject/forecast + hostname: forecast + networks: + - continuity-network + ports: + - '8087:80' + depends_on: + - rabbitmq + - eureka networks: continuity-network: driver: bridge diff --git a/settings.gradle b/settings.gradle index 680a7d2c..29df0084 100644 --- a/settings.gradle +++ b/settings.gradle @@ -4,7 +4,9 @@ include "continuity.api" include "continuity.benchflow" include "continuity.cli" include "continuity.commons" +include "continuity.dsl" include "continuity.eureka" +include "continuity.forecast" include "continuity.idpa" include "continuity.idpa.annotation" include "continuity.idpa.application"