diff --git a/components/inspectit-ocelot-configurationserver-ui/src/components/views/configuration/ConfigurationView.js b/components/inspectit-ocelot-configurationserver-ui/src/components/views/configuration/ConfigurationView.js index 1a3498059d..5a44cd26b4 100644 --- a/components/inspectit-ocelot-configurationserver-ui/src/components/views/configuration/ConfigurationView.js +++ b/components/inspectit-ocelot-configurationserver-ui/src/components/views/configuration/ConfigurationView.js @@ -165,7 +165,7 @@ class ConfigurationView extends React.Component { * If no version is specified, the latest version will be selected. */ openFile = (filename, versionId = null) => { - if (this.props.selectedVersion != versionId) { + if (this.props.selectedVersion !== versionId) { this.props.selectVersion(versionId, false); } this.props.selectFile(filename); @@ -191,7 +191,6 @@ class ConfigurationView extends React.Component { canWrite, } = this.props; const showEditor = (selection || selectedDefaultConfigFile) && !isDirectory; - const { path, name } = this.parsePath(selection, selectedDefaultConfigFile); const icon = 'pi-' + (isDirectory ? 'folder' : 'file'); const showHeader = !!name; @@ -199,7 +198,6 @@ class ConfigurationView extends React.Component { const readOnly = !canWrite || !!selectedDefaultConfigFile || !isLatestVersion; const fileContentWithoutFirstLine = fileContent ? fileContent.split('\n').slice(1).join('\n') : ''; - return (
+
} > @@ -86,6 +101,10 @@ DownloadDialogue.propTypes = { contentType: PropTypes.string, /** The name of the data's context. E.g. the agent whose logs are being shown.*/ contextName: PropTypes.string, + /** Whether the cancel Button is disabled */ + disableDownloadCancelButton: PropTypes.bool, + /** Callback on dialog cancel */ + onCancel: PropTypes.func, }; DownloadDialogue.defaultProps = { diff --git a/components/inspectit-ocelot-configurationserver-ui/src/components/views/status/StatusTable.js b/components/inspectit-ocelot-configurationserver-ui/src/components/views/status/StatusTable.js index dd82d581ec..6be7443a78 100644 --- a/components/inspectit-ocelot-configurationserver-ui/src/components/views/status/StatusTable.js +++ b/components/inspectit-ocelot-configurationserver-ui/src/components/views/status/StatusTable.js @@ -53,23 +53,19 @@ class AgentMappingCell extends React.Component { display: flex; align-items: stretch; } - .mapping-name { flex: 1; margin-right: 0.5rem; } - .no-mapping { color: gray; font-style: italic; } - .show-attributes { float: right; cursor: pointer; color: #007ad9; } - .attributes { margin-top: 0.5rem; border-left: 0.25rem solid #ddd; @@ -174,7 +170,6 @@ class StatusTable extends React.Component { .this { position: relative; } - .this :global(.config-info-button) { width: 1.2rem; height: 1.2rem; @@ -184,7 +179,6 @@ class StatusTable extends React.Component { background: #ddd; border-color: #ddd; } - .this :global(.log-button) { width: 1.2rem; height: 1.2rem; @@ -194,7 +188,6 @@ class StatusTable extends React.Component { background: #ddd; border-color: #ddd; } - .this :global(.service-state-button) { width: 1.2rem; height: 1.2rem; @@ -204,7 +197,6 @@ class StatusTable extends React.Component { background: #ddd; border-color: #ddd; } - .this :global(.badge) { width: 1.2rem; height: 1.2rem; @@ -217,7 +209,6 @@ class StatusTable extends React.Component { justify-content: center; color: white; } - .this :global(.might-overflow) { max-width: 17.8rem; display: inline-block; @@ -301,8 +292,11 @@ class StatusTable extends React.Component { }; agentHealthTemplate = (rowData) => { + const { onShowHealthStateDialog } = this.props; + const { healthState, metaInformation } = rowData; + const { health } = healthState; + const { agentId } = metaInformation; const { onShowDownloadDialog } = this.props; - const { health, metaInformation } = rowData; let { agentCommandsEnabled, supportArchiveAvailable } = this.resolveServiceAvailability(metaInformation); @@ -326,12 +320,14 @@ class StatusTable extends React.Component { iconColor = '#e8c413'; break; } + return ( <> {health ? (
- - {healthInfo} +
onShowHealthStateDialog(agentId, healthState)}> + + {healthInfo} +
this.setDownloadDialogShown(false)} + visible={showDownloadDialog} + onHide={() => this.setShowDownloadDialog(false)} error={contentLoadingFailed} loading={isLoading} contentValue={contentValue} contentType={contentType} contextName={'Agent ' + agentId} + isDownloadDialogFooterHidden={isDownloadDialogFooterHidden} + onCancel={() => { + this.setShowDownloadDialog(false); + this.axiosAbortController.abort(); + }} + /> + this.setShowHealthStateDialog(false)} + contentValue={this.state.attributes} + contextName={'Agent ' + agentId} /> { - if (showDialog == false) { - controller.abort(); - controller = new AbortController(); // A new instance has to be created in order for new requests to be accepted - } + setShowDownloadDialog = (showDialog) => { + this.setState({ + showDownloadDialog: showDialog, + }); + }; + setShowHealthStateDialog = (showDialog) => { this.setState({ - isDownloadDialogShown: showDialog, + showHealthStateDialog: showDialog, }); }; showDownloadDialog = (agentId, attributes, contentType) => { - this.setDownloadDialogShown(true); + this.setShowDownloadDialog(true); this.setState( { agentId, @@ -334,36 +352,47 @@ class StatusView extends React.Component { this.downloadSupportArchive(agentId, attributes); break; default: - this.setDownloadDialogShown(false); + this.setShowDownloadDialog(false); break; } } ); }; + showHealthStateDialog = (agentId, attributes) => { + this.setShowHealthStateDialog(true); + this.setState({ + agentId, + attributes, + }); + }; + downloadSupportArchive = (agentId, agentVersion) => { this.setState( { isLoading: true, + isDownloadDialogFooterHidden: true, }, () => { axios .get('/agent/supportArchive', { + signal: this.axiosAbortController.signal, params: { 'agent-id': agentId }, - signal: controller.signal, }) .then((res) => { + downloadArchiveFromJson(res.data, agentId, agentVersion); + this.setShowDownloadDialog(false); this.setState({ isLoading: false, + isDownloadDialogFooterHidden: false, }); - downloadArchiveFromJson(res.data, agentId, agentVersion); - this.setDownloadDialogShown(false); }) .catch(() => { this.setState({ - contentValue: null, + contentValue: '', contentLoadingFailed: true, isLoading: false, + isDownloadDialogFooterHidden: false, }); }); } @@ -382,8 +411,8 @@ class StatusView extends React.Component { () => { axios .get('/configuration/agent-configuration', { + signal: this.axiosAbortController.signal, params: { ...requestParams }, - signal: controller.signal, }) .then((res) => { this.setState({ @@ -411,8 +440,8 @@ class StatusView extends React.Component { () => { axios .get('/command/logs', { + signal: this.axiosAbortController.signal, params: { 'agent-id': agentId }, - signal: controller.signal, }) .then((res) => { this.setState({ diff --git a/components/inspectit-ocelot-configurationserver-ui/src/components/views/status/dialogs/AgentHealthStateDialogue.js b/components/inspectit-ocelot-configurationserver-ui/src/components/views/status/dialogs/AgentHealthStateDialogue.js new file mode 100644 index 0000000000..b8d2d89ba1 --- /dev/null +++ b/components/inspectit-ocelot-configurationserver-ui/src/components/views/status/dialogs/AgentHealthStateDialogue.js @@ -0,0 +1,152 @@ +import React from 'react'; +import { Dialog } from 'primereact/dialog'; +import { Button } from 'primereact/button'; +import PropTypes from 'prop-types'; + +/** + * Dialog that shows the given content, applying syntax highlighting if available, and offering a download option + */ +const AgentHealthStateDialogue = ({ visible, onHide, error, contentValue, contextName }) => { + const { health, source, message, history } = contentValue; + const { healthInfo, iconColor } = resolveHealthState(health); + + let latestIncidentElement = <>There are no incidents.; + + if (source && message) { + latestIncidentElement = ( + <> +
+

Latest Incident:

+
+
+ {source}: {message} +
+
+ + ); + } + + const historyElements = []; + if (history && history.length > 0) { + historyElements.push( +
+

History:

+
+ ); + history.forEach((value) => { + const { time, source, health, message } = value; + const { healthInfo, iconColor } = resolveHealthState(health); + historyElements.push( +
+ {time} {healthInfo}{' '} + {source}:
{message} +
+
+ ); + }); + } + return ( + <> + + +
+ } + > + {error ? ( +
+ ) : ( + <> +
+

+ The agent is in an + {healthInfo} + state +

+
+ {latestIncidentElement} +
{historyElements}
+ + )} +
+ + ); +}; + +AgentHealthStateDialogue.propTypes = { + /** Whether a error is thrown */ + error: PropTypes.bool, + /** Whether the dialog is visible */ + visible: PropTypes.bool, + /** Callback on dialog hide */ + onHide: PropTypes.func, + /** The string value being displayed. E.g. the logs.*/ + contentValue: PropTypes.string, + /** The type of content. E.g. config or log.*/ + contentType: PropTypes.string, + /** The name of the data's context. E.g. the agent whose logs are being shown.*/ + contextName: PropTypes.string, +}; + +AgentHealthStateDialogue.defaultProps = { + error: false, + visible: true, + onHide: () => {}, + contentValue: 'No content found', + contentType: '', +}; + +const resolveHealthState = (health) => { + switch (health) { + case 'OK': + return { + healthInfo: ' OK ', + iconClass: 'pi-check-circle', + iconColor: '#0abd04', + }; + case 'WARNING': + return { + healthInfo: ' Warning ', + iconClass: 'pi-minus-circle', + iconColor: '#e8c413', + }; + } + return { + healthInfo: ' Error ', + iconClass: 'pi-times-circle', + iconColor: 'red', + }; +}; + +export default AgentHealthStateDialogue; diff --git a/components/inspectit-ocelot-configurationserver-ui/src/lib/axios-api.js b/components/inspectit-ocelot-configurationserver-ui/src/lib/axios-api.js index 0bff230596..3b4830d162 100644 --- a/components/inspectit-ocelot-configurationserver-ui/src/lib/axios-api.js +++ b/components/inspectit-ocelot-configurationserver-ui/src/lib/axios-api.js @@ -18,7 +18,7 @@ const axiosBearer = axios.create(commonConfiguration); // Request Interceptors /** - * Ensures requres are authenticated using the bearer token. + * Ensures requests are authenticated using the bearer token. */ axiosBearer.interceptors.request.use( function (config) { diff --git a/components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/agentstatus/AgentStatus.java b/components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/agentstatus/AgentStatus.java index 59a52df5a8..7469207d7e 100644 --- a/components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/agentstatus/AgentStatus.java +++ b/components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/agentstatus/AgentStatus.java @@ -4,7 +4,7 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import rocks.inspectit.ocelot.commons.models.health.AgentHealth; +import rocks.inspectit.ocelot.commons.models.health.AgentHealthState; import java.util.Date; import java.util.Map; @@ -47,5 +47,5 @@ public class AgentStatus { /** * The health status of the agent. */ - private AgentHealth health; + private AgentHealthState healthState; } diff --git a/components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/agentstatus/AgentStatusManager.java b/components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/agentstatus/AgentStatusManager.java index 6c556e7106..259ffeea81 100644 --- a/components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/agentstatus/AgentStatusManager.java +++ b/components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/agentstatus/AgentStatusManager.java @@ -1,5 +1,8 @@ package rocks.inspectit.ocelot.agentstatus; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -8,6 +11,7 @@ import org.springframework.stereotype.Component; import rocks.inspectit.ocelot.agentconfiguration.AgentConfiguration; import rocks.inspectit.ocelot.commons.models.health.AgentHealth; +import rocks.inspectit.ocelot.commons.models.health.AgentHealthState; import rocks.inspectit.ocelot.config.model.InspectitServerSettings; import javax.annotation.PostConstruct; @@ -79,19 +83,28 @@ public void notifyAgentConfigurationFetched(Map agentAttributes, } if (headers.containsKey(HEADER_AGENT_HEALTH)) { - AgentHealth agentHealth = AgentHealth.valueOf(headers.get(HEADER_AGENT_HEALTH)); - agentStatus.setHealth(agentHealth); - logHealthIfChanged(statusKey, agentHealth); + ObjectReader objectReader = new ObjectMapper().reader().forType(AgentHealthState.class); + + AgentHealthState agentHealthState = null; + try { + agentHealthState = objectReader.readValue(headers.get(HEADER_AGENT_HEALTH)); + } catch (JsonProcessingException e) { //If this exception occurs we assume the corresponding agent uses the legacy health indicator. + AgentHealth agentHealth = AgentHealth.valueOf(headers.get(HEADER_AGENT_HEALTH)); + agentHealthState = AgentHealthState.defaultState(); + agentHealthState.setHealth(agentHealth); + } + agentStatus.setHealthState(agentHealthState); + logHealthIfChanged(statusKey, agentHealthState); } attributesToAgentStatusCache.put(statusKey, agentStatus); } - private void logHealthIfChanged(Object statusKey, AgentHealth agentHealth) { + private void logHealthIfChanged(Object statusKey, AgentHealthState agentHealthState) { AgentStatus lastStatus = attributesToAgentStatusCache.getIfPresent(statusKey); - if (lastStatus == null || lastStatus.getHealth() != agentHealth) { - log.info("Health of agent {} changed to {}.", statusKey, agentHealth); + if (lastStatus == null || lastStatus.getHealthState().getHealth() != agentHealthState.getHealth()) { + log.info("Health of agent {} changed to {}.", statusKey, agentHealthState); } } diff --git a/components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/rest/yamlhighlighter/HighlightRulesMapController.java b/components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/rest/yamlhighlighter/HighlightRulesMapController.java index a5c925ea91..a93ba4f4a4 100644 --- a/components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/rest/yamlhighlighter/HighlightRulesMapController.java +++ b/components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/rest/yamlhighlighter/HighlightRulesMapController.java @@ -9,11 +9,9 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; import rocks.inspectit.ocelot.config.model.InspectitConfig; -import rocks.inspectit.ocelot.config.model.exporters.TransportProtocol; import rocks.inspectit.ocelot.config.model.instrumentation.actions.GenericActionSettings; import rocks.inspectit.ocelot.rest.AbstractBaseController; -import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.lang.reflect.ParameterizedType; import java.util.*; diff --git a/components/inspectit-ocelot-configurationserver/src/main/resources/application.yml b/components/inspectit-ocelot-configurationserver/src/main/resources/application.yml index 91aa4402e6..20b5b0640e 100644 --- a/components/inspectit-ocelot-configurationserver/src/main/resources/application.yml +++ b/components/inspectit-ocelot-configurationserver/src/main/resources/application.yml @@ -22,6 +22,7 @@ spring: # server properties - see https://docs.spring.io/spring-boot/docs/current/reference/html/appendix-application-properties.html#server-properties server: port: 8090 + max-http-header-size: 1MB inspectit-config-server: # the directory which is used as working directory diff --git a/components/inspectit-ocelot-configurationserver/src/test/java/rocks/inspectit/ocelot/rest/yamlhighlighter/HighlightRulesMapControllerIntTest.java b/components/inspectit-ocelot-configurationserver/src/test/java/rocks/inspectit/ocelot/rest/yamlhighlighter/HighlightRulesMapControllerIntTest.java index dd42c06432..5c7de942a1 100644 --- a/components/inspectit-ocelot-configurationserver/src/test/java/rocks/inspectit/ocelot/rest/yamlhighlighter/HighlightRulesMapControllerIntTest.java +++ b/components/inspectit-ocelot-configurationserver/src/test/java/rocks/inspectit/ocelot/rest/yamlhighlighter/HighlightRulesMapControllerIntTest.java @@ -16,7 +16,6 @@ import rocks.inspectit.ocelot.config.model.instrumentation.data.PropagationMode; import rocks.inspectit.ocelot.config.model.instrumentation.scope.AdvancedScopeSettings; - import java.util.List; import java.util.Map; diff --git a/inspectit-ocelot-config/build.gradle b/inspectit-ocelot-config/build.gradle index 7edbeee095..7b3508ea94 100644 --- a/inspectit-ocelot-config/build.gradle +++ b/inspectit-ocelot-config/build.gradle @@ -38,7 +38,6 @@ dependencies { libs.orgApacheCommonsCommonsLang3, libs.commonsIo, libs.comFasterxmlJacksonCoreJacksonDatabind, - // logging libs.chQosLogbackLogbackClassic, ) diff --git a/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/commons/models/health/AgentHealthIncident.java b/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/commons/models/health/AgentHealthIncident.java new file mode 100644 index 0000000000..2a55d81948 --- /dev/null +++ b/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/commons/models/health/AgentHealthIncident.java @@ -0,0 +1,28 @@ +package rocks.inspectit.ocelot.commons.models.health; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.LocalDateTime; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class AgentHealthIncident implements Comparable { + + private String time; + private AgentHealth health; + private String source; + private String message; + private boolean changedHealth; + + @Override + public int compareTo(Object o) { + if(!(o instanceof AgentHealthIncident)) { + return 1; + } + AgentHealthIncident healthIncident = (AgentHealthIncident) o; + return this.getHealth().compareTo(healthIncident.getHealth()); + } +} diff --git a/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/commons/models/health/AgentHealthState.java b/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/commons/models/health/AgentHealthState.java new file mode 100644 index 0000000000..6a190be850 --- /dev/null +++ b/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/commons/models/health/AgentHealthState.java @@ -0,0 +1,23 @@ +package rocks.inspectit.ocelot.commons.models.health; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Collections; +import java.util.List; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class AgentHealthState { + + private AgentHealth health; + private String source; + private String message; + private List history; + + public static AgentHealthState defaultState() { + return new AgentHealthState(AgentHealth.OK, "", "", Collections.emptyList()); + } +} diff --git a/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/metrics/MetricsSettings.java b/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/metrics/MetricsSettings.java index c0b086c555..c9aa327d6d 100644 --- a/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/metrics/MetricsSettings.java +++ b/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/metrics/MetricsSettings.java @@ -38,6 +38,11 @@ public class MetricsSettings { */ private Duration frequency; + /** + * Settings for controlling the amount of unique tag values + */ + private TagGuardSettings tagGuard; + @NotNull private Map<@NotBlank String, @NotNull @Valid MetricDefinitionSettings> definitions = Collections.emptyMap(); @@ -84,8 +89,6 @@ public class MetricsSettings { @NotNull private JmxMetricsRecorderSettings jmx; - - @AdditionalValidation public void noDuplicateViewNames(ViolationBuilder vios) { Map viewsToMeasuresMap = new HashMap<>(); diff --git a/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/metrics/TagGuardSettings.java b/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/metrics/TagGuardSettings.java new file mode 100644 index 0000000000..cf1c05518a --- /dev/null +++ b/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/metrics/TagGuardSettings.java @@ -0,0 +1,50 @@ +package rocks.inspectit.ocelot.config.model.metrics; + +import lombok.Data; +import lombok.NoArgsConstructor; +import rocks.inspectit.ocelot.config.validation.AdditionalValidations; + +import javax.validation.Valid; +import javax.validation.constraints.NotNull; +import java.time.Duration; +import java.util.Collections; +import java.util.Map; + +@Data +@NoArgsConstructor +@AdditionalValidations +public class TagGuardSettings { + + private boolean enabled; + + /** + * The schedule delay for the blocking task of the {@code MeasureTagValueGuard} + */ + private Duration scheduleDelay; + + /** + * File, which contains measures with their particular recorded tags and their tag values + */ + private String databaseFile; + + /** + * String, which should be used as tag value, if the defined limit of tag values is exceeded + */ + private String overflowReplacement; + + /** + * Default max values per tag for all measures that are not specified in {@link #maxValuesPerTagByMeasure} or {@link rocks.inspectit.ocelot.config.model.metrics.definition.MetricDefinitionSettings#maxValuesPerTag}. + */ + private int maxValuesPerTag; + + /** + * Map containing max values per tag by Measure, e.g., {{'method_duration': 1337}} + *
+ * max-values-per-tag-by-measure:
+ * method_duration: 1337
+ * http_in_responestime: 2000 + */ + @NotNull + private Map maxValuesPerTagByMeasure = Collections.emptyMap(); + +} diff --git a/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/metrics/definition/MetricDefinitionSettings.java b/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/metrics/definition/MetricDefinitionSettings.java index cea15ee6b2..62dcd4e99e 100644 --- a/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/metrics/definition/MetricDefinitionSettings.java +++ b/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/metrics/definition/MetricDefinitionSettings.java @@ -35,6 +35,9 @@ public enum MeasureType { @NotBlank private String unit; + @Builder.Default + private int maxValuesPerTag = -1; + @NotNull @Builder.Default private MetricDefinitionSettings.MeasureType type = MeasureType.DOUBLE; diff --git a/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/selfmonitoring/AgentHealthSettings.java b/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/selfmonitoring/AgentHealthSettings.java index 783c461d36..4449bff9ca 100644 --- a/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/selfmonitoring/AgentHealthSettings.java +++ b/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/selfmonitoring/AgentHealthSettings.java @@ -5,6 +5,7 @@ import lombok.NonNull; import javax.validation.constraints.AssertFalse; +import javax.validation.constraints.AssertTrue; import java.time.Duration; /** @@ -21,7 +22,29 @@ public class AgentHealthSettings { @NonNull private Duration validityPeriod; - @AssertFalse(message = "The specified period should not be negative!") + /** + * The amount of AgentHealthIncidents, which should be buffered. + */ + @NonNull + private Integer incidentBufferSize; + + /** + * The minimum delay how often the AgentHealthManager checks for invalid agent health events to clear health status. + */ + @NonNull + private Duration minHealthCheckDelay; + + @AssertTrue(message = "validityPeriod must be greater than minHealthCheckDelay!") + public boolean validityPeriodIsGreaterThanMinDelay() { + return validityPeriod.compareTo(minHealthCheckDelay) > 0; + } + + @AssertTrue(message = "minHealthCheckDelay must be at least 60 seconds!") + public boolean isMin60SecondsDelay() { + return minHealthCheckDelay.toMinutes() >= 1; + } + + @AssertFalse(message = "The specified period must not be negative!") public boolean isNegativeDuration() { return validityPeriod != null && validityPeriod.isNegative(); } diff --git a/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/tracing/TracingSettings.java b/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/tracing/TracingSettings.java index e1529f7797..f174b99dc9 100644 --- a/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/tracing/TracingSettings.java +++ b/inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/tracing/TracingSettings.java @@ -54,6 +54,12 @@ public enum AddCommonTags { @NotNull private AddCommonTags addCommonTags; + /** + * If enabled, metric tags will be added as attributes to tracing within the same rule + */ + @NotNull + private boolean addMetricTags; + /** * Settings for automatic tracing (stack trace sampling) */ @@ -79,7 +85,7 @@ public enum AddCommonTags { private long scheduleDelayMillis = 5000; /** - * I enabled 64 Bit Trace Ids are used instead of the default 128 Bit. + * If enabled, 64 Bit Trace Ids are used instead of the default 128 Bit. */ private boolean use64BitTraceIds = false; } diff --git a/inspectit-ocelot-config/src/main/resources/rocks/inspectit/ocelot/config/default/basics.yml b/inspectit-ocelot-config/src/main/resources/rocks/inspectit/ocelot/config/default/basics.yml index c49f28ceb2..986fdfe67b 100644 --- a/inspectit-ocelot-config/src/main/resources/rocks/inspectit/ocelot/config/default/basics.yml +++ b/inspectit-ocelot-config/src/main/resources/rocks/inspectit/ocelot/config/default/basics.yml @@ -36,6 +36,8 @@ inspectit: # defines when to add common tags as attributes to spans # options are: NEVER, ON_GLOBAL_ROOT, ON_LOCAL_ROOT, ALWAYS add-common-tags: ON_LOCAL_ROOT + # if enabled, metric tags will be added as attributes to tracing within the same rule + add-metric-tags: true # settings regarding automatic tracing (stack-trace-sampling) auto-tracing: frequency: 50ms @@ -78,6 +80,13 @@ inspectit: # - no views and measures are created enabled: true + tag-guard: + enabled: true + max-values-per-tag: 1000 + schedule-delay: 30s + database-file : ${inspectit.env.agent-dir}/${inspectit.service-name}/tag-guard-database.json + overflow-replacement: "TAG_LIMIT_EXCEEDED" + # logging settings logging: # path to a custom user-specified logback config file that should be used diff --git a/inspectit-ocelot-config/src/main/resources/rocks/inspectit/ocelot/config/default/self-monitoring.yml b/inspectit-ocelot-config/src/main/resources/rocks/inspectit/ocelot/config/default/self-monitoring.yml index 5912b02f20..a4d448bc8e 100644 --- a/inspectit-ocelot-config/src/main/resources/rocks/inspectit/ocelot/config/default/self-monitoring.yml +++ b/inspectit-ocelot-config/src/main/resources/rocks/inspectit/ocelot/config/default/self-monitoring.yml @@ -15,6 +15,14 @@ inspectit: # health changes due to instrumentation errors are valid until the next re-instrumentation validity-period: 1h + # The amount of agent health incidents, which should be buffered + incident-buffer-size: 10 + + # The minimum delay how often the AgentHealthManager checks for invalid agent health events to clear health status + # By default the delay is calculated based on the last agent health event + # Minimum value is 1m + min-health-check-delay: 1m + # the action tracing mode to use # options are: OFF, ONLY_ENABLED, ALL_WITHOUT_DEFAULT, ALL_WITH_DEFAULT action-tracing: ONLY_ENABLED diff --git a/inspectit-ocelot-core/${inspectit.env.agent-dir}/InspectIT Agent/tag-guard-database.json b/inspectit-ocelot-core/${inspectit.env.agent-dir}/InspectIT Agent/tag-guard-database.json new file mode 100644 index 0000000000..9e26dfeeb6 --- /dev/null +++ b/inspectit-ocelot-core/${inspectit.env.agent-dir}/InspectIT Agent/tag-guard-database.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/command/handler/impl/LogsCommandExecutor.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/command/handler/impl/LogsCommandExecutor.java index 7fc13c4034..a9c20a760f 100644 --- a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/command/handler/impl/LogsCommandExecutor.java +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/command/handler/impl/LogsCommandExecutor.java @@ -10,7 +10,7 @@ import rocks.inspectit.ocelot.commons.models.command.CommandResponse; import rocks.inspectit.ocelot.commons.models.command.impl.LogsCommand; import rocks.inspectit.ocelot.core.command.handler.CommandExecutor; -import rocks.inspectit.ocelot.core.selfmonitoring.LogPreloader; +import rocks.inspectit.ocelot.core.selfmonitoring.logs.LogPreloader; /** * Executor for executing {@link LogsCommand}s. diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/config/propertysources/http/HttpConfigurationPoller.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/config/propertysources/http/HttpConfigurationPoller.java index a3d4696cd3..212c0a6a1c 100644 --- a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/config/propertysources/http/HttpConfigurationPoller.java +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/config/propertysources/http/HttpConfigurationPoller.java @@ -1,13 +1,13 @@ package rocks.inspectit.ocelot.core.config.propertysources.http; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; +import rocks.inspectit.ocelot.commons.models.health.AgentHealthState; import rocks.inspectit.ocelot.config.model.InspectitConfig; import rocks.inspectit.ocelot.config.model.config.HttpConfigSettings; import rocks.inspectit.ocelot.core.config.InspectitEnvironment; -import rocks.inspectit.ocelot.core.selfmonitoring.event.AgentHealthChangedEvent; import rocks.inspectit.ocelot.core.service.DynamicallyActivatableService; import java.util.concurrent.ScheduledExecutorService; @@ -15,7 +15,7 @@ import java.util.concurrent.TimeUnit; /** - * Service for continuously triggering the updated of a agent configuration via HTTP. + * Service for continuously triggering the updating of an agent configuration via HTTP. */ @Service @Slf4j @@ -32,6 +32,7 @@ public class HttpConfigurationPoller extends DynamicallyActivatableService imple /** * The state of the used HTTP property source configuration. */ + @Getter private HttpPropertySourceState currentState; public HttpConfigurationPoller() { @@ -83,10 +84,14 @@ public void run() { } } - @EventListener - void agentHealthChanged(AgentHealthChangedEvent event) { + public void updateAgentHealthState(AgentHealthState agentHealth) { if (currentState != null) { - currentState.updateAgentHealth(event.getNewHealth()); + currentState.updateAgentHealthState(agentHealth); } } + + public AgentHealthState getCurrentAgentHealthState() { + if(currentState == null) return null; + return currentState.getAgentHealth(); + } } diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/config/propertysources/http/HttpPropertySourceState.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/config/propertysources/http/HttpPropertySourceState.java index 34219ee71b..468745b10d 100644 --- a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/config/propertysources/http/HttpPropertySourceState.java +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/config/propertysources/http/HttpPropertySourceState.java @@ -1,6 +1,9 @@ package rocks.inspectit.ocelot.core.config.propertysources.http; import io.github.resilience4j.retry.Retry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import lombok.Getter; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -19,7 +22,7 @@ import org.springframework.core.env.PropertiesPropertySource; import org.springframework.core.env.PropertySource; import rocks.inspectit.ocelot.bootstrap.AgentManager; -import rocks.inspectit.ocelot.commons.models.health.AgentHealth; +import rocks.inspectit.ocelot.commons.models.health.AgentHealthState; import rocks.inspectit.ocelot.config.model.config.HttpConfigSettings; import rocks.inspectit.ocelot.core.config.util.InvalidPropertiesException; import rocks.inspectit.ocelot.core.config.util.PropertyUtils; @@ -101,7 +104,8 @@ public class HttpPropertySourceState { @Getter private boolean firstFileWriteAttemptSuccessful = true; - private AgentHealth agentHealth = AgentHealth.OK; + @Getter + private AgentHealthState agentHealth = AgentHealthState.defaultState(); /** * Constructor. @@ -145,7 +149,7 @@ public boolean update(boolean fallBackToFile) { * * @param newHealth The new agent health */ - public void updateAgentHealth(@NonNull AgentHealth newHealth) { + public void updateAgentHealthState(@NonNull AgentHealthState newHealth) { agentHealth = newHealth; } @@ -280,13 +284,21 @@ private String fetchConfiguration(HttpClient client, HttpGet request) throws IOE private void setAgentMetaHeaders(HttpGet httpGet) { RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean(); + ObjectWriter ow = new ObjectMapper().writer().withDefaultPrettyPrinter(); + String agentHealthJson; + try { + agentHealthJson = ow.writeValueAsString(agentHealth); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + httpGet.setHeader(META_HEADER_PREFIX + "AGENT-ID", runtime.getName()); httpGet.setHeader(META_HEADER_PREFIX + "AGENT-VERSION", AgentManager.getAgentVersion()); httpGet.setHeader(META_HEADER_PREFIX + "JAVA-VERSION", System.getProperty("java.version")); httpGet.setHeader(META_HEADER_PREFIX + "VM-NAME", runtime.getVmName()); httpGet.setHeader(META_HEADER_PREFIX + "VM-VENDOR", runtime.getVmVendor()); httpGet.setHeader(META_HEADER_PREFIX + "START-TIME", String.valueOf(runtime.getStartTime())); - httpGet.setHeader(META_HEADER_PREFIX + "HEALTH", agentHealth.name()); + httpGet.setHeader(META_HEADER_PREFIX + "HEALTH", agentHealthJson); httpGet.setHeader(META_HEADER_PREFIX + "SERVICE-STATES-MAP", DynamicallyActivatableServiceObserver.asJson()); } diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/exporter/OtlpMetricsExporterService.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/exporter/OtlpMetricsExporterService.java index 308045565e..5f03a6e38c 100644 --- a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/exporter/OtlpMetricsExporterService.java +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/exporter/OtlpMetricsExporterService.java @@ -34,7 +34,7 @@ public class OtlpMetricsExporterService extends DynamicallyActivatableMetricsExp /** * The {@link MetricExporter} for exporting metrics via OTLP */ - MetricExporter metricExporter; + MetricExporter metricExporter; /** * The {@link PeriodicMetricReaderBuilder} for reading metrics to the log diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/config/MethodHookConfigurationResolver.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/config/MethodHookConfigurationResolver.java index 6963474b18..39542655c6 100644 --- a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/config/MethodHookConfigurationResolver.java +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/config/MethodHookConfigurationResolver.java @@ -155,7 +155,7 @@ private void resolveEndSpan(Set matchedRules, RuleTracingSe builder.endSpan(endSpan); if (endSpan) { builder.endSpanConditions(Optional.ofNullable(getAndDetectConflicts(rulesDefiningEndSpan, r -> r.getTracing() - .getEndSpanConditions(), ALWAYS_TRUE, "end span conditions")) + .getEndSpanConditions(), ALWAYS_TRUE, "end span conditions")) .orElse(new ConditionalActionSettings())); } } diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/MethodHookGenerator.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/MethodHookGenerator.java index 31246d5a1f..140613dbd0 100644 --- a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/MethodHookGenerator.java +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/MethodHookGenerator.java @@ -23,6 +23,7 @@ import rocks.inspectit.ocelot.core.instrumentation.hook.actions.model.MetricAccessor; import rocks.inspectit.ocelot.core.instrumentation.hook.actions.span.*; import rocks.inspectit.ocelot.core.instrumentation.hook.tags.CommonTagsToAttributesManager; +import rocks.inspectit.ocelot.core.metrics.MeasureTagValueGuard; import rocks.inspectit.ocelot.core.metrics.MeasuresAndViewsManager; import rocks.inspectit.ocelot.core.opentelemetry.trace.samplers.OcelotSamplerUtils; import rocks.inspectit.ocelot.core.privacy.obfuscation.ObfuscationManager; @@ -72,6 +73,9 @@ public class MethodHookGenerator { @Autowired private ActionScopeFactory actionScopeFactory; + @Autowired + private MeasureTagValueGuard tagValueGuard; + /** * Builds an executable method hook based on the given configuration. * @@ -105,7 +109,7 @@ public MethodHook buildHook(Class declaringClass, MethodDescription method, M builder.exitActions(buildActionCalls(config.getPreExitActions(), methodInfo)); builder.exitActions(buildActionCalls(config.getExitActions(), methodInfo)); if (tracingSettings != null) { - List actions = buildTracingExitActions(tracingSettings); + List actions = buildTracingExitActions(config); if (isTracingInternalActions() && config.isTraceExitHook()) { actions = wrapActionsWithTracing(actions); } @@ -186,7 +190,8 @@ private void configureSampling(RuleTracingSettings tracing, ContinueOrStartSpanA } @VisibleForTesting - List buildTracingExitActions(RuleTracingSettings tracing) { + List buildTracingExitActions(MethodHookConfiguration config) { + RuleTracingSettings tracing = config.getTracing(); val result = new ArrayList(); boolean isSpanStartedOrContinued = tracing.getStartSpan() || StringUtils.isNotBlank(tracing.getContinueSpan()); @@ -197,10 +202,24 @@ List buildTracingExitActions(RuleTracingSettings tracing) { result.add(new SetSpanStatusAction(accessor)); } - val attributes = tracing.getAttributes(); - if (!attributes.isEmpty()) { + Map tracingAttributes = tracing.getAttributes(); + Map attributes = tracingAttributes; + Map constantAttributes = new HashMap<>(); + + if(addMetricsToTracing()) { + Collection metrics = config.getMetrics(); + constantAttributes = collectMetricConstantTags(metrics); + attributes = collectMetricDataTags(metrics); + // write tracing attributes after metric tags, to allow overwriting of metric tags + attributes.putAll(tracingAttributes); + } + + if (!attributes.isEmpty() || !constantAttributes.isEmpty()) { Map attributeAccessors = new HashMap<>(); + constantAttributes.forEach((attribute, constant) -> attributeAccessors.put(attribute, variableAccessorFactory.getConstantAccessor(constant))); + // if necessary, overwrite constant attributes attributes.forEach((attribute, variable) -> attributeAccessors.put(attribute, variableAccessorFactory.getVariableAccessor(variable))); + IHookAction endTraceAction = new WriteSpanAttributesAction(attributeAccessors, obfuscationManager.obfuscatorySupplier()); IHookAction actionWithConditions = ConditionalHookAction.wrapWithConditionChecks(tracing.getAttributeConditions(), endTraceAction, variableAccessorFactory); result.add(actionWithConditions); @@ -215,14 +234,25 @@ List buildTracingExitActions(RuleTracingSettings tracing) { return result; } + private Map collectMetricDataTags(Collection metrics) { + Map dataTags = new HashMap<>(); + metrics.forEach(metric -> dataTags.putAll(metric.getDataTags())); + return dataTags; + } + + private Map collectMetricConstantTags(Collection metrics) { + Map constantTags = new HashMap<>(); + metrics.forEach(metric -> constantTags.putAll(metric.getConstantTags())); + return constantTags; + } + private Optional buildMetricsRecorder(MethodHookConfiguration config) { Collection metricRecordingSettings = config.getMetrics(); if (!metricRecordingSettings.isEmpty()) { List metricAccessors = metricRecordingSettings.stream() .map(this::buildMetricAccessor) .collect(Collectors.toList()); - - IHookAction recorder = new MetricsRecorder(metricAccessors, commonTagsManager, metricsManager); + IHookAction recorder = new MetricsRecorder(metricAccessors, commonTagsManager, metricsManager, tagValueGuard); if (isTracingInternalActions() && config.isTraceExitHook()) { recorder = TracingHookAction.wrap(recorder, null, "INTERNAL"); @@ -234,6 +264,13 @@ private Optional buildMetricsRecorder(MethodHookConfiguration confi } } + /** + * @return Returns whether metrics tags should be added to tracing as attributes + */ + private boolean addMetricsToTracing() { + return environment.getCurrentConfig().getTracing().isAddMetricTags(); + } + /** * @return Returns whether action tracing should be enabled for internal actions (e.g. {@link MetricsRecorder}). */ diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/MetricsRecorder.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/MetricsRecorder.java index b614b95db8..9bcd96d6ab 100644 --- a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/MetricsRecorder.java +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/MetricsRecorder.java @@ -1,19 +1,14 @@ package rocks.inspectit.ocelot.core.instrumentation.hook.actions; import io.opencensus.tags.TagContext; -import io.opencensus.tags.TagContextBuilder; -import io.opencensus.tags.TagKey; -import io.opencensus.tags.Tags; import lombok.Value; import lombok.extern.slf4j.Slf4j; -import rocks.inspectit.ocelot.core.instrumentation.context.InspectitContextImpl; import rocks.inspectit.ocelot.core.instrumentation.hook.actions.model.MetricAccessor; +import rocks.inspectit.ocelot.core.metrics.MeasureTagValueGuard; import rocks.inspectit.ocelot.core.metrics.MeasuresAndViewsManager; import rocks.inspectit.ocelot.core.tags.CommonTagsManager; -import rocks.inspectit.ocelot.core.tags.TagUtils; import java.util.List; -import java.util.Optional; /** * Hook action responsible for recording measurements at the exit of an instrumented method @@ -37,46 +32,22 @@ public class MetricsRecorder implements IHookAction { */ private MeasuresAndViewsManager metricsManager; + private MeasureTagValueGuard tagValueGuard; + @Override public void execute(ExecutionContext context) { // then iterate all metrics and enter new scope for metric collection for (MetricAccessor metricAccessor : metrics) { Object value = metricAccessor.getVariableAccessor().get(context); + // only record metrics where a value is present + // this allows to disable the recording of a metric depending on the results of action executions if (value instanceof Number) { - // only record metrics where a value is present - // this allows to disable the recording of a metric depending on the results of action executions - TagContext tagContext = getTagContext(context, metricAccessor); + TagContext tagContext = tagValueGuard.getTagContext(context, metricAccessor); metricsManager.tryRecordingMeasurement(metricAccessor.getName(), (Number) value, tagContext); } } } - private TagContext getTagContext(ExecutionContext context, MetricAccessor metricAccessor) { - InspectitContextImpl inspectitContext = context.getInspectitContext(); - - // create builder - TagContextBuilder builder = Tags.getTagger().emptyBuilder(); - - // first common tags to allow overwrite by constant or data tags - commonTagsManager.getCommonTagKeys() - .forEach(commonTagKey -> Optional.ofNullable(inspectitContext.getData(commonTagKey.getName())) - .ifPresent(value -> builder.putLocal(commonTagKey, TagUtils.createTagValue(commonTagKey.getName(), value - .toString())))); - - // then constant tags to allow overwrite by data - metricAccessor.getConstantTags() - .forEach((key, value) -> builder.putLocal(TagKey.create(key), TagUtils.createTagValue(key, value))); - - // go over data tags and match the value to the key from the contextTags (if available) - metricAccessor.getDataTagAccessors() - .forEach((key, accessor) -> Optional.ofNullable(accessor.get(context)) - .ifPresent(tagValue -> builder.putLocal(TagKey.create(key), TagUtils.createTagValue(key, tagValue - .toString())))); - - // build and return - return builder.build(); - } - @Override public String getName() { return "Metrics Recorder"; diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/span/ContinueOrStartSpanAction.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/span/ContinueOrStartSpanAction.java index 41615a0149..72cb9be2e9 100644 --- a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/span/ContinueOrStartSpanAction.java +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/span/ContinueOrStartSpanAction.java @@ -138,8 +138,9 @@ private void startSpan(ExecutionContext context) { hasLocalParent = !(currentSpan == Span.getInvalid() || !currentSpan.getSpanContext().isValid()); } - // This is necessary, since the lambda expression needs a final value + // This is necessary, since the lambda expression needs an effectively final value SpanContext finalRemoteParent = remoteParent; + Sampler sampler = getSampler(context); AutoCloseable spanCtx = Instances.logTraceCorrelator.startCorrelatedSpanScope(() -> stackTraceSampler.createAndEnterSpan(spanName, finalRemoteParent, sampler, spanKind, methodInfo, autoTrace)); ctx.setSpanScope(spanCtx); diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/span/WriteSpanAttributesAction.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/span/WriteSpanAttributesAction.java index a15b7cd3ce..17bf86519c 100644 --- a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/span/WriteSpanAttributesAction.java +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/span/WriteSpanAttributesAction.java @@ -1,10 +1,7 @@ package rocks.inspectit.ocelot.core.instrumentation.hook.actions.span; import io.opentelemetry.api.trace.Span; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Singular; -import lombok.val; +import lombok.*; import rocks.inspectit.ocelot.core.instrumentation.hook.VariableAccessor; import rocks.inspectit.ocelot.core.instrumentation.hook.actions.IHookAction; import rocks.inspectit.ocelot.core.privacy.obfuscation.IObfuscatory; @@ -20,6 +17,7 @@ public class WriteSpanAttributesAction implements IHookAction { @Singular + @Getter private final Map attributeAccessors; private final Supplier obfuscatorySupplier; diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/metrics/MeasureTagValueGuard.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/metrics/MeasureTagValueGuard.java new file mode 100644 index 0000000000..c670beb445 --- /dev/null +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/metrics/MeasureTagValueGuard.java @@ -0,0 +1,302 @@ +package rocks.inspectit.ocelot.core.metrics; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import io.opencensus.tags.TagContext; +import io.opencensus.tags.TagContextBuilder; +import io.opencensus.tags.TagKey; +import io.opencensus.tags.Tags; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.NonNull; +import lombok.Value; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import rocks.inspectit.ocelot.commons.models.health.AgentHealth; +import rocks.inspectit.ocelot.config.model.InspectitConfig; +import rocks.inspectit.ocelot.config.model.metrics.TagGuardSettings; +import rocks.inspectit.ocelot.config.model.metrics.definition.MetricDefinitionSettings; +import rocks.inspectit.ocelot.core.config.InspectitEnvironment; +import rocks.inspectit.ocelot.core.instrumentation.context.InspectitContextImpl; +import rocks.inspectit.ocelot.core.instrumentation.hook.actions.IHookAction; +import rocks.inspectit.ocelot.core.instrumentation.hook.actions.model.MetricAccessor; +import rocks.inspectit.ocelot.core.selfmonitoring.AgentHealthManager; +import rocks.inspectit.ocelot.core.tags.CommonTagsManager; +import rocks.inspectit.ocelot.core.tags.TagUtils; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +@Component +@Slf4j +public class MeasureTagValueGuard { + private static final String tagOverFlowMessageTemplate = "Overflow for tag %s"; + + @Autowired + private InspectitEnvironment env; + + @Autowired + private AgentHealthManager agentHealthManager; + + /** + * Common tags manager needed for gathering common tags when recording metrics. + */ + @Autowired + private CommonTagsManager commonTagsManager; + + @Autowired + private ScheduledExecutorService executor; + + private PersistedTagsReaderWriter fileReaderWriter; + + private volatile boolean isShuttingDown = false; + + private boolean hasTagValueOverflow = false; + + /** + * Map of measure names and their related set of tag keys, which are currently blocked. + */ + private final Map> blockedTagKeysByMeasure = Maps.newHashMap(); + + private Set latestTags = Collections.synchronizedSet(new HashSet<>()); + + private Future blockTagValuesFuture; + + @PostConstruct + protected void init() { + TagGuardSettings tagGuardSettings = env.getCurrentConfig().getMetrics().getTagGuard(); + if (!tagGuardSettings.isEnabled()) return; + + fileReaderWriter = new PersistedTagsReaderWriter(tagGuardSettings.getDatabaseFile(), new ObjectMapper()); + scheduleTagGuardJob(); + + log.info(String.format("TagValueGuard started with scheduleDelay %s and database file %s", tagGuardSettings.getScheduleDelay(), tagGuardSettings.getDatabaseFile())); + } + + private void scheduleTagGuardJob() { + Duration tagGuardScheduleDelay = env.getCurrentConfig().getMetrics().getTagGuard().getScheduleDelay(); + blockTagValuesFuture = executor.schedule(blockTagValuesTask, tagGuardScheduleDelay.toNanos(), TimeUnit.NANOSECONDS); + } + + @PreDestroy + protected void stop() { + if (!env.getCurrentConfig().getMetrics().getTagGuard().isEnabled()) return; + + isShuttingDown = true; + blockTagValuesFuture.cancel(true); + } + + /** + * Task, which reads the persisted tag values to determine, which tags should be blocked, because of exceeding + * the specific tag value limit. + * If new tags values have been created, they will be persisted. + */ + @VisibleForTesting + Runnable blockTagValuesTask = () -> { + if (!env.getCurrentConfig().getMetrics().getTagGuard().isEnabled()) return; + + // read current tag value database + Map>> availableTagsByMeasure = fileReaderWriter.read(); + + Set copy = latestTags; + latestTags = Collections.synchronizedSet(new HashSet<>()); + + // process new tags + copy.forEach(tagsHolder -> { + String measureName = tagsHolder.getMeasureName(); + Map newTags = tagsHolder.getTags(); + int maxValuesPerTag = getMaxValuesPerTag(measureName, env.getCurrentConfig()); + + Map> tagValuesByTagKey = availableTagsByMeasure.computeIfAbsent(measureName, k -> Maps.newHashMap()); + newTags.forEach((tagKey, tagValue) -> { + Set tagValues = tagValuesByTagKey.computeIfAbsent(tagKey, (x) -> new HashSet<>()); + // if tag value is new AND max values per tag is already reached + if (!tagValues.contains(tagValue) && tagValues.size() >= maxValuesPerTag) { + blockedTagKeysByMeasure.computeIfAbsent(measureName, measure -> Sets.newHashSet()).add(tagKey); + agentHealthManager.notifyAgentHealth(AgentHealth.ERROR, this.getClass(), this.getClass().getName(), + String.format(tagOverFlowMessageTemplate, tagKey)); + hasTagValueOverflow = true; + } else { + tagValues.add(tagValue); + } + }); + + }); + + fileReaderWriter.write(availableTagsByMeasure); + + // remove all blocked tags, if no values are stored in the database file + if(availableTagsByMeasure.isEmpty()) blockedTagKeysByMeasure.clear(); + + // independent of processing new tags, check if tags should be blocked or unblocked due to their tag value limit + availableTagsByMeasure.forEach((measureName, tags) -> { + int maxValuesPerTag = getMaxValuesPerTag(measureName, env.getCurrentConfig()); + tags.forEach((tagKey, tagValues) -> { + if(tagValues.size() >= maxValuesPerTag) { + boolean isNewBlockedTag = blockedTagKeysByMeasure.computeIfAbsent(measureName, measure -> Sets.newHashSet()) + .add(tagKey); + if(isNewBlockedTag) { + agentHealthManager.notifyAgentHealth(AgentHealth.ERROR, this.getClass(), this.getClass().getName(), + String.format(tagOverFlowMessageTemplate, tagKey)); + hasTagValueOverflow = true; + } + } else { + blockedTagKeysByMeasure.getOrDefault(measureName, Sets.newHashSet()).remove(tagKey); + } + }); + }); + + // invalidate incident, if tag overflow was detected, but no more tags are blocked + boolean noBlockedTagKeys = blockedTagKeysByMeasure.values().stream().allMatch(Set::isEmpty); + if(hasTagValueOverflow && noBlockedTagKeys) { + agentHealthManager.invalidateIncident(this.getClass(), "Overflow for tags resolved"); + hasTagValueOverflow = false; + } + + if (!isShuttingDown) scheduleTagGuardJob(); + }; + + /** + * Gets the max value amount per tag for the given measure by hierarchically extracting + * {@link MetricDefinitionSettings#maxValuesPerTag} (prio 1), + * {@link TagGuardSettings#maxValuesPerTagByMeasure} (prio 2) and + * {@link TagGuardSettings#maxValuesPerTag} (default). + * + * @param measureName the current measure + * @return The maximum amount of tag values for the given measure + */ + @VisibleForTesting + int getMaxValuesPerTag(String measureName, InspectitConfig config) { + int maxValuesPerTag = config.getMetrics().getDefinitions().get(measureName).getMaxValuesPerTag(); + + if (maxValuesPerTag > 0) return maxValuesPerTag; + + Map maxValuesPerTagPerMeasuresMap = config.getMetrics() + .getTagGuard() + .getMaxValuesPerTagByMeasure(); + return maxValuesPerTagPerMeasuresMap.getOrDefault(measureName, config.getMetrics() + .getTagGuard() + .getMaxValuesPerTag()); + } + + /** + * Creates the full tag context, including all specified tags, for the current measure + * @param context current context + * @param metricAccessor accessor for the measure as well as the particular tags + * @return TagContext including all tags for the current measure + */ + public TagContext getTagContext(IHookAction.ExecutionContext context, MetricAccessor metricAccessor) { + Map tags = Maps.newHashMap(); + String measureName = metricAccessor.getName(); + InspectitContextImpl inspectitContext = context.getInspectitContext(); + TagGuardSettings tagGuardSettings = env.getCurrentConfig().getMetrics().getTagGuard(); + + Set blockedTagKeys = blockedTagKeysByMeasure.getOrDefault(measureName, Sets.newHashSet()); + log.debug("Currently blocked tag keys for measure {}, due to exceeding the configured tag value limit: {}", + measureName, blockedTagKeys); + + // first common tags to allow to overwrite by constant or data tags + commonTagsManager.getCommonTagKeys().forEach(commonTagKey -> { + Optional.ofNullable(inspectitContext.getData(commonTagKey.getName())) + .ifPresent(value -> tags.put(commonTagKey.getName(), TagUtils.createTagValueAsString(commonTagKey.getName(), value.toString()))); + }); + + // then constant tags to allow to overwrite by data + metricAccessor.getConstantTags().forEach((key, value) -> { + if (tagGuardSettings.isEnabled() && blockedTagKeys.contains(key)) { + String overflowReplacement = env.getCurrentConfig().getMetrics().getTagGuard().getOverflowReplacement(); + tags.put(key, TagUtils.createTagValueAsString(key, overflowReplacement)); + } else { + tags.put(key, TagUtils.createTagValueAsString(key, value)); + } + }); + + // go over data tags and match the value to the key from the contextTags (if available) + metricAccessor.getDataTagAccessors().forEach((key, accessor) -> { + if (tagGuardSettings.isEnabled() && blockedTagKeys.contains(key)) { + String overflowReplacement = env.getCurrentConfig().getMetrics().getTagGuard().getOverflowReplacement(); + tags.put(key, TagUtils.createTagValueAsString(key, overflowReplacement)); + } else { + Optional.ofNullable(accessor.get(context)) + .ifPresent(tagValue -> tags.put(key, TagUtils.createTagValueAsString(key, tagValue.toString()))); + } + }); + + TagContextBuilder tagContextBuilder = Tags.getTagger().emptyBuilder(); + tags.forEach((key, value) -> tagContextBuilder.putLocal(TagKey.create(key), TagUtils.createTagValue(key, value))); + + // store the new tags for this measure as simple object and delay traversing trough tagKeys to async job + latestTags.add(new TagsHolder(measureName, tags)); + + return tagContextBuilder.build(); + + } + + @Value + @EqualsAndHashCode + private static class TagsHolder { + + String measureName; + + Map tags; + + } + + @AllArgsConstructor + static class PersistedTagsReaderWriter { + + @NonNull + private String fileName; + + @NonNull + private ObjectMapper mapper; + + public Map>> read() { + if (!StringUtils.isBlank(fileName)) { + Path path = Paths.get(fileName); + if (Files.exists(path)) { + try { + byte[] content = Files.readAllBytes(path); + @SuppressWarnings("unchecked") Map>> tags = mapper.readValue(content, new TypeReference>>>() { + }); + return tags; + } catch (Exception e) { + log.error("Error loading tag-guard database from persistence file '{}'", fileName, e); + } + } else { + log.info("Could not find tag-guard database file. File will be created during next write"); + } + } + return Maps.newHashMap(); + } + + public void write(Map>> tagValues) { + if (!StringUtils.isBlank(fileName)) { + try { + Path path = Paths.get(fileName); + Files.createDirectories(path.getParent()); + String tagValuesString = mapper.writeValueAsString(tagValues); + Files.write(path, tagValuesString.getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + log.error("Error writing tag-guard database to file '{}'", fileName, e); + } + } + } + } + +} diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthIncidentBuffer.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthIncidentBuffer.java new file mode 100644 index 0000000000..c15ecaba9c --- /dev/null +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthIncidentBuffer.java @@ -0,0 +1,56 @@ +package rocks.inspectit.ocelot.core.selfmonitoring; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.stereotype.Component; +import rocks.inspectit.ocelot.commons.models.health.AgentHealthIncident; +import rocks.inspectit.ocelot.core.config.InspectitEnvironment; +import rocks.inspectit.ocelot.core.selfmonitoring.event.models.AgentHealthIncidentAddedEvent; + +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * Buffer with queued AgentHealthIncidents. + * New incidents will be inserted at the beginning of the queue. + * As soon as incidents are put into a full queue, old incidents will be removed to create space + */ +@Component +public class AgentHealthIncidentBuffer { + + @Autowired + private ApplicationContext ctx; + @Autowired + private InspectitEnvironment env; + + private final ConcurrentLinkedQueue buffer = new ConcurrentLinkedQueue<>(); + + /** + * Add new incident to the buffer. + * If the buffer is full, remove the latest incident at first. + * The buffer size will be read from the current inspectIT configuration. + * @param incident new incident + */ + public void put(AgentHealthIncident incident) { + int bufferSize = env.getCurrentConfig().getSelfMonitoring().getAgentHealth().getIncidentBufferSize(); + while(buffer.size() >= bufferSize) buffer.poll(); + + buffer.offer(incident); + ctx.publishEvent(new AgentHealthIncidentAddedEvent(this, asList())); + } + + /** + * Creates a list from the internal queue. + * The list will be reversed, since the queue inserts new elements at the tail + * @return List of agent health incidents + */ + public List asList() { + List incidentList = new LinkedList<>(buffer); + Collections.reverse(incidentList); + return incidentList; + } + + public void clear() { + buffer.clear(); + } +} diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthManager.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthManager.java index bb2137a89f..fcce2277ca 100644 --- a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthManager.java +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthManager.java @@ -1,21 +1,21 @@ package rocks.inspectit.ocelot.core.selfmonitoring; -import ch.qos.logback.classic.spi.ILoggingEvent; import com.google.common.annotations.VisibleForTesting; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; import rocks.inspectit.ocelot.commons.models.health.AgentHealth; +import rocks.inspectit.ocelot.commons.models.health.AgentHealthIncident; import rocks.inspectit.ocelot.core.config.InspectitEnvironment; -import rocks.inspectit.ocelot.core.logging.logback.InternalProcessingAppender; -import rocks.inspectit.ocelot.core.selfmonitoring.event.AgentHealthChangedEvent; +import rocks.inspectit.ocelot.core.selfmonitoring.event.listener.LogWritingHealthEventListener; +import rocks.inspectit.ocelot.core.selfmonitoring.event.models.AgentHealthChangedEvent; import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.time.Duration; import java.time.LocalDateTime; import java.util.Comparator; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; @@ -26,22 +26,20 @@ */ @Component @Slf4j -@RequiredArgsConstructor -public class AgentHealthManager implements InternalProcessingAppender.LogEventConsumer { +public class AgentHealthManager { - private static final String LOG_CHANGE_STATUS = "The agent status changed from {} to {}."; - - private final ApplicationContext ctx; - - private final ScheduledExecutorService executor; - - private final InspectitEnvironment env; - - private final SelfMonitoringService selfMonitoringService; + @Autowired + private ApplicationContext ctx; + @Autowired + private ScheduledExecutorService executor; + @Autowired + private InspectitEnvironment env; + @Autowired + private AgentHealthIncidentBuffer healthIncidentBuffer; /** * Map of {@code eventClass -> agentHealth}, whereas the {@code agentHealth} is reset whenever an event of type - * {@code eventClass} occurs (see {@link #onInvalidationEvent(Object)}. + * {@code eventClass} occurs (see {@link #onInvalidationEvent(Object)}). * The resulting agent health is the most severe value in the map. */ private final Map, AgentHealth> invalidatableHealth = new ConcurrentHashMap<>(); @@ -54,85 +52,58 @@ public class AgentHealthManager implements InternalProcessingAppender.LogEventCo private AgentHealth lastNotifiedHealth = AgentHealth.OK; - @Override - public void onLoggingEvent(ILoggingEvent event, Class invalidator) { - if (AgentHealthManager.class.getCanonicalName().equals(event.getLoggerName())) { - // ignore own logs, which otherwise would tend to cause infinite loops - return; - } - - AgentHealth eventHealth = AgentHealth.fromLogLevel(event.getLevel()); - - if (invalidator == null) { - handleTimeoutHealth(eventHealth); - } else { - handleInvalidatableHealth(eventHealth, invalidator); - } - - triggerEventAndMetricIfHealthChanged(); - } - - private void handleInvalidatableHealth(AgentHealth eventHealth, Class invalidator) { - invalidatableHealth.merge(invalidator, eventHealth, AgentHealth::mostSevere); + @PostConstruct + @VisibleForTesting + void startHealthCheckScheduler() { + checkHealthAndSchedule(); } - private void handleTimeoutHealth(AgentHealth eventHealth) { - Duration validityPeriod = env.getCurrentConfig().getSelfMonitoring().getAgentHealth().getValidityPeriod(); - - if (eventHealth.isMoreSevereOrEqualTo(AgentHealth.WARNING)) { - generalHealthTimeouts.put(eventHealth, LocalDateTime.now().plus(validityPeriod)); - } + public List getIncidentHistory() { + return healthIncidentBuffer.asList(); } /** - * Returns the current agent health, which is the most severe out of instrumentation and general status. + * Notifies the AgentHealthManager about an eventHealth. + * The manager determines, whether the event is invalidatable or times out. * - * @return The current agent health + * @param eventHealth health of event + * @param invalidator class, which created the invalidatable eventHealth + * @param loggerName name of the logger, who created the event + * @param message message of the event */ - public AgentHealth getCurrentHealth() { - AgentHealth generalHealth = generalHealthTimeouts.entrySet() - .stream() - .filter((entry) -> entry.getValue().isAfter(LocalDateTime.now())) - .map(Map.Entry::getKey) - .max(Comparator.naturalOrder()) - .orElse(AgentHealth.OK); - - AgentHealth invHealth = invalidatableHealth.values() - .stream() - .reduce(AgentHealth::mostSevere) - .orElse(AgentHealth.OK); - - return AgentHealth.mostSevere(generalHealth, invHealth); + public void notifyAgentHealth(AgentHealth eventHealth, Class invalidator, String loggerName, String message) { + if (invalidator == null) + handleTimeoutHealth(eventHealth, loggerName, message); + else + handleInvalidatableHealth(eventHealth, invalidator, message); } - @Override - public void onInvalidationEvent(Object invalidator) { - invalidatableHealth.remove(invalidator.getClass()); - triggerEventAndMetricIfHealthChanged(); - } + private void handleInvalidatableHealth(AgentHealth eventHealth, Class invalidator, String eventMessage) { + invalidatableHealth.merge(invalidator, eventHealth, AgentHealth::mostSevere); - @PostConstruct - @VisibleForTesting - void registerAtAppender() { - InternalProcessingAppender.register(this); + boolean shouldCreateIncident = eventHealth.isMoreSevereOrEqualTo(AgentHealth.WARNING); + triggerAgentHealthChangedEvent(invalidator.getTypeName(), eventMessage, shouldCreateIncident); } - @PostConstruct - @VisibleForTesting - void startHealthCheckScheduler() { - checkHealthAndSchedule(); - } + private void handleTimeoutHealth(AgentHealth eventHealth, String loggerName, String eventMassage) { + Duration validityPeriod = env.getCurrentConfig().getSelfMonitoring().getAgentHealth().getValidityPeriod(); + boolean isNotInfo = eventHealth.isMoreSevereOrEqualTo(AgentHealth.WARNING); - @PostConstruct - @VisibleForTesting - void sendInitialHealthMetric() { - selfMonitoringService.recordMeasurement("health", AgentHealth.OK.ordinal()); + if (isNotInfo) { + generalHealthTimeouts.put(eventHealth, LocalDateTime.now().plus(validityPeriod)); + } + String fullEventMessage = eventMassage + ". This status is valid for " + validityPeriod; + triggerAgentHealthChangedEvent(loggerName, fullEventMessage, isNotInfo); } - @PreDestroy - @VisibleForTesting - void unregisterFromAppender() { - InternalProcessingAppender.unregister(this); + /** + * Invalidates an invalidatable eventHealth and creates a new AgentHealthIncident + * @param eventClass class, which created the invalidatable eventHealth + * @param eventMessage message of the event + */ + public void invalidateIncident(Class eventClass, String eventMessage) { + invalidatableHealth.remove(eventClass); + triggerAgentHealthChangedEvent(eventClass.getTypeName(), eventMessage); } /** @@ -143,48 +114,87 @@ void unregisterFromAppender() { *
  • exists -> run until that timeout is over
    • * */ - private void checkHealthAndSchedule() { - triggerEventAndMetricIfHealthChanged(); + @VisibleForTesting + void checkHealthAndSchedule() { + triggerAgentHealthChangedEvent(AgentHealthManager.class.getCanonicalName(), "Checking timed out agent healths"); Duration validityPeriod = env.getCurrentConfig().getSelfMonitoring().getAgentHealth().getValidityPeriod(); + Duration minDelay = env.getCurrentConfig().getSelfMonitoring().getAgentHealth().getMinHealthCheckDelay(); Duration delay = generalHealthTimeouts.values() .stream() - .filter(d -> d.isAfter(LocalDateTime.now())) + .filter(dateTime -> dateTime.isAfter(LocalDateTime.now())) .max(Comparator.naturalOrder()) - .map(d -> Duration.between(d, LocalDateTime.now())) + .map(dateTime -> { + Duration dif = Duration.between(dateTime, LocalDateTime.now()); + if (minDelay.compareTo(dif) > 0) return minDelay; + else return dif; + }) .orElse(validityPeriod); executor.schedule(this::checkHealthAndSchedule, delay.toMillis(), TimeUnit.MILLISECONDS); } - private void triggerEventAndMetricIfHealthChanged() { - if (getCurrentHealth() != lastNotifiedHealth) { - AgentHealthChangedEvent event = null; - synchronized (this) { - AgentHealth currHealth = getCurrentHealth(); - if (currHealth != lastNotifiedHealth) { - AgentHealth lastHealth = lastNotifiedHealth; - lastNotifiedHealth = currHealth; - - selfMonitoringService.recordMeasurement("health", currHealth.ordinal()); - - event = new AgentHealthChangedEvent(this, lastHealth, currHealth); - ctx.publishEvent(event); - } + /** + * Creates a new AgentHealthIncident, if specified, and also triggers an AgentHealthChangedEvent, + * if the agent health has changed + * + * @param incidentSource class, which caused the incident + * @param message message, describing the incident + * @param shouldCreateIncident whether to create a new AgentHealthIncident or not + */ + private void triggerAgentHealthChangedEvent(String incidentSource, String message, Boolean shouldCreateIncident) { + synchronized (this) { + boolean changedHealth = healthHasChanged(); + AgentHealth currentHealth = getCurrentHealth(); + + // Don't create incident for health event logs + boolean isLoggedHealthEvent = incidentSource.equals(LogWritingHealthEventListener.class.getName()); + + if(shouldCreateIncident && !isLoggedHealthEvent) { + AgentHealthIncident incident = new AgentHealthIncident( + LocalDateTime.now().toString(), currentHealth, incidentSource, message, changedHealth); + healthIncidentBuffer.put(incident); } - // It is important that logging happens outside the synchronized block above. - // Otherwise, a deadlock may happen since _this_ is via Interface - // InternalProcessingAppender.LogEventConsumer indirectly a part of the logback infrastructure - if (event != null) { - AgentHealth newHealth = event.getNewHealth(); - AgentHealth oldHealth = event.getOldHealth(); - if(newHealth.isMoreSevereOrEqualTo(oldHealth)) { - log.warn(LOG_CHANGE_STATUS, oldHealth, newHealth); - } else { - log.info(LOG_CHANGE_STATUS, oldHealth, newHealth); - } + if(changedHealth) { + AgentHealth lastHealth = lastNotifiedHealth; + lastNotifiedHealth = currentHealth; + AgentHealthChangedEvent event = new AgentHealthChangedEvent(this, lastHealth, currentHealth, message); + ctx.publishEvent(event); } } } + + private void triggerAgentHealthChangedEvent(String incidentSource, String message) { + triggerAgentHealthChangedEvent(incidentSource, message, true); + } + + /** + * Checks whether the current health has changed since last check. + * + * @return true if the health state changed + */ + private boolean healthHasChanged() { + return getCurrentHealth() != lastNotifiedHealth; + } + + /** + * Returns the current agent health, which is the most severe out of instrumentation and general status. + * + * @return The current agent health + */ + public AgentHealth getCurrentHealth() { + AgentHealth generalHealth = generalHealthTimeouts.entrySet() + .stream() + .filter((entry) -> entry.getValue().isAfter(LocalDateTime.now())) + .map(Map.Entry::getKey) + .max(Comparator.naturalOrder()) + .orElse(AgentHealth.OK); + + AgentHealth invHealth = invalidatableHealth.values() + .stream() + .reduce(AgentHealth::mostSevere) + .orElse(AgentHealth.OK); + return AgentHealth.mostSevere(generalHealth, invHealth); + } } diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/HealthEventListener.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/HealthEventListener.java new file mode 100644 index 0000000000..20832ab049 --- /dev/null +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/HealthEventListener.java @@ -0,0 +1,13 @@ +package rocks.inspectit.ocelot.core.selfmonitoring.event.listener; + +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Async; +import rocks.inspectit.ocelot.core.selfmonitoring.event.models.AgentHealthChangedEvent; + +public interface HealthEventListener { + + @Async + @EventListener + void onAgentHealthEvent(AgentHealthChangedEvent event); + +} diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/LogWritingHealthEventListener.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/LogWritingHealthEventListener.java new file mode 100644 index 0000000000..de1d1f2ec3 --- /dev/null +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/LogWritingHealthEventListener.java @@ -0,0 +1,23 @@ +package rocks.inspectit.ocelot.core.selfmonitoring.event.listener; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import rocks.inspectit.ocelot.core.selfmonitoring.event.models.AgentHealthChangedEvent; + +@Component +@Slf4j +public class LogWritingHealthEventListener implements HealthEventListener { + + private static final String LOG_CHANGE_STATUS = "The agent status changed from {} to {}. Reason: {}"; + + @Override + public void onAgentHealthEvent(AgentHealthChangedEvent event) { + if (event.getNewHealth().isMoreSevereOrEqualTo(event.getOldHealth())) { + log.warn(LOG_CHANGE_STATUS, event.getOldHealth(), event.getNewHealth(), event.getMessage()); + } else { + log.info(LOG_CHANGE_STATUS, event.getOldHealth(), event.getNewHealth(), event.getMessage()); + } + + } +} diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/MetricWritingHealthEventListener.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/MetricWritingHealthEventListener.java new file mode 100644 index 0000000000..02ee047c6c --- /dev/null +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/MetricWritingHealthEventListener.java @@ -0,0 +1,35 @@ +package rocks.inspectit.ocelot.core.selfmonitoring.event.listener; + +import com.google.common.annotations.VisibleForTesting; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import rocks.inspectit.ocelot.commons.models.health.AgentHealth; +import rocks.inspectit.ocelot.core.selfmonitoring.SelfMonitoringService; +import rocks.inspectit.ocelot.core.selfmonitoring.event.models.AgentHealthChangedEvent; + +import javax.annotation.PostConstruct; +import java.util.HashMap; + +@Component +public class MetricWritingHealthEventListener implements HealthEventListener { + + private static final String INITIAL_METRIC_MESSAGE = "Initial health metric sent"; + + @Autowired + private SelfMonitoringService selfMonitoringService; + + @PostConstruct + @VisibleForTesting + void sendInitialHealthMetric() { + HashMap tags = new HashMap<>(); + tags.put("message", INITIAL_METRIC_MESSAGE); + selfMonitoringService.recordMeasurement("health", AgentHealth.OK.ordinal(), tags); + } + @Override + public void onAgentHealthEvent(AgentHealthChangedEvent event) { + HashMap tags = new HashMap<>(); + tags.put("message", event.getMessage()); + tags.put("source", event.getSource().getClass().getName()); + selfMonitoringService.recordMeasurement("health", event.getNewHealth().ordinal(), tags); + } +} diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/PollerWritingHealthEventListener.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/PollerWritingHealthEventListener.java new file mode 100644 index 0000000000..a33028b975 --- /dev/null +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/PollerWritingHealthEventListener.java @@ -0,0 +1,42 @@ +package rocks.inspectit.ocelot.core.selfmonitoring.event.listener; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; +import rocks.inspectit.ocelot.commons.models.health.AgentHealthIncident; +import rocks.inspectit.ocelot.commons.models.health.AgentHealthState; +import rocks.inspectit.ocelot.core.config.propertysources.http.HttpConfigurationPoller; +import rocks.inspectit.ocelot.core.selfmonitoring.AgentHealthManager; +import rocks.inspectit.ocelot.core.selfmonitoring.event.models.AgentHealthChangedEvent; +import rocks.inspectit.ocelot.core.selfmonitoring.event.models.AgentHealthIncidentAddedEvent; + +import java.util.List; + +@Component +public class PollerWritingHealthEventListener implements HealthEventListener { + + @Autowired + HttpConfigurationPoller httpConfigurationPoller; + + @Autowired + AgentHealthManager agentHealthManager; + + @Override + public void onAgentHealthEvent(AgentHealthChangedEvent event) { + List incidentHistory = agentHealthManager.getIncidentHistory(); + + AgentHealthState state = new AgentHealthState(event.getNewHealth(), event.getSource().toString(), event.getMessage(), incidentHistory); + httpConfigurationPoller.updateAgentHealthState(state); + } + + @Async + @EventListener + public void onAgentHealthIncidentEvent(AgentHealthIncidentAddedEvent event) { + List incidentHistory = event.getCurrentIncidents(); + AgentHealthIncident latestIncident = incidentHistory.get(0); + + AgentHealthState state = new AgentHealthState(latestIncident.getHealth(), latestIncident.getSource(), latestIncident.getMessage(), incidentHistory); + httpConfigurationPoller.updateAgentHealthState(state); + } +} diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/AgentHealthChangedEvent.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/models/AgentHealthChangedEvent.java similarity index 67% rename from inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/AgentHealthChangedEvent.java rename to inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/models/AgentHealthChangedEvent.java index eb288a40c2..401ed4bef3 100644 --- a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/AgentHealthChangedEvent.java +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/models/AgentHealthChangedEvent.java @@ -1,10 +1,10 @@ -package rocks.inspectit.ocelot.core.selfmonitoring.event; +package rocks.inspectit.ocelot.core.selfmonitoring.event.models; import lombok.Getter; import lombok.NonNull; import org.springframework.context.ApplicationEvent; import rocks.inspectit.ocelot.commons.models.health.AgentHealth; -import rocks.inspectit.ocelot.core.selfmonitoring.LogPreloader; +import rocks.inspectit.ocelot.core.selfmonitoring.logs.LogPreloader; /** * Fired by {@link LogPreloader} whenever the agent health changed. @@ -23,10 +23,17 @@ public class AgentHealthChangedEvent extends ApplicationEvent { @Getter private AgentHealth newHealth; - public AgentHealthChangedEvent(Object source, @NonNull AgentHealth oldHealth, @NonNull AgentHealth newHealth) { + /** + * The message stating the cause of the event. + */ + @Getter + private String message; + + public AgentHealthChangedEvent(Object source, @NonNull AgentHealth oldHealth, @NonNull AgentHealth newHealth, String message) { super(source); this.oldHealth = oldHealth; this.newHealth = newHealth; + this.message = message; } } diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/models/AgentHealthIncidentAddedEvent.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/models/AgentHealthIncidentAddedEvent.java new file mode 100644 index 0000000000..a2acba676e --- /dev/null +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/event/models/AgentHealthIncidentAddedEvent.java @@ -0,0 +1,22 @@ +package rocks.inspectit.ocelot.core.selfmonitoring.event.models; + +import lombok.Getter; +import org.springframework.context.ApplicationEvent; +import rocks.inspectit.ocelot.commons.models.health.AgentHealthIncident; +import rocks.inspectit.ocelot.core.selfmonitoring.AgentHealthIncidentBuffer; + +import java.util.List; + +/** + * Fired by {@link AgentHealthIncidentBuffer} whenever a new incident has been added. + */ +public class AgentHealthIncidentAddedEvent extends ApplicationEvent { + + @Getter + private final List currentIncidents; + + public AgentHealthIncidentAddedEvent(Object source, List currentIncidents) { + super(source); + this.currentIncidents = currentIncidents; + } +} diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogHealthMonitor.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogHealthMonitor.java new file mode 100644 index 0000000000..27af2daace --- /dev/null +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogHealthMonitor.java @@ -0,0 +1,51 @@ +package rocks.inspectit.ocelot.core.selfmonitoring.logs; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import com.google.common.annotations.VisibleForTesting; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import rocks.inspectit.ocelot.commons.models.health.AgentHealth; +import rocks.inspectit.ocelot.core.logging.logback.InternalProcessingAppender; +import rocks.inspectit.ocelot.core.selfmonitoring.AgentHealthManager; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +@Component +@Slf4j +@RequiredArgsConstructor +public class LogHealthMonitor implements InternalProcessingAppender.LogEventConsumer { + + @Autowired + final AgentHealthManager agentHealthManager; + + @Override + public void onLoggingEvent(ILoggingEvent event, Class invalidator) { + if (AgentHealthManager.class.getCanonicalName().equals(event.getLoggerName())) { + // ignore own logs, which otherwise would tend to cause infinite loops + return; + } + AgentHealth eventHealth = AgentHealth.fromLogLevel(event.getLevel()); + agentHealthManager.notifyAgentHealth(eventHealth, invalidator, event.getLoggerName(), event.getFormattedMessage()); + } + + @Override + public void onInvalidationEvent(Object invalidator) { + agentHealthManager.invalidateIncident(invalidator.getClass(), "Invalidation due to invalidator event"); + } + + @PostConstruct + @VisibleForTesting + void registerAtAppender() { + InternalProcessingAppender.register(this); + } + + @PreDestroy + @VisibleForTesting + void unregisterFromAppender() { + InternalProcessingAppender.unregister(this); + } + +} diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/LogMetricsRecorder.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogMetricsRecorder.java similarity index 91% rename from inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/LogMetricsRecorder.java rename to inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogMetricsRecorder.java index a000c4f2e3..23a5215bc0 100644 --- a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/LogMetricsRecorder.java +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogMetricsRecorder.java @@ -1,9 +1,10 @@ -package rocks.inspectit.ocelot.core.selfmonitoring; +package rocks.inspectit.ocelot.core.selfmonitoring.logs; import ch.qos.logback.classic.spi.ILoggingEvent; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import rocks.inspectit.ocelot.core.logging.logback.InternalProcessingAppender; +import rocks.inspectit.ocelot.core.selfmonitoring.SelfMonitoringService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/LogPreloader.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogPreloader.java similarity index 99% rename from inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/LogPreloader.java rename to inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogPreloader.java index a26c16455f..59ecee0456 100644 --- a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/LogPreloader.java +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogPreloader.java @@ -1,4 +1,4 @@ -package rocks.inspectit.ocelot.core.selfmonitoring; +package rocks.inspectit.ocelot.core.selfmonitoring.logs; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; diff --git a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/tags/TagUtils.java b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/tags/TagUtils.java index 86d7e045a1..e98a3714d7 100644 --- a/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/tags/TagUtils.java +++ b/inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/tags/TagUtils.java @@ -5,6 +5,8 @@ import io.opencensus.tags.TagValue; import lombok.extern.slf4j.Slf4j; +import java.util.function.Function; + @Slf4j public final class TagUtils { @@ -49,11 +51,19 @@ private TagUtils() { * @return the created TagValue with 'v' or '<invalid>' */ public static TagValue createTagValue(String tagKey, String value) { + return resolveTageValue(tagKey, value, TagValue::create); + } + + public static String createTagValueAsString(String tagKey, String value) { + return resolveTageValue(tagKey, value, s -> s); + } + + private static T resolveTageValue(String tagKey, String value, Function creator) { if (isTagValueValid(value)) { - return TagValue.create(value); + return creator.apply(value); } printWarning(tagKey, value); - return TagValue.create(""); + return creator.apply(""); } private static boolean isTagValueValid(String value) { diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/command/handler/impl/LogsCommandExecutorTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/command/handler/impl/LogsCommandExecutorTest.java index f7e76a5989..7c5fb01c74 100644 --- a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/command/handler/impl/LogsCommandExecutorTest.java +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/command/handler/impl/LogsCommandExecutorTest.java @@ -18,7 +18,7 @@ import rocks.inspectit.ocelot.config.model.selfmonitoring.LogPreloadingSettings; import rocks.inspectit.ocelot.core.SpringTestBase; import rocks.inspectit.ocelot.core.config.InspectitEnvironment; -import rocks.inspectit.ocelot.core.selfmonitoring.LogPreloader; +import rocks.inspectit.ocelot.core.selfmonitoring.logs.LogPreloader; import java.io.File; import java.io.IOException; diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/exporter/ExporterServiceIntegrationTestBase.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/exporter/ExporterServiceIntegrationTestBase.java index c7eecbd1b8..08f6610883 100644 --- a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/exporter/ExporterServiceIntegrationTestBase.java +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/exporter/ExporterServiceIntegrationTestBase.java @@ -52,17 +52,19 @@ import static org.testcontainers.Testcontainers.exposeHostPorts; /** - * Base class for exporter integration tests. Verifies integration with the OpenTelemetry Collector. The Collector can be configured to accept the required data over gRPC or HTTP and exports the data over gRPC to a server running in process, allowing assertions to be made against the data. + * Base class for exporter integration tests. Verifies integration with the OpenTelemetry Collector. + * The Collector can be configured to accept the required data over gRPC or HTTP and exports the data over gRPC + * to a server running in process, allowing assertions to be made against the data. * This class is based on the {@link io.opentelemetry.integrationtest.OtlpExporterIntegrationTest} */ @Testcontainers(disabledWithoutDocker = true) -abstract class ExporterServiceIntegrationTestBase extends SpringTestBase { +public abstract class ExporterServiceIntegrationTestBase extends SpringTestBase { static final String COLLECTOR_TAG = "0.58.0"; static final String COLLECTOR_IMAGE = "otel/opentelemetry-collector-contrib:" + COLLECTOR_TAG; - static final Integer COLLECTOR_OTLP_GRPC_PORT = 4317; + protected static final Integer COLLECTOR_OTLP_GRPC_PORT = 4317; static final Integer COLLECTOR_OTLP_HTTP_PORT = 4318; @@ -154,7 +156,8 @@ static Tracer getOtelTracer() { } /** - * Gets the desired endpoint of the {@link #collector} constructed as 'http://{@link GenericContainer#getHost() collector.getHost()}:{@link GenericContainer#getMappedPort(int) collector.getMappedPort(port)}/path' + * Gets the desired endpoint of the {@link #collector} constructed as + * 'http://{@link GenericContainer#getHost() collector.getHost()}:{@link GenericContainer#getMappedPort(int) collector.getMappedPort(port)}/path' * * @param originalPort the port to get the actual mapped port for * @param path the path @@ -166,13 +169,14 @@ static String getEndpoint(Integer originalPort, String path) { } /** - * Gets the desired endpoint of the {@link #collector} constructed as 'http://{@link GenericContainer#getHost() collector.getHost()}:{@link GenericContainer#getMappedPort(int) collector.getMappedPort(port)}' + * Gets the desired endpoint of the {@link #collector} constructed as + * 'http://{@link GenericContainer#getHost() collector.getHost()}:{@link GenericContainer#getMappedPort(int) collector.getMappedPort(port)}' * * @param originalPort the port to get the actual mapped port for * * @return the constructed endpoint for the {@link #collector} */ - static String getEndpoint(Integer originalPort) { + protected static String getEndpoint(Integer originalPort) { return String.format("http://%s:%d", collector.getHost(), collector.getMappedPort(originalPort)); } @@ -204,23 +208,24 @@ void makeSpansAndFlush(String parentSpanName, String childSpanName) { * Records some dummy metrics and flushes them. */ void recordMetricsAndFlush() { - recordMetricsAndFlush(1, "my-key", "my-val"); + recordMetricsAndFlush("my-counter", 1, "my-key", "my-val"); } /** * Records a counter with the given value and tag * + * @param measureName the name of the measure * @param value the value to add to the counter * @param tagKey the key of the tag * @param tagVal the value of the tag */ - void recordMetricsAndFlush(int value, String tagKey, String tagVal) { + protected void recordMetricsAndFlush(String measureName, int value, String tagKey, String tagVal) { // get the meter and create a counter Meter meter = GlobalOpenTelemetry.getMeterProvider() .meterBuilder("rocks.inspectit.ocelot") .setInstrumentationVersion("0.0.1") .build(); - LongCounter counter = meter.counterBuilder("my-counter").setDescription("My counter").setUnit("1").build(); + LongCounter counter = meter.counterBuilder(measureName).setDescription("My counter").setUnit("1").build(); // record counter counter.add(value, Attributes.of(AttributeKey.stringKey(tagKey), tagVal)); @@ -230,13 +235,15 @@ void recordMetricsAndFlush(int value, String tagKey, String tagVal) { } /** - * Verifies that the metric with the given value and key/value attribute (tag) has been exported to and received by the {@link #grpcServer} + * Verifies that the metric with the given value and key/value attribute (tag) has been exported to and received + * by the {@link #grpcServer} * - * @param value - * @param tagKey - * @param tagVal + * @param measureName the name of the measure + * @param value the value of the measure + * @param tagKey the key of the tag + * @param tagVal the value of the tag */ - void awaitMetricsExported(int value, String tagKey, String tagVal) { + protected void awaitMetricsExported(String measureName, int value, String tagKey, String tagVal) { // create the attribute that we will use to verify that the metric has been written KeyValue attribute = KeyValue.newBuilder() .setKey(tagKey) @@ -247,12 +254,11 @@ void awaitMetricsExported(int value, String tagKey, String tagVal) { .untilAsserted(() -> assertThat(grpcServer.metricRequests.stream()).anyMatch(mReq -> mReq.getResourceMetricsList() .stream() .anyMatch(rm -> - // check for the "my-counter" metrics - rm.getInstrumentationLibraryMetrics(0).getMetrics(0).getName().equals("my-counter") + // check for the specified measure + rm.getInstrumentationLibraryMetrics(0) + .getMetricsList().stream() + .filter(metric -> metric.getName().equals(measureName)) // check for the specific attribute and value - && rm.getInstrumentationLibraryMetrics(0) - .getMetricsList() - .stream() .anyMatch(metric -> metric.getSum() .getDataPointsList() .stream() @@ -261,7 +267,9 @@ void awaitMetricsExported(int value, String tagKey, String tagVal) { } /** - * Waits for the spans to be exported to and received by the {@link #grpcServer}. This method asserts that Spans with the given names exist and that the child's {@link io.opentelemetry.proto.trace.v1.Span#getParentSpanId()} equals the parent's {@link io.opentelemetry.proto.trace.v1.Span#getSpanId()} + * Waits for the spans to be exported to and received by the {@link #grpcServer}. This method asserts that Spans + * with the given names exist and that the child's {@link io.opentelemetry.proto.trace.v1.Span#getParentSpanId()} + * equals the parent's {@link io.opentelemetry.proto.trace.v1.Span#getSpanId()} * * @param parentSpanName the name of the parent span * @param childSpanName the name of the child span @@ -305,7 +313,7 @@ void awaitSpansExported(String parentSpanName, String childSpanName) { /** * OpenTelemetry Protocol gRPC Server */ - static class OtlpGrpcServer extends ServerExtension { + public static class OtlpGrpcServer extends ServerExtension { final List traceRequests = new ArrayList<>(); @@ -319,6 +327,10 @@ private void reset() { logRequests.clear(); } + public List getMetricRequests() { + return metricRequests; + } + @Override protected void configure(ServerBuilder sb) { sb.service("/opentelemetry.proto.collector.trace.v1.TraceService/Export", new AbstractUnaryGrpcService() { @@ -358,4 +370,8 @@ protected CompletionStage handleMessage(ServiceRequestContext ctx, byte[ } } + public OtlpGrpcServer getGrpcServer() { + return grpcServer; + } + } diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/exporter/OtlpMetricsExporterServiceIntTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/exporter/OtlpMetricsExporterServiceIntTest.java index 062b6d4348..a344d4beb3 100644 --- a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/exporter/OtlpMetricsExporterServiceIntTest.java +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/exporter/OtlpMetricsExporterServiceIntTest.java @@ -39,12 +39,10 @@ public class OtlpMetricsExporterServiceIntTest extends ExporterServiceIntegratio @Autowired InspectitEnvironment environment; + String measure = "my-counter"; String tagKeyGrpc = "otlp-grpc-metrics-test"; - String tagKeyHttp = "otlp-grpc-metrics-test"; - String tagVal = "random-val"; - int metricVal = 1337; @BeforeEach @@ -66,9 +64,9 @@ void verifyMetricsWrittenGrpc() { .pollInterval(500, TimeUnit.MILLISECONDS) .untilAsserted(() -> assertThat(service.isEnabled()).isTrue()); - recordMetricsAndFlush(metricVal, tagKeyGrpc, tagVal); + recordMetricsAndFlush(measure, metricVal, tagKeyGrpc, tagVal); - awaitMetricsExported(metricVal, tagKeyGrpc, tagVal); + awaitMetricsExported(measure, metricVal, tagKeyGrpc, tagVal); } @DirtiesContext @@ -85,9 +83,9 @@ void verifyMetricsWrittenHttp() { .pollInterval(500, TimeUnit.MILLISECONDS) .untilAsserted(() -> assertThat(service.isEnabled()).isTrue()); - recordMetricsAndFlush(metricVal, tagKeyHttp, tagVal); + recordMetricsAndFlush(measure, metricVal, tagKeyHttp, tagVal); - awaitMetricsExported(metricVal, tagKeyHttp, tagVal); + awaitMetricsExported(measure, metricVal, tagKeyHttp, tagVal); } @@ -139,8 +137,8 @@ void testAggregationTemporalityCumulative(){ assertThat(service.isEnabled()).isTrue(); - recordMetricsAndFlush(1, "key", "val"); - recordMetricsAndFlush(2, "key", "val"); + recordMetricsAndFlush(measure, 1, "key", "val"); + recordMetricsAndFlush(measure, 2, "key", "val"); await().atMost(30, TimeUnit.SECONDS) .untilAsserted(() -> assertThat(grpcServer.metricRequests.stream()).anyMatch(mReq -> mReq.getResourceMetricsList() @@ -171,8 +169,8 @@ void testAggregationTemporalityDelta(){ assertThat(service.isEnabled()).isTrue(); - recordMetricsAndFlush(1, "key", "val"); - recordMetricsAndFlush(2, "key", "val"); + recordMetricsAndFlush(measure, 1, "key", "val"); + recordMetricsAndFlush(measure, 2, "key", "val"); await().atMost(30, TimeUnit.SECONDS) .untilAsserted(() -> assertThat(grpcServer.metricRequests.stream()).anyMatch(mReq -> mReq.getResourceMetricsList() diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/instrumentation/hook/MethodHookGeneratorTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/instrumentation/hook/MethodHookGeneratorTest.java index 970b7c84bf..5f643586ca 100644 --- a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/instrumentation/hook/MethodHookGeneratorTest.java +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/instrumentation/hook/MethodHookGeneratorTest.java @@ -1,6 +1,8 @@ package rocks.inspectit.ocelot.core.instrumentation.hook; +import com.google.common.collect.HashMultiset; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Multiset; import net.bytebuddy.description.method.MethodDescription; import net.bytebuddy.description.type.TypeDescription; import org.assertj.core.api.Assertions; @@ -27,8 +29,8 @@ import rocks.inspectit.ocelot.core.selfmonitoring.ActionScopeFactory; import rocks.inspectit.ocelot.core.testutils.Dummy; -import java.util.Collections; -import java.util.List; +import java.util.*; +import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; @@ -190,7 +192,8 @@ public void verifyNoActionsGeneratedIfNoSpanStartedOrContinued() { .endSpan(true) .build(); - List actions = generator.buildTracingExitActions(settings); + MethodHookConfiguration config = MethodHookConfiguration.builder().tracing(settings).build(); + List actions = generator.buildTracingExitActions(config); assertThat(actions).isEmpty(); } @@ -205,7 +208,8 @@ public void verifyActionsGeneratedIfSpanStarted() { .endSpan(true) .build(); - List actions = generator.buildTracingExitActions(settings); + MethodHookConfiguration config = MethodHookConfiguration.builder().tracing(settings).build(); + List actions = generator.buildTracingExitActions(config); assertThat(actions) .hasSize(3) @@ -224,7 +228,8 @@ public void verifyActionsGeneratedIfSpanContinued() { .endSpan(true) .build(); - List actions = generator.buildTracingExitActions(settings); + MethodHookConfiguration config = MethodHookConfiguration.builder().tracing(settings).build(); + List actions = generator.buildTracingExitActions(config); assertThat(actions) .hasSize(3) @@ -234,4 +239,240 @@ public void verifyActionsGeneratedIfSpanContinued() { } } + + @Nested + class collectMetricTagsInTracing { + + private RuleTracingSettings createTracingSettings(Map attributes) { + return RuleTracingSettings.builder() + .startSpan(true) + .continueSpan(null) + .attributes(attributes) + .endSpan(true) + .build(); + } + + private MetricRecordingSettings createMetricSettings(Map dataTags, Map constantTags) { + return MetricRecordingSettings.builder().metric("name").value("1.0") + .dataTags(dataTags).constantTags(constantTags).build(); + } + + @Test + public void verifyTagsHaveBeenAdded() { + when(environment.getCurrentConfig().getTracing().isAddMetricTags()).thenReturn(true); + VariableAccessor tracingAccessor = (context) -> "tracing-text"; + VariableAccessor dataAccessor = (context) -> "data-text"; + VariableAccessor constantAccessor = (context) -> "constant-text"; + when(variableAccessorFactory.getVariableAccessor("tracing-value")).thenReturn(tracingAccessor); + when(variableAccessorFactory.getVariableAccessor("data-value")).thenReturn(dataAccessor); + when(variableAccessorFactory.getConstantAccessor(any())).thenReturn(constantAccessor); + Map expectedResult = new HashMap<>(); + expectedResult.put("tracing-key", tracingAccessor); + expectedResult.put("data-tag", dataAccessor); + expectedResult.put("constant-tag", constantAccessor); + + Map attributes = ImmutableMap.of("tracing-key", "tracing-value"); + RuleTracingSettings tracing = createTracingSettings(attributes); + + Map dataTags = ImmutableMap.of("data-tag", "data-value"); + Map constantTags = ImmutableMap.of("constant-tag", "constant-value"); + MetricRecordingSettings metric = createMetricSettings(dataTags, constantTags); + Multiset metrics = HashMultiset.create(); + metrics.add(metric); + + MethodHookConfiguration config = MethodHookConfiguration.builder().tracing(tracing).metrics(metrics).build(); + List actions = generator.buildTracingExitActions(config); + + Optional maybeAction = actions.stream().filter(action -> action instanceof WriteSpanAttributesAction).findFirst(); + assertThat(maybeAction.isPresent()).isTrue(); + WriteSpanAttributesAction attributeAction = (WriteSpanAttributesAction) maybeAction.get(); + + Map accessors = attributeAction.getAttributeAccessors(); + assertThat(accessors.size()).isEqualTo(expectedResult.size()); + assertThat(accessors).containsAllEntriesOf(expectedResult); + } + + @Test + public void verifyNoTagsHaveBeenAdded() { + when(environment.getCurrentConfig().getTracing().isAddMetricTags()).thenReturn(false); + VariableAccessor tracingAccessor = (context) -> "tracing-text"; + when(variableAccessorFactory.getVariableAccessor("tracing-value")).thenReturn(tracingAccessor); + Map expectedResult = new HashMap<>(); + expectedResult.put("tracing-key", tracingAccessor); + + Map attributes = ImmutableMap.of("tracing-key", "tracing-value"); + RuleTracingSettings tracing = createTracingSettings(attributes); + + Map dataTags = ImmutableMap.of("data-tag", "data-value"); + Map constantTags = ImmutableMap.of("constant-tag", "constant-value"); + MetricRecordingSettings metric = createMetricSettings(dataTags, constantTags); + Multiset metrics = HashMultiset.create(); + metrics.add(metric); + + MethodHookConfiguration config = MethodHookConfiguration.builder().tracing(tracing).metrics(metrics).build(); + List actions = generator.buildTracingExitActions(config); + + Optional maybeAction = actions.stream().filter(action -> action instanceof WriteSpanAttributesAction).findFirst(); + assertThat(maybeAction.isPresent()).isTrue(); + WriteSpanAttributesAction attributeAction = (WriteSpanAttributesAction) maybeAction.get(); + + Map accessors = attributeAction.getAttributeAccessors(); + assertThat(accessors.size()).isEqualTo(expectedResult.size()); + assertThat(accessors).containsAllEntriesOf(expectedResult); + } + + @Test + void verifyTracingOverwritesMetrics() { + when(environment.getCurrentConfig().getTracing().isAddMetricTags()).thenReturn(true); + VariableAccessor tracingAccessor = (context) -> "tracing-text"; + when(variableAccessorFactory.getVariableAccessor("tracing-value")).thenReturn(tracingAccessor); + Map expectedResult = new HashMap<>(); + expectedResult.put("one-key", tracingAccessor); + + Map attributes = ImmutableMap.of("one-key", "tracing-value"); + RuleTracingSettings tracing = createTracingSettings(attributes); + + Map dataTags = ImmutableMap.of("one-key", "data-value"); + Map constantTags = ImmutableMap.of("one-key", "constant-value"); + MetricRecordingSettings metric = createMetricSettings(dataTags, constantTags); + Multiset metrics = HashMultiset.create(); + metrics.add(metric); + + MethodHookConfiguration config = MethodHookConfiguration.builder().tracing(tracing).metrics(metrics).build(); + List actions = generator.buildTracingExitActions(config); + + Optional maybeAction = actions.stream().filter(action -> action instanceof WriteSpanAttributesAction).findFirst(); + assertThat(maybeAction.isPresent()).isTrue(); + WriteSpanAttributesAction attributeAction = (WriteSpanAttributesAction) maybeAction.get(); + + Map accessors = attributeAction.getAttributeAccessors(); + assertThat(accessors.size()).isEqualTo(expectedResult.size()); + assertThat(accessors).containsAllEntriesOf(expectedResult); + } + + @Test + void verifyDataTagsOverwriteConstantTags() { + when(environment.getCurrentConfig().getTracing().isAddMetricTags()).thenReturn(true); + VariableAccessor dataAccessor = (context) -> "data-text"; + when(variableAccessorFactory.getVariableAccessor("data-value")).thenReturn(dataAccessor); + Map expectedResult = new HashMap<>(); + expectedResult.put("one-key", dataAccessor); + + Map attributes = Collections.emptyMap(); + RuleTracingSettings tracing = createTracingSettings(attributes); + + Map dataTags = ImmutableMap.of("one-key", "data-value"); + Map constantTags = ImmutableMap.of("one-key", "constant-value"); + MetricRecordingSettings metric = createMetricSettings(dataTags, constantTags); + Multiset metrics = HashMultiset.create(); + metrics.add(metric); + + MethodHookConfiguration config = MethodHookConfiguration.builder().tracing(tracing).metrics(metrics).build(); + List actions = generator.buildTracingExitActions(config); + + Optional maybeAction = actions.stream().filter(action -> action instanceof WriteSpanAttributesAction).findFirst(); + assertThat(maybeAction.isPresent()).isTrue(); + WriteSpanAttributesAction attributeAction = (WriteSpanAttributesAction) maybeAction.get(); + + Map accessors = attributeAction.getAttributeAccessors(); + assertThat(accessors.size()).isEqualTo(expectedResult.size()); + assertThat(accessors).containsAllEntriesOf(expectedResult); + } + + @Test + void verifyOnlyDataTags() { + when(environment.getCurrentConfig().getTracing().isAddMetricTags()).thenReturn(true); + VariableAccessor dataAccessor = (context) -> "data-text"; + when(variableAccessorFactory.getVariableAccessor("data-value")).thenReturn(dataAccessor); + Map expectedResult = new HashMap<>(); + expectedResult.put("data-tag", dataAccessor); + + Map attributes = Collections.emptyMap(); + RuleTracingSettings tracing = createTracingSettings(attributes); + + Map dataTags = ImmutableMap.of("data-tag", "data-value"); + Map constantTags = Collections.emptyMap(); + MetricRecordingSettings metric = createMetricSettings(dataTags, constantTags); + Multiset metrics = HashMultiset.create(); + metrics.add(metric); + + MethodHookConfiguration config = MethodHookConfiguration.builder().tracing(tracing).metrics(metrics).build(); + List actions = generator.buildTracingExitActions(config); + + Optional maybeAction = actions.stream().filter(action -> action instanceof WriteSpanAttributesAction).findFirst(); + assertThat(maybeAction.isPresent()).isTrue(); + WriteSpanAttributesAction attributeAction = (WriteSpanAttributesAction) maybeAction.get(); + + Map accessors = attributeAction.getAttributeAccessors(); + assertThat(accessors.size()).isEqualTo(expectedResult.size()); + assertThat(accessors).containsAllEntriesOf(expectedResult); + } + + @Test + void verifyOnlyConstantTags() { + when(environment.getCurrentConfig().getTracing().isAddMetricTags()).thenReturn(true); + VariableAccessor constantAccessor = (context) -> "constant-text"; + when(variableAccessorFactory.getConstantAccessor(any())).thenReturn(constantAccessor); + Map expectedResult = new HashMap<>(); + expectedResult.put("constant-tag", constantAccessor); + + Map attributes = Collections.emptyMap(); + RuleTracingSettings tracing = createTracingSettings(attributes); + + Map dataTags = Collections.emptyMap(); + Map constantTags = ImmutableMap.of("constant-tag", "constant-value"); + MetricRecordingSettings metric = createMetricSettings(dataTags, constantTags); + Multiset metrics = HashMultiset.create(); + metrics.add(metric); + + MethodHookConfiguration config = MethodHookConfiguration.builder().tracing(tracing).metrics(metrics).build(); + List actions = generator.buildTracingExitActions(config); + + Optional maybeAction = actions.stream().filter(action -> action instanceof WriteSpanAttributesAction).findFirst(); + assertThat(maybeAction.isPresent()).isTrue(); + WriteSpanAttributesAction attributeAction = (WriteSpanAttributesAction) maybeAction.get(); + + Map accessors = attributeAction.getAttributeAccessors(); + assertThat(accessors.size()).isEqualTo(expectedResult.size()); + assertThat(accessors).containsAllEntriesOf(expectedResult); + } + + @Test + void verifyTagsOfMultipleMetrics() { + when(environment.getCurrentConfig().getTracing().isAddMetricTags()).thenReturn(true); + VariableAccessor tracingAccessor = (context) -> "tracing-text"; + VariableAccessor dataAccessor1 = (context) -> "data-text-1"; + VariableAccessor dataAccessor2 = (context) -> "data-text-2"; + when(variableAccessorFactory.getVariableAccessor("tracing-value")).thenReturn(tracingAccessor); + when(variableAccessorFactory.getVariableAccessor("data-value-1")).thenReturn(dataAccessor1); + when(variableAccessorFactory.getVariableAccessor("data-value-2")).thenReturn(dataAccessor2); + Map expectedResult = new HashMap<>(); + expectedResult.put("tracing-key", tracingAccessor); + expectedResult.put("data-tag-1", dataAccessor1); + expectedResult.put("data-tag-2", dataAccessor2); + + Map attributes = ImmutableMap.of("tracing-key", "tracing-value"); + RuleTracingSettings tracing = createTracingSettings(attributes); + + Map dataTags1 = ImmutableMap.of("data-tag-1", "data-value-1"); + Map dataTags2 = ImmutableMap.of("data-tag-2", "data-value-2"); + Map constantTags = Collections.emptyMap(); + MetricRecordingSettings metric1 = createMetricSettings(dataTags1, constantTags); + MetricRecordingSettings metric2 = createMetricSettings(dataTags2, constantTags); + Multiset metrics = HashMultiset.create(); + metrics.add(metric1); + metrics.add(metric2); + + MethodHookConfiguration config = MethodHookConfiguration.builder().tracing(tracing).metrics(metrics).build(); + List actions = generator.buildTracingExitActions(config); + + Optional maybeAction = actions.stream().filter(action -> action instanceof WriteSpanAttributesAction).findFirst(); + assertThat(maybeAction.isPresent()).isTrue(); + WriteSpanAttributesAction attributeAction = (WriteSpanAttributesAction) maybeAction.get(); + + Map accessors = attributeAction.getAttributeAccessors(); + assertThat(accessors.size()).isEqualTo(expectedResult.size()); + assertThat(accessors).containsAllEntriesOf(expectedResult); + } + } } diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/MetricsRecorderTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/MetricsRecorderTest.java index 926e5eeb5d..c67ff2dbd5 100644 --- a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/MetricsRecorderTest.java +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/instrumentation/hook/actions/MetricsRecorderTest.java @@ -8,13 +8,14 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.Mockito; +import org.mockito.*; import org.mockito.junit.jupiter.MockitoExtension; +import rocks.inspectit.ocelot.core.SpringTestBase; +import rocks.inspectit.ocelot.core.config.InspectitEnvironment; import rocks.inspectit.ocelot.core.instrumentation.context.InspectitContextImpl; import rocks.inspectit.ocelot.core.instrumentation.hook.VariableAccessor; import rocks.inspectit.ocelot.core.instrumentation.hook.actions.model.MetricAccessor; +import rocks.inspectit.ocelot.core.metrics.MeasureTagValueGuard; import rocks.inspectit.ocelot.core.metrics.MeasuresAndViewsManager; import rocks.inspectit.ocelot.core.tags.CommonTagsManager; @@ -26,7 +27,7 @@ import static org.mockito.Mockito.*; @ExtendWith(MockitoExtension.class) -public class MetricsRecorderTest { +public class MetricsRecorderTest extends SpringTestBase { @Mock CommonTagsManager commonTagsManager; @@ -37,9 +38,16 @@ public class MetricsRecorderTest { @Mock IHookAction.ExecutionContext executionContext; + @Spy + @InjectMocks + MeasureTagValueGuard tagValueGuard; + @Mock InspectitContextImpl inspectitContext; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private InspectitEnvironment environment; + @BeforeEach void setupMock() { when(commonTagsManager.getCommonTagKeys()).thenReturn(Collections.emptyList()); @@ -53,9 +61,8 @@ class Execute { void verifyNullValueDataMetricIgnored() { VariableAccessor variableAccess = Mockito.mock(VariableAccessor.class); when(variableAccess.get(any())).thenReturn(null); - MetricAccessor metricAccessor = new MetricAccessor("my_metric", variableAccess, Collections.emptyMap(), Collections - .emptyMap()); - MetricsRecorder rec = new MetricsRecorder(Collections.singletonList(metricAccessor), commonTagsManager, metricsManager); + MetricAccessor metricAccessor = new MetricAccessor("my_metric", variableAccess, Collections.emptyMap(), Collections.emptyMap()); + MetricsRecorder rec = new MetricsRecorder(Collections.singletonList(metricAccessor), commonTagsManager, metricsManager, tagValueGuard); rec.execute(executionContext); @@ -78,12 +85,10 @@ void verifyInvalidDataTypeHandled() { VariableAccessor dataB = Mockito.mock(VariableAccessor.class); when(dataA.get(any())).thenReturn(100.0); when(dataB.get(any())).thenReturn("notanumber"); - MetricAccessor metricAccessorA = new MetricAccessor("my_metric1", dataA, Collections.emptyMap(), Collections - .emptyMap()); - MetricAccessor metricAccessorB = new MetricAccessor("my_metric2", dataB, Collections.emptyMap(), Collections - .emptyMap()); + MetricAccessor metricAccessorA = new MetricAccessor("my_metric1", dataA, Collections.emptyMap(), Collections.emptyMap()); + MetricAccessor metricAccessorB = new MetricAccessor("my_metric2", dataB, Collections.emptyMap(), Collections.emptyMap()); - MetricsRecorder rec = new MetricsRecorder(Arrays.asList(metricAccessorA, metricAccessorB), commonTagsManager, metricsManager); + MetricsRecorder rec = new MetricsRecorder(Arrays.asList(metricAccessorA, metricAccessorB), commonTagsManager, metricsManager, tagValueGuard); rec.execute(executionContext); @@ -110,9 +115,8 @@ void commonTagsIncluded() { VariableAccessor variableAccess = Mockito.mock(VariableAccessor.class); when(variableAccess.get(any())).thenReturn(100L); - MetricAccessor metricAccessor = new MetricAccessor("my_metric", variableAccess, Collections.emptyMap(), Collections - .emptyMap()); - MetricsRecorder rec = new MetricsRecorder(Collections.singletonList(metricAccessor), commonTagsManager, metricsManager); + MetricAccessor metricAccessor = new MetricAccessor("my_metric", variableAccess, Collections.emptyMap(), Collections.emptyMap()); + MetricsRecorder rec = new MetricsRecorder(Collections.singletonList(metricAccessor), commonTagsManager, metricsManager, tagValueGuard); rec.execute(executionContext); @@ -129,9 +133,8 @@ void constantTags() { VariableAccessor variableAccess = Mockito.mock(VariableAccessor.class); when(variableAccess.get(any())).thenReturn(100L); - MetricAccessor metricAccessor = new MetricAccessor("my_metric", variableAccess, Collections.singletonMap("constant", "tag"), Collections - .emptyMap()); - MetricsRecorder rec = new MetricsRecorder(Collections.singletonList(metricAccessor), commonTagsManager, metricsManager); + MetricAccessor metricAccessor = new MetricAccessor("my_metric", variableAccess, Collections.singletonMap("constant", "tag"), Collections.emptyMap()); + MetricsRecorder rec = new MetricsRecorder(Collections.singletonList(metricAccessor), commonTagsManager, metricsManager, tagValueGuard); rec.execute(executionContext); @@ -150,9 +153,8 @@ void dataTagsNotAvailable() { VariableAccessor variableAccess = Mockito.mock(VariableAccessor.class); when(variableAccess.get(any())).thenReturn(100L); - MetricAccessor metricAccessor = new MetricAccessor("my_metric", variableAccess, Collections.emptyMap(), Collections - .singletonMap("data", mockAccessor)); - MetricsRecorder rec = new MetricsRecorder(Collections.singletonList(metricAccessor), commonTagsManager, metricsManager); + MetricAccessor metricAccessor = new MetricAccessor("my_metric", variableAccess, Collections.emptyMap(), Collections.singletonMap("data", mockAccessor)); + MetricsRecorder rec = new MetricsRecorder(Collections.singletonList(metricAccessor), commonTagsManager, metricsManager, tagValueGuard); rec.execute(executionContext); @@ -169,9 +171,8 @@ void dataTags() { VariableAccessor variableAccess = Mockito.mock(VariableAccessor.class); when(variableAccess.get(any())).thenReturn(100L); - MetricAccessor metricAccessor = new MetricAccessor("my_metric", variableAccess, Collections.emptyMap(), Collections - .singletonMap("data", mockAccessor)); - MetricsRecorder rec = new MetricsRecorder(Collections.singletonList(metricAccessor), commonTagsManager, metricsManager); + MetricAccessor metricAccessor = new MetricAccessor("my_metric", variableAccess, Collections.emptyMap(), Collections.singletonMap("data", mockAccessor)); + MetricsRecorder rec = new MetricsRecorder(Collections.singletonList(metricAccessor), commonTagsManager, metricsManager, tagValueGuard); rec.execute(executionContext); @@ -207,20 +208,22 @@ void multipleAccessorsMixedTags() { dataTags2.put("existing2", mockAccessorC); MetricAccessor metricAccessorB = new MetricAccessor("my_metric2", dataB, Collections.singletonMap("cA", "200"), dataTags2); - MetricsRecorder rec = new MetricsRecorder(Arrays.asList(metricAccessorA, metricAccessorB), commonTagsManager, metricsManager); + MetricsRecorder rec = new MetricsRecorder(Arrays.asList(metricAccessorA, metricAccessorB), commonTagsManager, metricsManager, tagValueGuard); rec.execute(executionContext); InOrder inOrder = inOrder(metricsManager); // first recording - TagContext expected1 = Tags.getTagger().emptyBuilder() + TagContext expected1 = Tags.getTagger() + .emptyBuilder() .putLocal(TagKey.create("cA"), TagValue.create("100")) .putLocal(TagKey.create("existing"), TagValue.create("data1")) .build(); inOrder.verify(metricsManager) .tryRecordingMeasurement(eq("my_metric1"), eq((Number) 100.0d), eq(expected1)); // second recording - TagContext expected2 = Tags.getTagger().emptyBuilder() + TagContext expected2 = Tags.getTagger() + .emptyBuilder() .putLocal(TagKey.create("cA"), TagValue.create("200")) .putLocal(TagKey.create("existing1"), TagValue.create("12")) .putLocal(TagKey.create("existing2"), TagValue.create("false")) @@ -239,9 +242,8 @@ void dataOverwritesConstant() { VariableAccessor variableAccess = Mockito.mock(VariableAccessor.class); when(variableAccess.get(any())).thenReturn(100L); - MetricAccessor metricAccessor = new MetricAccessor("my_metric", variableAccess, Collections.singletonMap("data", "constant"), Collections - .singletonMap("data", mockAccessor)); - MetricsRecorder rec = new MetricsRecorder(Collections.singletonList(metricAccessor), commonTagsManager, metricsManager); + MetricAccessor metricAccessor = new MetricAccessor("my_metric", variableAccess, Collections.singletonMap("data", "constant"), Collections.singletonMap("data", mockAccessor)); + MetricsRecorder rec = new MetricsRecorder(Collections.singletonList(metricAccessor), commonTagsManager, metricsManager, tagValueGuard); rec.execute(executionContext); diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/logging/logback/InternalProcessingAppenderTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/logging/logback/InternalProcessingAppenderTest.java index 11cb8e6c65..7fa28dc987 100644 --- a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/logging/logback/InternalProcessingAppenderTest.java +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/logging/logback/InternalProcessingAppenderTest.java @@ -11,7 +11,7 @@ import org.slf4j.LoggerFactory; import rocks.inspectit.ocelot.core.instrumentation.InstrumentationManager; import rocks.inspectit.ocelot.core.instrumentation.config.event.InstrumentationConfigurationChangedEvent; -import rocks.inspectit.ocelot.core.selfmonitoring.LogMetricsRecorderTest; +import rocks.inspectit.ocelot.core.selfmonitoring.logs.LogMetricsRecorderTest; import static org.mockito.Mockito.*; diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/metrics/MeasureTagValueGuardIntTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/metrics/MeasureTagValueGuardIntTest.java new file mode 100644 index 0000000000..99f5167390 --- /dev/null +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/metrics/MeasureTagValueGuardIntTest.java @@ -0,0 +1,113 @@ +package rocks.inspectit.ocelot.core.metrics; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.annotation.DirtiesContext; +import rocks.inspectit.ocelot.config.model.exporters.ExporterEnabledState; +import rocks.inspectit.ocelot.config.model.exporters.TransportProtocol; +import rocks.inspectit.ocelot.config.model.metrics.MetricsSettings; +import rocks.inspectit.ocelot.config.model.metrics.definition.MetricDefinitionSettings; +import rocks.inspectit.ocelot.config.model.metrics.definition.MetricDefinitionSettings.MeasureType; +import rocks.inspectit.ocelot.config.model.metrics.definition.ViewDefinitionSettings; +import rocks.inspectit.ocelot.config.model.metrics.definition.ViewDefinitionSettings.Aggregation; +import rocks.inspectit.ocelot.core.config.InspectitEnvironment; +import rocks.inspectit.ocelot.core.exporter.ExporterServiceIntegrationTestBase; +import rocks.inspectit.ocelot.core.instrumentation.config.model.propagation.PropagationMetaData; +import rocks.inspectit.ocelot.core.instrumentation.context.InspectitContextImpl; +import rocks.inspectit.ocelot.core.instrumentation.hook.VariableAccessor; +import rocks.inspectit.ocelot.core.instrumentation.hook.actions.IHookAction.ExecutionContext; +import rocks.inspectit.ocelot.core.instrumentation.hook.actions.MetricsRecorder; +import rocks.inspectit.ocelot.core.instrumentation.hook.actions.model.MetricAccessor; +import rocks.inspectit.ocelot.core.metrics.percentiles.PercentileViewManager; +import rocks.inspectit.ocelot.core.tags.CommonTagsManager; + +import java.time.Duration; +import java.util.*; + +/** + * Integration Test Class for {@link MeasureTagValueGuard} + */ +@DirtiesContext +public class MeasureTagValueGuardIntTest extends ExporterServiceIntegrationTestBase { + + @Autowired + InspectitEnvironment env; + @Autowired + CommonTagsManager commonTagsManager; + @Autowired + MeasuresAndViewsManager metricsManager; + @Autowired + MeasureTagValueGuard tagValueGuard; + @Autowired + PercentileViewManager percentileViewManager; + + static final String MEASURE_NAME = "my-counter"; + static final int VALUE = 42; + static final String TAG_KEY = "test-tag-key"; + static final String TAG_VALUE_1 = "test-tag-value-1"; + static final String TAG_VALUE_2 = "test-tag-value-2"; + static final String OVERFLOW = "overflow"; + + private ExecutionContext createExecutionContext() { + InspectitContextImpl ctx = InspectitContextImpl.createFromCurrent(new HashMap<>(), PropagationMetaData.builder().build(), false); + return new ExecutionContext(null, this, "return", null, null, + ctx, null); + } + + /** + * Update properties for OpenTelemetry-Collector & Tag-Guard + * Create metric-definition for MEASURE_NAME + */ + @BeforeEach + void updateProperties() { + ViewDefinitionSettings viewDefinition = new ViewDefinitionSettings(); + viewDefinition.setAggregation(Aggregation.SUM); + viewDefinition.setTags(Collections.singletonMap(TAG_KEY, true)); + + MetricDefinitionSettings metricDefinition = new MetricDefinitionSettings(); + metricDefinition.setEnabled(true); + metricDefinition.setUnit("1"); + metricDefinition.setType(MeasureType.LONG); + metricDefinition.setViews(Collections.singletonMap(MEASURE_NAME, viewDefinition)); + + MetricsSettings metricsSettings = new MetricsSettings(); + metricsSettings.setDefinitions(Collections.singletonMap(MEASURE_NAME, metricDefinition)); + + updateProperties(mps -> { + mps.setProperty("inspectit.exporters.metrics.otlp.endpoint", getEndpoint(COLLECTOR_OTLP_GRPC_PORT)); + mps.setProperty("inspectit.exporters.metrics.otlp.export-interval", "500ms"); + mps.setProperty("inspectit.exporters.metrics.otlp.enabled", ExporterEnabledState.ENABLED); + mps.setProperty("inspectit.exporters.metrics.otlp.protocol", TransportProtocol.GRPC); + mps.setProperty("inspectit.metrics.tag-guard.enabled", true); + mps.setProperty("inspectit.metrics.tag-guard.max-values-per-tag", 1); + mps.setProperty("inspectit.metrics.tag-guard.schedule-delay", Duration.ofMillis(500)); + mps.setProperty("inspectit.metrics.tag-guard.overflow-replacement", OVERFLOW); + mps.setProperty("inspectit.metrics.definitions." + MEASURE_NAME, metricDefinition); + }); + } + + @Test + void verifyTagValueOverflowReplacement() { + VariableAccessor variableAccessor = (context) -> VALUE; + Map dataTags = new HashMap<>(); + dataTags.put(TAG_KEY, (context) -> TAG_VALUE_1); + MetricAccessor metricAccessor = new MetricAccessor(MEASURE_NAME, variableAccessor, new HashMap<>(), dataTags); + List metrics = new LinkedList<>(); + metrics.add(metricAccessor); + + MetricsRecorder metricsRecorder = new MetricsRecorder(metrics, commonTagsManager, metricsManager, tagValueGuard); + ExecutionContext executionContext = createExecutionContext(); + + metricsRecorder.execute(executionContext); + awaitMetricsExported(MEASURE_NAME, VALUE, TAG_KEY, TAG_VALUE_1); + + // for some reason, the ScheduledExecutorService is not working inside tests + tagValueGuard.blockTagValuesTask.run(); + + dataTags.put(TAG_KEY, (context) -> TAG_VALUE_2); + metricsRecorder.execute(executionContext); + // tag should have been replaced, due to overflow + awaitMetricsExported(MEASURE_NAME, VALUE, TAG_KEY, OVERFLOW); + } +} diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/metrics/MeasureTagValueGuardTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/metrics/MeasureTagValueGuardTest.java new file mode 100644 index 0000000000..d26b4b00c9 --- /dev/null +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/metrics/MeasureTagValueGuardTest.java @@ -0,0 +1,302 @@ +package rocks.inspectit.ocelot.core.metrics; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Maps; +import io.opencensus.tags.TagContext; +import io.opencensus.tags.TagKey; +import io.opencensus.tags.TagValue; +import io.opencensus.tags.Tags; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Answers; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import rocks.inspectit.ocelot.config.model.metrics.TagGuardSettings; +import rocks.inspectit.ocelot.config.model.metrics.definition.MetricDefinitionSettings; +import rocks.inspectit.ocelot.core.config.InspectitEnvironment; +import rocks.inspectit.ocelot.core.instrumentation.config.model.propagation.PropagationMetaData; +import rocks.inspectit.ocelot.core.instrumentation.context.InspectitContextImpl; +import rocks.inspectit.ocelot.core.instrumentation.hook.VariableAccessor; +import rocks.inspectit.ocelot.core.instrumentation.hook.actions.IHookAction.ExecutionContext; + +import rocks.inspectit.ocelot.core.instrumentation.hook.actions.model.MetricAccessor; +import rocks.inspectit.ocelot.core.selfmonitoring.AgentHealthManager; +import rocks.inspectit.ocelot.core.tags.CommonTagsManager; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.ScheduledExecutorService; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class MeasureTagValueGuardTest { + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private InspectitEnvironment environment; + @Mock + private CommonTagsManager commonTagsManager; + @Mock + private AgentHealthManager agentHealthManager; + @Mock + private ScheduledExecutorService executor; + @Mock + MeasureTagValueGuard.PersistedTagsReaderWriter readerWriter; + @InjectMocks + private MeasureTagValueGuard guard = new MeasureTagValueGuard(); + + private ExecutionContext context; + private final static int defaultMaxValuePerTag = 42; + private final static String OVERFLOW = "overflow"; + + /** + * Helper method to configure tag value limits as well as metrics settings before testing + * @param maxValuesPerTagByMeasure Map with measures and their tag value limits + * @param settings MetricDefinitionSettings, which should be applied for "measure" + */ + private void setupTagGuard(Map maxValuesPerTagByMeasure, MetricDefinitionSettings settings) { + TagGuardSettings tagGuardSettings = new TagGuardSettings(); + tagGuardSettings.setEnabled(true); + tagGuardSettings.setScheduleDelay(Duration.ofSeconds(1)); + tagGuardSettings.setOverflowReplacement(OVERFLOW); + tagGuardSettings.setMaxValuesPerTag(defaultMaxValuePerTag); + if (maxValuesPerTagByMeasure != null) + tagGuardSettings.setMaxValuesPerTagByMeasure(maxValuesPerTagByMeasure); + + when(environment.getCurrentConfig().getMetrics().getTagGuard()).thenReturn(tagGuardSettings); + + if (settings != null) + when(environment.getCurrentConfig() + .getMetrics() + .getDefinitions() + .get("measure")).thenReturn(settings); + } + + @Nested + public class ReaderWrite { + + private String generateTempFilePath() { + try { + Path tempFile = Files.createTempFile("inspectit", ""); + System.out.println(tempFile); + Files.delete(tempFile); + tempFile.toFile().deleteOnExit(); + return tempFile.toString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private Map>> createTagValues() { + Set tagValue = new HashSet<>(); + tagValue.add("value1"); + tagValue.add("value2"); + tagValue.add("value3"); + + Map> tagKeys2Values = Maps.newHashMap(); + tagKeys2Values.put("tagKey_1", tagValue); + + Map>> measure2TagKeys = Maps.newHashMap(); + measure2TagKeys.put("measure_1", tagKeys2Values); + + return measure2TagKeys; + } + + @Test + public void testReadWriteTagsFromDisk() { + String tempFileName = generateTempFilePath(); + + MeasureTagValueGuard.PersistedTagsReaderWriter readerWriter = new MeasureTagValueGuard.PersistedTagsReaderWriter(tempFileName, new ObjectMapper()); + Map>> tagValues = createTagValues(); + readerWriter.write(tagValues); + Map>> loaded = readerWriter.read(); + + assertThat(loaded).flatExtracting("measure_1") + .flatExtracting("tagKey_1") + .containsExactlyInAnyOrder("value1", "value2", "value3"); + + } + } + + @Nested + public class getMaxValuesPerTag { + + @Test + public void getMaxValuesPerTagByDefault() { + setupTagGuard(null, null); + + assertThat(guard.getMaxValuesPerTag("measure", environment.getCurrentConfig())).isEqualTo(defaultMaxValuePerTag); + } + + @Test + public void getMaxValuesPerTagByMeasure() { + Map maxValuesPerTagByMeasure = new HashMap<>(); + maxValuesPerTagByMeasure.put("measure", 43); + setupTagGuard(maxValuesPerTagByMeasure, null); + + assertThat(guard.getMaxValuesPerTag("measure", environment.getCurrentConfig())).isEqualTo(43); + assertThat(guard.getMaxValuesPerTag("measure1", environment.getCurrentConfig())).isEqualTo(defaultMaxValuePerTag); + } + + @Test + public void getMaxValuesPerTagByMetricDefinitionSettings() { + MetricDefinitionSettings settings = new MetricDefinitionSettings(); + settings.setMaxValuesPerTag(43); + setupTagGuard(null, settings); + + assertThat(guard.getMaxValuesPerTag("measure", environment.getCurrentConfig())).isEqualTo(43); + assertThat(guard.getMaxValuesPerTag("measure1", environment.getCurrentConfig())).isEqualTo(defaultMaxValuePerTag); + } + + @Test + public void getMaxValuesPerTagWhenAllSettingsAreSet() { + Map maxValuesPerTagByMeasure = new HashMap<>(); + maxValuesPerTagByMeasure.put("measure", 43); + maxValuesPerTagByMeasure.put("measure2", 48); + + MetricDefinitionSettings settings = new MetricDefinitionSettings(); + settings.setMaxValuesPerTag(44); + + setupTagGuard(maxValuesPerTagByMeasure, settings); + + assertThat(guard.getMaxValuesPerTag("measure", environment.getCurrentConfig())).isEqualTo(44); + assertThat(guard.getMaxValuesPerTag("measure2", environment.getCurrentConfig())).isEqualTo(48); + assertThat(guard.getMaxValuesPerTag("measure3", environment.getCurrentConfig())).isEqualTo(defaultMaxValuePerTag); + } + } + + @Nested + public class getTagContext { + + static final String TAG_KEY = "test-tag-key"; + static final String TAG_VALUE_1 = "test-tag-value-1"; + static final String TAG_VALUE_2 = "test-tag-value-2"; + private MetricAccessor metricAccessor1; + private MetricAccessor metricAccessor2; + + private ExecutionContext createExecutionContext() { + InspectitContextImpl ctx = InspectitContextImpl.createFromCurrent(new HashMap<>(), PropagationMetaData.builder().build(), false); + return new ExecutionContext(null, this, "return", null, null, + ctx, null); + } + + @BeforeEach + void setUp() { + VariableAccessor metricValueAccess = Mockito.mock(VariableAccessor.class); + metricAccessor1 = new MetricAccessor("measure", metricValueAccess, Collections.emptyMap(), + Collections.singletonMap(TAG_KEY, (context) -> TAG_VALUE_1)); + metricAccessor2 = new MetricAccessor("measure", metricValueAccess, Collections.emptyMap(), + Collections.singletonMap(TAG_KEY, (context) -> TAG_VALUE_2)); + + context = createExecutionContext(); + + when(readerWriter.read()).thenReturn(new HashMap<>()); + when(commonTagsManager.getCommonTagKeys()).thenReturn(Collections.emptyList()); + } + + @Test + void verifyOverflow() { + Map maxValuesPerTagByMeasure = new HashMap<>(); + maxValuesPerTagByMeasure.put("measure", 1); + setupTagGuard(maxValuesPerTagByMeasure, null); + + TagContext expectedTagContext = Tags.getTagger() + .emptyBuilder() + .putLocal(TagKey.create(TAG_KEY), TagValue.create(TAG_VALUE_1)) + .build(); + + TagContext expectedOverflow = Tags.getTagger() + .emptyBuilder() + .putLocal(TagKey.create(TAG_KEY), TagValue.create(OVERFLOW)) + .build(); + + // first tag value should be accepted + TagContext tagContext = guard.getTagContext(context, metricAccessor1); + guard.blockTagValuesTask.run(); + // second tag value will exceed the limit + TagContext overflow = guard.getTagContext(context, metricAccessor2); + + assertThat(tagContext.equals(expectedTagContext)).isTrue(); + assertThat(overflow.equals(expectedOverflow)).isTrue(); + } + + @Test + void verifyOverflowResolvedAfterLimitIncrease() { + Map maxValuesPerTagByMeasure = new HashMap<>(); + maxValuesPerTagByMeasure.put("measure", 1); + setupTagGuard(maxValuesPerTagByMeasure, null); + + TagContext expectedTagContext1 = Tags.getTagger() + .emptyBuilder() + .putLocal(TagKey.create(TAG_KEY), TagValue.create(TAG_VALUE_1)) + .build(); + + TagContext expectedTagContext2 = Tags.getTagger() + .emptyBuilder() + .putLocal(TagKey.create(TAG_KEY), TagValue.create(TAG_VALUE_2)) + .build(); + + TagContext expectedOverflow = Tags.getTagger() + .emptyBuilder() + .putLocal(TagKey.create(TAG_KEY), TagValue.create(OVERFLOW)) + .build(); + + // first tag value should be accepted + TagContext tagContext1 = guard.getTagContext(context, metricAccessor1); + guard.blockTagValuesTask.run(); + // second tag value will exceed the limit + TagContext overflow = guard.getTagContext(context, metricAccessor2); + // increase tag limit to resolve overflow + maxValuesPerTagByMeasure.put("measure", 5); + setupTagGuard(maxValuesPerTagByMeasure, null); + guard.blockTagValuesTask.run(); + // second tag value should be accepted + TagContext tagContext2 = guard.getTagContext(context, metricAccessor2); + + assertThat(tagContext1.equals(expectedTagContext1)).isTrue(); + assertThat(overflow.equals(expectedOverflow)).isTrue(); + assertThat(tagContext2.equals(expectedTagContext2)).isTrue(); + } + + @Test + void verifyOverflowNotResolvedAfterLimitIncrease() { + Map maxValuesPerTagByMeasure = new HashMap<>(); + maxValuesPerTagByMeasure.put("measure", 1); + setupTagGuard(maxValuesPerTagByMeasure, null); + + TagContext expectedTagContext1 = Tags.getTagger() + .emptyBuilder() + .putLocal(TagKey.create(TAG_KEY), TagValue.create(TAG_VALUE_1)) + .build(); + + TagContext expectedOverflow = Tags.getTagger() + .emptyBuilder() + .putLocal(TagKey.create(TAG_KEY), TagValue.create(OVERFLOW)) + .build(); + + // first tag value should be accepted + TagContext tagContext1 = guard.getTagContext(context, metricAccessor1); + guard.blockTagValuesTask.run(); + // second tag value will exceed the limit + TagContext overflow1 = guard.getTagContext(context, metricAccessor2); + // increase tag limit to resolve overflow + maxValuesPerTagByMeasure.put("measure", 2); + setupTagGuard(maxValuesPerTagByMeasure, null); + guard.blockTagValuesTask.run(); + // second tag value should be accepted + TagContext overflow2 = guard.getTagContext(context, metricAccessor2); + + assertThat(tagContext1.equals(expectedTagContext1)).isTrue(); + assertThat(overflow1.equals(expectedOverflow)).isTrue(); + assertThat(overflow2.equals(expectedOverflow)).isTrue(); + } + } +} diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthIncidentBufferTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthIncidentBufferTest.java new file mode 100644 index 0000000000..81dc50af44 --- /dev/null +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthIncidentBufferTest.java @@ -0,0 +1,55 @@ +package rocks.inspectit.ocelot.core.selfmonitoring; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Answers; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.ApplicationContext; +import rocks.inspectit.ocelot.commons.models.health.AgentHealth; +import rocks.inspectit.ocelot.commons.models.health.AgentHealthIncident; +import rocks.inspectit.ocelot.core.SpringTestBase; +import rocks.inspectit.ocelot.core.config.InspectitEnvironment; +import rocks.inspectit.ocelot.core.selfmonitoring.event.models.AgentHealthIncidentAddedEvent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +public class AgentHealthIncidentBufferTest { + + @InjectMocks + private AgentHealthIncidentBuffer incidentBuffer; + @Mock + private ApplicationContext ctx; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private InspectitEnvironment env; + + private AgentHealthIncident incident; + + @BeforeEach + void setUp() { + when(env.getCurrentConfig().getSelfMonitoring().getAgentHealth().getIncidentBufferSize()).thenReturn(2); + incident = new AgentHealthIncident("2001-01-01", AgentHealth.WARNING, this.getClass().getCanonicalName(), "Mock message", true); + } + @Test + void verifyBufferSize() { + incidentBuffer.put(incident); + incidentBuffer.put(incident); + incidentBuffer.put(incident); + + verify(ctx, times(3)).publishEvent(any(AgentHealthIncidentAddedEvent.class)); + } + + @Test + void verifyEventPublisher() { + incidentBuffer.put(incident); + incidentBuffer.put(incident); + incidentBuffer.put(incident); + + verify(ctx, times(3)).publishEvent(any(AgentHealthIncidentAddedEvent.class)); + } +} diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthManagerDeadlockGh1597IntTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthManagerDeadlockGh1597IntTest.java index 6395640231..97e705b78a 100644 --- a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthManagerDeadlockGh1597IntTest.java +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthManagerDeadlockGh1597IntTest.java @@ -1,9 +1,7 @@ package rocks.inspectit.ocelot.core.selfmonitoring; import org.awaitility.Awaitility; -import org.checkerframework.checker.units.qual.A; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -11,8 +9,6 @@ import rocks.inspectit.ocelot.core.SpringTestBase; import rocks.inspectit.ocelot.core.logging.logback.LogbackInitializer; -import java.util.Random; -import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -28,7 +24,7 @@ public class AgentHealthManagerDeadlockGh1597IntTest extends SpringTestBase { private AgentHealthManager cut; @Test - void testLogging() throws Exception { + void testLogging() { // This installs InternalProcessingAppender which together with AgentHealthManager caused a deadlock LogbackInitializer.initDefaultLogging(); @@ -53,7 +49,7 @@ void testLogging() throws Exception { Thread invalidationThread = new Thread(() -> { long start = System.currentTimeMillis(); while (System.currentTimeMillis() - start < millisToRun) { - cut.onInvalidationEvent(cut.getClass()); + cut.invalidateIncident(cut.getClass(), "Invalidation due to invalidator event"); } isInvalidationThreadDone.getAndSet(true); }); diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthManagerIntTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthManagerIntTest.java new file mode 100644 index 0000000000..bff61db4c8 --- /dev/null +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthManagerIntTest.java @@ -0,0 +1,107 @@ +package rocks.inspectit.ocelot.core.selfmonitoring; + +import org.junit.jupiter.api.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.TestPropertySource; +import rocks.inspectit.ocelot.commons.models.health.AgentHealth; +import rocks.inspectit.ocelot.core.SpringTestBase; +import rocks.inspectit.ocelot.core.config.propertysources.http.HttpConfigurationPoller; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Integration tests {@link AgentHealthManager} + */ +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_CLASS) +@TestPropertySource(properties = "inspectit.self-monitoring.agent-health.validity-period:60s") +public class AgentHealthManagerIntTest extends SpringTestBase { + + @Autowired + private AgentHealthManager healthManager; + @Autowired + private HttpConfigurationPoller configurationPoller; + + /** + * Period how long a TimeOut AgentHealth is valid (+1s buffer) + */ + private final long validityPeriod = 61000; + + @Nested + class InvalidatableHealth { + + @Test + void verifyAgentHealthUpdating() { + AgentHealth currentManagerHealth = healthManager.getCurrentHealth(); + AgentHealth currentHealth = configurationPoller.getCurrentAgentHealthState().getHealth(); + assertEquals(currentHealth, currentManagerHealth); + assertEquals(currentHealth, AgentHealth.OK); + + healthManager.notifyAgentHealth(AgentHealth.WARNING, this.getClass(), this.getClass().getName(), "Mock message"); + + currentHealth = configurationPoller.getCurrentAgentHealthState().getHealth(); + currentManagerHealth = healthManager.getCurrentHealth(); + assertEquals(currentHealth, currentManagerHealth); + assertEquals(currentHealth, AgentHealth.WARNING); + healthManager.notifyAgentHealth(AgentHealth.ERROR, this.getClass(), this.getClass().getName(), "Mock message"); + + currentHealth = configurationPoller.getCurrentAgentHealthState().getHealth(); + currentManagerHealth = healthManager.getCurrentHealth(); + assertEquals(currentHealth, currentManagerHealth); + assertEquals(currentHealth, AgentHealth.ERROR); + } + + @Test + void verifyAgentHealthInvalidation() throws InterruptedException { + AgentHealth currentHealth = configurationPoller.getCurrentAgentHealthState().getHealth(); + AgentHealth currentManagerHealth = healthManager.getCurrentHealth(); + assertEquals(currentHealth, currentManagerHealth); + assertEquals(currentHealth, AgentHealth.OK); + + healthManager.notifyAgentHealth(AgentHealth.ERROR, this.getClass(), this.getClass().getName(), "Mock message"); + + currentHealth = configurationPoller.getCurrentAgentHealthState().getHealth(); + currentManagerHealth = healthManager.getCurrentHealth(); + assertEquals(currentHealth, currentManagerHealth); + assertEquals(currentHealth, AgentHealth.ERROR); + + healthManager.invalidateIncident(this.getClass(), "Mock invalidation"); + // simulate scheduler + Thread.sleep(validityPeriod); + healthManager.checkHealthAndSchedule(); + + currentHealth = configurationPoller.getCurrentAgentHealthState().getHealth(); + currentManagerHealth = healthManager.getCurrentHealth(); + assertEquals(currentHealth, currentManagerHealth); + assertEquals(currentHealth, AgentHealth.OK); + } + } + + @Nested + class TimeoutHealth { + + @Test + void verifyAgentHealthUpdating() throws InterruptedException { + AgentHealth currentHealth = configurationPoller.getCurrentAgentHealthState().getHealth(); + AgentHealth currentManagerHealth = healthManager.getCurrentHealth(); + assertEquals(currentHealth, currentManagerHealth); + assertEquals(currentHealth, AgentHealth.OK); + + healthManager.notifyAgentHealth(AgentHealth.WARNING, null, this.getClass().getName(), "Mock message"); + + currentHealth = configurationPoller.getCurrentAgentHealthState().getHealth(); + currentManagerHealth = healthManager.getCurrentHealth(); + assertEquals(currentHealth, currentManagerHealth); + assertEquals(currentHealth, AgentHealth.WARNING); + + // simulate scheduler + Thread.sleep(validityPeriod); + healthManager.checkHealthAndSchedule(); + + currentHealth = configurationPoller.getCurrentAgentHealthState().getHealth(); + currentManagerHealth = healthManager.getCurrentHealth(); + assertEquals(currentHealth, currentManagerHealth); + assertEquals(currentHealth, AgentHealth.OK); + } + } +} diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthManagerTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthManagerTest.java index 1ade277b13..b8436574c8 100644 --- a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthManagerTest.java +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/AgentHealthManagerTest.java @@ -1,30 +1,20 @@ package rocks.inspectit.ocelot.core.selfmonitoring; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.Logger; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.classic.spi.LoggingEvent; import org.junit.jupiter.api.*; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; +import org.mockito.Answers; +import org.mockito.InjectMocks; +import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; import rocks.inspectit.ocelot.commons.models.health.AgentHealth; -import rocks.inspectit.ocelot.config.model.InspectitConfig; -import rocks.inspectit.ocelot.config.model.selfmonitoring.AgentHealthSettings; -import rocks.inspectit.ocelot.config.model.selfmonitoring.SelfMonitoringSettings; +import rocks.inspectit.ocelot.commons.models.health.AgentHealthIncident; import rocks.inspectit.ocelot.core.config.InspectitEnvironment; -import rocks.inspectit.ocelot.core.instrumentation.config.event.InstrumentationConfigurationChangedEvent; -import rocks.inspectit.ocelot.core.selfmonitoring.event.AgentHealthChangedEvent; +import rocks.inspectit.ocelot.core.selfmonitoring.event.models.AgentHealthChangedEvent; import java.time.Duration; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.*; /** @@ -33,126 +23,117 @@ @ExtendWith(MockitoExtension.class) public class AgentHealthManagerTest { - private static final long VALIDITY_PERIOD_MILLIS = 500; - - private static InspectitConfig config; - - private ScheduledExecutorService executorService; - + @InjectMocks + private AgentHealthManager healthManager; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) private InspectitEnvironment environment; + @Mock + private ScheduledExecutorService executor; + @Mock + private ApplicationContext ctx; + @Mock + private AgentHealthIncidentBuffer incidentBuffer; - private ApplicationContext context; - - private AgentHealthManager healthManager; + @Nested + class InvalidatableHealth { - @BeforeAll - static void createInspectitConfig() { - config = new InspectitConfig(); - AgentHealthSettings agentHealth = new AgentHealthSettings(); - agentHealth.setValidityPeriod(Duration.ofMillis(VALIDITY_PERIOD_MILLIS)); - SelfMonitoringSettings selfMonitoring = new SelfMonitoringSettings(); - selfMonitoring.setAgentHealth(agentHealth); - config.setSelfMonitoring(selfMonitoring); - } + @Test + void verifyAgentHealthChangedEvent() { + healthManager.notifyAgentHealth(AgentHealth.WARNING, this.getClass(), this.getClass().getName(), "Mock message"); - @BeforeEach - void setupStatusManager() { - executorService = new ScheduledThreadPoolExecutor(1); + verify(ctx).publishEvent(any(AgentHealthChangedEvent.class)); + } - environment = mock(InspectitEnvironment.class); - when(environment.getCurrentConfig()).thenReturn(config); + @Test + void verifyAgentHealthIncidentAddedEvent() { + healthManager.notifyAgentHealth(AgentHealth.WARNING, this.getClass(), this.getClass().getName(), "Mock message"); - context = mock(ApplicationContext.class); + verify(incidentBuffer).put(any(AgentHealthIncident.class)); + } - healthManager = new AgentHealthManager(context, executorService, environment, mock(SelfMonitoringService.class)); - healthManager.startHealthCheckScheduler(); - } + @Test + void verifyNoAgentHealthIncidentAddedEvent() { + healthManager.notifyAgentHealth(AgentHealth.OK, this.getClass(), this.getClass().getName(), "Mock message"); - @AfterEach - void shutdownExecutorService() { - executorService.shutdown(); - } + verifyNoInteractions(incidentBuffer); + } - private ILoggingEvent createLoggingEvent(Level level) { - return new LoggingEvent("com.dummy.Method", (Logger) LoggerFactory.getLogger(AgentHealthManagerTest.class), level, "Dummy Info", new Throwable(), new String[]{}); - } + @Test + void verifyInvalidateAgentHealthIncident() { + healthManager.notifyAgentHealth(AgentHealth.ERROR, this.getClass(), this.getClass().getName(), "Mock message"); + healthManager.invalidateIncident(this.getClass(), "Mock invalidation"); - private void verifyExactlyOneEventWasPublished(AgentHealth status) { - ArgumentCaptor statusCaptor = ArgumentCaptor.forClass(AgentHealthChangedEvent.class); - verify(context).publishEvent(statusCaptor.capture()); - assertThat(statusCaptor.getValue().getNewHealth()).isEqualTo(status); - verifyNoMoreInteractions(context); + verify(ctx, times(2)).publishEvent(any(AgentHealthChangedEvent.class)); + verify(incidentBuffer, times(2)).put(any(AgentHealthIncident.class)); + } } - @Nested - class OnLogEvent { + class TimeoutHealth { - @Test - void logInstrumentationEvents() { - assertThat(healthManager.getCurrentHealth()).withFailMessage("Initial status shall be OK") - .isEqualTo(AgentHealth.OK); + @BeforeEach + void setUpValidityPeriod() { + when(environment.getCurrentConfig().getSelfMonitoring().getAgentHealth().getValidityPeriod()) + .thenReturn(Duration.ofSeconds(5)); + } - healthManager.onLoggingEvent(createLoggingEvent(Level.INFO), InstrumentationConfigurationChangedEvent.class); - healthManager.onLoggingEvent(createLoggingEvent(Level.DEBUG), InstrumentationConfigurationChangedEvent.class); - assertThat(healthManager.getCurrentHealth()).withFailMessage("INFO and DEBUG messages shall not change the status") - .isEqualTo(AgentHealth.OK); + @Test + void verifyAgentHealthChangedEvent() { + healthManager.notifyAgentHealth(AgentHealth.WARNING, null, this.getClass().getName(), "Mock message"); - verifyNoInteractions(context); + verify(ctx).publishEvent(any(AgentHealthChangedEvent.class)); + } - healthManager.onLoggingEvent(createLoggingEvent(Level.WARN), InstrumentationConfigurationChangedEvent.class); - assertThat(healthManager.getCurrentHealth()).withFailMessage("Status after WARN message shall be WARNING") - .isEqualTo(AgentHealth.WARNING); - verifyExactlyOneEventWasPublished(AgentHealth.WARNING); + @Test + void verifyAgentHealthIncidentAddedEvent() { + healthManager.notifyAgentHealth(AgentHealth.WARNING, null, this.getClass().getName(), "Mock message"); - clearInvocations(context); + verify(incidentBuffer).put(any(AgentHealthIncident.class)); + } - healthManager.onLoggingEvent(createLoggingEvent(Level.ERROR), InstrumentationConfigurationChangedEvent.class); - assertThat(healthManager.getCurrentHealth()).withFailMessage("Status after ERROR message shall be ERROR") - .isEqualTo(AgentHealth.ERROR); - verifyExactlyOneEventWasPublished(AgentHealth.ERROR); + @Test + void verifyNoAgentHealthIncidentAddedEvent() { + healthManager.notifyAgentHealth(AgentHealth.OK, null, this.getClass().getName(), "Mock message"); - clearInvocations(context); + verifyNoInteractions(incidentBuffer); + } - healthManager.onLoggingEvent(createLoggingEvent(Level.INFO), InstrumentationConfigurationChangedEvent.class); - assertThat(healthManager.getCurrentHealth()).withFailMessage("INFO messages shall not change the status") - .isEqualTo(AgentHealth.ERROR); - verifyNoMoreInteractions(context); + @Test + void verifyAgentHealthTimeout() throws InterruptedException { + when(environment.getCurrentConfig().getSelfMonitoring().getAgentHealth().getValidityPeriod()) + .thenReturn(Duration.ofSeconds(5)); - clearInvocations(context); + healthManager.notifyAgentHealth(AgentHealth.ERROR, null, this.getClass().getName(), "Mock message"); + // Wait 6s for time out (= 5s validityPeriod + 1s buffer) + Thread.sleep(6000); - healthManager.onInvalidationEvent(new InstrumentationConfigurationChangedEvent(this, null, null)); - assertThat(healthManager.getCurrentHealth()).withFailMessage("When new instrumentation was triggered, status shall be OK") - .isEqualTo(AgentHealth.OK); - verifyExactlyOneEventWasPublished(AgentHealth.OK); + assertEquals(healthManager.getCurrentHealth(), AgentHealth.OK); } @Test - void logGeneralEvents() { - assertThat(healthManager.getCurrentHealth()).withFailMessage("Initial status shall be OK") - .isEqualTo(AgentHealth.OK); + void verifyCheckAgentHealth() throws InterruptedException { + when(environment.getCurrentConfig().getSelfMonitoring().getAgentHealth().getMinHealthCheckDelay()) + .thenReturn(Duration.ofSeconds(1)); - healthManager.onLoggingEvent(createLoggingEvent(Level.INFO), null); - healthManager.onLoggingEvent(createLoggingEvent(Level.DEBUG), null); - assertThat(healthManager.getCurrentHealth()).withFailMessage("INFO and DEBUG messages shall not change the status") - .isEqualTo(AgentHealth.OK); + healthManager.notifyAgentHealth(AgentHealth.ERROR, null, this.getClass().getName(), "Mock message"); + // Wait 6s for time out (= 5s validityPeriod + 1s buffer) + Thread.sleep(6000); - verifyNoInteractions(context); + healthManager.checkHealthAndSchedule(); - healthManager.onLoggingEvent(createLoggingEvent(Level.ERROR), null); - assertThat(healthManager.getCurrentHealth()).withFailMessage("Status after ERROR message shall be ERROR") - .isEqualTo(AgentHealth.ERROR); - verifyExactlyOneEventWasPublished(AgentHealth.ERROR); + verify(ctx, times(2)).publishEvent(any(AgentHealthChangedEvent.class)); + verify(incidentBuffer, times(2)).put(any(AgentHealthIncident.class)); + } - clearInvocations(context); + @Test + void verifyCheckAgentHealthTooEarly() { + when(environment.getCurrentConfig().getSelfMonitoring().getAgentHealth().getMinHealthCheckDelay()) + .thenReturn(Duration.ofSeconds(1)); - await().atMost(VALIDITY_PERIOD_MILLIS * 2, TimeUnit.MILLISECONDS) - .untilAsserted(() -> assertThat(healthManager.getCurrentHealth()).withFailMessage("ERROR status should jump back to OK after timeout") - .isEqualTo(AgentHealth.OK)); + healthManager.notifyAgentHealth(AgentHealth.ERROR, null, this.getClass().getName(), "Mock message"); + healthManager.checkHealthAndSchedule(); - await().atMost(VALIDITY_PERIOD_MILLIS * 2, TimeUnit.MILLISECONDS) - .untilAsserted(() -> verifyExactlyOneEventWasPublished(AgentHealth.OK)); + verify(ctx, times(1)).publishEvent(any(AgentHealthChangedEvent.class)); + verify(incidentBuffer, times(2)).put(any(AgentHealthIncident.class)); } - } - } diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/LogWritingHealthEventListenerTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/LogWritingHealthEventListenerTest.java new file mode 100644 index 0000000000..727d10ebff --- /dev/null +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/LogWritingHealthEventListenerTest.java @@ -0,0 +1,44 @@ +package rocks.inspectit.ocelot.core.selfmonitoring.event.listener; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.junit.jupiter.MockitoExtension; +import org.slf4j.LoggerFactory; +import rocks.inspectit.ocelot.commons.models.health.AgentHealth; +import rocks.inspectit.ocelot.core.selfmonitoring.event.models.AgentHealthChangedEvent; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.verify; + +public class LogWritingHealthEventListenerTest { + + private final LogWritingHealthEventListener logWritingHealthEventListener = new LogWritingHealthEventListener(); + + private ListAppender listAppender; + + @BeforeEach + void setUp() { + listAppender = new ListAppender<>(); + listAppender.start(); + + Logger logger = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); + logger.addAppender(listAppender); + } + + @Test + void verifyAgentHealthLogging() { + String eventMessage = "Mock message"; + AgentHealthChangedEvent event = new AgentHealthChangedEvent(this, AgentHealth.OK, AgentHealth.WARNING, eventMessage); + String expectedFullMessage = String.format("The agent status changed from %s to %s. Reason: %s", + AgentHealth.OK, AgentHealth.WARNING, eventMessage); + + logWritingHealthEventListener.onAgentHealthEvent(event); + + assertTrue(listAppender.list.stream().anyMatch(logEvent -> logEvent.getFormattedMessage().contains(expectedFullMessage))); + } +} diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/MetricWritingHealthEventListenerTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/MetricWritingHealthEventListenerTest.java new file mode 100644 index 0000000000..1a7cafa011 --- /dev/null +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/MetricWritingHealthEventListenerTest.java @@ -0,0 +1,48 @@ +package rocks.inspectit.ocelot.core.selfmonitoring.event.listener; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import rocks.inspectit.ocelot.commons.models.health.AgentHealth; +import rocks.inspectit.ocelot.core.selfmonitoring.SelfMonitoringService; +import rocks.inspectit.ocelot.core.selfmonitoring.event.models.AgentHealthChangedEvent; + +import java.util.HashMap; + +import static org.mockito.Mockito.verify; + +@ExtendWith(MockitoExtension.class) +public class MetricWritingHealthEventListenerTest { + + private static final String INITIAL_METRIC_MESSAGE = "Initial health metric sent"; + + @InjectMocks + private MetricWritingHealthEventListener metricWritingHealthEventListener; + + @Mock + private SelfMonitoringService selfMonitoringService; + + @Test + void sendInitialHealthMetric() { + HashMap tags = new HashMap<>(); + tags.put("message", INITIAL_METRIC_MESSAGE); + + metricWritingHealthEventListener.sendInitialHealthMetric(); + + verify(selfMonitoringService).recordMeasurement("health", AgentHealth.OK.ordinal(), tags); + } + + @Test + void recordNewHealthMeasurement() { + AgentHealthChangedEvent event = new AgentHealthChangedEvent(this, AgentHealth.OK, AgentHealth.WARNING, "Mock Message"); + HashMap tags = new HashMap<>(); + tags.put("message", event.getMessage()); + tags.put("source", event.getSource().getClass().getName()); + + metricWritingHealthEventListener.onAgentHealthEvent(event); + + verify(selfMonitoringService).recordMeasurement("health", event.getNewHealth().ordinal(), tags); + } +} diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/PollerWritingHealthEventListenerTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/PollerWritingHealthEventListenerTest.java new file mode 100644 index 0000000000..da70c229d0 --- /dev/null +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/event/listener/PollerWritingHealthEventListenerTest.java @@ -0,0 +1,60 @@ +package rocks.inspectit.ocelot.core.selfmonitoring.event.listener; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import rocks.inspectit.ocelot.commons.models.health.AgentHealth; +import rocks.inspectit.ocelot.commons.models.health.AgentHealthIncident; +import rocks.inspectit.ocelot.commons.models.health.AgentHealthState; +import rocks.inspectit.ocelot.core.config.propertysources.http.HttpConfigurationPoller; +import rocks.inspectit.ocelot.core.selfmonitoring.AgentHealthManager; +import rocks.inspectit.ocelot.core.selfmonitoring.event.models.AgentHealthChangedEvent; +import rocks.inspectit.ocelot.core.selfmonitoring.event.models.AgentHealthIncidentAddedEvent; + +import java.util.Collections; +import java.util.List; + +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class PollerWritingHealthEventListenerTest { + + @InjectMocks + private PollerWritingHealthEventListener pollerWritingHealthEventListener; + @Mock + private HttpConfigurationPoller httpConfigurationPoller; + @Mock + private AgentHealthManager agentHealthManager; + + private List lastIncidents; + + @BeforeEach + void setUpIncidents() { + AgentHealthIncident incident = new AgentHealthIncident("2000-01-01", AgentHealth.WARNING, this.getClass().getCanonicalName(), "Mock message", true); + lastIncidents = Collections.singletonList(incident); + } + + @Test + void verifyAgentHealthUpdateOnChangedHealth() { + when(agentHealthManager.getIncidentHistory()).thenReturn(lastIncidents); + AgentHealthChangedEvent event = new AgentHealthChangedEvent(this, AgentHealth.OK, AgentHealth.WARNING, "Mock message"); + + pollerWritingHealthEventListener.onAgentHealthEvent(event); + AgentHealthState healthState = new AgentHealthState(AgentHealth.WARNING, this.toString(), "Mock message", lastIncidents); + + verify(httpConfigurationPoller).updateAgentHealthState(healthState); + } + @Test + void verifyAgentHealthUpdateOnAddedIncident() { + AgentHealthIncidentAddedEvent event = new AgentHealthIncidentAddedEvent(this, lastIncidents); + + pollerWritingHealthEventListener.onAgentHealthIncidentEvent(event); + AgentHealthState healthState = new AgentHealthState(AgentHealth.WARNING, this.getClass().getCanonicalName(), "Mock message", lastIncidents); + + verify(httpConfigurationPoller).updateAgentHealthState(healthState); + } +} diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogHealthMonitorTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogHealthMonitorTest.java new file mode 100644 index 0000000000..cccfd3dae0 --- /dev/null +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogHealthMonitorTest.java @@ -0,0 +1,101 @@ +package rocks.inspectit.ocelot.core.selfmonitoring.logs; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.LoggingEvent; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.slf4j.LoggerFactory; +import rocks.inspectit.ocelot.commons.models.health.AgentHealth; +import rocks.inspectit.ocelot.core.config.PropertySourcesReloadEvent; +import rocks.inspectit.ocelot.core.selfmonitoring.AgentHealthManager; + +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +@ExtendWith(MockitoExtension.class) +public class LogHealthMonitorTest { + + @InjectMocks + private LogHealthMonitor healthMonitor; + @Mock + private AgentHealthManager healthManager; + + private ILoggingEvent createLoggingEvent(Class loggedClass, Level logLevel) { + return new LoggingEvent("com.dummy.Method", (Logger) LoggerFactory.getLogger(loggedClass), logLevel, "Dummy Info", new Throwable(), new String[]{}); + } + + @Test + void ignoreLogsFromAgentHealthManagerClass() { + ILoggingEvent loggingEvent = createLoggingEvent(AgentHealthManager.class, Level.INFO); + + healthMonitor.onLoggingEvent(loggingEvent, null); + + verifyNoMoreInteractions(healthManager); + } + + @Test + void verifyNotifyAgentHealthOnInfo() { + Class loggerClass = LogHealthMonitor.class; + ILoggingEvent loggingEvent = createLoggingEvent(loggerClass, Level.INFO); + Class invalidatorMock = PropertySourcesReloadEvent.class; + AgentHealth eventHealth = AgentHealth.OK; + + healthMonitor.onLoggingEvent(loggingEvent, invalidatorMock); + + verify(healthManager).notifyAgentHealth(eventHealth, invalidatorMock, loggerClass.getCanonicalName(), loggingEvent.getFormattedMessage()); + } + + @Test + void verifyNotifyAgentHealthOnWarn() { + Class loggerClass = LogHealthMonitor.class; + ILoggingEvent loggingEvent = createLoggingEvent(loggerClass, Level.WARN); + Class invalidatorMock = PropertySourcesReloadEvent.class; + AgentHealth eventHealth = AgentHealth.WARNING; + + healthMonitor.onLoggingEvent(loggingEvent, invalidatorMock); + + verify(healthManager).notifyAgentHealth(eventHealth, invalidatorMock, loggerClass.getCanonicalName(), loggingEvent.getFormattedMessage()); + } + + @Test + void verifyNotifyAgentHealthOnError() { + Class loggerClass = LogHealthMonitor.class; + ILoggingEvent loggingEvent = createLoggingEvent(loggerClass, Level.ERROR); + Class invalidatorMock = PropertySourcesReloadEvent.class; + AgentHealth eventHealth = AgentHealth.ERROR; + + healthMonitor.onLoggingEvent(loggingEvent, invalidatorMock); + + verify(healthManager).notifyAgentHealth(eventHealth, invalidatorMock, loggerClass.getCanonicalName(), loggingEvent.getFormattedMessage()); + } + + @Test + void verifyNotifyAgentHealthOnTrace() { + Class loggerClass = LogHealthMonitor.class; + ILoggingEvent loggingEvent = createLoggingEvent(loggerClass, Level.TRACE); + Class invalidatorMock = PropertySourcesReloadEvent.class; + AgentHealth eventHealth = AgentHealth.OK; + + healthMonitor.onLoggingEvent(loggingEvent, invalidatorMock); + + verify(healthManager).notifyAgentHealth(eventHealth, invalidatorMock, loggerClass.getCanonicalName(), loggingEvent.getFormattedMessage()); + } + + @Test + void verifyNotifyAgentHealthOnDebug() { + Class loggerClass = LogHealthMonitor.class; + ILoggingEvent loggingEvent = createLoggingEvent(loggerClass, Level.DEBUG); + Class invalidatorMock = PropertySourcesReloadEvent.class; + AgentHealth eventHealth = AgentHealth.OK; + + healthMonitor.onLoggingEvent(loggingEvent, invalidatorMock); + + verify(healthManager).notifyAgentHealth(eventHealth, invalidatorMock, loggerClass.getCanonicalName(), loggingEvent.getFormattedMessage()); + } +} diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/LogMetricsIntTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogMetricsIntTest.java similarity index 96% rename from inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/LogMetricsIntTest.java rename to inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogMetricsIntTest.java index 676454eece..1bb25069cb 100644 --- a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/LogMetricsIntTest.java +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogMetricsIntTest.java @@ -1,4 +1,4 @@ -package rocks.inspectit.ocelot.core.selfmonitoring; +package rocks.inspectit.ocelot.core.selfmonitoring.logs; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.spi.ILoggingEvent; @@ -12,6 +12,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import rocks.inspectit.ocelot.core.logging.logback.InternalProcessingAppender; +import rocks.inspectit.ocelot.core.selfmonitoring.SelfMonitoringService; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/LogMetricsRecorderTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogMetricsRecorderTest.java similarity index 93% rename from inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/LogMetricsRecorderTest.java rename to inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogMetricsRecorderTest.java index cd09e988c1..07840e491d 100644 --- a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/LogMetricsRecorderTest.java +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogMetricsRecorderTest.java @@ -1,4 +1,4 @@ -package rocks.inspectit.ocelot.core.selfmonitoring; +package rocks.inspectit.ocelot.core.selfmonitoring.logs; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; @@ -10,6 +10,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.LoggerFactory; +import rocks.inspectit.ocelot.core.selfmonitoring.SelfMonitoringService; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/LogPreloaderTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogPreloaderTest.java similarity index 98% rename from inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/LogPreloaderTest.java rename to inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogPreloaderTest.java index 14d6348ac0..c9974b8c8c 100644 --- a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/LogPreloaderTest.java +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/logs/LogPreloaderTest.java @@ -1,4 +1,4 @@ -package rocks.inspectit.ocelot.core.selfmonitoring; +package rocks.inspectit.ocelot.core.selfmonitoring.logs; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; diff --git a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/service/DynamicallyActivatableServiceObserverTest.java b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/service/DynamicallyActivatableServiceObserverTest.java index 57c65a25cb..e6dd0414ec 100644 --- a/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/service/DynamicallyActivatableServiceObserverTest.java +++ b/inspectit-ocelot-core/src/test/java/rocks/inspectit/ocelot/core/selfmonitoring/service/DynamicallyActivatableServiceObserverTest.java @@ -5,7 +5,7 @@ import rocks.inspectit.ocelot.core.command.AgentCommandService; import rocks.inspectit.ocelot.core.exporter.JaegerExporterService; import rocks.inspectit.ocelot.core.exporter.PrometheusExporterService; -import rocks.inspectit.ocelot.core.selfmonitoring.LogPreloader; +import rocks.inspectit.ocelot.core.selfmonitoring.logs.LogPreloader; import rocks.inspectit.ocelot.core.service.DynamicallyActivatableService; import java.util.ArrayList; diff --git a/inspectit-ocelot-documentation/docs/assets/agent-health-incidents.png b/inspectit-ocelot-documentation/docs/assets/agent-health-incidents.png new file mode 100644 index 0000000000..36c8670a22 Binary files /dev/null and b/inspectit-ocelot-documentation/docs/assets/agent-health-incidents.png differ diff --git a/inspectit-ocelot-documentation/docs/config-server/agent-mappings.md b/inspectit-ocelot-documentation/docs/config-server/agent-mappings.md index 2dfd3b79ee..58932a7555 100644 --- a/inspectit-ocelot-documentation/docs/config-server/agent-mappings.md +++ b/inspectit-ocelot-documentation/docs/config-server/agent-mappings.md @@ -9,7 +9,7 @@ Furthermore, you can specify which branch (`WORKSPACE` or `LIVE`) the mapping sh It's important to note that the first matching agent mapping will be used to determine which configuration is shipped to an agent. Additional agent mappings which may also match the attributes list sent by an agent will be ignored. -See section [HTTP-based Configuration](configuration/external-configuration-sources.md#http-based-configuration) for +See section [HTTP-based Configuration](configuration/external-configuration-sources.md#http-based-configuration) for information on how to specify which attributes will be sent by an agent. An agent mapping consists of the following properties: @@ -55,7 +55,7 @@ The `sourceBranch` property of an individual agent mapping determines, which bra ![Different Source Branches on Agent Mappings Page](assets/agent_mappings_source_branch.png) -You can define, which source branch should be used at start-up for the agent mappings +You can define, which source branch should be used at start-up for the agent mappings in the application properties of the configuration server: ```YAML @@ -93,4 +93,3 @@ The following agent mapping will deliver all configuration files located in the service: "customer-service" sourceBranch: "WORKSPACE" ``` - diff --git a/inspectit-ocelot-documentation/docs/metrics/tag-guard.md b/inspectit-ocelot-documentation/docs/metrics/tag-guard.md new file mode 100644 index 0000000000..b6fdd01f4c --- /dev/null +++ b/inspectit-ocelot-documentation/docs/metrics/tag-guard.md @@ -0,0 +1,98 @@ +--- +id: tag-guard +title: Tag-Guard +--- + +Since version `2.6.0` it is possible to limit the amount of tag values of metrics. +This can be useful for controlling the amount of tag values, which will be written to your time series database +(e.g. InfluxDB or Prometheus). A high amount of unique tag values for a metric will result in a high cardinality, +which in turn might lead to performance or memory issues in your time series database. + +The recorded tag values for each measure of an agent will be stored inside a local JSON file. This file serves +as a tag-guard-database and helps to check, if tag values exceeded their limit. + +### Configuring Tag-Guard + +You can set the Tag-Guard configuration in `inspectit.metrics.tag-guard`. + +| Property | Default | Description | +|----------------------------------|--------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------| +| `.enabled` | `true` | Only when the tag-guard is enabled, the tag value limits will be checked. +| `.database-file` | `${inspectit.env.agent-dir}/${inspectit.service-name}/tag-guard-database.json` | The location of the JSON file with all recorded tag values. | +| `.schedule-delay` | `30s` | The delay for the scheduler, which will regularly compare the tag-guard-database with the configured tag value limits. | +| `.overflow-replacement` | `TAG_LIMIT_EXCEEDED` | After exceeding it's configured tag value limit, every tag will use this overflow replacement as value. | +| `.max-values-per-tag` | `1000` | The global tag value limit for every measure. | +| `.max-values-per-tag-by-measure` | `{}` | A map with measure names as key and their specific tag value limit as value. | + +There are three ways to define a tag value limit for measures. They are prioritized in the following order: + +1. Inside a metric definition for a particular measure +2. Globally for specific measures via `may-values-per-tag-by-measure` +3. Globally for all measures via `max-values-per-tag` + +This means that a tag value limit inside a metric definition will overwrite all other tag value limits +for the particular metric. A configured tag value limit in `max-values-per-tag-by-measure` will only overwrite the +globally set tag value limit in `max-values-per-tag` for the particular measure, but not a configured tag value limit +inside the metric definition. Let's look at an example: + +```yaml +inspectit: + metrics: + tag-guard: + max-values-per-tag: 1000 + max-values-per-tag-by-measure: + my_metric: 200 +``` + +In this configuration the global tag value limit is set to 1000, which means that every measure can only record 1000 unique +tag values for each tag. However, this does not apply to the measure `my_metric`, because the global tag value limit is +overwritten by `max-values-per-tag-by-measure` with 200. Thus, the measure `my_metric` can only record a maximum of 200 unique +tag values for each tag. + +Now, let's add another configuration: + +```yaml +inspectit: + metrics: + definitions: + 'my_metric': + tag-guard: 100 +``` + +This metric definition will overwrite the tag value limit specified in `max-values-per-tag-by-measure` for the measure `my_metric`, +resulting in a tag value limit of 100. Every other measure still uses the globally configured tag value +limit of 1000. + + +### Agent Health Monitoring + +If the tag value for a specific agent is exceeded, the Tag-Guard scheduler will detect an overflow and change +the agent health to `ERROR`. +Additionally, an agent health incident will be created, mentioning which tag-key has exceeded its tag value limit. +In the [Agent Status Table View](../config-server/status-table-view.md) of the Configuration-Server you can click on the +health state icon of a particular agent to view its last agent health incidents. You can set the amount of buffered incidents +with `inspectit.self-monitoring.agent-health.incident-buffer-size`. A list of incidents could look like this: + +![List of agent health incidents](assets/agent-health-incidents.png) + + +### How To Fix A Tag Value Overflow + +If a tag value limit was exceeded, there are two options to resolve the agent health `ERROR`. + +The **first option** would be to increase the tag value limit. Probably the limit has been estimated too low and thus has +to be increased. After increasing the tag value limit, the tag-guard-database scheduler will resolve the `ERROR`. + +The **second option** would be to adjust your configuration or application so the tag value limit should not be exceeded anymore. +After the adjustment, you will have to "reset" the recorded tag values in the tag-guard-database to resolve the `ERROR`. +One way to reset the tag-guard-database is to delete the local JSON file. However, this will delete all recorded tag values +and might not be the preferred solution.
    +A more preferable solution would be to only reset the tag values for a specific tag of a measure, +which has exceeded its tag value limit. +To do this, you could use the _**jq command-line JSON processor**_, which has to be installed on your system manually. +For example, you could use the following command, if you would like to delete all recorded tag values for the tag _my_tag_ inside the measure _my_metric_: + +- Unix: `jq '.my_metric.my_tag = []' tag-guard-database.json > temp.json && mv temp.json tag-guard-database.json` +- Windows: `jq ".my_metric.my_tag = []" tag-guard-database.json > temp.json && move temp.json tag-guard-database.json` + +In future versions of inspectIT there might be an option to reset specific tag values directly in the Configuration-Server UI. diff --git a/inspectit-ocelot-documentation/docs/tracing/tracing.md b/inspectit-ocelot-documentation/docs/tracing/tracing.md index 01b5770e68..4560f3f025 100644 --- a/inspectit-ocelot-documentation/docs/tracing/tracing.md +++ b/inspectit-ocelot-documentation/docs/tracing/tracing.md @@ -75,7 +75,7 @@ inspectit: propagation-format: B3 # the format for propagating correlation headers ``` -Currently the following formats are supported for sending correlation information: +Currently, the following formats are supported for sending correlation information: | Property | Format | Description |---|---|---| @@ -87,6 +87,61 @@ Currently the following formats are supported for sending correlation informatio It is important to note that this configuration refers to the format of the correlation information used to **send this data**. When processing correlation information that the agent receives, it automatically uses the correct format. ::: +### Adding Metric Tags + +It is possible to include all metrics tags of the current rule scope as tracing attributes. +This way it isn't necessary to define key-value pairs twice for metrics as well as tracing. +However, it is only possible to use metric tags as tracing attributes, but not vice versa! + +You can disable this feature in the tracing configuration: + +```YAML +inspectit: + tracing: + add-metric-tags: true +``` + +In this example, both tags of the metric `my_counter` will be used as attributes for the tracing within this rule. + +```YAML +rules: + 'r_example': + include: + 'r_myRule': true + entry: + 'my_data': + action: 'a_getData' + metrics: + my_counter: + value: 1 + data-tags: + 'example': 'my_data' + constant-tags: + 'scope': 'EXAMPLE' +``` + +Each tag key can only be used once within one trace. Thus, if a tag key has been assigned multiple values within one rule, +the acquired tag value will be determined hierarchically. Tag keys defined in `metrics.data-tags` will overwrite tag keys +defined in `metrics.constant-tags`. Tag keys defined in `tracing.attributes` will always overwrite tag keys defined in `metrics`. +In the example below, the tracing attributes will use 'trace' as value for 'myTag' and 'yourData' as value for 'yourTag'. + + +```YAML +rules: + 'r_example': + tracing: + attributes: + 'myTag': 'trace' + metrics: + my_counter: + data-tags: + 'myTag': 'myData' + 'yourTag': 'yourData' + constant-tags: + 'yourTag': 'CONSTANT' +``` + + ### Using 64-Bit Trace IDs Since version 2.0.0, the inspectIT Ocelot Agent is able to generate trace IDs with a size of 64 bits instead of the 128 bit trace IDs used by default by the agent. @@ -100,4 +155,4 @@ inspectit: :::important Please note that some propagation formats do not support 64-bit Ids, such as the W3C "Trace Context". In this case the 64-bit trace IDs are padded with leading zeros. -::: \ No newline at end of file +::: diff --git a/inspectit-ocelot-documentation/website/sidebars.json b/inspectit-ocelot-documentation/website/sidebars.json index 53bfd828d1..1f32b88ff1 100644 --- a/inspectit-ocelot-documentation/website/sidebars.json +++ b/inspectit-ocelot-documentation/website/sidebars.json @@ -20,7 +20,8 @@ "metrics/metric-exporters", "metrics/common-tags", "metrics/custom-metrics", - "metrics/self-monitoring" + "metrics/self-monitoring", + "metrics/tag-guard" ], "Tracing": [ "tracing/tracing",