Skip to content

Commit

Permalink
Improve sampling override behavior
Browse files Browse the repository at this point in the history
  • Loading branch information
@trask
trask committed Jan 14, 2025
1 parent da24299 commit 59be9a0
Show file tree
Hide file tree
Showing 7 changed files with 234 additions and 56 deletions.
28 changes: 18 additions & 10 deletions ...main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,14 @@
package com.microsoft.applicationinsights.agent.internal.exporter;

import static com.azure.monitor.opentelemetry.exporter.implementation.utils.AzureMonitorMsgId.EXPORTER_MAPPING_ERROR;
import static com.microsoft.applicationinsights.agent.internal.exporter.ExporterUtils.shouldSample;

import com.azure.monitor.opentelemetry.exporter.implementation.AiSemanticAttributes;
import com.azure.monitor.opentelemetry.exporter.implementation.LogDataMapper;
import com.azure.monitor.opentelemetry.exporter.implementation.logging.OperationLogger;
import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem;
import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.QuickPulse;
import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverride;
import com.microsoft.applicationinsights.agent.internal.sampling.AiSampler;
import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides;
import com.microsoft.applicationinsights.agent.internal.telemetry.BatchItemProcessor;
import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient;
Expand All @@ -21,6 +22,7 @@
import io.opentelemetry.sdk.common.CompletableResultCode;
import io.opentelemetry.sdk.logs.data.LogRecordData;
import io.opentelemetry.sdk.logs.export.LogRecordExporter;
import io.opentelemetry.sdk.trace.samplers.SamplingDecision;
import io.opentelemetry.semconv.SemanticAttributes;
import java.util.Collection;
import java.util.List;
Expand Down Expand Up @@ -108,24 +110,30 @@ private CompletableResultCode internalExport(Collection<LogRecordData> logs) {
stack != null ? exceptionSamplingOverrides : logSamplingOverrides;

SpanContext spanContext = log.getSpanContext();
Double parentSpanSampleRate = log.getAttributes().get(AiSemanticAttributes.SAMPLE_RATE);

Double samplingPercentage = samplingOverrides.getOverridePercentage(log.getAttributes());
AiSampler sampler = samplingOverrides.getOverride(log.getAttributes());

if (samplingPercentage != null && !shouldSample(spanContext, samplingPercentage)) {
continue;
}

if (samplingPercentage == null
&& spanContext.isValid()
&& !spanContext.getTraceFlags().isSampled()) {
if (sampler == null && spanContext.isValid() && !spanContext.getTraceFlags().isSampled()) {
// if there is no sampling override, and the log is part of an unsampled trace, then don't
// capture it
continue;
}

Double sampleRate = parentSpanSampleRate;
if (sampler != null) {
if (sampler.shouldSampleLog(spanContext, parentSpanSampleRate).getDecision()
== SamplingDecision.DROP) {
continue;
}
// sampling override percentage takes precedence
sampleRate = sampler.getParentlessDependencySamplingPercentage().get();
}

logger.debug("exporting log: {}", log);

TelemetryItem telemetryItem = mapper.map(log, stack, samplingPercentage);
// TODO (trask) no longer need to check AiSemanticAttributes.SAMPLE_RATE in map() method
TelemetryItem telemetryItem = mapper.map(log, stack, sampleRate);
telemetryItemConsumer.accept(telemetryItem);

exportingLogLogger.recordSuccess();
Expand Down
16 changes: 11 additions & 5 deletions ...src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import static java.util.concurrent.TimeUnit.MINUTES;

import com.azure.core.util.logging.ClientLogger;
import com.azure.monitor.opentelemetry.exporter.implementation.AiSemanticAttributes;
import com.azure.monitor.opentelemetry.exporter.implementation.AzureMonitorExporterProviderKeys;
import com.azure.monitor.opentelemetry.exporter.implementation.AzureMonitorLogRecordExporterProvider;
import com.azure.monitor.opentelemetry.exporter.implementation.AzureMonitorMetricExporterProvider;
Expand Down Expand Up @@ -37,14 +38,14 @@
import com.microsoft.applicationinsights.agent.internal.exporter.AgentLogExporter;
import com.microsoft.applicationinsights.agent.internal.exporter.AgentMetricExporter;
import com.microsoft.applicationinsights.agent.internal.exporter.AgentSpanExporter;
import com.microsoft.applicationinsights.agent.internal.exporter.ExporterUtils;
import com.microsoft.applicationinsights.agent.internal.httpclient.LazyHttpClient;
import com.microsoft.applicationinsights.agent.internal.legacyheaders.AiLegacyHeaderSpanProcessor;
import com.microsoft.applicationinsights.agent.internal.processors.ExporterWithLogProcessor;
import com.microsoft.applicationinsights.agent.internal.processors.ExporterWithSpanProcessor;
import com.microsoft.applicationinsights.agent.internal.processors.LogExporterWithAttributeProcessor;
import com.microsoft.applicationinsights.agent.internal.processors.SpanExporterWithAttributeProcessor;
import com.microsoft.applicationinsights.agent.internal.profiler.triggers.AlertTriggerSpanProcessor;
import com.microsoft.applicationinsights.agent.internal.sampling.AiSampler;
import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides;
import com.microsoft.applicationinsights.agent.internal.telemetry.BatchItemProcessor;
import com.microsoft.applicationinsights.agent.internal.telemetry.MetricFilter;
Expand All @@ -66,6 +67,7 @@
import io.opentelemetry.sdk.metrics.internal.view.AiViewRegistry;
import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder;
import io.opentelemetry.sdk.trace.export.SpanExporter;
import io.opentelemetry.sdk.trace.samplers.SamplingDecision;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
Expand Down Expand Up @@ -579,10 +581,14 @@ private static SpanExporter createSpanExporter(
return false;
},
(span, event) -> {
Double samplingPercentage =
exceptionSamplingOverrides.getOverridePercentage(event.getAttributes());
return samplingPercentage != null
&& !ExporterUtils.shouldSample(span.getSpanContext(), samplingPercentage);
AiSampler sampler = exceptionSamplingOverrides.getOverride(event.getAttributes());
return sampler != null
&& sampler
.shouldSampleLog(
span.getSpanContext(),
span.getAttributes().get(AiSemanticAttributes.SAMPLE_RATE))
.getDecision()
== SamplingDecision.DROP;
});

BatchItemProcessor batchItemProcessor = telemetryClient.getGeneralBatchItemProcessor();
Expand Down
87 changes: 68 additions & 19 deletions ...ng/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,32 +30,48 @@ public class AiSampler implements Sampler {
private static final double SAMPLE_RATE_TO_DISABLE_INGESTION_SAMPLING = 99.99;

private final boolean ingestionSamplingEnabled;
private final boolean localParentBased;
private final boolean sampleWhenLocalParentSampled;
private final boolean dropWhenLocalParentDropped;
private final SamplingPercentage requestSamplingPercentage;
// when localParentBased=false, then this applies to all dependencies, not only parentless
private final SamplingPercentage parentlessDependencySamplingPercentage;
private final Cache<Double, SamplingResult> recordAndSampleWithSampleRateMap = Cache.bounded(100);

public AiSampler(
public static AiSampler create(
SamplingPercentage requestSamplingPercentage,
SamplingPercentage parentlessDependencySamplingPercentage,
boolean ingestionSamplingEnabled) {
this(
return new AiSampler(
requestSamplingPercentage,
parentlessDependencySamplingPercentage,
ingestionSamplingEnabled,
true,
true);
}

public AiSampler(
public static AiSampler createSamplingOverride(
SamplingPercentage samplingPercentage,
boolean sampleWhenLocalParentSampled,
boolean dropWhenLocalParentDropped) {
return new AiSampler(
samplingPercentage,
samplingPercentage,
false,
sampleWhenLocalParentSampled,
dropWhenLocalParentDropped);
}

private AiSampler(
SamplingPercentage requestSamplingPercentage,
SamplingPercentage parentlessDependencySamplingPercentage,
boolean ingestionSamplingEnabled,
boolean localParentBased) {
boolean sampleWhenLocalParentSampled,
boolean dropWhenLocalParentDropped) {
this.requestSamplingPercentage = requestSamplingPercentage;
this.parentlessDependencySamplingPercentage = parentlessDependencySamplingPercentage;
this.ingestionSamplingEnabled = ingestionSamplingEnabled;
this.localParentBased = localParentBased;
this.sampleWhenLocalParentSampled = sampleWhenLocalParentSampled;
this.dropWhenLocalParentDropped = dropWhenLocalParentDropped;
}

@Override
Expand All @@ -66,8 +82,42 @@ public SamplingResult shouldSample(
SpanKind spanKind,
Attributes attributes,
List<LinkData> parentLinks) {
if (localParentBased) {
SamplingResult samplingResult = useLocalParentDecisionIfPossible(parentContext);

Span parentSpan = Span.fromContext(parentContext);
SpanContext parentSpanContext = parentSpan.getSpanContext();
Double parentSpanSampleRate = null;
if (parentSpan instanceof ReadableSpan) {
parentSpanSampleRate =
((ReadableSpan) parentSpan).getAttribute(AiSemanticAttributes.SAMPLE_RATE);
}

return internalShouldSample(
parentSpanContext, parentSpanSampleRate, traceId, spanKind, attributes);
}

public SamplingResult shouldSampleLog(SpanContext spanContext, @Nullable Double spanSampleRate) {
return internalShouldSample(
spanContext,
spanSampleRate,
spanContext.getTraceId(),
SpanKind.INTERNAL, // unused
Attributes.empty());
}

public SamplingPercentage getParentlessDependencySamplingPercentage() {
return parentlessDependencySamplingPercentage;
}

private SamplingResult internalShouldSample(
SpanContext parentSpanContext,
@Nullable Double parentSpanSampleRate,
String traceId,
SpanKind spanKind,
Attributes attributes) {

if (sampleWhenLocalParentSampled || dropWhenLocalParentDropped) {
SamplingResult samplingResult =
useLocalParentDecisionIfPossible(parentSpanContext, parentSpanSampleRate);
if (samplingResult != null) {
return samplingResult;
}
Expand All @@ -78,7 +128,6 @@ public SamplingResult shouldSample(
// optimization for fixed-rate sampling
sp = requestSamplingPercentage.get();
} else {
SpanContext parentSpanContext = Span.fromContext(parentContext).getSpanContext();
boolean isRequest = RequestChecker.isRequest(spanKind, parentSpanContext, attributes::get);
sp =
isRequest
Expand Down Expand Up @@ -113,26 +162,26 @@ public SamplingResult shouldSample(
return samplingResult;
}

@SuppressWarnings("SystemOut")
@Nullable
private static SamplingResult useLocalParentDecisionIfPossible(Context parentContext) {
private SamplingResult useLocalParentDecisionIfPossible(
SpanContext parentSpanContext, @Nullable Double parentSpanSampleRate) {
// remote parent-based sampling messes up item counts since item count is not propagated in
// tracestate (yet), but local parent-based sampling doesn't have this issue since we are
// propagating item count locally
Span parentSpan = Span.fromContext(parentContext);
SpanContext parentSpanContext = parentSpan.getSpanContext();
if (!parentSpanContext.isValid() || parentSpanContext.isRemote()) {
System.out.println("useLocalParentDecisionIfPossible1");
return null;
}
if (!parentSpanContext.isSampled()) {
return SamplingResult.drop();
System.out.println("useLocalParentDecisionIfPossible2: " + dropWhenLocalParentDropped);
return dropWhenLocalParentDropped ? SamplingResult.drop() : null;
}
if (parentSpan instanceof ReadableSpan) {
Double parentSampleRate =
((ReadableSpan) parentSpan).getAttribute(AiSemanticAttributes.SAMPLE_RATE);
if (parentSampleRate != null) {
return new RecordAndSampleWithItemCount(parentSampleRate);
}
if (sampleWhenLocalParentSampled && parentSpanSampleRate != null) {
System.out.println("useLocalParentDecisionIfPossible3");
return new RecordAndSampleWithItemCount(parentSpanSampleRate);
}
System.out.println("useLocalParentDecisionIfPossible5");
return null;
}

Expand Down
4 changes: 2 additions & 2 deletions ...ing/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,14 @@ public static Sampler getSampler(
SamplingPercentage.rateLimited(sampling.requestsPerSecond);
SamplingPercentage parentlessDependencySamplingPercentage = SamplingPercentage.fixed(100);
sampler =
new AiSampler(
AiSampler.create(
requestSamplingPercentage,
parentlessDependencySamplingPercentage,
samplingPreview.ingestionSamplingEnabled);
} else if (sampling.percentage != null) {
SamplingPercentage samplingPercentage = SamplingPercentage.fixed(sampling.percentage);
sampler =
new AiSampler(
AiSampler.create(
samplingPercentage, samplingPercentage, samplingPreview.ingestionSamplingEnabled);
} else {
throw new AssertionError("ConfigurationBuilder should have set the default sampling");
Expand Down
29 changes: 9 additions & 20 deletions ...ain/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverrideAttribute;
import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.sdk.trace.samplers.Sampler;
import io.opentelemetry.semconv.SemanticAttributes;
import java.util.ArrayList;
import java.util.List;
Expand All @@ -32,7 +31,7 @@ public SamplingOverrides(List<SamplingOverride> overrides) {
}

@Nullable
public Sampler getOverride(Attributes attributes) {
public AiSampler getOverride(Attributes attributes) {
LazyHttpUrl lazyHttpUrl = new LazyHttpUrl(attributes);
LazyHttpTarget lazyHttpTarget = new LazyHttpTarget(attributes);
for (MatcherGroup matcherGroups : matcherGroups) {
Expand All @@ -43,20 +42,9 @@ public Sampler getOverride(Attributes attributes) {
return null;
}

// used to do sampling inside the log exporter
@Nullable
public Double getOverridePercentage(Attributes attributes) {
for (MatcherGroup matcherGroups : matcherGroups) {
if (matcherGroups.matches(attributes, null, null)) {
return matcherGroups.getPercentage();
}
}
return null;
}

private static class MatcherGroup {
private final List<TempPredicate> predicates;
private final Sampler sampler;
private final AiSampler sampler;
// for now only support fixed percentage, but could extend sampling overrides to support
// rate-limited sampling
private final SamplingPercentage samplingPercentage;
Expand All @@ -70,17 +58,18 @@ private MatcherGroup(SamplingOverride override) {
}
}
samplingPercentage = SamplingPercentage.fixed(override.percentage);
sampler = new AiSampler(samplingPercentage, samplingPercentage, false, false);
// setting sampleWhenLocalParentSampled = (override.percentage == 100) would end up the same
boolean sampleWhenLocalParentSampled = false;
boolean dropWhenLocalParentDropped = override.percentage < 100;
sampler =
AiSampler.createSamplingOverride(
samplingPercentage, sampleWhenLocalParentSampled, dropWhenLocalParentDropped);
}

Sampler getSampler() {
AiSampler getSampler() {
return sampler;
}

double getPercentage() {
return samplingPercentage.get();
}

private boolean matches(
Attributes attributes,
@Nullable LazyHttpUrl lazyHttpUrl,
Expand Down
Loading

0 comments on commit 59be9a0

Please sign in to comment.