diff --git a/.chloggen/feat_azuremonitorreceiver_filter_metrics.yaml b/.chloggen/feat_azuremonitorreceiver_filter_metrics.yaml new file mode 100644 index 000000000000..4ab9ea46addf --- /dev/null +++ b/.chloggen/feat_azuremonitorreceiver_filter_metrics.yaml @@ -0,0 +1,28 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: azuremonitorreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Adds filtering by metric and/or aggregation + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [37420] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] + diff --git a/receiver/azuremonitorreceiver/README.md b/receiver/azuremonitorreceiver/README.md index 2396b8a4ae7b..0aea8e39a4f4 100644 --- a/receiver/azuremonitorreceiver/README.md +++ b/receiver/azuremonitorreceiver/README.md @@ -25,6 +25,7 @@ The following settings are optional: - `auth` (default = service_principal): Specifies the used authentication method. Supported values are `service_principal`, `workload_identity`, `managed_identity`, `default_credentials`. - `resource_groups` (default = none): Filter metrics for specific resource groups, not setting a value will scrape metrics for all resources in the subscription. - `services` (default = none): Filter metrics for specific services, not setting a value will scrape metrics for all services integrated with Azure Monitor. +- `metrics` (default = none): Filter specific metrics (e.g., `metrics: ["MetricName"]`) and aggregations (e.g., `metrics: ["MetricName/Total"]`). - `cache_resources` (default = 86400): List of resources will be cached for the provided amount of time in seconds. - `cache_resources_definitions` (default = 86400): List of metrics definitions will be cached for the provided amount of time in seconds. - `maximum_number_of_metrics_in_a_call` (default = 20): Maximum number of metrics to fetch in per API call, current limit in Azure is 20 (as of 03/27/2023). @@ -48,6 +49,10 @@ Authenticating using managed identities has the following optional settings: - `client_id` +### Filtering metrics + +The `metrics` configuration setting is designed to constrain scraping to particular metrics and their specific aggregations. It accepts an array of metrics, where each metric can optionally include an aggregation method following a `/` (slash): `MetricName/Aggregation`. The metric name should correspond to a supported Azure Monitor metric for the designated resource groups and services. The aggregation method can be any aggregation compatible with Azure Monitor (e.g., Average, Minimum, Maximum, Total, Count). The case of the metric name and aggregation does not affect the functionality. + ### Example Configurations Using [Service Principal](https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication?tabs=bash#service-principal-with-a-secret) for authentication: @@ -66,6 +71,10 @@ receivers: services: - "${service1}" - "${service2}" + metrics: + - "${metric1}" + - "${metric2}/${aggregator1}" + - "${metric2}/${aggregator2}" collection_interval: 60s initial_delay: 1s ``` @@ -101,6 +110,18 @@ receivers: auth: "default_credentials" ``` +Scrapping limited metrics and aggregations: + +```yaml +receivers: + azuremonitor: + subscription_id: "${subscription_id}" + auth: "default_credentials" + metrics: + - metric1 # This will scrape all known aggregations for "metric1" + - metric2/total # This will include "metric2" with the "Total" aggregation in the scraping + - metric3/average # This will include "metric3" with the "Average" aggregation in the scraping +``` ## Metrics diff --git a/receiver/azuremonitorreceiver/config.go b/receiver/azuremonitorreceiver/config.go index a7d5da3479e1..3d1c60b1453d 100644 --- a/receiver/azuremonitorreceiver/config.go +++ b/receiver/azuremonitorreceiver/config.go @@ -241,6 +241,7 @@ type Config struct { FederatedTokenFile string `mapstructure:"federated_token_file"` ResourceGroups []string `mapstructure:"resource_groups"` Services []string `mapstructure:"services"` + Metrics []string `mapstructure:"metrics"` CacheResources float64 `mapstructure:"cache_resources"` CacheResourcesDefinitions float64 `mapstructure:"cache_resources_definitions"` MaximumNumberOfMetricsInACall int `mapstructure:"maximum_number_of_metrics_in_a_call"` diff --git a/receiver/azuremonitorreceiver/scraper.go b/receiver/azuremonitorreceiver/scraper.go index 99d24600faa5..e895f65e0783 100644 --- a/receiver/azuremonitorreceiver/scraper.go +++ b/receiver/azuremonitorreceiver/scraper.go @@ -67,8 +67,9 @@ type azureResource struct { } type metricsCompositeKey struct { - dimensions string // comma separated sorted dimensions - timeGrain string + dimensions string // comma separated sorted dimensions + aggregations string // comma separated sorted aggregations + timeGrain string } type azureResourceMetrics struct { @@ -337,9 +338,17 @@ func (s *azureScraper) getResourceMetricsDefinitions(ctx context.Context, resour } for _, v := range nextResult.Value { - timeGrain := *v.MetricAvailabilities[0].TimeGrain name := *v.Name.Value - compositeKey := metricsCompositeKey{timeGrain: timeGrain} + metricAggregations := getMetricAggregations(name, s.cfg.Metrics) + if len(metricAggregations) == 0 { + continue + } + + timeGrain := *v.MetricAvailabilities[0].TimeGrain + compositeKey := metricsCompositeKey{ + timeGrain: timeGrain, + aggregations: strings.Join(metricAggregations, ","), + } if len(v.Dimensions) > 0 { var dimensionsSlice []string @@ -388,6 +397,7 @@ func (s *azureScraper) getResourceMetricsValues(ctx context.Context, resourceID metricsByGrain.metrics, compositeKey.dimensions, compositeKey.timeGrain, + compositeKey.aggregations, start, end, s.cfg.MaximumNumberOfRecordsPerResource, @@ -435,6 +445,7 @@ func getResourceMetricsValuesRequestOptions( metrics []string, dimensionsStr string, timeGrain string, + aggregations string, start int, end int, top int32, @@ -444,7 +455,7 @@ func getResourceMetricsValuesRequestOptions( Metricnames: &resType, Interval: to.Ptr(timeGrain), Timespan: to.Ptr(timeGrain), - Aggregation: to.Ptr(strings.Join(aggregations, ",")), + Aggregation: to.Ptr(aggregations), Top: to.Ptr(top), } @@ -500,3 +511,24 @@ func (s *azureScraper) processTimeseriesData( } } } + +func getMetricAggregations(name string, filters []string) []string { + if len(filters) == 0 { + return aggregations + } + + out := []string{} + for _, filter := range filters { + if strings.EqualFold(name, filter) { + return aggregations + } + + for _, aggregation := range aggregations { + if strings.EqualFold(name+"/"+aggregation, filter) { + out = append(out, aggregation) + } + } + } + + return out +} diff --git a/receiver/azuremonitorreceiver/scraper_test.go b/receiver/azuremonitorreceiver/scraper_test.go index cd53f3abf940..7b8fea13b997 100644 --- a/receiver/azuremonitorreceiver/scraper_test.go +++ b/receiver/azuremonitorreceiver/scraper_test.go @@ -244,12 +244,83 @@ func (mdcm *metricsDefinitionsClientMock) NewListPager(resourceURI string, _ *ar }) } -type metricsValuesClientMock struct { - lists map[string]map[string]armmonitor.MetricsClientListResponse +type metricsValuesClientMock struct{} + +func (mvcm metricsValuesClientMock) List(_ context.Context, _ string, options *armmonitor.MetricsClientListOptions) (armmonitor.MetricsClientListResponse, error) { + var unit1 armmonitor.Unit = "unit1" + + amMetrics := []*armmonitor.Metric{} + for _, name := range strings.Split(*options.Metricnames, ",") { + var metaValues []*armmonitor.MetadataValue + if options.Filter != nil { + metaValues = mvcm.getAMMetadataValues(*options.Filter) + } + + amMetric := &armmonitor.Metric{ + Name: &armmonitor.LocalizableString{ + Value: &name, + }, + Unit: &unit1, + Timeseries: []*armmonitor.TimeSeriesElement{ + { + Data: []*armmonitor.MetricValue{ + mvcm.getAMDataPoints(*options.Aggregation), + }, + Metadatavalues: metaValues, + }, + }, + } + amMetrics = append(amMetrics, amMetric) + + if name == "metric7" { + amMetric.Timeseries[0].Data[0] = mvcm.getAMDataPoints("Count") + } + } + + return armmonitor.MetricsClientListResponse{ + Response: armmonitor.Response{Value: amMetrics}, + }, nil } -func (mvcm metricsValuesClientMock) List(_ context.Context, resourceURI string, options *armmonitor.MetricsClientListOptions) (armmonitor.MetricsClientListResponse, error) { - return mvcm.lists[resourceURI][*options.Metricnames], nil +func (mvcm metricsValuesClientMock) getAMDataPoints(aggregations string) *armmonitor.MetricValue { + var value1 float64 = 1 + + amPoints := &armmonitor.MetricValue{} + for _, aggregation := range strings.Split(aggregations, ",") { + switch aggregation { + case "Average": + amPoints.Average = &value1 + case "Count": + amPoints.Count = &value1 + case "Maximum": + amPoints.Maximum = &value1 + case "Minimum": + amPoints.Minimum = &value1 + case "Total": + amPoints.Total = &value1 + } + } + + return amPoints +} + +func (mvcm metricsValuesClientMock) getAMMetadataValues(filter string) []*armmonitor.MetadataValue { + var out []*armmonitor.MetadataValue + knownDimensions := []string{"dimension1", "dimension2"} + dimensionValue := "dimension value" + + for _, dimension := range knownDimensions { + if strings.Contains(filter, dimension) { + out = append(out, &armmonitor.MetadataValue{ + Name: &armmonitor.LocalizableString{ + Value: &dimension, + }, + Value: &dimensionValue, + }) + } + } + + return out } func TestAzureScraperScrape(t *testing.T) { @@ -266,6 +337,10 @@ func TestAzureScraperScrape(t *testing.T) { cfgTagsEnabled.AppendTagsAsAttributes = true cfgTagsEnabled.MaximumNumberOfMetricsInACall = 2 + cfgLimitedMertics := createDefaultConfig().(*Config) + cfgLimitedMertics.MaximumNumberOfMetricsInACall = 2 + cfgLimitedMertics.Metrics = []string{"metric1", "metric3/total", "metric4/average", "metric4/minimum", "metric4/maximum"} + tests := []struct { name string fields fields @@ -290,6 +365,15 @@ func TestAzureScraperScrape(t *testing.T) { ctx: context.Background(), }, }, + { + name: "metrics_filtered", + fields: fields{ + cfg: cfgLimitedMertics, + }, + args: args{ + ctx: context.Background(), + }, + }, } for _, tt := range tests { @@ -308,9 +392,7 @@ func TestAzureScraperScrape(t *testing.T) { pages: pages, } - metricsValuesClientMock := &metricsValuesClientMock{ - lists: getMetricsValuesMockData(), - } + metricsValuesClientMock := &metricsValuesClientMock{} s := &azureScraper{ cfg: tt.fields.cfg, @@ -512,217 +594,6 @@ func getMetricsDefinitionsMockData() (map[string]int, map[string][]armmonitor.Me return counters, pages } -func getMetricsValuesMockData() map[string]map[string]armmonitor.MetricsClientListResponse { - name1, name2, name3, name4, name5, name6, name7, dimension1, dimension2, dimensionValue := "metric1", "metric2", - "metric3", "metric4", "metric5", "metric6", "metric7", "dimension1", "dimension2", "dimension value" - var unit1 armmonitor.Unit = "unit1" - var value1 float64 = 1 - - return map[string]map[string]armmonitor.MetricsClientListResponse{ - "/resourceGroups/group1/resourceId1": { - strings.Join([]string{name1, name2}, ","): { - Response: armmonitor.Response{ - Value: []*armmonitor.Metric{ - { - Name: &armmonitor.LocalizableString{ - Value: &name1, - }, - Unit: &unit1, - Timeseries: []*armmonitor.TimeSeriesElement{ - { - Data: []*armmonitor.MetricValue{ - { - Average: &value1, - Count: &value1, - Maximum: &value1, - Minimum: &value1, - Total: &value1, - }, - }, - }, - }, - }, - { - Name: &armmonitor.LocalizableString{ - Value: &name2, - }, - Unit: &unit1, - Timeseries: []*armmonitor.TimeSeriesElement{ - { - Data: []*armmonitor.MetricValue{ - { - Average: &value1, - Count: &value1, - Maximum: &value1, - Minimum: &value1, - Total: &value1, - }, - }, - }, - }, - }, - }, - }, - }, - name3: { - Response: armmonitor.Response{ - Value: []*armmonitor.Metric{ - { - Name: &armmonitor.LocalizableString{ - Value: &name3, - }, - Unit: &unit1, - Timeseries: []*armmonitor.TimeSeriesElement{ - { - Data: []*armmonitor.MetricValue{ - { - Average: &value1, - Count: &value1, - Maximum: &value1, - Minimum: &value1, - Total: &value1, - }, - }, - }, - }, - }, - }, - }, - }, - }, - "/resourceGroups/group1/resourceId2": { - name4: { - Response: armmonitor.Response{ - Value: []*armmonitor.Metric{ - { - Name: &armmonitor.LocalizableString{ - Value: &name4, - }, - Unit: &unit1, - Timeseries: []*armmonitor.TimeSeriesElement{ - { - Data: []*armmonitor.MetricValue{ - { - Average: &value1, - Count: &value1, - Maximum: &value1, - Minimum: &value1, - Total: &value1, - }, - }, - }, - }, - }, - }, - }, - }, - name5: { - Response: armmonitor.Response{ - Value: []*armmonitor.Metric{ - { - Name: &armmonitor.LocalizableString{ - Value: &name5, - }, - Unit: &unit1, - Timeseries: []*armmonitor.TimeSeriesElement{ - { - Data: []*armmonitor.MetricValue{ - { - Average: &value1, - Count: &value1, - Maximum: &value1, - Minimum: &value1, - Total: &value1, - }, - }, - Metadatavalues: []*armmonitor.MetadataValue{ - { - Name: &armmonitor.LocalizableString{ - Value: &dimension1, - }, - Value: &dimensionValue, - }, - { - Name: &armmonitor.LocalizableString{ - Value: &dimension2, - }, - Value: &dimensionValue, - }, - }, - }, - }, - }, - }, - }, - }, - name6: { - Response: armmonitor.Response{ - Value: []*armmonitor.Metric{ - { - Name: &armmonitor.LocalizableString{ - Value: &name6, - }, - Unit: &unit1, - Timeseries: []*armmonitor.TimeSeriesElement{ - { - Data: []*armmonitor.MetricValue{ - { - Average: &value1, - Count: &value1, - Maximum: &value1, - Minimum: &value1, - Total: &value1, - }, - }, - Metadatavalues: []*armmonitor.MetadataValue{ - { - Name: &armmonitor.LocalizableString{ - Value: &dimension1, - }, - Value: &dimensionValue, - }, - }, - }, - }, - }, - }, - }, - }, - }, - "/resourceGroups/group1/resourceId3": { - name7: { - Response: armmonitor.Response{ - Value: []*armmonitor.Metric{ - { - Name: &armmonitor.LocalizableString{ - Value: &name7, - }, - Unit: &unit1, - Timeseries: []*armmonitor.TimeSeriesElement{ - { - Data: []*armmonitor.MetricValue{ - { - Count: &value1, - }, - }, - Metadatavalues: []*armmonitor.MetadataValue{ - { - Name: &armmonitor.LocalizableString{ - Value: &dimension1, - }, - Value: &dimensionValue, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } -} - func TestAzureScraperClientOptions(t *testing.T) { type fields struct { cfg *Config @@ -783,3 +654,50 @@ func TestAzureScraperClientOptions(t *testing.T) { }) } } + +func TestGetMetricAggregations(t *testing.T) { + testMetricName := "MetricName" + tests := []struct { + name string + filters []string + want []string + }{ + { + "filters_empty", + []string{}, + aggregations, + }, + { + "filters_include_metric", + []string{"foo", testMetricName, "bar"}, + aggregations, + }, + { + "filters_include_metric_ignore_case", + []string{"foo", strings.ToLower(testMetricName), "bar"}, + aggregations, + }, + { + "filters_include_metric_aggregation", + []string{"foo/count", testMetricName + "/" + aggregations[0], "bar/total"}, + []string{aggregations[0]}, + }, + { + "filters_include_metric_aggregation_ignore_case", + []string{"foo/count", testMetricName + "/" + strings.ToLower(aggregations[0]), "bar/total"}, + []string{aggregations[0]}, + }, + { + "filters_include_metric_multiple_aggregations", + []string{"foo/count", testMetricName + "/" + aggregations[0], testMetricName + "/" + aggregations[2]}, + []string{aggregations[0], aggregations[2]}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getMetricAggregations(testMetricName, tt.filters) + require.Equal(t, tt.want, got) + }) + } +} diff --git a/receiver/azuremonitorreceiver/testdata/expected_metrics/metrics_filtered.yaml b/receiver/azuremonitorreceiver/testdata/expected_metrics/metrics_filtered.yaml new file mode 100644 index 000000000000..6e472e89d053 --- /dev/null +++ b/receiver/azuremonitorreceiver/testdata/expected_metrics/metrics_filtered.yaml @@ -0,0 +1,221 @@ +resourceMetrics: + - resource: + attributes: + - key: azuremonitor.subscription_id + value: + stringValue: "" + - key: azuremonitor.tenant_id + value: + stringValue: "" + scopeMetrics: + - metrics: + - gauge: + dataPoints: + - asDouble: 1 + attributes: + - key: azuremonitor.resource_id + value: + stringValue: /resourceGroups/group1/resourceId1 + - key: location + value: + stringValue: location1 + - key: name + value: + stringValue: name1 + - key: resource_group + value: + stringValue: group1 + - key: type + value: + stringValue: type1 + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: azure_metric1_maximum + unit: unit1 + - gauge: + dataPoints: + - asDouble: 1 + attributes: + - key: azuremonitor.resource_id + value: + stringValue: /resourceGroups/group1/resourceId2 + - key: location + value: + stringValue: location1 + - key: name + value: + stringValue: name1 + - key: resource_group + value: + stringValue: group1 + - key: type + value: + stringValue: type1 + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: azure_metric4_minimum + unit: unit1 + - gauge: + dataPoints: + - asDouble: 1 + attributes: + - key: azuremonitor.resource_id + value: + stringValue: /resourceGroups/group1/resourceId1 + - key: location + value: + stringValue: location1 + - key: name + value: + stringValue: name1 + - key: resource_group + value: + stringValue: group1 + - key: type + value: + stringValue: type1 + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: azure_metric1_total + unit: unit1 + - gauge: + dataPoints: + - asDouble: 1 + attributes: + - key: azuremonitor.resource_id + value: + stringValue: /resourceGroups/group1/resourceId1 + - key: location + value: + stringValue: location1 + - key: name + value: + stringValue: name1 + - key: resource_group + value: + stringValue: group1 + - key: type + value: + stringValue: type1 + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: azure_metric3_total + unit: unit1 + - gauge: + dataPoints: + - asDouble: 1 + attributes: + - key: azuremonitor.resource_id + value: + stringValue: /resourceGroups/group1/resourceId2 + - key: location + value: + stringValue: location1 + - key: name + value: + stringValue: name1 + - key: resource_group + value: + stringValue: group1 + - key: type + value: + stringValue: type1 + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: azure_metric4_average + unit: unit1 + - gauge: + dataPoints: + - asDouble: 1 + attributes: + - key: azuremonitor.resource_id + value: + stringValue: /resourceGroups/group1/resourceId2 + - key: location + value: + stringValue: location1 + - key: name + value: + stringValue: name1 + - key: resource_group + value: + stringValue: group1 + - key: type + value: + stringValue: type1 + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: azure_metric4_maximum + unit: unit1 + - gauge: + dataPoints: + - asDouble: 1 + attributes: + - key: azuremonitor.resource_id + value: + stringValue: /resourceGroups/group1/resourceId1 + - key: location + value: + stringValue: location1 + - key: name + value: + stringValue: name1 + - key: resource_group + value: + stringValue: group1 + - key: type + value: + stringValue: type1 + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: azure_metric1_count + unit: unit1 + - gauge: + dataPoints: + - asDouble: 1 + attributes: + - key: azuremonitor.resource_id + value: + stringValue: /resourceGroups/group1/resourceId1 + - key: location + value: + stringValue: location1 + - key: name + value: + stringValue: name1 + - key: resource_group + value: + stringValue: group1 + - key: type + value: + stringValue: type1 + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: azure_metric1_minimum + unit: unit1 + - gauge: + dataPoints: + - asDouble: 1 + attributes: + - key: azuremonitor.resource_id + value: + stringValue: /resourceGroups/group1/resourceId1 + - key: location + value: + stringValue: location1 + - key: name + value: + stringValue: name1 + - key: resource_group + value: + stringValue: group1 + - key: type + value: + stringValue: type1 + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: azure_metric1_average + unit: unit1 + scope: + name: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/azuremonitorreceiver + version: latest