[ci] Scrape and verify metrics at the end of e2e tests

.github/actions/verify-metrics-snapshot/action.yaml

@@ -0,0 +1,82 @@
+# Copyright (c) 2023 The Jaeger Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+
+
+name: 'Verify Metric Snapshot and Upload Metrics'
+description: 'Upload or cache the metrics data after verification'
+inputs:
+  snapshot:
+    description: 'Path to the metric file'
+    required: true
+  artifact_key:
+    description: 'Artifact key used for uploading and fetching artifacts'
+    required: true
+  cache_key:
+    description: 'Cache key used for uploading and fetching the correct cached metric'
+    required: true
+runs:
+  using: 'composite'
+  steps:
+    - name: Upload Metrics Artifact
+      uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+      with:
+        name: ${{ inputs.artifact_key }}
+        path: ./.metrics/${{ inputs.snapshot }}
+        retention-days: 7
+
+    - name: Get v2 tag 
+      shell: bash
+      run: |
+        v2_tag=$(make echo-v2)
+        echo "v2_tag=$v2_tag" >> $GITHUB_ENV
+        echo "$v2_tag"
+
+      # The github cache restore successfully restores when cache saved has same key and same path.
+      # Hence to restore release metric with name relese_{metric_name} , the name must be changed to the same.
+    - name: Change file name before caching
+      if: github.ref_name == 'main'
+      shell: bash
+      run: |
+        mv ./.metrics/${{ inputs.snapshot }} ./.metrics/release_${{ inputs.snapshot}}
+
+    - name: Cache scraped metrics for tagged release for longer retention
+      id: tagged-metrics
+      if: github.ref_name == 'main'
+      uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57
+      with: 
+        path: ./.metrics/release_${{ inputs.snapshot}}
+        key: ${{ env.v2_tag }}_${{ inputs.artifact_key }}_${{ github.run_id }}
+        restore-keys: |
+          ${{ env.v2_tag }}_${{ inputs.artifact_key }}_
+
+
+    - name: Download the cached tagged metrics
+      id: download-release-snapshot
+      if: github.ref_name != 'main'
+      uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57
+      with:
+        path: ./.metrics/release_${{ inputs.snapshot}}
+        key: ${{ env.v2_tag }}_${{ inputs.artifact_key }}_${{ github.run_id }}
+
+    - name: Calculate diff between the snapshots 
+      id: compare-snapshots
+      if: ${{ (steps.download-release-snapshot.outputs.cache-hit == 'true') && (github.ref_name != 'main') }}
+      shell: bash
+      run: |
+        python3 -m pip install prometheus-client
+        python3 ./scripts/e2e/compare_metrics.py --file1 ./.metrics/${{ inputs.snapshot }} --file2 ./.metrics/release_${{ inputs.snapshot}} --output ./.metrics/diff_${{ inputs.snapshot }}
+        if [ $? -eq 1 ]; then
+        echo "🛑 Differences found in metrics"
+        exit 1
+        fi
+
+    - name: Upload the diff artifact
+      if: ${{ (github.ref_name != 'main') && (steps.compare-snapshots.outcome == 'failure') }}
+      uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+      with:
+        name: diff_${{ inputs.artifact_key }}
+        path: ./.metrics/diff_${{ inputs.snapshot }}
+        retention-days: 7
+
+

.github/workflows/ci-e2e-all.yml

@@ -36,7 +36,3 @@ jobs:
   opensearch:
     uses: ./.github/workflows/ci-e2e-opensearch.yml
 
-
-
-
-

.github/workflows/ci-e2e-badger.yaml

@@ -25,7 +25,9 @@ jobs:
         egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
 
     - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
-
+      with:
+        fetch-depth: 0
+
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
         go-version: 1.23.x
@@ -41,6 +43,12 @@ jobs:
             ;;
         esac
 
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: matrix.version == 'v2'
+      with:
+        snapshot: badger_metrics.txt
+        artifact_key: badger-metrics-${{ matrix.version }}.txt
+
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:

.github/workflows/ci-e2e-cassandra.yml

@@ -38,6 +38,8 @@ jobs:
         egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
 
     - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      with:
+        fetch-depth: 0
 
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
@@ -48,9 +50,17 @@ jobs:
       run: bash scripts/e2e/cassandra.sh ${{ matrix.version.major }} ${{ matrix.version.schema }} ${{ matrix.jaeger-version }}
       env:
         SKIP_APPLY_SCHEMA: ${{ matrix.create-schema == 'auto' && true || false }}
+
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: matrix.jaeger-version == 'v2'
+      with:
+        snapshot: cassandra_metrics.txt
+        artifact_key: cassandra_metrics_${{ matrix.version.major }}_${{ matrix.version.schema }}_${{ matrix.jaeger-version }}_${{ matrix.create-schema }}.txt
 
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:
         files: cover.out
         flags: cassandra-${{ matrix.version.major }}-${{ matrix.jaeger-version }}-${{ matrix.create-schema }}
+
+

.github/workflows/ci-e2e-elasticsearch.yml

@@ -40,11 +40,8 @@ jobs:
     - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       with:
         submodules: true
-
-    - name: Fetch git tags
-      run: |
-        git fetch --prune --unshallow --tags
-
+        fetch-depth: 0
+
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
         go-version: 1.23.x
@@ -58,6 +55,12 @@ jobs:
     - name: Run ${{ matrix.version.distribution }} integration tests
       id: test-execution
       run: bash scripts/e2e/elasticsearch.sh ${{ matrix.version.distribution }} ${{ matrix.version.major }} ${{ matrix.version.jaeger }}
+
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: matrix.version.jaeger == 'v2'
+      with:
+        snapshot: elasticsearch_metrics.txt
+        artifact_key: elasticsearch-metrics-${{ matrix.version.major }}_${{ matrix.version.jaeger}}.txt
 
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov

.github/workflows/ci-e2e-grpc.yml

@@ -25,6 +25,8 @@ jobs:
         egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
 
     - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      with:
+        fetch-depth: 0
 
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
@@ -41,6 +43,12 @@ jobs:
             ;;
         esac
 
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: matrix.version == 'v2'
+      with:
+        snapshot: grpc_metrics.txt
+        artifact_key: grpc-metrics-${{ matrix.version }}.txt
+
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:

.github/workflows/ci-e2e-kafka.yml

@@ -27,6 +27,8 @@ jobs:
         egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
 
     - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      with:
+        fetch-depth: 0 
 
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
@@ -35,6 +37,12 @@ jobs:
     - name: Run kafka integration tests
       id: test-execution
       run: bash scripts/e2e/kafka.sh -j ${{ matrix.jaeger-version }} -v ${{ matrix.kafka-version }}
+
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: matrix.jaeger-version == 'v2'
+      with:
+        snapshot: kafka_metrics.txt
+        artifact_key: kafka-metrics-${{ matrix.jaeger-version }}.txt
 
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov

.github/workflows/ci-e2e-memory.yaml

@@ -21,15 +21,22 @@ jobs:
         egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
 
     - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
-
+      with:
+        fetch-depth: 0
+
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
         go-version: 1.23.x
 
     - name: Run Memory storage integration tests
       run: |
         STORAGE=memory_v2 make jaeger-v2-storage-integration-test
-
+
+    - uses: ./.github/actions/verify-metrics-snapshot
+      with:
+        snapshot: memory_metrics.txt
+        artifact_key: memory-metrics.txt
+
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:

.github/workflows/ci-e2e-opensearch.yml

@@ -37,10 +37,7 @@ jobs:
     - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       with:
         submodules: true
-
-    - name: Fetch git tags
-      run: |
-        git fetch --prune --unshallow --tags
+        fetch-depth: 0
 
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
@@ -52,6 +49,12 @@ jobs:
       id: test-execution
       run: bash scripts/e2e/elasticsearch.sh ${{ matrix.version.distribution }} ${{ matrix.version.major }} ${{ matrix.version.jaeger }}
 
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: matrix.version.jaeger == 'v2'
+      with:
+        snapshot: opensearch_metrics.txt
+        artifact_key: opensearch-metrics-${{ matrix.version.major }}.txt
+
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:

.gitignore

@@ -50,3 +50,4 @@ sha256sum.combined.txt
 resource.syso
 .gocache
 test-results.json
+.metrics/

cmd/jaeger/internal/integration/e2e_integration.go

@@ -99,11 +99,32 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) {
 	require.NoError(t, err)
 
 	t.Cleanup(func() {
+		scrapeMetrics(t, storage)
 		require.NoError(t, s.TraceReader.(io.Closer).Close())
 		require.NoError(t, s.TraceWriter.(io.Closer).Close())
 	})
 }
 
+func scrapeMetrics(t *testing.T, storage string) {
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://localhost:8888/metrics", nil)
+	require.NoError(t, err)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	outputDir := "../../../../.metrics"
+	require.NoError(t, os.MkdirAll(outputDir, os.ModePerm))
+
+	metricsFile, err := os.Create(fmt.Sprintf("%s/%v_metrics.txt", outputDir, storage))
+	require.NoError(t, err)
+	defer metricsFile.Close()
+
+	_, err = io.Copy(metricsFile, resp.Body)
+	require.NoError(t, err)
+}
+
 func createStorageCleanerConfig(t *testing.T, configFile string, storage string) string {
 	data, err := os.ReadFile(configFile)
 	require.NoError(t, err)

scripts/e2e/compare_metrics.py

@@ -0,0 +1,77 @@
+# Copyright (c) 2024 The Jaeger Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+from difflib import unified_diff
+from bisect import insort
+from prometheus_client.parser import text_string_to_metric_families
+
+def read_metric_file(file_path):
+    with open(file_path, 'r') as f:
+        return f.readlines()
+
+def parse_metrics(content):
+    metrics = []
+    for family in text_string_to_metric_families(content):
+        for sample in family.samples:
+            labels = dict(sample.labels)
+            #simply pop undesirable metric labels
+            labels.pop('service_instance_id',None)
+            label_pairs = sorted(labels.items(), key=lambda x: x[0])
+            label_str = ','.join(f'{k}="{v}"' for k,v in label_pairs)
+            metric = f"{family.name}{{{label_str}}}"
+            insort(metrics , metric)
+
+    return metrics
+
+
+def generate_diff(file1_content, file2_content):
+    if isinstance(file1_content, list):
+        file1_content = ''.join(file1_content)
+    if isinstance(file2_content, list):
+        file2_content = ''.join(file2_content)
+
+    metrics1 = parse_metrics(file1_content)
+    metrics2 = parse_metrics(file2_content)
+
+    diff = unified_diff(metrics1, metrics2,lineterm='',n=0)
+
+    return '\n'.join(diff)
+
+def write_diff_file(diff_lines, output_path):
+
+    with open(output_path, 'w') as f:
+        f.write(diff_lines)
+        f.write('\n')  # Add final newline
+        print(f"Diff file successfully written to: {output_path}")
+
+def main():
+    parser = argparse.ArgumentParser(description='Generate diff between two Jaeger metric files')
+    parser.add_argument('--file1', help='Path to first metric file')
+    parser.add_argument('--file2', help='Path to second metric file')
+    parser.add_argument('--output', '-o', default='metrics_diff.txt',
+                       help='Output diff file path (default: metrics_diff.txt)')
+
+    args = parser.parse_args()
+
+    # Read input files
+    file1_lines = read_metric_file(args.file1)
+    file2_lines = read_metric_file(args.file2)
+
+    # Generate diff
+    diff_lines = generate_diff(file1_lines, file2_lines)
+
+    # Check if there are any differences
+    if diff_lines:
+        print("differences found between the metric files.")
+        print("=== Metrics Comparison Results ===")
+        print(diff_lines)
+        write_diff_file(diff_lines, args.output)
+
+        return 1    
+
+    print("no difference found")
+    return 0
+
+if __name__ == '__main__':
+    main()