From 588aebdad44997981bf99599e823c00ed51d2f73 Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Sat, 5 Feb 2022 09:31:22 -0800 Subject: [PATCH] 1.4.12 (#714) * 1.4.12 * fixed limit/request test Co-authored-by: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> --- .github/workflows/ci_build_test.yaml | 6 ++++ PLUGIN_VERSIONS.sh | 2 +- ci_scripts/deploy_connector.sh | 5 +-- .../splunk-kubernetes-logging/values.yaml | 2 +- .../templates/daemonset.yaml | 1 + .../deploymentMetricsAggregator.yaml | 1 + .../splunk-connect-for-kubernetes/values.yaml | 2 +- .../splunk-kubernetes-logging/daemonset.yaml | 2 +- .../splunk-kubernetes-metrics/daemonset.yaml | 1 + .../deploymentMetricsAggregator.yaml | 1 + test/common.py | 34 ++++++++----------- test/conftest.py | 17 +++++++++- .../test_metric_aggr_plugin.py | 12 +++---- 13 files changed, 54 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci_build_test.yaml b/.github/workflows/ci_build_test.yaml index c85c7a72..270d5d48 100644 --- a/.github/workflows/ci_build_test.yaml +++ b/.github/workflows/ci_build_test.yaml @@ -131,6 +131,12 @@ jobs: run: | kubectl apply -f test/test_setup.yaml sleep 80 + echo "logging pod logs" + kubectl logs -l app=splunk-kubernetes-logging + echo "metrics pod logs" + kubectl logs -l app=splunk-kubernetes-metrics + echo "objects pod logs" + kubectl logs -l app=splunk-kubernetes-objects - uses: actions/setup-python@v2 with: diff --git a/PLUGIN_VERSIONS.sh b/PLUGIN_VERSIONS.sh index 44bd4a17..c8e399b5 100755 --- a/PLUGIN_VERSIONS.sh +++ b/PLUGIN_VERSIONS.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -FLUENTD_HEC_VERSION=1.2.9 +FLUENTD_HEC_VERSION=1.2.10 K8S_METRICS_VERISION=1.1.9 K8S_METRICS_AGGR_VERSION=1.1.9 KUBE_OBJECT_VERSION=1.1.9 diff --git a/ci_scripts/deploy_connector.sh b/ci_scripts/deploy_connector.sh index 2b6abc21..3967cfa1 100755 --- a/ci_scripts/deploy_connector.sh +++ b/ci_scripts/deploy_connector.sh @@ -14,6 +14,7 @@ helm install ci-sck --set global.splunk.hec.token=$CI_SPLUNK_HEC_TOKEN \ --set kubelet.serviceMonitor.https=true \ -f ci_scripts/sck_values.yml helm-artifacts/splunk-connect-for-kubernetes*.tgz #wait for deployment to finish -until kubectl get pod | grep Running | [[ $(wc -l) == 4 ]]; do - sleep 1; +# 2 logging, 2 metrics, 1 aggregator, 1 object +until kubectl get pod | grep Running | [[ $(wc -l) == 7 ]]; do + sleep 5; done \ No newline at end of file diff --git a/helm-chart/splunk-connect-for-kubernetes/charts/splunk-kubernetes-logging/values.yaml b/helm-chart/splunk-connect-for-kubernetes/charts/splunk-kubernetes-logging/values.yaml index 438f734e..07e743be 100644 --- a/helm-chart/splunk-connect-for-kubernetes/charts/splunk-kubernetes-logging/values.yaml +++ b/helm-chart/splunk-connect-for-kubernetes/charts/splunk-kubernetes-logging/values.yaml @@ -305,7 +305,7 @@ image: # The name of the image to pull name: splunk/fluentd-hec # The tag of the image to pull - tag: 1.2.9 + tag: 1.2.10 # The policy that specifies when the user wants the images to be pulled pullPolicy: IfNotPresent # Indicates if the image should be pulled using authentication from a secret diff --git a/helm-chart/splunk-connect-for-kubernetes/charts/splunk-kubernetes-metrics/templates/daemonset.yaml b/helm-chart/splunk-connect-for-kubernetes/charts/splunk-kubernetes-metrics/templates/daemonset.yaml index 28cbdab8..19847e4f 100644 --- a/helm-chart/splunk-connect-for-kubernetes/charts/splunk-kubernetes-metrics/templates/daemonset.yaml +++ b/helm-chart/splunk-connect-for-kubernetes/charts/splunk-kubernetes-metrics/templates/daemonset.yaml @@ -15,6 +15,7 @@ spec: selector: matchLabels: name: {{ template "splunk-kubernetes-metrics.fullname" . }} + component: collector template: metadata: name: {{ template "splunk-kubernetes-metrics.fullname" . }} diff --git a/helm-chart/splunk-connect-for-kubernetes/charts/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml b/helm-chart/splunk-connect-for-kubernetes/charts/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml index f7bd08a4..7cc2c52e 100644 --- a/helm-chart/splunk-connect-for-kubernetes/charts/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml +++ b/helm-chart/splunk-connect-for-kubernetes/charts/splunk-kubernetes-metrics/templates/deploymentMetricsAggregator.yaml @@ -16,6 +16,7 @@ spec: matchLabels: app: {{ template "splunk-kubernetes-metrics.name" . }} release: {{ .Release.Name }} + component: aggregator template: metadata: labels: diff --git a/helm-chart/splunk-connect-for-kubernetes/values.yaml b/helm-chart/splunk-connect-for-kubernetes/values.yaml index dadb3a06..0108d844 100644 --- a/helm-chart/splunk-connect-for-kubernetes/values.yaml +++ b/helm-chart/splunk-connect-for-kubernetes/values.yaml @@ -372,7 +372,7 @@ splunk-kubernetes-logging: # The name of the image to pull name: splunk/fluentd-hec # The tag of the image to pull - tag: 1.2.9 + tag: 1.2.10 # The policy that specifies when the user wants the images to be pulled pullPolicy: IfNotPresent # Indicates if the image should be pulled using authentication from a secret diff --git a/manifests/splunk-kubernetes-logging/daemonset.yaml b/manifests/splunk-kubernetes-logging/daemonset.yaml index 5c0cd418..04b3d891 100644 --- a/manifests/splunk-kubernetes-logging/daemonset.yaml +++ b/manifests/splunk-kubernetes-logging/daemonset.yaml @@ -28,7 +28,7 @@ spec: key: node-role.kubernetes.io/master containers: - name: splunk-fluentd-k8s-logs - image: docker.io/splunk/fluentd-hec:1.2.9 + image: docker.io/splunk/fluentd-hec:1.2.10 imagePullPolicy: IfNotPresent securityContext: runAsUser: 0 diff --git a/manifests/splunk-kubernetes-metrics/daemonset.yaml b/manifests/splunk-kubernetes-metrics/daemonset.yaml index 4a51c54d..f01fc31b 100644 --- a/manifests/splunk-kubernetes-metrics/daemonset.yaml +++ b/manifests/splunk-kubernetes-metrics/daemonset.yaml @@ -14,6 +14,7 @@ spec: selector: matchLabels: name: splunk-kubernetes-metrics + component: collector template: metadata: name: splunk-kubernetes-metrics diff --git a/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml b/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml index b7fff1a4..8213adfa 100644 --- a/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml +++ b/manifests/splunk-kubernetes-metrics/deploymentMetricsAggregator.yaml @@ -14,6 +14,7 @@ spec: selector: matchLabels: app: splunk-kubernetes-metrics + component: aggregator template: metadata: labels: diff --git a/test/common.py b/test/common.py index 210a9098..3e5a461b 100644 --- a/test/common.py +++ b/test/common.py @@ -36,15 +36,13 @@ logger.addHandler(handler) -def check_events_from_splunk( - index="circleci_events", - start_time="-1h@h", - end_time="now", - url="", - user="", - query="", - password="", -): +def check_events_from_splunk(index="ci_events", + start_time="-1h@h", + end_time="now", + url="", + user="", + query="", + password=""): """ send a search request to splunk and return the events from the result """ @@ -53,16 +51,14 @@ def check_events_from_splunk( return events - -def check_metrics_from_splunk( - index="circleci_metrics", - start_time="-1h@h", - end_time="now", - url="", - user="", - password="", - metric_name="", -): +def check_metrics_from_splunk(index="ci_metrics", + start_time="-1h@h", + end_time="now", + url="", + user="", + password="", + metric_name=""): + """ send a search api request to splunk to check for values associated with a given metric and dimension """ diff --git a/test/conftest.py b/test/conftest.py index 7e724304..c73c22e5 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -17,7 +17,7 @@ import pytest import time import os - +from .common import check_events_from_splunk def pytest_addoption(parser): parser.addoption( @@ -32,6 +32,21 @@ def pytest_addoption(parser): ) parser.addoption("--nodes-count", help="splunk username", default="1") +# Print events ingested in splunk for debugging +def pytest_unconfigure(config): + indexes = ["main", "ci_events", "ns-anno", "pod-anno"] + for index in indexes: + search_query = "index=" + index + " | fields *" + events = check_events_from_splunk(start_time="-1h@h", + url=config.getoption("--splunkd-url"), + user=config.getoption("--splunk-user"), + query=["search {0}".format( + search_query)], + password=config.getoption("--splunk-password")) + print("index=" + index + " event count=" + str(len(events))) + for event in events: + print(event) + @pytest.fixture(scope="function") def setup(request): diff --git a/test/k8s_metrics_tests/test_metric_aggr_plugin.py b/test/k8s_metrics_tests/test_metric_aggr_plugin.py index 5a501c72..df3984e0 100644 --- a/test/k8s_metrics_tests/test_metric_aggr_plugin.py +++ b/test/k8s_metrics_tests/test_metric_aggr_plugin.py @@ -68,7 +68,7 @@ def test_container_limit_and_requests(setup, index_metrics): url=setup["splunkd_url"], user=setup["splunk_user"], password=setup["splunk_password"], - func="avg", + func="latest", ) actual_data = container_limit_request[resource][metric] for container_name in containers: @@ -108,7 +108,7 @@ def test_pod_and_cluster_limit_and_requests(setup, index_metrics): url=setup["splunkd_url"], user=setup["splunk_user"], password=setup["splunk_password"], - func="avg", + func="latest", ) actual_data = container_limit_request[resource][metric] @@ -142,7 +142,7 @@ def test_pod_and_cluster_limit_and_requests(setup, index_metrics): url=setup["splunkd_url"], user=setup["splunk_user"], password=setup["splunk_password"], - func="avg", + func="latest", ) logger.info( "{} metric received by splunk: {}".format( @@ -167,7 +167,7 @@ def test_namespace_limit_and_requests(setup, index_metrics): url=setup["splunkd_url"], user=setup["splunk_user"], password=setup["splunk_password"], - func="avg", + func="latest", ) logger.info( "Splunk received {} events for metric {}".format( @@ -200,7 +200,7 @@ def test_multi_container_pod_limit_and_request(setup, index_metrics): url=setup["splunkd_url"], user=setup["splunk_user"], password=setup["splunk_password"], - func="avg", + func="latest", ) if not "pod-wo-index-wo-ns-index-dup" in container_events: pytest.fail("pod 'pod-wo-index-wo-ns-index-dup' not found" ) @@ -215,7 +215,7 @@ def test_multi_container_pod_limit_and_request(setup, index_metrics): url=setup["splunkd_url"], user=setup["splunk_user"], password=setup["splunk_password"], - func="avg", + func="latest", ) for pod_name, metric_value in pod_events.items():