From 058014dff0c2e6cab9dc02e110a8c44d4db6b122 Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Thu, 11 Jul 2024 10:05:05 +0200 Subject: [PATCH 1/5] Added support for persistent buffering in receiver/gateway --- .../templates/config/_otel-collector.tpl | 9 +++++ .../_otel-k8s-cluster-receiver-config.tpl | 8 ++++- .../deployment-cluster-receiver.yaml | 35 +++++++++++++++++++ .../templates/deployment-gateway.yaml | 28 +++++++++++++++ 4 files changed, 79 insertions(+), 1 deletion(-) diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl index 180c320470..3653bc36e5 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl @@ -12,6 +12,12 @@ extensions: endpoint: {{ include "splunk-otel-collector.o11yApiUrl" . }} {{- end }} + {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} + file_storage/persistent_queue_gateway: + directory: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/gateway + timeout: 0 + {{- end }} + zpages: @@ -142,6 +148,9 @@ service: {{- if (eq (include "splunk-otel-collector.splunkO11yEnabled" .) "true") }} - http_forwarder {{- end }} + {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} + - file_storage/persistent_queue_gateway + {{- end }} # The default pipelines should not need to be changed. You can add any custom pipeline instead. # In order to disable a default pipeline just set it to `null` in gateway.config overrides. diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl index 2bbd23e2a3..7141acd92b 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl @@ -7,7 +7,6 @@ The values can be overridden in .Values.clusterReceiver.config extensions: health_check: - {{- if eq (include "splunk-otel-collector.distribution" .) "eks/fargate" }} # k8s_observer w/ pod and node detection for eks/fargate deployment k8s_observer: @@ -16,6 +15,13 @@ extensions: observe_nodes: true {{- end }} + {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} + file_storage/persistent_queue_receiver: + directory: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver + timeout: 0 + {{- end }} + + receivers: # Prometheus receiver scraping metrics from the pod itself {{- include "splunk-otel-collector.prometheusInternalMetrics" "k8s-cluster-receiver" | nindent 2}} diff --git a/helm-charts/splunk-otel-collector/templates/deployment-cluster-receiver.yaml b/helm-charts/splunk-otel-collector/templates/deployment-cluster-receiver.yaml index d65f1c707b..b945a8d11f 100644 --- a/helm-charts/splunk-otel-collector/templates/deployment-cluster-receiver.yaml +++ b/helm-charts/splunk-otel-collector/templates/deployment-cluster-receiver.yaml @@ -104,6 +104,31 @@ spec: mountPath: /splunk-messages - mountPath: /conf name: collector-configmap + - name: patch-log-dirs + image: {{ template "splunk-otel-collector.image.initPatchLogDirs" . }} + imagePullPolicy: {{ .Values.image.initPatchLogDirs.pullPolicy }} + command: ['sh', '-c', ' + mkdir -p {{ .Values.logsCollection.checkpointPath }}; + chown -Rv {{ $clusterReceiver.securityContext.runAsUser | default 999 }}:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }} {{ .Values.logsCollection.checkpointPath }}; + chmod -v g+rwxs {{ .Values.logsCollection.checkpointPath }}; + + {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} + mkdir -p {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver; + chown -Rv {{ $clusterReceiver.securityContext.runAsUser | default 999 }}:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }} {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver; + chmod -v g+rwxs {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver; + setfacl -n -Rm d:m::rx,m::rx,d:g:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }}:rx,g:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }}:rx {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver; + {{- end }}'] + securityContext: + runAsUser: 0 + resources: + {{- toYaml $clusterReceiver.resources | nindent 12 }} + volumeMounts: + - name: checkpoint + mountPath: {{ .Values.logsCollection.checkpointPath }} + {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} + - name: persistent-queue + mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver + {{- end }} {{- end }} containers: - name: otel-collector @@ -189,6 +214,10 @@ spec: mountPath: /otel/etc readOnly: true {{- end }} + {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} + - name: persistent-queue + mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver + {{- end }} - mountPath: {{ .Values.isWindows | ternary "C:\\conf" "/conf" }} name: collector-configmap {{- if eq (include "splunk-otel-collector.distribution" .) "eks/fargate" }} @@ -217,6 +246,12 @@ spec: secret: secretName: {{ template "splunk-otel-collector.secret" . }} {{- end }} + {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} + - name: persistent-queue + hostPath: + path: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver + type: DirectoryOrCreate + {{- end }} {{- if eq (include "splunk-otel-collector.distribution" .) "eks/fargate" }} - name: init-eks-fargate-cluster-receiver-script configMap: diff --git a/helm-charts/splunk-otel-collector/templates/deployment-gateway.yaml b/helm-charts/splunk-otel-collector/templates/deployment-gateway.yaml index 86866587f4..78dde1a8e1 100644 --- a/helm-charts/splunk-otel-collector/templates/deployment-gateway.yaml +++ b/helm-charts/splunk-otel-collector/templates/deployment-gateway.yaml @@ -62,6 +62,24 @@ spec: securityContext: {{- include "splunk-otel-collector.securityContext" (dict "isWindows" .Values.isWindows "securityContext" $gateway.securityContext) | nindent 8 }} {{- end }} + initContainers: + - name: patch-log-dirs + image: {{ template "splunk-otel-collector.image.initPatchLogDirs" . }} + imagePullPolicy: {{ .Values.image.initPatchLogDirs.pullPolicy }} + command: ['sh', '-c', ' + {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} + mkdir -p {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/gateway; + chown -Rv {{ $gateway.securityContext.runAsUser | default 999 }}:{{ $gateway.securityContext.runAsGroup | default 999 }} {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/gateway; + chmod -v g+rwxs {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/gateway; + setfacl -n -Rm d:m::rx,m::rx,d:g:{{ $gateway.securityContext.runAsGroup | default 999 }}:rx,g:{{ $gateway.securityContext.runAsGroup | default 999 }}:rx {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/gateway; + {{- end }}'] + securityContext: + runAsUser: 0 + volumeMounts: + {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} + - name: persistent-queue + mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/gateway + {{- end }} containers: - name: otel-collector command: @@ -151,6 +169,10 @@ spec: {{- end }} - mountPath: {{ .Values.isWindows | ternary "C:\\conf" "/conf" }} name: collector-configmap + {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} + - name: persistent-queue + mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/gateway + {{- end }} {{- if $gateway.extraVolumeMounts }} {{- toYaml $gateway.extraVolumeMounts | nindent 8 }} {{- end }} @@ -167,6 +189,12 @@ spec: secret: secretName: {{ template "splunk-otel-collector.secret" . }} {{- end }} + {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} + - name: persistent-queue + hostPath: + path: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/gateway + type: DirectoryOrCreate + {{- end }} {{- if $gateway.extraVolumes }} {{- toYaml $gateway.extraVolumes | nindent 6 }} {{- end }} From 503912f7a09ac68f74ecb4e60c96e68c863afd79 Mon Sep 17 00:00:00 2001 From: Szymon Bylica Date: Mon, 15 Jul 2024 12:05:00 +0000 Subject: [PATCH 2/5] Fixed receiver tpl --- .../config/_otel-k8s-cluster-receiver-config.tpl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl index 7141acd92b..b77bfabbc7 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl @@ -211,11 +211,14 @@ service: telemetry: metrics: address: 0.0.0.0:8889 - {{- if eq (include "splunk-otel-collector.distribution" .) "eks/fargate" }} - extensions: [health_check, k8s_observer] - {{- else }} - extensions: [health_check] - {{- end }} + extensions: + - health_check + {{- if eq (include "splunk-otel-collector.distribution" .) "eks/fargate" }} + - k8s_observer + {{- end }} + {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} + - file_storage/persistent_queue_receiver + {{- end }} pipelines: {{- if or (eq (include "splunk-otel-collector.o11yMetricsEnabled" $) "true") (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }} # k8s metrics pipeline From 57a97b40629fc1f5d252141f1cc5088987d26820 Mon Sep 17 00:00:00 2001 From: Szymon Bylica <142112966+sbylica-splunk@users.noreply.github.com> Date: Thu, 18 Jul 2024 12:28:22 +0200 Subject: [PATCH 3/5] Update helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl Co-authored-by: jvoravong <47871238+jvoravong@users.noreply.github.com> --- .../templates/config/_otel-k8s-cluster-receiver-config.tpl | 1 - 1 file changed, 1 deletion(-) diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl index b77bfabbc7..871dac6435 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl @@ -21,7 +21,6 @@ extensions: timeout: 0 {{- end }} - receivers: # Prometheus receiver scraping metrics from the pod itself {{- include "splunk-otel-collector.prometheusInternalMetrics" "k8s-cluster-receiver" | nindent 2}} From 25b578e8dec19f75c70c7074fc1644ced03a9fd9 Mon Sep 17 00:00:00 2001 From: Szymon Bylica <142112966+sbylica-splunk@users.noreply.github.com> Date: Thu, 18 Jul 2024 12:28:34 +0200 Subject: [PATCH 4/5] Update helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl Co-authored-by: jvoravong <47871238+jvoravong@users.noreply.github.com> --- .../templates/config/_otel-k8s-cluster-receiver-config.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl index 871dac6435..0e02d94e38 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl @@ -17,7 +17,7 @@ extensions: {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} file_storage/persistent_queue_receiver: - directory: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver + directory: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/clusterReceiver timeout: 0 {{- end }} From 9a8b3f75431bab4bf39719b34c2a7d30abdf5e7b Mon Sep 17 00:00:00 2001 From: Szymon Bylica Date: Thu, 18 Jul 2024 10:31:46 +0000 Subject: [PATCH 5/5] changed dir to clusterReceiver --- .../templates/deployment-cluster-receiver.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/helm-charts/splunk-otel-collector/templates/deployment-cluster-receiver.yaml b/helm-charts/splunk-otel-collector/templates/deployment-cluster-receiver.yaml index b945a8d11f..d637608c83 100644 --- a/helm-charts/splunk-otel-collector/templates/deployment-cluster-receiver.yaml +++ b/helm-charts/splunk-otel-collector/templates/deployment-cluster-receiver.yaml @@ -113,10 +113,10 @@ spec: chmod -v g+rwxs {{ .Values.logsCollection.checkpointPath }}; {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} - mkdir -p {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver; - chown -Rv {{ $clusterReceiver.securityContext.runAsUser | default 999 }}:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }} {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver; - chmod -v g+rwxs {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver; - setfacl -n -Rm d:m::rx,m::rx,d:g:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }}:rx,g:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }}:rx {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver; + mkdir -p {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/clusterReceiver; + chown -Rv {{ $clusterReceiver.securityContext.runAsUser | default 999 }}:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }} {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/clusterReceiver; + chmod -v g+rwxs {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/clusterReceiver; + setfacl -n -Rm d:m::rx,m::rx,d:g:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }}:rx,g:{{ $clusterReceiver.securityContext.runAsGroup | default 999 }}:rx {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/clusterReceiver; {{- end }}'] securityContext: runAsUser: 0 @@ -127,7 +127,7 @@ spec: mountPath: {{ .Values.logsCollection.checkpointPath }} {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} - name: persistent-queue - mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver + mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/clusterReceiver {{- end }} {{- end }} containers: @@ -216,7 +216,7 @@ spec: {{- end }} {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} - name: persistent-queue - mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver + mountPath: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/clusterReceiver {{- end }} - mountPath: {{ .Values.isWindows | ternary "C:\\conf" "/conf" }} name: collector-configmap @@ -249,7 +249,7 @@ spec: {{- if .Values.splunkPlatform.sendingQueue.persistentQueue.enabled }} - name: persistent-queue hostPath: - path: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/receiver + path: {{ .Values.splunkPlatform.sendingQueue.persistentQueue.storagePath }}/clusterReceiver type: DirectoryOrCreate {{- end }} {{- if eq (include "splunk-otel-collector.distribution" .) "eks/fargate" }}