Add support for time based partitioning for compactor and store-gateway

charts/thanos/CHANGELOG.md

@@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Changed the default `configmap-reload` image variant to `cgr.dev/chainguard/configmap-reload:latest`.
 - Added support for specifying custom Kubernetes cluster domain
+- Add support for time based partitioning for [compactor](https://thanos.io/tip/operating/compactor-backlog.md/#scale-the-compactor:~:text=To%20scale%20the%20compactor%20horizontally%2C%20you%20can%20run%20multiple%20compactor%20instances%20with%20different%20min%2Dtime%20and%20max%2Dtime%20flags%20so%20that%20each%20compactor%20will%20only%20work%20on%20the%20blocks%20that%20are%20within%20the%20time%20range.) and [store gateway](https://thanos.io/tip/components/store.md/#time-based-partitioning) 
 
 ## [v1.14.5] - 2023-10-20
 

charts/thanos/README.md

@@ -75,6 +75,8 @@ The following table lists the configurable parameters of the _Thanos_ chart and
 | `compact.affinity`                                             | Affinity settings for compact pod assignment. If an explicit label selector is not provided for pod affinity or pod anti-affinity one will be created from the pod selector labels.          | `{}`                                       |
 | `compact.topologySpreadConstraints`                            | Topology spread constraints for compact pod assignment. If an explicit label selector is not provided one will be created from the pod selector labels.                                      | `[]`                                       |
 | `compact.tolerations`                                          | Toleration labels for compact pod assignment.                                                                                                                                                | `[]`                                       |
+| `compact.sharded.enabled`                                      | If `true`, enables sharding for compactor.                                              | `false`
+| `compact.sharded.timePartitioning`                             | Accepts an array of values for min and max time. min-time : Start of time range limit to compact. Thanos Compactor will compact only blocks, which happened later than this value. max-time : End of time range limit to compact. Thanos Compactor will compact only blocks, which happened earlier than this value. Option can be a constant time in RFC3339 format or time duration relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y. Example : "min-time": -30d -> Compaction covering 0 - 30 days
 | `query.serviceAccount.create`                                  | If `true`, create a new service account for the query pod.                                                                                                                                   | `true`                                     |
 | `query.serviceAccount.labels`                                  | Labels to add to the query service account.                                                                                                                                                  | `{}`                                       |
 | `query.serviceAccount.annotations`                             | Annotations to add to the query service account.                                                                                                                                             | `{}`                                       |
@@ -313,3 +315,5 @@ The following table lists the configurable parameters of the _Thanos_ chart and
 | `storeGateway.affinity`                                        | Affinity settings for store gateway pod assignment. If an explicit label selector is not provided for pod affinity or pod anti-affinity one will be created from the pod selector labels.    | `{}`                                       |
 | `storeGateway.topologySpreadConstraints`                       | Topology spread constraints for store gateway pod assignment. If an explicit label selector is not provided one will be created from the pod selector labels.                                | `[]`                                       |
 | `storeGateway.tolerations`                                     | Toleration labels for store gateway pod assignment.                                                                                                                                          | `[]`                                       |
+| `storeGateway.sharded.enabled`                                      | If `true`, enables sharding for compactor.                                              | `false`
+| `storeGateway.sharded.timePartitioning`                             | Accepts an array of values for min and max time. min-time : Start of time range limit to serve. Thanos Store will serve only metrics, which happened later than this value. max-time : End of time range limit to serve. Thanos Store will serve only blocks, which happened earlier than this value. Option can be a constant time in RFC3339 format or time duration relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y. Example : "min-time": -30d -> Store Gateway covering 0 - 30 days

charts/thanos/templates/compact/statefulset-shards.yaml

@@ -0,0 +1,174 @@
+{{- if and .Values.compact.enabled .Values.compact.sharded.enabled -}}
+{{- $shards := int 0 }}
+{{- $shards = len .Values.compact.sharded.timePartitioning }}
+{{- range $index, $_ := until $shards }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ printf "%s-%s" (include "thanos.compact.fullname" $) (toString $index) }}
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    {{- include "thanos.compact.labels" $ | nindent 4 }}
+spec:
+  serviceName: {{ printf "%s-headless" (include "thanos.compact.fullname" $) }}
+  replicas: 1
+  {{- with $.Values.compact.updateStrategy }}
+  updateStrategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "thanos.compact.selectorLabels" $ | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "thanos.compact.selectorLabels" $ | nindent 8 }}
+      {{- with $.Values.compact.podLabels }}
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- if or $.Values.objstoreConfig.create (gt (len $.Values.compact.podAnnotations) 0) }}
+      annotations:
+      {{- if $.Values.objstoreConfig.create }}
+        checksum/config: {{ include "thanos.objstoreConfigHash" $ }}
+      {{- end }}
+      {{- with $.Values.compact.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- end }}
+    spec:
+      {{- with $.Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "thanos.compact.serviceAccountName" $ }}
+      {{- with $.Values.compact.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with $.Values.compact.priorityClassName }}
+      priorityClassName: {{ . | quote }}
+      {{- end }}
+      {{- with $.Values.compact.terminationGracePeriodSeconds }}
+      terminationGracePeriodSeconds: {{ . }}
+      {{- end }}
+      containers:
+        - name: thanos-compact
+          {{- with $.Values.compact.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          image: {{ include "thanos.image" $ }}
+          imagePullPolicy: {{ $.Values.image.pullPolicy }}
+          env:
+            - name: NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: HOST_IP_ADDRESS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+          {{- with $.Values.compact.extraEnv }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          args:
+            - compact
+            - --wait
+            - --log.level={{ $.Values.logLevel }}
+            - --log.format={{ $.Values.logFormat }}
+            - --http-address=0.0.0.0:10902
+            - --data-dir=/var/thanos/compact
+            - --objstore.config-file=/etc/thanos/objstore.yaml
+          {{- range (concat $.Values.additionalReplicaLabels (ternary (list "rule_replica") (list) $.Values.rule.enabled ) (ternary (list "receive_replica") (list) $.Values.receive.enabled )) | uniq }}
+            - --deduplication.replica-label={{ . }}
+          {{- end }}
+            {{- $partion := (slice $.Values.compact.sharded.timePartitioning  $index) | first }}
+            {{- if $partion.max }}
+            - --max-time={{ $partion.max }}
+            {{- end }}
+            {{- if $partion.min }}
+            - --min-time={{ $partion.min }}
+            {{- end }}
+          {{- if $.Values.compact.replicaDeduplication }}
+            - --deduplication.func=penalty
+          {{- end }}
+          {{- with $.Values.compact.extraArgs }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          ports:
+            - name: http
+              protocol: TCP
+              containerPort: 10902
+          livenessProbe:
+            {{- toYaml $.Values.compact.livenessProbe | nindent 12 }}
+          readinessProbe:
+            {{- toYaml $.Values.compact.readinessProbe | nindent 12 }}
+          volumeMounts:
+            - mountPath: /etc/thanos/objstore.yaml
+              name: objstore-config
+              subPath: {{ $.Values.objstoreConfig.key }}
+              readOnly: true
+            - mountPath: /var/thanos/compact
+              name: data
+              readOnly: false
+          {{- with $.Values.compact.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      volumes:
+        - name: objstore-config
+          secret:
+            secretName: {{ include "thanos.objstoreConfigSecretName" $ }}
+      {{- if not $.Values.compact.persistence.enabled }}
+        - name: data
+          emptyDir: {}
+      {{- end }}
+      {{- with $.Values.compact.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with $.Values.compact.affinity }}
+      {{- $_ := include "thanos.compact.patchAffinity" $ }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with $.Values.compact.topologySpreadConstraints }}
+      {{- $_ := include "thanos.compact.patchTopologySpreadConstraints" $ }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with $.Values.compact.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+  {{- if $.Values.compact.persistence.enabled }}
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+        labels:
+          {{- include "thanos.compact.selectorLabels" $ | nindent 10 }}
+        {{- with $.Values.compact.persistence.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+      spec:
+      {{- with $.Values.compact.persistence.storageClass }}
+      {{- if (eq "-" .) }}
+        storageClassName: ""
+      {{- else }}
+        storageClassName: {{ . }}
+      {{- end }}
+      {{- end }}
+        accessModes:
+          - {{ $.Values.compact.persistence.accessMode }}
+        resources:
+          requests:
+            storage: {{ $.Values.compact.persistence.size }}
+  {{- if semverCompare ">= 1.27-0" .Capabilities.KubeVersion.Version }}
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: {{ ternary "Retain" "Delete" $.Values.compact.persistence.retainDeleted }}
+    whenScaled: {{ ternary "Retain" "Delete" $.Values.compact.persistence.retainScaled }}
+  {{- end }}
+  {{- end }}
+{{- end -}}
+{{- end }}

charts/thanos/templates/compact/statefulset.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.compact.enabled -}}
+{{- if and .Values.compact.enabled (not .Values.compact.sharded.enabled) -}}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:

charts/thanos/templates/query/deployment.yaml

@@ -68,7 +68,16 @@ spec:
           {{- range (concat .Values.additionalReplicaLabels .Values.query.replicaLabels (ternary (list "rule_replica") (list) .Values.rule.enabled ) (ternary (list "receive_replica") (list) .Values.receive.enabled )) | uniq }}
             - --query.replica-label={{ . }}
           {{- end }}
+          {{- if (not .Values.storeGateway.sharded.enabled) -}}
             - {{ printf "--endpoint=dnssrv+_grpc._tcp.%s-headless.%s.svc.%s" (include "thanos.storeGateway.fullname" .) .Release.Namespace .Values.clusterDomain}}
+          {{- end }}
+          {{- if .Values.storeGateway.sharded.enabled -}}
+          {{- $shards := int 0 }}
+          {{- $shards = len .Values.storeGateway.sharded.timePartitioning }}
+          {{- range $index, $_ := until $shards }}
+            - {{ printf "--endpoint=dnssrv+_grpc._tcp.%s-headless-%s.%s.svc.%s" (include "thanos.storeGateway.fullname" $) (toString $index) $.Release.Namespace $.Values.clusterDomain }}
+          {{- end }}
+          {{- end }}
           {{- if .Values.rule.enabled }}
             - {{ printf "--endpoint=dnssrv+_grpc._tcp.%s-headless.%s.svc.%s" (include "thanos.rule.fullname" .) .Release.Namespace .Values.clusterDomain}}
           {{- end }}

charts/thanos/templates/store-gateway/service-headless-shards.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.storeGateway.sharded.enabled -}}
+{{- $shards := int 0 }}
+{{- $shards = len .Values.storeGateway.sharded.timePartitioning }}
+{{- range $index, $_ := until $shards }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ printf "%s-headless-%s" (include "thanos.storeGateway.fullname" $) (toString $index)}}
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    {{- include "thanos.labels" $ | nindent 4 }}
+  {{- with $.Values.storeGateway.service.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  type: ClusterIP
+  clusterIP: None
+  ports:
+    - name: grpc
+      port: 10901
+      targetPort: grpc
+    - name: http
+      protocol: TCP
+      port: 10902
+      targetPort: http
+  selector:
+    {{- include "thanos.storeGateway.selectorLabels" $ | nindent 4 }}
+    app.kubernetes.io/component: {{ printf "%s-%s" (include "thanos.storeGateway.fullname" $) (toString $index) }}
+---
+{{- end }}
+{{- end }}

charts/thanos/templates/store-gateway/service-headless.yaml

@@ -1,3 +1,4 @@
+{{- if (not .Values.storeGateway.sharded.enabled) -}}
 apiVersion: v1
 kind: Service
 metadata:
@@ -22,3 +23,4 @@ spec:
       targetPort: http
   selector:
     {{- include "thanos.storeGateway.selectorLabels" . | nindent 4 }}
+{{- end }}