diff --git a/charts/thanos/CHANGELOG.md b/charts/thanos/CHANGELOG.md index 53e546ec..81009837 100644 --- a/charts/thanos/CHANGELOG.md +++ b/charts/thanos/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Changed the default `configmap-reload` image variant to `cgr.dev/chainguard/configmap-reload:latest`. - Added support for specifying custom Kubernetes cluster domain +- Add support for time based partitioning for [compactor](https://thanos.io/tip/operating/compactor-backlog.md/#scale-the-compactor:~:text=To%20scale%20the%20compactor%20horizontally%2C%20you%20can%20run%20multiple%20compactor%20instances%20with%20different%20min%2Dtime%20and%20max%2Dtime%20flags%20so%20that%20each%20compactor%20will%20only%20work%20on%20the%20blocks%20that%20are%20within%20the%20time%20range.) and [store gateway](https://thanos.io/tip/components/store.md/#time-based-partitioning) ## [v1.14.5] - 2023-10-20 diff --git a/charts/thanos/README.md b/charts/thanos/README.md index 8d7db93a..258bece9 100644 --- a/charts/thanos/README.md +++ b/charts/thanos/README.md @@ -75,6 +75,8 @@ The following table lists the configurable parameters of the _Thanos_ chart and | `compact.affinity` | Affinity settings for compact pod assignment. If an explicit label selector is not provided for pod affinity or pod anti-affinity one will be created from the pod selector labels. | `{}` | | `compact.topologySpreadConstraints` | Topology spread constraints for compact pod assignment. If an explicit label selector is not provided one will be created from the pod selector labels. | `[]` | | `compact.tolerations` | Toleration labels for compact pod assignment. | `[]` | +| `compact.sharded.enabled` | If `true`, enables sharding for compactor. | `false` +| `compact.sharded.timePartitioning` | Accepts an array of values for min and max time. min-time : Start of time range limit to compact. Thanos Compactor will compact only blocks, which happened later than this value. max-time : End of time range limit to compact. Thanos Compactor will compact only blocks, which happened earlier than this value. Option can be a constant time in RFC3339 format or time duration relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y. Example : "min-time": -30d -> Compaction covering 0 - 30 days | `query.serviceAccount.create` | If `true`, create a new service account for the query pod. | `true` | | `query.serviceAccount.labels` | Labels to add to the query service account. | `{}` | | `query.serviceAccount.annotations` | Annotations to add to the query service account. | `{}` | @@ -313,3 +315,5 @@ The following table lists the configurable parameters of the _Thanos_ chart and | `storeGateway.affinity` | Affinity settings for store gateway pod assignment. If an explicit label selector is not provided for pod affinity or pod anti-affinity one will be created from the pod selector labels. | `{}` | | `storeGateway.topologySpreadConstraints` | Topology spread constraints for store gateway pod assignment. If an explicit label selector is not provided one will be created from the pod selector labels. | `[]` | | `storeGateway.tolerations` | Toleration labels for store gateway pod assignment. | `[]` | +| `storeGateway.sharded.enabled` | If `true`, enables sharding for compactor. | `false` +| `storeGateway.sharded.timePartitioning` | Accepts an array of values for min and max time. min-time : Start of time range limit to serve. Thanos Store will serve only metrics, which happened later than this value. max-time : End of time range limit to serve. Thanos Store will serve only blocks, which happened earlier than this value. Option can be a constant time in RFC3339 format or time duration relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y. Example : "min-time": -30d -> Store Gateway covering 0 - 30 days \ No newline at end of file diff --git a/charts/thanos/templates/compact/statefulset-shards.yaml b/charts/thanos/templates/compact/statefulset-shards.yaml new file mode 100644 index 00000000..170ea909 --- /dev/null +++ b/charts/thanos/templates/compact/statefulset-shards.yaml @@ -0,0 +1,174 @@ +{{- if and .Values.compact.enabled .Values.compact.sharded.enabled -}} +{{- $shards := int 0 }} +{{- $shards = len .Values.compact.sharded.timePartitioning }} +{{- range $index, $_ := until $shards }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ printf "%s-%s" (include "thanos.compact.fullname" $) (toString $index) }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "thanos.compact.labels" $ | nindent 4 }} +spec: + serviceName: {{ printf "%s-headless" (include "thanos.compact.fullname" $) }} + replicas: 1 + {{- with $.Values.compact.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "thanos.compact.selectorLabels" $ | nindent 6 }} + template: + metadata: + labels: + {{- include "thanos.compact.selectorLabels" $ | nindent 8 }} + {{- with $.Values.compact.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if or $.Values.objstoreConfig.create (gt (len $.Values.compact.podAnnotations) 0) }} + annotations: + {{- if $.Values.objstoreConfig.create }} + checksum/config: {{ include "thanos.objstoreConfigHash" $ }} + {{- end }} + {{- with $.Values.compact.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + spec: + {{- with $.Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "thanos.compact.serviceAccountName" $ }} + {{- with $.Values.compact.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.compact.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} + {{- with $.Values.compact.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + containers: + - name: thanos-compact + {{- with $.Values.compact.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: {{ include "thanos.image" $ }} + imagePullPolicy: {{ $.Values.image.pullPolicy }} + env: + - name: NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: HOST_IP_ADDRESS + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- with $.Values.compact.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + args: + - compact + - --wait + - --log.level={{ $.Values.logLevel }} + - --log.format={{ $.Values.logFormat }} + - --http-address=0.0.0.0:10902 + - --data-dir=/var/thanos/compact + - --objstore.config-file=/etc/thanos/objstore.yaml + {{- range (concat $.Values.additionalReplicaLabels (ternary (list "rule_replica") (list) $.Values.rule.enabled ) (ternary (list "receive_replica") (list) $.Values.receive.enabled )) | uniq }} + - --deduplication.replica-label={{ . }} + {{- end }} + {{- $partion := (slice $.Values.compact.sharded.timePartitioning $index) | first }} + {{- if $partion.max }} + - --max-time={{ $partion.max }} + {{- end }} + {{- if $partion.min }} + - --min-time={{ $partion.min }} + {{- end }} + {{- if $.Values.compact.replicaDeduplication }} + - --deduplication.func=penalty + {{- end }} + {{- with $.Values.compact.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + protocol: TCP + containerPort: 10902 + livenessProbe: + {{- toYaml $.Values.compact.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml $.Values.compact.readinessProbe | nindent 12 }} + volumeMounts: + - mountPath: /etc/thanos/objstore.yaml + name: objstore-config + subPath: {{ $.Values.objstoreConfig.key }} + readOnly: true + - mountPath: /var/thanos/compact + name: data + readOnly: false + {{- with $.Values.compact.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: objstore-config + secret: + secretName: {{ include "thanos.objstoreConfigSecretName" $ }} + {{- if not $.Values.compact.persistence.enabled }} + - name: data + emptyDir: {} + {{- end }} + {{- with $.Values.compact.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.compact.affinity }} + {{- $_ := include "thanos.compact.patchAffinity" $ }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.compact.topologySpreadConstraints }} + {{- $_ := include "thanos.compact.patchTopologySpreadConstraints" $ }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.compact.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if $.Values.compact.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: data + labels: + {{- include "thanos.compact.selectorLabels" $ | nindent 10 }} + {{- with $.Values.compact.persistence.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} + spec: + {{- with $.Values.compact.persistence.storageClass }} + {{- if (eq "-" .) }} + storageClassName: "" + {{- else }} + storageClassName: {{ . }} + {{- end }} + {{- end }} + accessModes: + - {{ $.Values.compact.persistence.accessMode }} + resources: + requests: + storage: {{ $.Values.compact.persistence.size }} + {{- if semverCompare ">= 1.27-0" .Capabilities.KubeVersion.Version }} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ ternary "Retain" "Delete" $.Values.compact.persistence.retainDeleted }} + whenScaled: {{ ternary "Retain" "Delete" $.Values.compact.persistence.retainScaled }} + {{- end }} + {{- end }} +{{- end -}} +{{- end }} diff --git a/charts/thanos/templates/compact/statefulset.yaml b/charts/thanos/templates/compact/statefulset.yaml index d9112dda..16fd38a1 100644 --- a/charts/thanos/templates/compact/statefulset.yaml +++ b/charts/thanos/templates/compact/statefulset.yaml @@ -1,4 +1,4 @@ -{{- if .Values.compact.enabled -}} +{{- if and .Values.compact.enabled (not .Values.compact.sharded.enabled) -}} apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/charts/thanos/templates/query/deployment.yaml b/charts/thanos/templates/query/deployment.yaml index 164556fd..9c7742bd 100644 --- a/charts/thanos/templates/query/deployment.yaml +++ b/charts/thanos/templates/query/deployment.yaml @@ -68,7 +68,16 @@ spec: {{- range (concat .Values.additionalReplicaLabels .Values.query.replicaLabels (ternary (list "rule_replica") (list) .Values.rule.enabled ) (ternary (list "receive_replica") (list) .Values.receive.enabled )) | uniq }} - --query.replica-label={{ . }} {{- end }} + {{- if (not .Values.storeGateway.sharded.enabled) -}} - {{ printf "--endpoint=dnssrv+_grpc._tcp.%s-headless.%s.svc.%s" (include "thanos.storeGateway.fullname" .) .Release.Namespace .Values.clusterDomain}} + {{- end }} + {{- if .Values.storeGateway.sharded.enabled -}} + {{- $shards := int 0 }} + {{- $shards = len .Values.storeGateway.sharded.timePartitioning }} + {{- range $index, $_ := until $shards }} + - {{ printf "--endpoint=dnssrv+_grpc._tcp.%s-headless-%s.%s.svc.%s" (include "thanos.storeGateway.fullname" $) (toString $index) $.Release.Namespace $.Values.clusterDomain }} + {{- end }} + {{- end }} {{- if .Values.rule.enabled }} - {{ printf "--endpoint=dnssrv+_grpc._tcp.%s-headless.%s.svc.%s" (include "thanos.rule.fullname" .) .Release.Namespace .Values.clusterDomain}} {{- end }} diff --git a/charts/thanos/templates/store-gateway/service-headless-shards.yaml b/charts/thanos/templates/store-gateway/service-headless-shards.yaml new file mode 100644 index 00000000..c1944cf4 --- /dev/null +++ b/charts/thanos/templates/store-gateway/service-headless-shards.yaml @@ -0,0 +1,32 @@ +{{- if .Values.storeGateway.sharded.enabled -}} +{{- $shards := int 0 }} +{{- $shards = len .Values.storeGateway.sharded.timePartitioning }} +{{- range $index, $_ := until $shards }} +apiVersion: v1 +kind: Service +metadata: + name: {{ printf "%s-headless-%s" (include "thanos.storeGateway.fullname" $) (toString $index)}} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "thanos.labels" $ | nindent 4 }} + {{- with $.Values.storeGateway.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: grpc + port: 10901 + targetPort: grpc + - name: http + protocol: TCP + port: 10902 + targetPort: http + selector: + {{- include "thanos.storeGateway.selectorLabels" $ | nindent 4 }} + app.kubernetes.io/component: {{ printf "%s-%s" (include "thanos.storeGateway.fullname" $) (toString $index) }} +--- +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/thanos/templates/store-gateway/service-headless.yaml b/charts/thanos/templates/store-gateway/service-headless.yaml index 7e849b02..30ee1771 100644 --- a/charts/thanos/templates/store-gateway/service-headless.yaml +++ b/charts/thanos/templates/store-gateway/service-headless.yaml @@ -1,3 +1,4 @@ +{{- if (not .Values.storeGateway.sharded.enabled) -}} apiVersion: v1 kind: Service metadata: @@ -22,3 +23,4 @@ spec: targetPort: http selector: {{- include "thanos.storeGateway.selectorLabels" . | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/charts/thanos/templates/store-gateway/statefulset-shards.yaml b/charts/thanos/templates/store-gateway/statefulset-shards.yaml new file mode 100644 index 00000000..8dc4211d --- /dev/null +++ b/charts/thanos/templates/store-gateway/statefulset-shards.yaml @@ -0,0 +1,175 @@ +{{- if .Values.storeGateway.sharded.enabled -}} +{{- $shards := int 0 }} +{{- $shards = len .Values.storeGateway.sharded.timePartitioning }} +{{- range $index, $_ := until $shards }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ printf "%s-%s" (include "thanos.storeGateway.fullname" $) (toString $index) }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "thanos.labels" $ | nindent 4 }} + app.kubernetes.io/component: {{ printf "%s-%s" (include "thanos.storeGateway.fullname" $) (toString $index) }} +spec: + serviceName: {{ printf "%s-headless-%s" (include "thanos.storeGateway.fullname" $) (toString $index) }} + replicas: {{ $.Values.storeGateway.replicas }} + {{- with $.Values.storeGateway.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "thanos.selectorLabels" $ | nindent 6 }} + app.kubernetes.io/component: {{ printf "%s-%s" (include "thanos.storeGateway.fullname" $) (toString $index) }} + template: + metadata: + labels: + {{- include "thanos.storeGateway.selectorLabels" $ | nindent 8 }} + app.kubernetes.io/component: {{ printf "%s-%s" (include "thanos.storeGateway.fullname" $) (toString $index) }} + {{- with $.Values.storeGateway.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if or $.Values.objstoreConfig.create (gt (len $.Values.storeGateway.podAnnotations) 0) }} + annotations: + {{- if $.Values.objstoreConfig.create }} + checksum/config: {{ include "thanos.objstoreConfigHash" $ }} + {{- end }} + {{- with $.Values.storeGateway.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + spec: + {{- with $.Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "thanos.storeGateway.serviceAccountName" $ }} + {{- with $.Values.storeGateway.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.storeGateway.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} + {{- with $.Values.storeGateway.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + containers: + - name: thanos-store-gateway + {{- with $.Values.storeGateway.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: {{ include "thanos.image" $ }} + imagePullPolicy: {{ $.Values.image.pullPolicy }} + env: + - name: NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: HOST_IP_ADDRESS + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- with $.Values.storeGateway.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + args: + - store + - --log.level={{ $.Values.logLevel }} + - --log.format={{ $.Values.logFormat }} + - --grpc-address=0.0.0.0:10901 + - --http-address=0.0.0.0:10902 + - --data-dir=/var/thanos/store + - --objstore.config-file=/etc/thanos/objstore.yaml + {{- with $.Values.storeGateway.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- $partion := (slice $.Values.storeGateway.sharded.timePartitioning $index) | first }} + {{- if $partion.max }} + - --max-time={{ $partion.max }} + {{- end }} + {{- if $partion.min }} + - --min-time={{ $partion.min }} + {{- end }} + ports: + - name: grpc + protocol: TCP + containerPort: 10901 + - name: http + protocol: TCP + containerPort: 10902 + livenessProbe: + {{- toYaml $.Values.storeGateway.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml $.Values.storeGateway.readinessProbe | nindent 12 }} + volumeMounts: + - mountPath: /etc/thanos/objstore.yaml + name: objstore-config + subPath: {{ $.Values.objstoreConfig.key }} + readOnly: true + - mountPath: /var/thanos/store + name: data + readOnly: false + {{- with $.Values.storeGateway.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: objstore-config + secret: + secretName: {{ include "thanos.objstoreConfigSecretName" $ }} + {{- if not $.Values.storeGateway.persistence.enabled }} + - name: data + emptyDir: {} + {{- end }} + {{- with $.Values.storeGateway.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.storeGateway.affinity }} + {{- $_ := include "thanos.storeGateway.patchAffinity" $ }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.storeGateway.topologySpreadConstraints }} + {{- $_ := include "thanos.storeGateway.patchTopologySpreadConstraints" $ }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.storeGateway.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if $.Values.storeGateway.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: data + labels: + {{- include "thanos.storeGateway.selectorLabels" $ | nindent 10 }} + {{- with $.Values.storeGateway.persistence.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} + spec: + {{- with $.Values.storeGateway.persistence.storageClass }} + {{- if (eq "-" .) }} + storageClassName: "" + {{- else }} + storageClassName: {{ . }} + {{- end }} + {{- end }} + accessModes: + - {{ $.Values.storeGateway.persistence.accessMode }} + resources: + requests: + storage: {{ $.Values.storeGateway.persistence.size }} + {{- if semverCompare ">= 1.27-0" .Capabilities.KubeVersion.Version }} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ ternary "Retain" "Delete" $.Values.storeGateway.persistence.retainDeleted }} + whenScaled: {{ ternary "Retain" "Delete" $.Values.storeGateway.persistence.retainScaled }} + {{- end }} + {{- end }} +--- +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/thanos/templates/store-gateway/statefulset.yaml b/charts/thanos/templates/store-gateway/statefulset.yaml index b02d4d43..513b02fb 100644 --- a/charts/thanos/templates/store-gateway/statefulset.yaml +++ b/charts/thanos/templates/store-gateway/statefulset.yaml @@ -1,3 +1,4 @@ +{{- if (not .Values.storeGateway.sharded.enabled) -}} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -156,3 +157,4 @@ spec: whenScaled: {{ ternary "Retain" "Delete" .Values.storeGateway.persistence.retainScaled }} {{- end }} {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/thanos/values.yaml b/charts/thanos/values.yaml index 0d147621..85a1ddb3 100644 --- a/charts/thanos/values.yaml +++ b/charts/thanos/values.yaml @@ -42,6 +42,12 @@ clusterDomain: cluster.local compact: enabled: false + sharded: + enabled: false + timePartitioning: + - min: "" + max: "" + serviceAccount: create: true labels: {} @@ -601,6 +607,12 @@ storeGateway: name: "" automountToken: false + sharded: + enabled: false + timePartitioning: + - min: "" + max: "" + service: annotations: {}