chore: added storage_proxy option

Added the api_gateway.storage_proxy option that is going to replace api_gateway.minio specific configuration. This is a generic storage gateway service, that could work for both minio and Amazon S3 Ticket: MC-6856 Signed-off-by: Roberto Giovanardi <[email protected]>
mendersoftware · Jan 30, 2024 · 8f2d37b · 8f2d37b
1 parent 3cfbfdb
commit 8f2d37b
Show file tree

Hide file tree

Showing 5 changed files with 74 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -301,8 +301,11 @@ The following table lists the parameters for the `api-gateway` component and the
 | `api_gateway.service.httpNodePort` | Node port for the HTTP service | `nil` |
 | `api_gateway.service.httpsNodePort` | Node port for the HTTPS service | `nil` |
 | `api_gateway.env.SSL` | SSL termination flag | `true` |
-| `api_gateway.minio.enabled` | Enable routing of S3 requests to the minio service | `true` |
-| `api_gateway.minio.url` | URL of the minio service | `http://minio:9000` |
+| `api_gateway.minio.enabled` | Enable routing of S3 requests to the minio service. **[Deprecated from 5.5.0, use `api_gateway.storage_proxy`` instead]** | `true` |
+| `api_gateway.minio.url` | URL of the minio service. **[Deprecated from 5.5.0, use `api_gateway.storage_proxy`` instead]** | `http://minio:9000` |
+| `api_gateway.storage_proxy.enabled` | Enable storage proxy to the S3/minio service | `false` |
+| `api_gateway.storage_proxy.url` | URL of the storage proxy. Should point to the AWS Bucket/Minio URL | `nil` |
+| `api_gateway.storage_proxy.customRule` | Custom rules for the storage proxy. | ``HostRegexp(`{domain:^artifacts.*$}`)`` |
 | `api_gateway.rateLimit.average` | See the [Traefik rate limit configuration options](https://doc.traefik.io/traefik/v2.6/middlewares/http/ratelimit/#configuration-options) | `100` |
 | `api_gateway.rateLimit.burst` | See the [Traefik rate limit configuration options](https://doc.traefik.io/traefik/v2.6/middlewares/http/ratelimit/#configuration-options) | `100` |
 | `api_gateway.rateLimit.period` | See the [Traefik rate limit configuration options](https://doc.traefik.io/traefik/v2.6/middlewares/http/ratelimit/#configuration-options) | `1s` |
@@ -317,7 +320,7 @@ The following table lists the parameters for the `api-gateway` component and the
 | `api_gateway.containerSecurityContext.runAsUser` | User ID for the container | `65534` |
 | `api_gateway.compression` | Enable Traefik Compression | `true` |
 | `api_gateway.security_redirect` | Custom redirect to a company security page | `null` |
-| `api_gateway.minio.customRule` | Custom redirect for MinIO. Uses the default one if not specified | `null` |
+| `api_gateway.minio.customRule` | Custom redirect for MinIO. Uses the default one if not specified. **[Deprecated from 5.5.0, use `api_gateway.storage_proxy`` instead]** | `null` |
 | `api_gateway.hpa` | HorizontalPodAutoscaler support | `nil` |
 | `api_gateway.hpa.enabled` | HorizontalPodAutoscaler enabled | `nil` |
 | `api_gateway.hpa.minReplicas` | HorizontalPodAutoscaler minReplicas | `nil` |

diff --git a/mender/CHANGELOG.md b/mender/CHANGELOG.md
@@ -1,8 +1,20 @@
 # Mender Helm chart
 
-## master
+## 5.5.0
 * Fix mongodb uri creation when using the mongodb subchart and replicast architecture
 * Added customEnv option to set default or per-service custom env variables
+* Added generic `storage_proxy` service, that could
+  work for both minio and Amazon S3, and it's going to replace the `api_gateway.minio` configuration.
+* **Deprecations**:
+  * `api_gateway.minio` is deprecated in favor of `api_gateway.storage_proxy`.
+    This entry could be used, but it is no longer maintained, and could be removed
+    in future releases.  
+    **How to upgrade**: 
+    * set `api_gateway.minio.enabled=false`
+    * set `api_gateway.storage_proxy.enabled=true`
+    * set `api_gateway.storage_proxy.url` to the external storage url that you want to map externally. For example `https://fleetstorage.example.com`.
+      If you leave it empty, it uses the Amazon S3 external URL.
+
 
 ## Version 5.4.1
 * Upgrade to Mender version `3.7.1`.

diff --git a/mender/templates/_helpers.tpl b/mender/templates/_helpers.tpl
@@ -290,3 +290,29 @@ Synopsis:
 {{- println "" }}
 {{- end }}
 {{- end -}}
+
+{{/*
+Define mender.storageProxyUrl
+*/}}
+{{- define "mender.storageProxyUrl" -}}
+{{- $dot := (ternary . .dot (empty .dot)) -}}
+{{- with $dot.Values.api_gateway.storage_proxy }}
+  {{- if .url }}
+    {{- printf "%s" .url }}
+  {{- else if eq $dot.Values.global.storage "aws" }}
+    {{- printf "https://%s.s3.%s.amazonaws.com" $dot.Values.global.s3.AWS_BUCKET $dot.Values.global.s3.AWS_REGION}}
+  {{- else }}
+    {{- required "A valid storage proxy URL is required" $dot.Values.api_gateway.storage_proxy.url }}
+  {{- end }}
+{{- else }}
+{{- printf "" }}
+{{- end }}
+{{- end }}
+
+{{/*
+Storage Proxy Rule
+*/}}
+{{- define "mender.storageProxyRule" -}}
+  {{- default "HostRegexp(`{domain:^artifacts.*$}`)" .Values.api_gateway.storage_proxy.customRule | quote }}
+{{- end -}}
+
diff --git a/mender/templates/api-gateway/configmap.yaml b/mender/templates/api-gateway/configmap.yaml
@@ -417,7 +417,9 @@ data:
           rule: "!Path(`/api/management/{ver:v[0-9]+}/useradm/auth/logout`)&&PathPrefix(`/api/management/{ver:v[0-9]+}/useradm/{ep:(auth|oauth2)}`)"
           service: useradm
           tls: {{ $isTls }}
-{{- if .Values.api_gateway.minio.enabled }}
+{{- if and .Values.api_gateway.storage_proxy.enabled .Values.api_gateway.minio.enabled }}
+    {{- fail "You cannot set both api_gateway.storage_proxy and api_gateway.minio. Choose one" }}
+{{- else if and .Values.api_gateway.minio.enabled (not .Values.api_gateway.storage_proxy.enabled) }}
 
         #
         # minio
@@ -430,6 +432,19 @@ data:
           priority: 900
           service: minio
           tls: {{ $isTls }}
+{{- else if and .Values.api_gateway.storage_proxy.enabled (not .Values.api_gateway.minio.enabled) }}
+
+        #
+        # storage_proxy
+        #
+        storage_proxy:
+          entrypoints: {{ $scheme }}
+          middlewares:
+          - ratelimit
+          rule: {{ include "mender.storageProxyRule" . }}
+          priority: 65535
+          service: storage_proxy
+          tls: {{ $isTls }}
 {{- end }}
 
       #
@@ -496,12 +511,20 @@ data:
           loadBalancer:
             servers:
             - url: "http://{{ .Values.useradm.service.name }}:{{ .Values.useradm.service.port }}"
-{{- if .Values.api_gateway.minio.enabled }}
+{{- if and .Values.api_gateway.storage_proxy.enabled .Values.api_gateway.minio.enabled }}
+    {{- fail "You cannot set both api_gateway.storage_proxy and api_gateway.minio. Choose one" }}
+{{- else if and .Values.api_gateway.minio.enabled (not .Values.api_gateway.storage_proxy.enabled) }}
 
         minio:
           loadBalancer:
             servers:
             - url: "{{ .Values.api_gateway.minio.url }}"
+{{- else if and .Values.api_gateway.storage_proxy.enabled (not .Values.api_gateway.minio.enabled) }}
+
+        storage_proxy:
+          loadBalancer:
+            servers:
+            - url: {{ include "mender.storageProxyUrl" . }}
 {{- end }}
 
       #

diff --git a/mender/values.yaml b/mender/values.yaml
@@ -202,6 +202,10 @@ api_gateway:
     podAnnotations: {}
     url: "http://minio:9000"
     customRule: null
+  storage_proxy:
+    enabled: false
+    url: ""
+    customRule: "HostRegexp(`{domain:^artifacts.*$}`)"
   compression: true
   security_redirect: null
   rateLimit: