Skip to content

Commit

Permalink
feat(apps/prod/tekton/configs): use aws-cli to upload to qiniu (#1390)
Browse files Browse the repository at this point in the history
prepare for compatible with future object storage providers.

Signed-off-by: wuhuizuo <[email protected]>

---------

Signed-off-by: wuhuizuo <[email protected]>
  • Loading branch information
wuhuizuo authored Jan 16, 2025
1 parent d222217 commit dfd87cb
Show file tree
Hide file tree
Showing 18 changed files with 110 additions and 97 deletions.
23 changes: 14 additions & 9 deletions apps/prod/tekton/configs/pipelines/pingcap-release-ga.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@ spec:
- name: oci-registry
default: hub.pingcap.net
description: The OCI registry stored the origin artifacts.
- name: qiniu-creds-secret-name
default: qiniu-creds
workspaces:
# TODO: why need to push dm offline pkgs to another bucket?
- name: aws-secrets
optional: true
- name: aws-secrets-tencent
optional: true
- name: github
description: for creating github release
tasks:
Expand Down Expand Up @@ -72,12 +72,13 @@ spec:
- "compose-offline-pkgs-amd64-community"
taskRef:
name: pingcap-upload-offline-package
workspaces:
- { name: aws-secrets-tencent, workspace: aws-secrets-tencent }
params:
- { name: version, value: "$(params.version)" }
- { name: edition, value: community }
- { name: os, value: linux }
- { name: arch, value: amd64 }
- { name: env-secret-name, value: "$(params.qiniu-creds-secret-name)" }
- name: compose-offline-pkgs-amd64-enterprise
runAfter: [wait-delivery-tiup]
taskRef: { name: pingcap-compose-offline-pkgs }
Expand All @@ -98,7 +99,8 @@ spec:
- { name: edition, value: enterprise }
- { name: os, value: linux }
- { name: arch, value: amd64 }
- { name: env-secret-name, value: "$(params.qiniu-creds-secret-name)" }
workspaces:
- { name: aws-secrets-tencent, workspace: aws-secrets-tencent }
- name: compose-offline-pkgs-amd64-dm
runAfter: [wait-delivery-tiup]
taskRef: { name: pingcap-compose-offline-pkgs }
Expand All @@ -116,12 +118,12 @@ spec:
name: pingcap-upload-offline-package
workspaces:
- { name: aws-secrets, workspace: aws-secrets }
- { name: aws-secrets-tencent, workspace: aws-secrets-tencent }
params:
- { name: version, value: "$(params.version)" }
- { name: edition, value: dm }
- { name: os, value: linux }
- { name: arch, value: amd64 }
- { name: env-secret-name, value: "$(params.qiniu-creds-secret-name)" }
- name: compose-offline-pkgs-arm64-community
runAfter: [wait-delivery-tiup]
taskRef: { name: pingcap-compose-offline-pkgs }
Expand All @@ -137,12 +139,13 @@ spec:
- "compose-offline-pkgs-arm64-community"
taskRef:
name: pingcap-upload-offline-package
workspaces:
- { name: aws-secrets-tencent, workspace: aws-secrets-tencent }
params:
- { name: version, value: "$(params.version)" }
- { name: edition, value: community }
- { name: os, value: linux }
- { name: arch, value: arm64 }
- { name: env-secret-name, value: "$(params.qiniu-creds-secret-name)" }
- name: compose-offline-pkgs-arm64-enterprise
runAfter: [wait-delivery-tiup]
taskRef: { name: pingcap-compose-offline-pkgs }
Expand All @@ -158,12 +161,13 @@ spec:
- "compose-offline-pkgs-arm64-enterprise"
taskRef:
name: pingcap-upload-offline-package
workspaces:
- { name: aws-secrets-tencent, workspace: aws-secrets-tencent }
params:
- { name: version, value: "$(params.version)" }
- { name: edition, value: enterprise }
- { name: os, value: linux }
- { name: arch, value: arm64 }
- { name: env-secret-name, value: "$(params.qiniu-creds-secret-name)" }
- name: compose-offline-pkgs-arm64-dm
runAfter: [wait-delivery-tiup]
taskRef: { name: pingcap-compose-offline-pkgs }
Expand All @@ -181,17 +185,18 @@ spec:
name: pingcap-upload-offline-package
workspaces:
- { name: aws-secrets, workspace: aws-secrets }
- { name: aws-secrets-tencent, workspace: aws-secrets-tencent }
params:
- { name: version, value: "$(params.version)" }
- { name: edition, value: dm }
- { name: os, value: linux }
- { name: arch, value: arm64 }
- { name: env-secret-name, value: "$(params.qiniu-creds-secret-name)" }
- name: upload-plugin-tarballs
runAfter:
- ga-tag-oci-artifacts
taskRef:
name: pingcap-upload-enterprise-plugins
workspaces:
- { name: aws-secrets-tencent, workspace: aws-secrets-tencent }
params:
- { name: version, value: "$(params.version)" }
- { name: env-secret-name, value: "$(params.qiniu-creds-secret-name)" }
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ spec:
- name: git-ref
- name: git-sha
- name: build-image
default: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
default: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
- name: save-file
default: build-package-artifacts.sh
steps:
Expand All @@ -48,4 +48,3 @@ spec:
$(workspaces.output.path)/$(params.save-file)
cat $(workspaces.output.path)/$(params.save-file)
3 changes: 1 addition & 2 deletions apps/prod/tekton/configs/tasks/gen-build-images-scripts.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ spec:
- name: git-ref
- name: git-sha
- name: build-image
default: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
default: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
- name: save-file
default: build-package-images.sh
steps:
Expand All @@ -48,4 +48,3 @@ spec:
$(workspaces.output.path)/$(params.save-file)
cat $(workspaces.output.path)/$(params.save-file)
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ spec:
description: tags pushed
steps:
- name: prepare-manifest
image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
workingDir: /workspace
script: |
#! /usr/bin/env bash
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@ metadata:
labels:
app.kubernetes.io/version: "0.1"
annotations:
tekton.dev/platforms: "linux/amd64,linux/arm64"
tekton.dev/platforms: "linux/amd64,linux/arm64"
spec:
params:
- name: image_url
description: The image full url for pull
stepTemplate:
image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
steps:
- name: add-nightly-tag
script: |
Expand All @@ -29,27 +29,27 @@ spec:
- name: add-latest-tag
script: |
pushed_repo="$(echo $(params.image_url) | cut -d ':' -f 1)"
pushed_tag="$(echo $(params.image_url) | cut -d ':' -f 2)"
pushed_repo="$(echo $(params.image_url) | cut -d ':' -f 1)"
pushed_tag="$(echo $(params.image_url) | cut -d ':' -f 2)"
# Check if the tag is the latest version and add "latest"
if [ "$(echo $pushed_tag | grep -E '^v\d+\.\d+\.+\d+$')" ]; then
echo "Checking if tag is the latest version"
# Retrieve existing tags
EXISTING_TAGS=$(oras repo tags ${pushed_repo} | grep -E '^v\d+\.\d+\.+\d+$')
# Check if the tag is the latest version and add "latest"
if [ "$(echo $pushed_tag | grep -E '^v\d+\.\d+\.+\d+$')" ]; then
echo "Checking if tag is the latest version"
# Retrieve existing tags
EXISTING_TAGS=$(oras repo tags ${pushed_repo} | grep -E '^v\d+\.\d+\.+\d+$')
# Determine if the tag is the latest version
LATEST_TAG=$(echo "$EXISTING_TAGS" | sort -V | tail -n 1)
if [ "$LATEST_TAG" == "$pushed_tag" ]; then
echo "🚀 Adding latest tag"
oras tag $(params.image_url) latest
echo "✅ Added latest tag"
else
echo "🤚 Tag is not the latest version"
fi
# Determine if the tag is the latest version
LATEST_TAG=$(echo "$EXISTING_TAGS" | sort -V | tail -n 1)
if [ "$LATEST_TAG" == "$pushed_tag" ]; then
echo "🚀 Adding latest tag"
oras tag $(params.image_url) latest
echo "✅ Added latest tag"
else
echo "🤚 Pushed tag is not a version tag"
echo "🤚 Tag is not the latest version"
fi
else
echo "🤚 Pushed tag is not a version tag"
fi
workspaces:
- description: Includes a docker `config.json`
mountPath: /root/.docker
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,20 +38,20 @@ spec:
- name: git-ref
- name: git-sha
- name: builder-image
default: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
default: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
- name: release-dir
default: build
- name: push
type: string
default: 'false'
default: "false"
- name: registry
default: hub.pingcap.net
- name: boskos-server-url
description: The URL of the running boskos server
default: "http://boskos.test-pods.svc.cluster.local"
steps:
- name: generate-build-script
image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
script: |
git clone --depth=1 --branch=main https://github.com/PingCAP-QE/artifacts.git /workspace/artifacts
Expand Down Expand Up @@ -134,7 +134,7 @@ spec:
--boskos.owner $(context.taskRun.name) \
--releaseDir $(params.release-dir)
- name: publish
image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
workingDir: $(workspaces.source.path)/$(params.component)
script: |
script="/workspace/build-package-artifacts.sh"
Expand Down
12 changes: 6 additions & 6 deletions apps/prod/tekton/configs/tasks/pingcap-build-binaries-linux.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -44,17 +44,17 @@ spec:
- name: git-ref
- name: git-sha
- name: builder-image
default: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
default: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
- name: release-dir
default: build
- name: push
type: string
default: 'false'
default: "false"
- name: registry
default: hub.pingcap.net
steps:
- name: generate-build-script
image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
script: |
git clone --depth=1 --branch=main https://github.com/PingCAP-QE/artifacts.git /workspace/artifacts
Expand Down Expand Up @@ -88,13 +88,13 @@ spec:
workingDir: $(workspaces.source.path)/$(params.component)
env:
- name: CARGO_NET_GIT_FETCH_WITH_CLI
value: 'true'
value: "true"
# - name: RUSTUP_DIST_SERVER
# value: https://rsproxy.cn
# - name: RUSTUP_UPDATE_ROOT
# value: https://rsproxy.cn/rustup
- name: GOPROXY
value: 'http://goproxy.apps.svc,direct'
value: "http://goproxy.apps.svc,direct"
- name: CARGO_HOME
value: /workspace/.cargo
- name: NPM_CONFIG_REGISTRY
Expand All @@ -110,7 +110,7 @@ spec:
"$script" -b -a -w "$(params.release-dir)"
- name: publish
image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
workingDir: $(workspaces.source.path)/$(params.component)
script: |
script="/workspace/build-package-artifacts.sh"
Expand Down
6 changes: 3 additions & 3 deletions apps/prod/tekton/configs/tasks/pingcap-build-images.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,13 @@ spec:
- name: release-dir
default: build
- name: build
default: 'false'
default: "false"
description: Does it need to build the binaries before packing the images?
- name: registry
default: hub.pingcap.net
steps:
- name: generate
image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
script: |
git clone --depth=1 --branch=main https://github.com/PingCAP-QE/artifacts.git /workspace/artifacts
Expand Down Expand Up @@ -102,7 +102,7 @@ spec:
"$script" -w "$(params.release-dir)" -k ${KANIKO_EXECUTOR} -o $(results.pushed.path)
fi
- name: add-more-tags
image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
script: |
script="/workspace/build-package-images.sh"
if [ ! -f "$script" ]; then
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,14 @@ spec:
supports: 'community' or 'enterprise'.
- name: push
type: string
default: 'false'
default: "false"
- name: registry
default: hub.pingcap.net
- name: tiup-mirror
description: Product is http://tiup.pingcap.net:8987, Staging is http://tiup.pingcap.net:8988,
default: http://tiup.pingcap.net:8988
stepTemplate:
image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
env:
- name: OUT_SCRIPT
value: /workspace/compose-offline-packages-artifacts.sh
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ spec:
default: ""
steps:
- name: get
image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-60-gb6f8928
image: ghcr.io/pingcap-qe/cd/utils/release:v2024.10.8-71-gf13e219
script: |
if [ -n "$(params.force-builder-image)" ]; then
echo "use force builder image: $(params.force-builder-image)"
Expand Down
Loading

0 comments on commit dfd87cb

Please sign in to comment.