From 5c61ef1eb942ffc776d841341045251e60e00814 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Tue, 4 Jun 2024 12:53:32 +0200 Subject: [PATCH 001/186] cleanup pipeline from all loggregator things * remove anchor containing all ops-files for loggregator deployment * remove acceptance-job * run performance tests with log-cache sending metrics to syslog-server --- ci/autoscaler/pipeline.yml | 70 +------------------------------------- 1 file changed, 1 insertion(+), 69 deletions(-) diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index d93fc3973b..ac7f18da2a 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -1,17 +1,5 @@ --- anchors: - app-autoscaler-ops-files: &app-autoscaler-ops-files - OPS_FILES: | - operations/add-releases.yml - operations/instance-identity-cert-from-cf.yml - operations/add-postgres-variables.yml - operations/enable-nats-tls.yml - operations/loggregator-certs-from-cf.yml - operations/append-deployment-name-to-loggregator-nozzle-shard-id.yml - operations/add-extra-plan.yml - operations/set-release-version.yml - operations/enable-scheduler-logging.yml - app-autoscaler-ops-files-log-cache-metron: &app-autoscaler-ops-files-log-cache-metron OPS_FILES: | operations/add-releases.yml @@ -57,7 +45,6 @@ anchors: groups: - name: all jobs: - - acceptance - acceptance-log-cache-metron - acceptance-log-cache-syslog - cleanup-autoscaler-deployments @@ -73,7 +60,6 @@ groups: - update-java - name: autoscaler-release jobs: - - acceptance - acceptance-log-cache-metron - acceptance-log-cache-syslog - draft @@ -226,59 +212,6 @@ jobs: file: ci/ci/autoscaler/tasks/run-integration-tests.yml timeout: 45m -- name: acceptance - public: true - build_logs_to_retain: 100 - serial: true - on_success: - task: cleanup - file: ci/ci/autoscaler/tasks/cleanup-autoscaler.yml - params: &acceptance-params - DEPLOYMENT_NAME: ((acceptance_deployment_name)) - plan: - - in_parallel: - - get: bbl-state - - get: app-autoscaler-release - passed: [unit-tests, integration-tests] - trigger: true - - get: ci - - task: make-prerelease - file: ci/ci/autoscaler/tasks/make/make.yaml - params: - # ⚠️ Here it is used that make officially guarantees to reach the goals in the provided order. - TARGETS: generate-fakes generate-openapi-generated-clients-and-servers go-mod-tidy go-mod-vendor db scheduler - timeout: 15m - - task: deploy-autoscaler - file: ci/ci/autoscaler/tasks/deploy-autoscaler.yml - params: - <<: *acceptance-params - <<: *app-autoscaler-ops-files - timeout: 30m - - task: register-broker - file: ci/ci/autoscaler/tasks/register-broker.yml - params: - <<: *acceptance-params - timeout: 5m - - in_parallel: - - task: autoscaler-acceptance-api - file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml - params: - <<: *acceptance-params - SUITES: api - timeout: 15m - - task: autoscaler-acceptance-app - file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml - params: - <<: *acceptance-params - SUITES: app - timeout: 45m - - task: autoscaler-acceptance-broker - file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml - params: - <<: *acceptance-params - SUITES: broker - timeout: 15m - - name: acceptance-log-cache-metron public: true build_logs_to_retain: 100 @@ -413,7 +346,7 @@ jobs: file: ci/ci/autoscaler/tasks/deploy-autoscaler.yml params: <<: *performance-env - <<: *app-autoscaler-ops-files + <<: *app-autoscaler-ops-files-log-cache-syslog timeout: 30m - task: register-broker file: ci/ci/autoscaler/tasks/register-broker.yml @@ -445,7 +378,6 @@ jobs: - get: ci - get: app-autoscaler-release passed: - - acceptance - acceptance-log-cache-metron - acceptance-log-cache-syslog trigger: true From 0b1999763af972554bd59ce6b28b9175b1e8d7f0 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:11:56 +0200 Subject: [PATCH 002/186] format make-file a bit --- ci/Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ci/Makefile b/ci/Makefile index 3a5c795c41..4317bd8c3a 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -1,11 +1,7 @@ - .PHONY: set-autoscaler-pipeline - set-autoscaler-pipeline: @./autoscaler/set-pipeline.sh .PHONY: unpause-pipeline - unpause-pipeline: @./scripts/unpause-pipeline.sh - From 907e134645d004d5e1e60824cf0e973cacd7f7a2 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:12:31 +0200 Subject: [PATCH 003/186] fix pipeline creation when git-branch contains slashes --- ci/autoscaler/set-pipeline.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/autoscaler/set-pipeline.sh b/ci/autoscaler/set-pipeline.sh index 3142b37252..e793fa88bb 100755 --- a/ci/autoscaler/set-pipeline.sh +++ b/ci/autoscaler/set-pipeline.sh @@ -75,7 +75,10 @@ function main(){ export PIPELINE_NAME="app-autoscaler-release" set_pipeline $PIPELINE_NAME else - export PIPELINE_NAME="app-autoscaler-release-${CURRENT_BRANCH}" + local current_branch_without_slashes + current_branch_without_slashes="$(echo "${CURRENT_BRANCH}" | sed 's/\//-/g')" # Concourse can't handle slashes in pipeline names + + export PIPELINE_NAME="app-autoscaler-release-${current_branch_without_slashes}" set_pipeline "$PIPELINE_NAME" pause_jobs "$PIPELINE_NAME" unpause_job "$PIPELINE_NAME/set-pipeline" From c18b1fb422b2b81fed317b6104a3bef896e5a2f5 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:35:29 +0200 Subject: [PATCH 004/186] remove non-working script (yq ".releases[].url" is always null) --- templates/fix-releases.sh | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 templates/fix-releases.sh diff --git a/templates/fix-releases.sh b/templates/fix-releases.sh deleted file mode 100644 index ee005213e5..0000000000 --- a/templates/fix-releases.sh +++ /dev/null @@ -1,6 +0,0 @@ -#! /bin/bash -for release in $(yq ".releases[].url" templates/app-autoscaler-deployment.yml); -do - bosh upload-release --fix "${release}" -done - From 8ff7babedc52b5e320f4fa7a2a34ca0e53c27f0d Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:35:50 +0200 Subject: [PATCH 005/186] exchange manifest in issue template --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 90e8e3edb9..cc3ef31229 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -14,7 +14,7 @@ Describe your issue here. * Version of app-autoscaler-release * Specify the database (postgres/mysql) along-with it's version * How did you deploy the release? - * Is your manifest based on [app-autoscaler-deployment.yml](https://github.com/cloudfoundry/app-autoscaler-release/blob/main/templates/app-autoscaler-deployment.yml)? + * Is your manifest based on [app-autoscaler.yml](https://github.com/cloudfoundry/app-autoscaler-release/blob/main/templates/app-autoscaler.yml)? * Which, if any, [ops-files](https://github.com/cloudfoundry/app-autoscaler-release/tree/main/example/operation) did you apply? * How did you run the acceptance-tests? * If you did not run them, please consider running them and attaching the output as well. From 7a97c496df51c4362450d55f71978dc58e1aed88 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:43:45 +0200 Subject: [PATCH 006/186] make app-autoscaler the default manifest during manifest-tests --- templates/manifest_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/manifest_tests.sh b/templates/manifest_tests.sh index 60c28b08d7..abdcedebfd 100755 --- a/templates/manifest_tests.sh +++ b/templates/manifest_tests.sh @@ -2,7 +2,7 @@ set -euo pipefail -manifest_path="${MANIFEST_PATH:-../templates/app-autoscaler-deployment.yml}" +manifest_path="${MANIFEST_PATH:-../templates/app-autoscaler.yml}" operation_dir_path="${OPERATION_DIR_PATH:-../example/operation}" scalingengine_instance_group="${SCALINGENGINE_INSTANCE_GROUP:-scalingengine}" scheduler_instance_group="${SCHEDULER_INSTANCE_GROUP:-scheduler}" From 010a70ae95e5ee07d2f16e71cf8568cc6aa5f10e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 13:40:52 +0000 Subject: [PATCH 007/186] fix(deps): update module golang.org/x/oauth2 to v0.21.0 --- src/changelog/go.mod | 2 +- src/changelog/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/changelog/go.mod b/src/changelog/go.mod index e234623981..7978d175bb 100644 --- a/src/changelog/go.mod +++ b/src/changelog/go.mod @@ -6,7 +6,7 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/shurcooL/githubv4 v0.0.0-20240429030203-be2daab69064 github.com/stretchr/testify v1.9.0 - golang.org/x/oauth2 v0.20.0 + golang.org/x/oauth2 v0.21.0 ) require ( diff --git a/src/changelog/go.sum b/src/changelog/go.sum index 072781d32a..45eaaee3d5 100644 --- a/src/changelog/go.sum +++ b/src/changelog/go.sum @@ -12,8 +12,8 @@ github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZV github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= -golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 26f9bd1dddf2e74a1ef4beba815339513ecc8676 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Wed, 5 Jun 2024 16:39:33 +0200 Subject: [PATCH 008/186] test(acceptance): add missing tests for `breach_duration_secs` and `cool_down_secs` (#2962) --- src/acceptance/app/lead_times_test.go | 60 +++++++++++++++++++++++++++ src/acceptance/helpers/helpers.go | 4 +- 2 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 src/acceptance/app/lead_times_test.go diff --git a/src/acceptance/app/lead_times_test.go b/src/acceptance/app/lead_times_test.go new file mode 100644 index 0000000000..876a15e34b --- /dev/null +++ b/src/acceptance/app/lead_times_test.go @@ -0,0 +1,60 @@ +package app_test + +import ( + . "acceptance/helpers" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Autoscaler lead times for scaling", func() { + var ( + policy string + err error + ) + BeforeEach(func() { + policy = GenerateDynamicScaleOutAndInPolicy(1, 2, "test_metric", 500, 500) + appName = CreateTestApp(cfg, "labeled-go_app", 1) + appGUID, err = GetAppGuid(cfg, appName) + Expect(err).NotTo(HaveOccurred()) + instanceName = CreatePolicy(cfg, appName, appGUID, policy) + StartApp(appName, cfg.CfPushTimeoutDuration()) + }) + AfterEach(AppAfterEach) + + When("breach_duration_secs and cool_down_secs are set", func() { + It("should do first scaling after breach_duration_secs have passed and second scaling after cool_down_secs have passed", func() { + breachDuration := TestBreachDurationSeconds * time.Second + coolDown := TestCoolDownSeconds * time.Second + scalingTimewindow := 130 * time.Second // be friendly and allow some time for "internal autoscaler processes" (metric polling interval etc.) to take place before actual scaling happens + + sendMetricForScaleOutAndReturnNumInstancesFunc := sendMetricToAutoscaler(cfg, appGUID, appName, 510, false) + sendMetricForScaleInAndReturnNumInstancesFunc := sendMetricToAutoscaler(cfg, appGUID, appName, 490, false) + + By("checking that no scaling out happens before breach_duration_secs have passed") + Consistently(sendMetricForScaleOutAndReturnNumInstancesFunc). + WithTimeout(breachDuration). + WithPolling(time.Second). + Should(Equal(1)) + + By("checking that scale out happens in a certain time window after breach_duration_secs have passed") + Eventually(sendMetricForScaleOutAndReturnNumInstancesFunc). + WithTimeout(scalingTimewindow). + WithPolling(time.Second). + Should(Equal(2)) + + By("checking that no scale in happens before cool_down_secs have passed") + Consistently(sendMetricForScaleInAndReturnNumInstancesFunc). + WithTimeout(coolDown). + WithPolling(time.Second). + Should(Equal(2)) + + By("checking that scale in happens in a certain time window after cool_down_secs have passed") + Eventually(sendMetricForScaleInAndReturnNumInstancesFunc). + WithTimeout(scalingTimewindow). + WithPolling(time.Second). + Should(Equal(1)) + }) + }) +}) diff --git a/src/acceptance/helpers/helpers.go b/src/acceptance/helpers/helpers.go index f6232e836a..f2a1ac5c2c 100644 --- a/src/acceptance/helpers/helpers.go +++ b/src/acceptance/helpers/helpers.go @@ -111,8 +111,8 @@ func DisableServiceAccess(cfg *config.Config, setup *workflowhelpers.Reproducibl if cfg.ShouldEnableServiceAccess() { workflowhelpers.AsUser(setup.AdminUserContext(), cfg.DefaultTimeoutDuration(), func() { orgName := setup.GetOrganizationName() - enableServiceAccess := cf.Cf("disable-service-access", cfg.ServiceName, "-b", cfg.ServiceBroker, "-o", orgName).Wait(cfg.DefaultTimeoutDuration()) - Expect(enableServiceAccess).To(Exit(0), fmt.Sprintf("Failed to disable service %s for org %s", cfg.ServiceName, orgName)) + disableServiceAccess := cf.Cf("disable-service-access", cfg.ServiceName, "-b", cfg.ServiceBroker, "-o", orgName).Wait(cfg.DefaultTimeoutDuration()) + Expect(disableServiceAccess).To(Exit(0), fmt.Sprintf("Failed to disable service %s for org %s", cfg.ServiceName, orgName)) }) } } From bf075c7cb6774b15806081b0aa734d8af561b692 Mon Sep 17 00:00:00 2001 From: Silvestre Zabala Date: Wed, 22 May 2024 11:18:25 +0200 Subject: [PATCH 009/186] feat(devbox): Add Cloud MTA Build Tool to devbox # Issue For building MTAs the Cloud MTA Build Tool is required. # Fix Add it via local flake --- devbox.json | 79 +++++++++++++++++++++--------------------- local-flake/flake.nix | 24 +++++++++++++ scripts/asdf2devbox.py | 6 +--- 3 files changed, 65 insertions(+), 44 deletions(-) diff --git a/devbox.json b/devbox.json index 963a033274..5e0feda74e 100644 --- a/devbox.json +++ b/devbox.json @@ -1,43 +1,44 @@ { - "packages": [ - "path:local-flake#bosh-bootloader", - "path:local-flake#app-autoscaler-cli-plugin", - "path:local-flake#log-cache-cli-plugin", - "path:local-flake#uaac", - "delve@latest", - "gh@latest", - "go-tools@latest", - "gopls@latest", - "nodejs@latest", - "nodePackages.yaml-language-server@latest", - "rubocop@latest", - "rubyPackages.solargraph@latest", - "swagger-cli@latest", - "actionlint@1.6.27", - "direnv@2.34.0", - "gnumake@4.4", - "maven@3.8.6", - "pre-commit@latest", - "bosh-cli@7.3.1", - "golangci-lint@1.57.2", - "yq-go@4.43.1", - "which@latest", - "jq@latest", - "fly@7.10.0", - "python@latest", - "cloudfoundry-cli@8.7.10", - "shellcheck@0.10.0", - "go@1.21.5", - "ruby@3.3.1", - "bundix@latest", - "oha@latest", - "credhub-cli@2.9.29", - "gum@0.13.0", - "act@latest", - "google-cloud-sdk@latest", - "ginkgo@2.19.0", - "temurin-bin-17@latest" - ], + "packages": { + "path:local-flake#bosh-bootloader": "", + "path:local-flake#app-autoscaler-cli-plugin": "", + "path:local-flake#log-cache-cli-plugin": "", + "path:local-flake#cloud-mta-build-tool": "", + "path:local-flake#uaac": "", + "credhub-cli": "2.9.29", + "delve": "latest", + "gh": "latest", + "go-tools": "latest", + "gopls": "latest", + "nodejs": "latest", + "nodePackages.yaml-language-server": "latest", + "rubocop": "latest", + "rubyPackages.solargraph": "latest", + "swagger-cli": "latest", + "actionlint": "1.6.27", + "direnv": "2.34.0", + "gnumake": "4.4", + "maven": "3.8.6", + "pre-commit": "latest", + "bosh-cli": "7.3.1", + "golangci-lint": "1.57.2", + "yq-go": "4.43.1", + "which": "latest", + "jq": "latest", + "fly": "7.10.0", + "python": "latest", + "cloudfoundry-cli": "8.7.10", + "shellcheck": "0.10.0", + "go": "1.21.5", + "ruby": "3.3.1", + "ginkgo": "2.19.0", + "bundix": "latest", + "oha": "latest", + "gum": "0.13.0", + "act": "latest", + "google-cloud-sdk": "latest", + "temurin-bin-17": "latest" + }, "shell": { "init_hook": [ "cf install-plugin -f $(which app-autoscaler-cli-plugin)", diff --git a/local-flake/flake.nix b/local-flake/flake.nix index ff108fd774..61804f71c9 100644 --- a/local-flake/flake.nix +++ b/local-flake/flake.nix @@ -58,6 +58,30 @@ ldflags = ["-s" "-w" "-X main.version=${version}"]; }; + cloud-mta-build-tool = nixpkgsFor.${system}.buildGoModule rec { + pname = "Cloud MTA Build Tool"; + version = "1.2.26"; + + src = nixpkgsFor.${system}.fetchFromGitHub { + owner = "SAP"; + repo = "cloud-mta-build-tool"; + rev = "v${version}"; + hash = "sha256-DKZ9Nj/sNC9dRjyiu4MKjLrIJWluYlZzUHWqEqtrNt4="; + }; + + vendorHash = "sha256-h8LPsuxvbr/aRhH1vR1fYgBot37yrfiemZTJMKj0zbk="; + + ldflags = ["-s" "-w" "-X main.Version=${version}"]; + + doCheck = false; + + postInstall = '' + pushd "$out/bin" &> /dev/null + ln -s 'cloud-mta-build-tool' 'mbt' + popd + ''; + }; + uaac = nixpkgsFor.${system}.bundlerApp rec { pname = "cf-uaac"; gemdir = ./.; diff --git a/scripts/asdf2devbox.py b/scripts/asdf2devbox.py index c8fec97260..49759d6eb6 100755 --- a/scripts/asdf2devbox.py +++ b/scripts/asdf2devbox.py @@ -11,10 +11,7 @@ def get_installed_version(package): with open(os.path.join(script_dir, '..', 'devbox.json'), 'r') as f: data = json.load(f) - for pkg in data['packages']: - if pkg.startswith(f"{package}@"): - return pkg.split('@')[1] - return None + return data['packages'][package] # Read the .tool-versions file and process each line if __name__ == "__main__": @@ -27,7 +24,6 @@ def get_installed_version(package): "bosh": "bosh-cli", "cf": "cloudfoundry-cli", "concourse": "fly", - "concourse": "fly", "credhub": "credhub-cli", "gcloud": "google-cloud-sdk", "golang": "go", From af28435b751631d38b00cf0b9dffac1d666b9162 Mon Sep 17 00:00:00 2001 From: app-autoscaler-ci-bot Date: Thu, 6 Jun 2024 10:27:18 +0100 Subject: [PATCH 010/186] Update golang version to 1.21.6 --- .final_builds/packages/golang-1-linux/index.yml | 4 ++++ .tool-versions | 2 +- .../app-autoscaler/concourse/pipelines/api-tester/src/go.mod | 2 +- go.work | 2 +- packages/golang-1-linux/spec.lock | 2 +- packages/golang-1-linux/vendored-commit | 2 +- packages/golang-1-linux/version | 2 +- src/acceptance/assets/app/go_app/go.mod | 2 +- src/acceptance/go.mod | 2 +- src/autoscaler/go.mod | 2 +- src/changelog/go.mod | 2 +- src/changeloglockcleaner/go.mod | 2 +- 12 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.final_builds/packages/golang-1-linux/index.yml b/.final_builds/packages/golang-1-linux/index.yml index 36e31e3962..595621a462 100644 --- a/.final_builds/packages/golang-1-linux/index.yml +++ b/.final_builds/packages/golang-1-linux/index.yml @@ -71,6 +71,10 @@ builds: version: b8021c5bdc377a4807fb1caad8416215032698ae356825afb427c7efc5ef76c1 blobstore_id: 1085db26-f5ec-42d3-7389-5750b941602a sha1: sha256:2a2785460af04aee276c3c2b38dd914c8e1cbefbf52cb14da130207e82e86289 + c2342901fca75f4c7ec3f32e6a757e923089c6c50d8eb3effd2c25eac1009e31: + version: c2342901fca75f4c7ec3f32e6a757e923089c6c50d8eb3effd2c25eac1009e31 + blobstore_id: b6da37e4-e3d9-451f-639a-17fd1690c6df + sha1: sha256:d354b084ed879adc3218f89b9109b474a68c3eafb5c0ea204c4b77963e4736b0 da6d4c66ebbf23ea0f9bf496e44b9f1f21cae9afa18842617d5b4b257cdfcfbb: version: da6d4c66ebbf23ea0f9bf496e44b9f1f21cae9afa18842617d5b4b257cdfcfbb blobstore_id: 8608be79-d102-4dd7-65f6-279f7bec2063 diff --git a/.tool-versions b/.tool-versions index d20c5c43e9..00e3964005 100644 --- a/.tool-versions +++ b/.tool-versions @@ -7,7 +7,7 @@ concourse 7.10.0 direnv 2.34.0 gcloud 473.0.0 ginkgo 2.19.0 -golang 1.21.5 +golang 1.21.6 golangci-lint 1.57.2 java temurin-21.0.3+9.0.LTS make 4.4 diff --git a/ci/terragrunt/app-autoscaler/concourse/pipelines/api-tester/src/go.mod b/ci/terragrunt/app-autoscaler/concourse/pipelines/api-tester/src/go.mod index 697d476ebc..6e984b878e 100644 --- a/ci/terragrunt/app-autoscaler/concourse/pipelines/api-tester/src/go.mod +++ b/ci/terragrunt/app-autoscaler/concourse/pipelines/api-tester/src/go.mod @@ -1,3 +1,3 @@ module apitester -go 1.21.5 +go 1.21.6 diff --git a/go.work b/go.work index 6dd2e0c8ba..373b4f353f 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.21.5 +go 1.21.6 use ( ./src/acceptance diff --git a/packages/golang-1-linux/spec.lock b/packages/golang-1-linux/spec.lock index 6a23a78647..c93c2bab17 100644 --- a/packages/golang-1-linux/spec.lock +++ b/packages/golang-1-linux/spec.lock @@ -1,2 +1,2 @@ name: golang-1-linux -fingerprint: a53d60ed79d278964faa9bc17836ed3bc5f084c0f3d41110c8a6e0b2de5f2dc0 +fingerprint: c2342901fca75f4c7ec3f32e6a757e923089c6c50d8eb3effd2c25eac1009e31 diff --git a/packages/golang-1-linux/vendored-commit b/packages/golang-1-linux/vendored-commit index 71e8079258..659d8e61fd 100644 --- a/packages/golang-1-linux/vendored-commit +++ b/packages/golang-1-linux/vendored-commit @@ -1 +1 @@ -5dcfbfc3789c847c524993c9c4fbeb085c0f87ec +257d975e17337d944396b59285fea374706c682a diff --git a/packages/golang-1-linux/version b/packages/golang-1-linux/version index d6a49eb373..6e83421e4f 100644 --- a/packages/golang-1-linux/version +++ b/packages/golang-1-linux/version @@ -1 +1 @@ -1.21.5 \ No newline at end of file +1.21.6 \ No newline at end of file diff --git a/src/acceptance/assets/app/go_app/go.mod b/src/acceptance/assets/app/go_app/go.mod index 5b5f8069a7..f2505fa1ee 100644 --- a/src/acceptance/assets/app/go_app/go.mod +++ b/src/acceptance/assets/app/go_app/go.mod @@ -1,6 +1,6 @@ module code.cloudfoundry.org/app-autoscaler-release/src/acceptance/assets/app/go_app -go 1.21.5 +go 1.21.6 require ( github.com/cloudfoundry-community/go-cfenv v1.18.0 diff --git a/src/acceptance/go.mod b/src/acceptance/go.mod index f9c379d755..78151acf88 100644 --- a/src/acceptance/go.mod +++ b/src/acceptance/go.mod @@ -1,6 +1,6 @@ module acceptance -go 1.21.5 +go 1.21.6 require ( github.com/cloudfoundry/cf-test-helpers/v2 v2.9.0 diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 56f13c5f21..ef00c1c62c 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -1,6 +1,6 @@ module code.cloudfoundry.org/app-autoscaler/src/autoscaler -go 1.21.5 +go 1.21.6 require ( code.cloudfoundry.org/cfhttp/v2 v2.1.0 diff --git a/src/changelog/go.mod b/src/changelog/go.mod index 7978d175bb..cbfddc60be 100644 --- a/src/changelog/go.mod +++ b/src/changelog/go.mod @@ -1,6 +1,6 @@ module changelog -go 1.21.5 +go 1.21.6 require ( github.com/Masterminds/semver/v3 v3.2.1 diff --git a/src/changeloglockcleaner/go.mod b/src/changeloglockcleaner/go.mod index 1a4b72cc30..ea0eca4c92 100644 --- a/src/changeloglockcleaner/go.mod +++ b/src/changeloglockcleaner/go.mod @@ -1,6 +1,6 @@ module changeloglockcleaner -go 1.21.5 +go 1.21.6 require ( github.com/go-sql-driver/mysql v1.8.1 From 530c98dd55d1af5b111f341a53bc8718a303fbad Mon Sep 17 00:00:00 2001 From: app-autoscaler-ci-bot Date: Thu, 6 Jun 2024 09:33:27 +0000 Subject: [PATCH 011/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20scripts/asdf2devbox.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- devbox.json | 4 ++-- devbox.lock | 48 ++++++++++++++++++++++++------------------------ 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/devbox.json b/devbox.json index 5e0feda74e..1767886ad7 100644 --- a/devbox.json +++ b/devbox.json @@ -29,14 +29,14 @@ "python": "latest", "cloudfoundry-cli": "8.7.10", "shellcheck": "0.10.0", - "go": "1.21.5", "ruby": "3.3.1", "ginkgo": "2.19.0", "bundix": "latest", "oha": "latest", "gum": "0.13.0", - "act": "latest", + "act": "0.2.63", "google-cloud-sdk": "latest", + "go": "1.21.6", "temurin-bin-17": "latest" }, "shell": { diff --git a/devbox.lock b/devbox.lock index 0e9921deb1..490b9eab58 100644 --- a/devbox.lock +++ b/devbox.lock @@ -1,51 +1,51 @@ { "lockfile_version": "1", "packages": { - "act@latest": { - "last_modified": "2024-05-29T10:04:41Z", - "resolved": "github:NixOS/nixpkgs/ac82a513e55582291805d6f09d35b6d8b60637a1#act", + "act@0.2.63": { + "last_modified": "2024-06-03T07:19:07Z", + "resolved": "github:NixOS/nixpkgs/4a4ecb0ab415c9fccfb005567a215e6a9564cdf5#act", "source": "devbox-search", - "version": "0.2.62", + "version": "0.2.63", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/anliky2rlgzcicrhn1yyv9q4nkz7p8wp-act-0.2.62", + "path": "/nix/store/fhn4kf6qa88l2pl0n202b4mplyc5pcyf-act-0.2.63", "default": true } ], - "store_path": "/nix/store/anliky2rlgzcicrhn1yyv9q4nkz7p8wp-act-0.2.62" + "store_path": "/nix/store/fhn4kf6qa88l2pl0n202b4mplyc5pcyf-act-0.2.63" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/jz4lqgv3amm5vg2kd9qxcw43vzrbcnxv-act-0.2.62", + "path": "/nix/store/5pmgbyh36x428ivx8dg2rq3rvaicjb66-act-0.2.63", "default": true } ], - "store_path": "/nix/store/jz4lqgv3amm5vg2kd9qxcw43vzrbcnxv-act-0.2.62" + "store_path": "/nix/store/5pmgbyh36x428ivx8dg2rq3rvaicjb66-act-0.2.63" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/g4zs1xipwgxx1prvi7bpvymibhiq4cm5-act-0.2.62", + "path": "/nix/store/iwjzs1r05xrdk4xsw6a2c33436s08x18-act-0.2.63", "default": true } ], - "store_path": "/nix/store/g4zs1xipwgxx1prvi7bpvymibhiq4cm5-act-0.2.62" + "store_path": "/nix/store/iwjzs1r05xrdk4xsw6a2c33436s08x18-act-0.2.63" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/xl5x5p5axgrqr9h6p2a3idpmcgmj9407-act-0.2.62", + "path": "/nix/store/4d5qhzbrwm9k9yxnwlkwjn8740mh2hlz-act-0.2.63", "default": true } ], - "store_path": "/nix/store/xl5x5p5axgrqr9h6p2a3idpmcgmj9407-act-0.2.62" + "store_path": "/nix/store/4d5qhzbrwm9k9yxnwlkwjn8740mh2hlz-act-0.2.63" } } }, @@ -457,51 +457,51 @@ } } }, - "go@1.21.5": { - "last_modified": "2024-01-14T03:55:27Z", - "resolved": "github:NixOS/nixpkgs/dd5621df6dcb90122b50da5ec31c411a0de3e538#go", + "go@1.21.6": { + "last_modified": "2024-02-10T18:15:24Z", + "resolved": "github:NixOS/nixpkgs/10b813040df67c4039086db0f6eaf65c536886c6#go", "source": "devbox-search", - "version": "1.21.5", + "version": "1.21.6", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/z6p2j8shdwi74kbm86jwdh03vxq91l0q-go-1.21.5", + "path": "/nix/store/44g95npjwgjyrxm1sx54g22w5sfmkz1i-go-1.21.6", "default": true } ], - "store_path": "/nix/store/z6p2j8shdwi74kbm86jwdh03vxq91l0q-go-1.21.5" + "store_path": "/nix/store/44g95npjwgjyrxm1sx54g22w5sfmkz1i-go-1.21.6" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/1p00q366rk2j7w9lnflrl7xaks7wfq5d-go-1.21.5", + "path": "/nix/store/bq5scmlrr463qwx4yfbid9p3w7009r57-go-1.21.6", "default": true } ], - "store_path": "/nix/store/1p00q366rk2j7w9lnflrl7xaks7wfq5d-go-1.21.5" + "store_path": "/nix/store/bq5scmlrr463qwx4yfbid9p3w7009r57-go-1.21.6" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/brv7d6mlrclkzywf1vaf35wqhq4c0c82-go-1.21.5", + "path": "/nix/store/f0mzci3szxjf0cws1a1vlls6d4zbd8s4-go-1.21.6", "default": true } ], - "store_path": "/nix/store/brv7d6mlrclkzywf1vaf35wqhq4c0c82-go-1.21.5" + "store_path": "/nix/store/f0mzci3szxjf0cws1a1vlls6d4zbd8s4-go-1.21.6" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/rccpgf2bnh58hhh4908p78i1rln9gzvp-go-1.21.5", + "path": "/nix/store/zg65r8ys8y5865lcwmmybrq5gn30n1az-go-1.21.6", "default": true } ], - "store_path": "/nix/store/rccpgf2bnh58hhh4908p78i1rln9gzvp-go-1.21.5" + "store_path": "/nix/store/zg65r8ys8y5865lcwmmybrq5gn30n1az-go-1.21.6" } } }, From 83e55daa52ff9611a00896cde38c3d85594685a8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 10:28:50 +0000 Subject: [PATCH 012/186] fix(deps): update code.cloudfoundry.org/loggregator-agent-release/src digest to 482a07a --- src/autoscaler/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index ef00c1c62c..b9a726a153 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,7 +8,7 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240604192159-0c056d8148bf + code.cloudfoundry.org/loggregator-agent-release/src 482a07ab6dbc code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65 dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 From 0e0afa973c392d5f88fd055a77d1f057d0a18171 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 11:35:30 +0100 Subject: [PATCH 013/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/autoscaler/go.mod | 16 ++++++++-------- src/autoscaler/go.sum | 32 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index b9a726a153..6bc479c1a7 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,7 +8,7 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src 482a07ab6dbc + code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240605163137-482a07ab6dbc code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65 dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 @@ -46,15 +46,15 @@ require ( go.opentelemetry.io/otel/trace v1.26.0 golang.org/x/crypto v0.24.0 golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc - golang.org/x/net v0.25.0 + golang.org/x/net v0.26.0 golang.org/x/time v0.5.0 google.golang.org/grpc v1.64.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - code.cloudfoundry.org/go-diodes v0.0.0-20240515174142-71582f284718 // indirect - code.cloudfoundry.org/go-metric-registry v0.0.0-20240523160243-6c152ef80e25 // indirect + code.cloudfoundry.org/go-diodes v0.0.0-20240604201846-c756bfed2ed3 // indirect + code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -96,13 +96,13 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/mod v0.17.0 // indirect + golang.org/x/mod v0.18.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect + golang.org/x/tools v0.22.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 707272e824..e76131babf 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -596,18 +596,18 @@ code.cloudfoundry.org/cfhttp/v2 v2.1.0 h1:HbQ5H2R+HEKG/rcB6Gk3okeC3h2fAC4PPnLQoM code.cloudfoundry.org/cfhttp/v2 v2.1.0/go.mod h1:k9R36Y/9dUc9OsX4dfDuEjHZ7Q00ttklKQj6HD6h6+U= code.cloudfoundry.org/clock v1.1.0 h1:XLzC6W3Ah/Y7ht1rmZ6+QfPdt1iGWEAAtIZXgiaj57c= code.cloudfoundry.org/clock v1.1.0/go.mod h1:yA3fxddT9RINQL2XHS7PS+OXxKCGhfrZmlNUCIM6AKo= -code.cloudfoundry.org/go-diodes v0.0.0-20240515174142-71582f284718 h1:6wazSuHaJjatGy8pvchSy6L+4M67WPfjgK9yh7cJLMs= -code.cloudfoundry.org/go-diodes v0.0.0-20240515174142-71582f284718/go.mod h1:eVHabU/rcpC5ocoIAXvnaySkdH6+PgORRVlY5l3SDys= +code.cloudfoundry.org/go-diodes v0.0.0-20240604201846-c756bfed2ed3 h1:4WCYwJmqSfV7ChDohsJB8Z0aDVklIE+n8OTBJxpif0c= +code.cloudfoundry.org/go-diodes v0.0.0-20240604201846-c756bfed2ed3/go.mod h1:8O5g1DEzJU9ktEmykKPhY4mZOM/dBENWVHKVInuuch8= code.cloudfoundry.org/go-log-cache/v2 v2.0.7 h1:yR/JjQ/RscO1n4xVAT9HDYcpx5ET/3Cq2/RhpJml6ZU= code.cloudfoundry.org/go-log-cache/v2 v2.0.7/go.mod h1:6KQe2FeeaqRheD5vCvpyTa80YoJojB/r21E54mT97Mc= code.cloudfoundry.org/go-loggregator/v9 v9.2.1 h1:S6Lgg5UJbhh2bt2TGQxs6R00CF8PrUA3GFPYDxy56Fk= code.cloudfoundry.org/go-loggregator/v9 v9.2.1/go.mod h1:FTFFruqGeOhVCDFvyLgl8EV8YW63NNwRzLhxJcporu8= -code.cloudfoundry.org/go-metric-registry v0.0.0-20240523160243-6c152ef80e25 h1:p1RGKEtKpZJ4CtjLoQudjEpvMLi+l2MZCyheWthmaNg= -code.cloudfoundry.org/go-metric-registry v0.0.0-20240523160243-6c152ef80e25/go.mod h1:C+P03mf5ZSJVpcigPZ3tH5Vb66HIfCTKxatA4llc298= +code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a h1:XpebbxgIBBy7SrwIGW+gREZuAtnJ9PHWC4Y+k7yje2I= +code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a/go.mod h1:/Be8VtLiCeMUoYdUzFtmW8GGkk89HAy3zD79KUXzbhs= code.cloudfoundry.org/lager/v3 v3.0.3 h1:/UTmadZfIaKuT/whEinSxK1mzRfNu1uPfvjFfGqiwzM= code.cloudfoundry.org/lager/v3 v3.0.3/go.mod h1:Zn5q1SrIuuHjEUE7xerMKt3ztunrJQCZETAo7rV0CH8= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240604192159-0c056d8148bf h1:EmmABS4FL7hQNwPpyO4AEZ43EFFA8vOOq7njrt8qyHU= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240604192159-0c056d8148bf/go.mod h1:3g7KeOBzN+XZBvuRVkX41LBMRTG+HNgtA6DKJjFdrC0= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240605163137-482a07ab6dbc h1:KzDejtGhap8QSBkGMGVQ+u9HupyXgBRsr8PQlWLPq6g= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240605163137-482a07ab6dbc/go.mod h1:wvherEAITeIr/OoqKfSZa8ngKJiglUjfgfmCSW9OGUc= code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65 h1:0wXfAs/32wbpMKDGiV3XEHev1l9dLgIPjda+WRwJhCc= code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65/go.mod h1:R4odSd3iXVYlIlesDHaQi9IZx2gXQryfvJmJSjPS/ug= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= @@ -1159,8 +1159,8 @@ golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1224,8 +1224,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1463,8 +1463,8 @@ golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1677,10 +1677,10 @@ google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= -google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= From bf62a520c6f1f7cdde039c56ceb2c0d23b503d97 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 21:21:49 +0000 Subject: [PATCH 014/186] fix(deps): update code.cloudfoundry.org/loggregator-agent-release/src digest to b2e6ea6 --- src/autoscaler/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 6bc479c1a7..078075ca95 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,7 +8,7 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240605163137-482a07ab6dbc + code.cloudfoundry.org/loggregator-agent-release/src b2e6ea608429 code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65 dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 From 331d4ed1392179f6a94bcddb0bebf771026b0005 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 22:28:35 +0100 Subject: [PATCH 015/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/autoscaler/go.mod | 2 +- src/autoscaler/go.sum | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 078075ca95..7bfd9db2d9 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,7 +8,7 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src b2e6ea608429 + code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240606143910-b2e6ea608429 code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65 dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index e76131babf..5425895564 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -606,8 +606,8 @@ code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a h1:X code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a/go.mod h1:/Be8VtLiCeMUoYdUzFtmW8GGkk89HAy3zD79KUXzbhs= code.cloudfoundry.org/lager/v3 v3.0.3 h1:/UTmadZfIaKuT/whEinSxK1mzRfNu1uPfvjFfGqiwzM= code.cloudfoundry.org/lager/v3 v3.0.3/go.mod h1:Zn5q1SrIuuHjEUE7xerMKt3ztunrJQCZETAo7rV0CH8= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240605163137-482a07ab6dbc h1:KzDejtGhap8QSBkGMGVQ+u9HupyXgBRsr8PQlWLPq6g= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240605163137-482a07ab6dbc/go.mod h1:wvherEAITeIr/OoqKfSZa8ngKJiglUjfgfmCSW9OGUc= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240606143910-b2e6ea608429 h1:wZjrTg9Xf3urmfVip8TKz8NLFR4VEcu87aGwBOWDVOE= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240606143910-b2e6ea608429/go.mod h1:5YRZP5jmXZ+VEuVT0qhXKuchToXgE1r1L9K1d2KQE8c= code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65 h1:0wXfAs/32wbpMKDGiV3XEHev1l9dLgIPjda+WRwJhCc= code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65/go.mod h1:R4odSd3iXVYlIlesDHaQi9IZx2gXQryfvJmJSjPS/ug= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= @@ -1077,8 +1077,8 @@ go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZu go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -go.step.sm/crypto v0.46.0 h1:cuVZMpDbmEsUX+atC24+VineQr4gO+zO46MxbIVai4Y= -go.step.sm/crypto v0.46.0/go.mod h1:hcr0oTS2vGRTGSZxoVYxE+RRcsd4xP3rpqt3wPwYGqc= +go.step.sm/crypto v0.47.0 h1:LWxiKWiN0Y/A5+dq+fTIAvFYAL8oe3PQmCurjtn6ZBU= +go.step.sm/crypto v0.47.0/go.mod h1:0NMEfYrFfV5jqs8aJY5wRqIShBV8y/fyDLTseyv5xhY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= From eead4b0851f271f56ce2a0a21f75ea72659c02e6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 12:53:51 +0000 Subject: [PATCH 016/186] fix(deps): update golang.org/x/exp digest to fc45aab --- src/autoscaler/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 7bfd9db2d9..00213e2a4c 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -45,7 +45,7 @@ require ( go.opentelemetry.io/otel/sdk v1.26.0 go.opentelemetry.io/otel/trace v1.26.0 golang.org/x/crypto v0.24.0 - golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc + golang.org/x/exp fc45aab8b7f8 golang.org/x/net v0.26.0 golang.org/x/time v0.5.0 google.golang.org/grpc v1.64.0 From 4cef41ba9619e3c3ac7703cf7b4b8d054ef7a717 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 14:00:48 +0100 Subject: [PATCH 017/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/autoscaler/go.mod | 2 +- src/autoscaler/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 00213e2a4c..329de275dd 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -45,7 +45,7 @@ require ( go.opentelemetry.io/otel/sdk v1.26.0 go.opentelemetry.io/otel/trace v1.26.0 golang.org/x/crypto v0.24.0 - golang.org/x/exp fc45aab8b7f8 + golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 golang.org/x/net v0.26.0 golang.org/x/time v0.5.0 google.golang.org/grpc v1.64.0 diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 5425895564..15c770200a 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -1112,8 +1112,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc h1:O9NuF4s+E/PvMIy+9IUZB9znFwUIXEWSstNjek6VpVg= -golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM= +golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= From ed22e247dbe052da0114b6322f8e74d68d933b55 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 08:41:11 +0000 Subject: [PATCH 018/186] fix(deps): update module github.com/prometheus/procfs to v0.15.1 --- src/acceptance/assets/app/go_app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/acceptance/assets/app/go_app/go.mod b/src/acceptance/assets/app/go_app/go.mod index f2505fa1ee..9480e2b758 100644 --- a/src/acceptance/assets/app/go_app/go.mod +++ b/src/acceptance/assets/app/go_app/go.mod @@ -17,7 +17,7 @@ require ( github.com/ogen-go/ogen v1.1.1 github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 - github.com/prometheus/procfs v0.14.0 + github.com/prometheus/procfs v0.15.1 github.com/steinfletcher/apitest v1.5.15 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.51.0 go.opentelemetry.io/otel v1.26.0 From b9bfa69b20838e1f7f2672808ca748c0155c8259 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 09:46:41 +0100 Subject: [PATCH 019/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/acceptance/assets/app/go_app/go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/acceptance/assets/app/go_app/go.sum b/src/acceptance/assets/app/go_app/go.sum index e30bccbb50..863cb6bbfd 100644 --- a/src/acceptance/assets/app/go_app/go.sum +++ b/src/acceptance/assets/app/go_app/go.sum @@ -106,8 +106,8 @@ github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/procfs v0.14.0 h1:Lw4VdGGoKEZilJsayHf0B+9YgLGREba2C6xr+Fdfq6s= -github.com/prometheus/procfs v0.14.0/go.mod h1:XL+Iwz8k8ZabyZfMFHPiilCniixqQarAy5Mu67pHlNQ= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= From 19369da7892d18e3a6664f8a3b315ad4a397bfee Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 11:53:41 +0000 Subject: [PATCH 020/186] fix(deps): update module github.com/steinfletcher/apitest to v1.5.16 --- src/acceptance/assets/app/go_app/go.mod | 2 +- src/autoscaler/go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/acceptance/assets/app/go_app/go.mod b/src/acceptance/assets/app/go_app/go.mod index 9480e2b758..e0112aa24b 100644 --- a/src/acceptance/assets/app/go_app/go.mod +++ b/src/acceptance/assets/app/go_app/go.mod @@ -18,7 +18,7 @@ require ( github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 github.com/prometheus/procfs v0.15.1 - github.com/steinfletcher/apitest v1.5.15 + github.com/steinfletcher/apitest v1.5.16 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.51.0 go.opentelemetry.io/otel v1.26.0 go.opentelemetry.io/otel/metric v1.26.0 diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 329de275dd..5f4fa13412 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -33,7 +33,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.1 github.com/rubyist/circuitbreaker v2.2.1+incompatible - github.com/steinfletcher/apitest v1.5.15 + github.com/steinfletcher/apitest v1.5.16 github.com/stretchr/testify v1.9.0 github.com/tedsuo/ifrit v0.0.0-20230516164442-7862c310ad26 github.com/uptrace/opentelemetry-go-extra/otelsql v0.2.4 From ac9e13bba43097a4d958ec5b22a9b435433cba37 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 13:00:26 +0100 Subject: [PATCH 021/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/acceptance/assets/app/go_app/go.sum | 4 ++-- src/autoscaler/go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/acceptance/assets/app/go_app/go.sum b/src/acceptance/assets/app/go_app/go.sum index 863cb6bbfd..91cbbc1b03 100644 --- a/src/acceptance/assets/app/go_app/go.sum +++ b/src/acceptance/assets/app/go_app/go.sum @@ -115,8 +115,8 @@ github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/steinfletcher/apitest v1.5.15 h1:AAdTN0yMbf0VMH/PMt9uB2I7jljepO6i+5uhm1PjH3c= -github.com/steinfletcher/apitest v1.5.15/go.mod h1:mF+KnYaIkuHM0C4JgGzkIIOJAEjo+EA5tTjJ+bHXnQc= +github.com/steinfletcher/apitest v1.5.16 h1:J/ZoBmhgdzH4qfxPSw9kaXRBgzy3OsCoKh1gcc1h2zM= +github.com/steinfletcher/apitest v1.5.16/go.mod h1:mF+KnYaIkuHM0C4JgGzkIIOJAEjo+EA5tTjJ+bHXnQc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 15c770200a..9384c98980 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -1013,8 +1013,8 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/square/certstrap v1.3.0 h1:N9P0ZRA+DjT8pq5fGDj0z3FjafRKnBDypP0QHpMlaAk= github.com/square/certstrap v1.3.0/go.mod h1:wGZo9eE1B7WX2GKBn0htJ+B3OuRl2UsdCFySNooy9hU= -github.com/steinfletcher/apitest v1.5.15 h1:AAdTN0yMbf0VMH/PMt9uB2I7jljepO6i+5uhm1PjH3c= -github.com/steinfletcher/apitest v1.5.15/go.mod h1:mF+KnYaIkuHM0C4JgGzkIIOJAEjo+EA5tTjJ+bHXnQc= +github.com/steinfletcher/apitest v1.5.16 h1:J/ZoBmhgdzH4qfxPSw9kaXRBgzy3OsCoKh1gcc1h2zM= +github.com/steinfletcher/apitest v1.5.16/go.mod h1:mF+KnYaIkuHM0C4JgGzkIIOJAEjo+EA5tTjJ+bHXnQc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= From b1ad201c0a7de050f86b4a7872d0610effb30f64 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 13:08:16 +0000 Subject: [PATCH 022/186] fix(deps): update module github.com/go-logr/logr to v1.4.2 --- src/acceptance/assets/app/go_app/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/acceptance/assets/app/go_app/go.mod b/src/acceptance/assets/app/go_app/go.mod index e0112aa24b..7aa5352a6a 100644 --- a/src/acceptance/assets/app/go_app/go.mod +++ b/src/acceptance/assets/app/go_app/go.mod @@ -9,7 +9,7 @@ require ( github.com/gin-gonic/gin v1.10.0 github.com/go-faster/errors v0.7.1 github.com/go-faster/jx v1.1.0 - github.com/go-logr/logr v1.4.1 + github.com/go-logr/logr v1.4.2 github.com/go-logr/zapr v1.3.0 github.com/json-iterator/go v1.1.12 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 From 82290b871532e27e91b71074e3cd11d321a5da55 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 15:02:03 +0100 Subject: [PATCH 023/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/acceptance/assets/app/go_app/go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/acceptance/assets/app/go_app/go.sum b/src/acceptance/assets/app/go_app/go.sum index 91cbbc1b03..0b2c31c116 100644 --- a/src/acceptance/assets/app/go_app/go.sum +++ b/src/acceptance/assets/app/go_app/go.sum @@ -35,8 +35,8 @@ github.com/go-faster/jx v1.1.0/go.mod h1:vKDNikrKoyUmpzaJ0OkIkRQClNHFX/nF3dnTJZb github.com/go-faster/yaml v0.4.6 h1:lOK/EhI04gCpPgPhgt0bChS6bvw7G3WwI8xxVe0sw9I= github.com/go-faster/yaml v0.4.6/go.mod h1:390dRIvV4zbnO7qC9FGo6YYutc+wyyUSHBgbXL52eXk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= From abbcfeab9a434ee84dc91ad3e07680ce2494f323 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Tue, 4 Jun 2024 15:03:33 +0200 Subject: [PATCH 024/186] cleanup readme and reduce it to a bare minimum --- README.md | 466 +++++++----------------------------------------------- 1 file changed, 61 insertions(+), 405 deletions(-) diff --git a/README.md b/README.md index 82880de77e..70c52987bb 100644 --- a/README.md +++ b/README.md @@ -1,408 +1,64 @@ -# App-AutoScaler [![Build Status](https://github.com/cloudfoundry/app-autoscaler/actions/workflows/postgres.yaml/badge.svg)](https://github.com/cloudfoundry/app-autoscaler/actions/workflows/postgres.yaml) [![Build Status](https://github.com/cloudfoundry/app-autoscaler/actions/workflows/mysql.yaml/badge.svg)](https://github.com/cloudfoundry/app-autoscaler/actions/workflows/mysql.yaml) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=cloudfoundry_app-autoscaler-release&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=cloudfoundry_app-autoscaler-release) - -The `App-AutoScaler` provides the capability to adjust the computation resources for Cloud Foundry applications through - - * Dynamic scaling based on application performance metrics - * Scheduled scaling based on time - -The `App-AutoScaler` has the following components: - -* `api` : provides public APIs to manage scaling policy -* `servicebroker`: implements the [Cloud Foundry service broker API][k] -* `metricsgateway` : collects and filters loggregator events via loggregator v2 API -* `metricsserver`: transforms loggregator events to app-autoscaler performance metrics ( metricsgateway + metricsserver is a replacement of metricscollector) -* `metricsforwarder`: receives and forwards custom metrics to loggregator via v2 ingress API -* `eventgenerator`: aggregates memory metrics, evaluates scaling rules and triggers events for dynamic scaling -* `scheduler`: manages the schedules in scaling policy and trigger events for scheduled scaling -* `scalingengine`: takes the scaling actions based on dynamic scaling rules or schedules - -## Development - -### System requirements - -* Java 11 or above -* Docker -* [Apache Maven][b] 3 -* [Cloud Foundry cf command line][f] 7 or 8 -* Go 1.17 or above - -### Database requirement - -The `App-AutoScaler` supports Postgres and MySQL. It uses Postgres as the default backend -data store. These are run up locally with docker images so ensure that docker is working on -your system before running up the tests. - -### Setup -**Note:** all of the setup is encapsulated in the makefile targets. So you can run the test targets (test|integration) directly -and it will setup and start the tests. - -To set up the development, firstly clone this project - -```shell -$ git clone https://github.com/cloudfoundry/app-autoscaler.git -``` - -Generate [scheduler test certs](https://github.com/cloudfoundry/app-autoscaler/blob/main/scheduler/README.md#generate-certificates) - - -#### Initialize the Database - -**Note:** The makefile will init the database if it has not already been run before running the tests. - -* **Postgres** - - ```shell - make init-db - ``` - -* **MySQL** - - ```shell - make init-db db_type=mysql - ``` - -#### Generate TLS Certificates - -Create the certificates. - -**Note**: - - * on macos it will install `certstrap` automatically but on other OS's it needs to be pre-installed - * The makefile will create the certificates if it has not already been run before running the tests. - -```shell -make test-certs -``` - -### Unit tests -The default database is postgres - - * **Postgres**: - - ```shell - make test - ``` - -To use a specific postgres version: - -```shell -make clean #Only if you're changing versions to refresh the running docker image. -make test POSTGRES_TAG=x.y -``` - -where: - - * x is the major version - * y is the minor version ( this can be left out to get the most recent patch) - * **MySQL**: - - ```shell - make test db_type=mysql - ``` - -To use a specific MySQL version: - -```shell -make clean #Only if you're changing versions to refresh the running docker image. -make test db_type=mysql MYSQL_TAG=x.y -``` - -where: - - * x is the major version - * y is the minor version ( this can be left out to get the most recent patch) - - -### Integration tests -The default database is postgres - - * **Postgres**: - - ```shell - make integration - ``` - -To use a specific postgres version: - -```shell -make clean #Only if you're changing versions to refresh the running docker image. -make integration POSTGRES_TAG=x.y -``` - -where: - - * x is the major version - * y is the minor version ( this can be left out to get the most recent patch) - * **MySQL**: - - ```shell - make integration db_type=mysql - ``` - -To use a specific MySQL version: - -```shell -make clean #Only if you're changing versions to refresh the running docker image. -make integration db_type=mysql MYSQL_TAG=x.y -``` - -where: - - * x is the major version - * y is the minor version ( this can be left out to get the most recent patch) - -### Build App-AutoScaler - -```shell -make build -``` - -### Clean up - -You can use the `make clean` to remove: - - * database ( postgres or mysql) - * autoscaler build artifacts - -### Coding Standards - -Autoscaler uses Golangci and Checkstyle for its code base. Refer to [style-guide](style-guide/README.md) - -## Bosh Release for app-autoscaler service - -## Purpose - -The purpose of this bosh release is to deploy and setup the [app-autoscaler](https://github.com/cloudfoundry-incubator/app-autoscaler) service. - -## Usage - -### Bosh Lite Deployment - -* Install [Bosh-cli-v2](https://bosh.io/docs/cli-v2.html#install) -* Install and start [BOSH-Deployment](https://github.com/cloudfoundry/bosh-deployment), following its [README](https://github.com/cloudfoundry/bosh-deployment/blob/master/README.md). -* Install [CF-deployment](https://github.com/cloudfoundry/cf-deployment#deploying-cf) -* Create a new autoscaler client - UAA CLI is required to here to create a new UAA client id. - * Install the UAA CLI, `uaac`. - - ```sh - gem install cf-uaac - ``` - - * Obtain `uaa_admin_client_secret` - - ```sh - bosh interpolate --path /uaa_admin_client_secret /path/to/cf-deployment/deployment-vars.yml - ``` - - * Use the `uaac target uaa.YOUR-DOMAIN` command to target your UAA server and obtain an access token for the admin client. - - ```sh - uaac target uaa.bosh-lite.com --skip-ssl-validation - uaac token client get admin -s - ``` - - * Create a new autoscaler client - - ```sh - uaac client add "autoscaler_client_id" \ - --authorized_grant_types "client_credentials" \ - --authorities "cloud_controller.read,cloud_controller.admin,uaa.resource" \ - --secret - ``` - -* Create and upload App-Autoscaler release - - ```sh - git clone https://github.com/cloudfoundry/app-autoscaler-release - cd app-autoscaler-release - make go-mod-tidy vendor db scheduler - bosh create-release - bosh -e YOUR_ENV upload-release - ``` - -* Deploy app-autoscaler with the newly created autoscaler client - - In the latest App-Autoscaler v2.0 release , App-Autoscaler retrieves application's metrics with [loggregator V2 API](https://github.com/cloudfoundry/loggregator-api/blob/master/README.md) via gRPC over mutual TLS connection. - - So the valid TLS certification to access Loggregator Reverse Log Proxy is required here. When deploying in bosh-lite, the most easy way is to provide loggregator certificates generated by `cf-deployments`. - - ```sh - bosh -e YOUR_ENV -d app-autoscaler \ - deploy templates/app-autoscaler-deployment.yml \ - --vars-store=bosh-lite/deployments/vars/autoscaler-deployment-vars.yml \ - -l \ - -v system_domain=bosh-lite.com \ - -v cf_client_id=autoscaler_client_id \ - -v cf_client_secret= \ - -v skip_ssl_validation=true - ``` - -* Deploy autoscaler with cf deployment mysql database - - **Notes**: It is blocked by the pull request [cf-deployment #881](https://github.com/cloudfoundry/cf-deployment/pull/881) temporarily. If you would like to use the cf mysql, please apply the `set-autoscaler-db.yml` in the pull request when deploy cf deployment. - - The lastest Autoscaler release add the support for mysql database, Autoscaler can connect the same mysql database with cf deployment. Use the operation file `example/operation/cf-mysql-db.yml` which including the cf database host , password and tls.ca cert. - - ```sh - bosh -e YOUR_ENV -d app-autoscaler \ - deploy templates/app-autoscaler-deployment.yml \ - --vars-store=bosh-lite/deployments/vars/autoscaler-deployment-vars.yml \ - -l \ - -v system_domain=bosh-lite.com \ - -v cf_client_id=autoscaler_client_id \ - -v cf_client_secret= \ - -v skip_ssl_validation=true \ - -o example/operation/cf-mysql-db.yml - ``` - -* Deploy autoscaler with external postgres database and mysql database - - ```sh - bosh -e YOUR_ENV -d app-autoscaler \ - deploy templates/app-autoscaler-deployment.yml \ - --vars-store=bosh-lite/deployments/vars/autoscaler-deployment-vars.yml \ - -l \ - -l \ - -v system_domain=bosh-lite.com \ - -v cf_client_id=autoscaler_client_id \ - -v cf_client_secret= \ - -v skip_ssl_validation=true \ - -o example/operation/external-db.yml - ``` - ->** The DATABASE_VAR_FILE should look like as below - - ```sh - database: - name: - host: - port: - scheme: - username: - password: - sslmode: - tls: - ca: | - -----BEGIN CERTIFICATE----- - - -----END CERTIFICATE----- - ``` - -The table below shows the description of all the variables: - -Property | Description --------- | ------------- -database.name | The database name. -database.host | The database server ip address or hostname. -database.port | The database server port. -database.scheme | The database scheme. Currently Autoscaler supports "postgres" and "mysql". -database.username | The username of the database specified above in "database.name". -database.password | The password of the user specified above in "database.username". -database.sslmode | There are 6 values allowed for "postgres": disable, allow, prefer, require, verify-ca and verify-full. Please refer to [Postgres SSL definition](https://www.postgresql.org/docs/current/libpq-ssl.html) when define `database_sslmode`. For "mysql", there are 7 values allowed: false, true, skip-verify, preferred, verify-ca, verify_identity.Please refer to [Mysql SSL definition(Golang)](https://github.com/go-sql-driver/mysql#tls) and [Mysql Connector SSL](https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-reference-using-ssl.html) -database.tls.ca | PEM-encoded certification authority for secure TLS communication. Only required when sslmode is verify-ca or verify-full(postgres) or verify_identity(mysql) and can be omitted for other sslmode. - -## Run linting - -Linting can be run through make: - -``` -make lint -``` - -Autofix can be trigger by providing the following options: - - -``` -OPTS=--fix RUBOCOP_OPTS=-A make lint -``` - -## Register service - -Log in to Cloud Foundry with admin user, and use the following commands to register `app-autoscaler` service - -```sh -cf create-service-broker autoscaler -``` - -* `brokerUserName`: the user name to authenticate with service broker. It's default value is `autoscaler_service_broker_user`. -* `brokerPassword`: the password to authenticate with service broker. It will be stored in the file passed to the --vars-store flag (bosh-lite/deployments/vars/autoscaler-deployment-vars.yml in the example). You can find them by searching for `autoscaler_service_broker_password`. -* `brokerURL`: the URL of the service broker - -All these parameters are configured in the bosh deployment. If you are using default values of deployment manifest, register the service with the commands below. - -```sh -cf create-service-broker autoscaler autoscaler_service_broker_user `bosh int ./bosh-lite/deployments/vars/autoscaler-deployment-vars.yml --path /autoscaler_service_broker_password` https://autoscalerservicebroker.bosh-lite.com -``` - -## Acceptance test - -Refer to [AutoScaler UAT guide](src/acceptance/README.md) to run acceptance test. - -## Use service - -To use the service to auto-scale your applications, log in to Cloud Foundry with admin user, and use the following command to enable service access to all or specific orgs. - -```sh -cf enable-service-access autoscaler [-o ORG] -``` - -The following commands don't require admin rights, but user needs to be Space Developer. Create the service instance, and then bind your application to the service instance with the policy as parameter. - -```sh -cf create-service autoscaler autoscaler-free-plan -cf bind-service -c -``` - -## Remove the service - -Log in to Cloud Foundry with admin user, and use the following commands to remove all the service instances and the service broker of `app-autoscaler` from Cloud Foundry. - -```sh -cf purge-service-offering autoscaler -cf delete-service-broker autoscaler -``` - -## Monitoring the service - -The app-autoscaler provides a number of health endpoints that are available externally that can be used to check the state of each component. Each health endpoint is protected with basic auth (apart from the api server), the usernames are listed in the table below, but the passwords are available in credhub. - -Component | Health URL | Username | Password Key | ---------- | -----------| ---------| -------------| -eventgenerator|https://autoscaler-eventgenerator.((system_domain))/health|eventgenerator|/autoscaler_eventgenerator_health_password| -metricsforwarder|https://autoscaler-metricsforwarder.((system_domain))/health|metricsforwarder|/autoscaler_metricsforwarder_health_password| -metricsgateway|https://autoscaler-metricsgateway.((system_domain))/health|metricsgateway|/autoscaler_metricsgateway_health_password| -metricsserver|https://autoscaler-metricsserver.((system_domain))/health|metricsserver|/autoscaler_metricsserver_health_password| -scalingengine|https://autoscaler-scalingengine.((system_domain))/health|scalingengine|/autoscaler_scalingengine_health_password| -operator|https://autoscaler-operator.((system_domain))/health|operator|/autoscaler_operator_health_password| -scheduler|https://autoscaler-scheduler.((system_domain))/health|scheduler|/autoscaler_scheduler_health_password| - -These endpoints can be disabled by using the ops file `example/operations/disable-basicauth-on-health-endpoints.yml` - -You can follow the development progress on [Pivotal Tracker][t]. - -## Deploy and offer Autoscaler as a service - -Go to [app-autoscaler-release][r] project for how to BOSH deploy `App-AutoScaler` - -## Use Autoscaler service - -Refer to [user guide][u] for the details of how to use the Auto-Scaler service, including policy definition, supported metrics, public API specification and command line tool. +# Application Autoscaler + +The Application Autoscaler provides the capability to adjust the computation resources for Cloud Foundry applications +through + +* dynamic scaling based on application performance metrics +* scheduled scaling based on time + +## Local Development + +### Prerequisites + +* [Docker](https://www.docker.com/products/docker-desktop/) to spin up the required databases +* [devbox](https://github.com/jetify-com/devbox) to start a shell with all required tools +* clone of https://github.com/cloudfoundry/app-autoscaler-env-bbl-state next to the clone of this repo to access systems +* [direnv](https://direnv.net/) to spin up the shell properly before running the make-targets + +### Make Targets + +| Category | Description | Target | +|-------------------|----------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| +| mock | generate mocks | `make generate-fakes` | +| unit-tests | run against PostgreSQL | `make test` | +| unit-tests | run against specific PostgreSQL version | 
make clean #Only if you're changing versions to refresh the running docker image
make test POSTGRES_TAG=x.y
| +| unit-tests | run against MySQL | `make test db_type=mysql` | +| unit-tests | run against specific MySQL version | 
make clean #Only if you're changing versions to refresh the running docker image
make test db_type=mysql MYSQL_TAG=x.y
| +| integration-tests | run against PostgreSQL | `make integration` | +| integration-tests | run against specific PostgreSQL version | 
make clean #Only if you're changing versions to refresh the running docker image
make integration POSTGRES_TAG=x.y
| +| integration-tests | run against MySQL | `make integration db_type=mysql` | +| integration-tests | run against specific MySQL version | 
make clean #Only if you're changing versions to refresh the running docker image
make integration db_type=mysql MYSQL_TAG=x.y
| +| acceptance-tests | run acceptance-tests, see [AutoScaler UAT guide](src/acceptance/README.md) for details | `make acceptance-tests` | +| lint | check code style | `make lint` | +| lint | check code style and apply auto-fixes | `OPTS=--fix RUBOCOP_OPTS=-A make lint` | +| build | compile project | `make build` | +| deploy | deploy Application Autoscaler and register the service broker in CF | `make deploy-autoscaler` | +| cleanup | remove build artifacts | `make clean` | + +## Use Application Autoscaler Service + +Refer to [user guide](docs/Readme.md) for the details of how to use the Auto-Scaler service, including policy +definition, supported metrics, public API specification and command line tool. + +## Monitor Microservices + +The app-autoscaler provides a number of health endpoints that are available externally that can be used to check the +state of each component. Each health endpoint is protected with basic auth (apart from the api server), the usernames +are listed in the table below, but the passwords are available in credhub. + +| Component | Health URL | Username | Password Key | +|------------------|--------------------------------------------------------------|------------------|----------------------------------------------| +| eventgenerator | https://autoscaler-eventgenerator.((system_domain))/health | eventgenerator | /autoscaler_eventgenerator_health_password | +| metricsforwarder | https://autoscaler-metricsforwarder.((system_domain))/health | metricsforwarder | /autoscaler_metricsforwarder_health_password | +| metricsgateway | https://autoscaler-metricsgateway.((system_domain))/health | metricsgateway | /autoscaler_metricsgateway_health_password | +| metricsserver | https://autoscaler-metricsserver.((system_domain))/health | metricsserver | /autoscaler_metricsserver_health_password | +| scalingengine | https://autoscaler-scalingengine.((system_domain))/health | scalingengine | /autoscaler_scalingengine_health_password | +| operator | https://autoscaler-operator.((system_domain))/health | operator | /autoscaler_operator_health_password | +| scheduler | https://autoscaler-scheduler.((system_domain))/health | scheduler | /autoscaler_scheduler_health_password | + +These endpoints can be disabled by using the ops +file [`example/operations/disable-basicauth-on-health-endpoints.yml`](operations/disable-basicauth-on-health-endpoints.yml) ## License -This project is released under version 2.0 of the [Apache License][l]. - - -[b]: https://maven.apache.org/ -[c]: http://couchdb.apache.org/ -[d]: http://www.eclipse.org/m2e/ -[e]: http://www.cloudant.com -[f]: https://github.com/cloudfoundry/cli/releases -[k]: http://docs.cloudfoundry.org/services/api.html -[l]: LICENSE -[t]: https://www.pivotaltracker.com/projects/1566795 -[p]: https://www.postgresql.org/ -[r]: https://github.com/cloudfoundry/app-autoscaler-release -[u]: docs/Readme.md -[m]: https://www.mysql.com/ +This project is released under version 2.0 of the [Apache License](LICENSE). From 29e8ee0cf1139c0bb9a0aaad8c5d396fff0d751e Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Tue, 4 Jun 2024 17:01:33 +0200 Subject: [PATCH 025/186] remove unused app-autoscaler-deployment.yml manifest --- templates/app-autoscaler-deployment.yml | 853 ------------------------ 1 file changed, 853 deletions(-) delete mode 100644 templates/app-autoscaler-deployment.yml diff --git a/templates/app-autoscaler-deployment.yml b/templates/app-autoscaler-deployment.yml deleted file mode 100644 index 8be6b9c98f..0000000000 --- a/templates/app-autoscaler-deployment.yml +++ /dev/null @@ -1,853 +0,0 @@ ---- -# This file contains the documentary template of the example-manifest. - -name: app-autoscaler - -# Release Details -releases: -- name: app-autoscaler - version: latest -- name: postgres - version: latest -- name: bosh-dns-aliases - version: latest -- name: routing - version: latest -- name: loggregator-agent - version: latest -- name: bpm - version: latest - -features: - use_dns_addresses: true -addons: -- name: bosh-dns-aliases - jobs: - - name: bosh-dns-aliases - release: bosh-dns-aliases - properties: - aliases: - - domain: autoscalerpostgres.service.cf.internal - targets: - - query: '*' - instance_group: postgres_autoscaler - deployment: app-autoscaler - network: default - domain: bosh - - domain: apiserver.service.cf.internal - targets: - - query: '*' - instance_group: asapi - deployment: app-autoscaler - network: default - domain: bosh - - domain: autoscalerscheduler.service.cf.internal - targets: - - query: '*' - instance_group: asactors - deployment: app-autoscaler - network: default - domain: bosh - - domain: servicebroker.service.cf.internal - targets: - - query: '*' - instance_group: asapi - deployment: app-autoscaler - network: default - domain: bosh - - domain: eventgenerator.service.cf.internal - targets: - - query: '*' - instance_group: asmetrics - deployment: app-autoscaler - network: default - domain: bosh - - domain: scalingengine.service.cf.internal - targets: - - query: '*' - instance_group: asactors - deployment: app-autoscaler - network: default - domain: bosh - - domain: reverse-log-proxy.service.cf.internal - targets: - - query: '*' - instance_group: log-api - deployment: cf - network: default - domain: bosh - - domain: metricsgateway.service.cf.internal - targets: - - query: '*' - instance_group: asnozzle - deployment: app-autoscaler - network: default - domain: bosh - - domain: metricsserver.service.cf.internal - targets: - - query: '*' - instance_group: asmetrics - deployment: app-autoscaler - network: default - domain: bosh - - domain: nats.service.cf.internal - targets: - - deployment: cf - domain: bosh - instance_group: nats - network: default - query: '*' - - domain: _.nats.service.cf.internal - targets: - - deployment: cf - domain: bosh - instance_group: nats - network: default - query: _ -- name: bpm - jobs: - - name: bpm - release: bpm -# Stemcell details -stemcells: -- alias: default - os: ubuntu-jammy - version: latest -- alias: xenial - os: ubuntu-xenial - version: latest - -# Canary details -update: - canaries: 1 - canary_watch_time: 1000-300000 - max_in_flight: 1 - update_watch_time: 1000-300000 - -# Instance Group details -instance_groups: - # Postgres Instance Group -- name: postgres_autoscaler - azs: - - z1 - instances: 1 - update: - serial: true - stemcell: xenial - vm_type: small - networks: - - name: default - jobs: - - name: postgres - release: postgres - properties: - databases: &database - sslmode: verify-full - tls: - ca: ((postgres_ca.ca)) - certificate: ((postgres_server.certificate)) - private_key: ((postgres_server.private_key)) - databases: - - name: autoscaler - tag: default - db_scheme: postgres - port: 5432 - roles: - - name: postgres - password: "((database_password))" - tag: default - connection_config: &databaseConnectionConfig - max_open_connections: 100 - max_idle_connections: 10 - connection_max_lifetime: 60s -# asactors Instance Group: scalingengine&scheduler&operator -- name: asactors - azs: - - z1 - instances: 1 - networks: - - name: default - stemcell: default - vm_type: minimal - update: - max_in_flight: 1 - serial: true - jobs: - - name: scalingengine - release: app-autoscaler - properties: - autoscaler: - scalingengine_db: *database - scheduler_db: *database - policy_db: *database - scalingengine_db_connection_config: *databaseConnectionConfig - scheduler_db_connection_config: *databaseConnectionConfig - policy_db_connection_config: *databaseConnectionConfig - cf: &cf_credentials - api: "https://api.((system_domain))" - grant_type: client_credentials - client_id: "((cf_client_id))" - secret: "((cf_client_secret))" - skip_ssl_validation: "((skip_ssl_validation))" - scalingengine: - logging: - level: info - http_client_timeout: 60s - server: - port: &scalingEnginePort 6104 - health: - port: &scalingEngineHealthPort 6204 - username: scalingengine - password: ((autoscaler_scalingengine_health_password)) - defaultCoolDownSecs: 300 - lockSize: 32 - ca_cert: ((scalingengine_ca.ca)) - server_cert: ((scalingengine_server.certificate)) - server_key: ((scalingengine_server.private_key)) - - name: scheduler - release: app-autoscaler - properties: - autoscaler: - scheduler: - port: &schedulerPort 6102 - health: - port: &schedulerHealthPort 6202 - basicAuthEnabled: true - username: scheduler - password: ((autoscaler_scheduler_health_password)) - http_client_timeout: 60 - job_reschedule_interval_millisecond: 10000 - job_reschedule_maxcount: 6 - notification_reschedule_maxcount: 3 - ca_cert: ((scheduler_ca.ca)) - server_cert: ((scheduler_server.certificate)) - server_key: ((scheduler_server.private_key)) - scaling_engine: - ca_cert: ((scalingengine_ca.ca)) - client_cert: ((scalingengine_client.certificate)) - client_key: ((scalingengine_client.private_key)) - scheduler_db: *database - policy_db: *database - - name: operator - release: app-autoscaler - properties: - autoscaler: - cf: *cf_credentials - policy_db: *database - appmetrics_db: *database - instancemetrics_db: *database - scalingengine_db: *database - policy_db_connection_config: *databaseConnectionConfig - appmetrics_db_connection_config: *databaseConnectionConfig - instancemetrics_db_connection_config: *databaseConnectionConfig - scalingengine_db_connection_config: *databaseConnectionConfig - lock_db: *database - lock_db_connection_config: *databaseConnectionConfig - operator: - app_sync_interval: 24h - scaling_engine: - port: *scalingEnginePort - ca_cert: ((scalingengine_ca.ca)) - client_cert: ((scalingengine_client.certificate)) - client_key: ((scalingengine_client.private_key)) - scheduler: - host: autoscalerscheduler.service.cf.internal - port: *schedulerPort - ca_cert: ((scheduler_ca.ca)) - client_cert: ((scheduler_client.certificate)) - client_key: ((scheduler_client.private_key)) - db_lock: - ttl: 15s - retry_interval: 5s - logging: - level: info - http_client_timeout: 60s - health: - port: &operatorHealthPort 6208 - username: operator - password: ((autoscaler_operator_health_password)) - - name: route_registrar - release: routing - consumes: - nats: {from: nats, deployment: cf} - properties: - route_registrar: - routes: - - name: autoscaler_scalingengine_health - registration_interval: 20s - port: *scalingEngineHealthPort - tags: - component: autoscaler_scalingengine_health - uris: - - autoscaler-scalingengine.((system_domain)) - - name: autoscaler_operator_health - registration_interval: 20s - port: *operatorHealthPort - tags: - component: autoscaler_operator_health - uris: - - autoscaler-operator.((system_domain)) - - name: autoscaler_scheduler_health - registration_interval: 20s - port: *schedulerHealthPort - tags: - component: autoscaler_scheduler_health - uris: - - autoscaler-scheduler.((system_domain)) -# asmetrics Instance Group: metricsserver&eventgenerator -- name: asmetrics - azs: - - z1 - instances: 1 - networks: - - name: default - stemcell: default - vm_type: small - update: - max_in_flight: 1 - serial: true - jobs: - - name: metricsserver - release: app-autoscaler - properties: - autoscaler: - instancemetrics_db: *database - policy_db: *database - instancemetrics_db_connection_config: *databaseConnectionConfig - policy_db_connection_config: *databaseConnectionConfig - metricsserver: - logging: - level: info - server: - port: &metricsserverPort 6103 - http_client_timeout: 60s - ca_cert: ((metricsserver_server.ca)) - server_cert: ((metricsserver_server.certificate)) - server_key: ((metricsserver_server.private_key)) - collector: - port: 7103 - keep_alive_time: 60s - refresh_interval: 60s - collect_interval: 60s - save_interval: 5s - persist_metrics: true - envelope_processor_count: 5 - metric_cache_size_per_app: 1000 - envelope_channel_size: 1000 - metric_channel_size: 1000 - health: - port: &metricsserverHealthPort 6303 - username: metricsserver - password: ((autoscaler_metricsserver_health_password)) - - name: eventgenerator - release: app-autoscaler - properties: - autoscaler: - appmetrics_db: *database - policy_db: *database - appmetrics_db_connection_config: *databaseConnectionConfig - policy_db_connection_config: *databaseConnectionConfig - eventgenerator: - logging: - level: info - http_client_timeout: 60s - server: - port: &eventGeneratorPort 6105 - health: - port: &eventGeneratorHealthPort 6205 - username: eventgenerator - password: ((autoscaler_eventgenerator_health_password)) - ca_cert: ((eventgenerator_ca.ca)) - server_cert: ((eventgenerator_server.certificate)) - server_key: ((eventgenerator_server.private_key)) - aggregator: - aggregator_execute_interval: 40s - policy_poller_interval: 60s - metric_poller_count: 20 - app_monitor_channel_size: 200 - save_interval: 5s - app_metric_channel_size: 1000 - evaluator: - evaluation_manager_execute_interval: 60s - evaluator_count: 20 - trigger_array_channel_size: 200 - defaultStatWindowSecs: 120 - defaultBreachDurationSecs: 120 - circuitBreaker: - back_off_initial_interval: 5m - back_off_max_interval: 120m - consecutive_failure_count: 5 - scaling_engine: - host: scalingengine.service.cf.internal - port: *scalingEnginePort - ca_cert: ((scalingengine_ca.ca)) - client_cert: ((scalingengine_client.certificate)) - client_key: ((scalingengine_client.private_key)) - metricscollector: - host: metricsserver.service.cf.internal - port: *metricsserverPort - ca_cert: ((metricsserver_ca.ca)) - client_cert: ((metricsserver_client.certificate)) - client_key: ((metricsserver_client.private_key)) - - name: route_registrar - release: routing - consumes: - nats: {from: nats, deployment: cf} - properties: - route_registrar: - routes: - - name: autoscaler_eventgenerator_health - registration_interval: 20s - port: *eventGeneratorHealthPort - tags: - component: autoscaler_eventgenerator_health - uris: - - autoscaler-eventgenerator.((system_domain)) - - name: autoscaler_metricsserver_health - registration_interval: 20s - port: *metricsserverHealthPort - tags: - component: autoscaler_metricsserver_health - uris: - - autoscaler-metricsserver.((system_domain)) -- name: asnozzle - azs: - - z1 - instances: 1 - networks: - - name: default - stemcell: default - vm_type: small - update: - max_in_flight: 1 - serial: true - jobs: - - name: metricsgateway - release: app-autoscaler - properties: - autoscaler: - policy_db: *database - policy_db_connection_config: *databaseConnectionConfig - metricsgateway: - logging: - level: info - envelop_chan_size: 1000 - nozzle_count: 3 - app_manager: - app_refresh_interval: 5s - emitter: - buffer_size: 500 - keep_alive_interval: 5s - handshake_timeout: 1s - max_setup_retry_count: 3 - max_close_retry_count: 3 - retry_delay: 1s - metricsserver_client: - cert: ((metricsserver_client.certificate)) - key: ((metricsserver_client.private_key)) - ca_cert: ((metricsserver_client.ca)) - nozzle: - rlp_addr: "reverse-log-proxy.service.cf.internal:8082" - shard_id: "CF_AUTOSCALER" - loggregator_rlp_tls: - cert: ((loggregator_tls_rlp.certificate)) - key: ((loggregator_tls_rlp.private_key)) - ca_cert: ((loggregator_ca.certificate)) - health: - port: &metricsgatewayHealthPort 6503 - username: metricsgateway - password: ((autoscaler_metricsgateway_health_password)) - - name: route_registrar - release: routing - consumes: - nats: {from: nats, deployment: cf} - properties: - route_registrar: - routes: - - name: autoscaler_metricsgateway_health - registration_interval: 20s - port: *metricsgatewayHealthPort - tags: - component: autoscaler_metricsgateway_health - uris: - - autoscaler-metricsgateway.((system_domain)) -# asapi Instance Group : apiserver&servicebroker -- name: asapi - azs: - - z1 - instances: 1 - networks: - - name: default - stemcell: default - vm_type: minimal - update: - max_in_flight: 1 - serial: true - jobs: - - name: golangapiserver - release: app-autoscaler - properties: - autoscaler: - apiserver: - logging: - level: info - broker: - server: - port: &brokerServerPort 6102 - catalog: - services: - - id: autoscaler-guid - name: autoscaler - description: Automatically increase or decrease the number of application instances based on a policy you define. - bindable: true - plans: - - id: autoscaler-free-plan-id - name: autoscaler-free-plan - description: This is the free service plan for the Auto-Scaling service. - dashboard_redirect_uri: "" - username: autoscaler_service_broker_user - password: ((autoscaler_service_broker_password)) - public_api: - server: - port: &publicApiServerPort 6101 - scheduler: - host: autoscalerscheduler.service.cf.internal - ca_cert: ((scheduler_ca.ca)) - client_cert: ((scheduler_client.certificate)) - client_key: ((scheduler_client.private_key)) - scaling_engine: - ca_cert: ((scalingengine_ca.ca)) - client_cert: ((scalingengine_client.certificate)) - client_key: ((scalingengine_client.private_key)) - event_generator: - ca_cert: ((eventgenerator_ca.ca)) - client_cert: ((eventgenerator_client.certificate)) - client_key: ((eventgenerator_client.private_key)) - metrics_forwarder: - host: autoscalermetrics.((system_domain)) - policy_db: *database - policy_db_connection_config: *databaseConnectionConfig - binding_db: *database - binding_db_connection_config: *databaseConnectionConfig - cf: *cf_credentials - - name: metricsforwarder - release: app-autoscaler - properties: - autoscaler: - metricsforwarder: - logging: - level: info - server: - port: &metricsforwarderServerPort 6201 - loggregator: - metron_address: "127.0.0.1:3458" - tls: - ca_cert: ((loggregator_tls_agent.ca)) - cert: ((loggregator_tls_agent.certificate)) - key: ((loggregator_tls_agent.private_key)) - cache_ttl: 900s - cache_cleanup_interval: 6h - policy_poller_interval: 60s - health: - port: &metricsforwarderHealthPort 6403 - username: metricsforwarder - password: ((autoscaler_metricsforwarder_health_password)) - policy_db: *database - policy_db_connection_config: *databaseConnectionConfig - - name: route_registrar - release: routing - consumes: - nats: {from: nats, deployment: cf} - properties: - route_registrar: - routes: - - name: api_server - registration_interval: 20s - port: *publicApiServerPort - tags: - component: api_server - uris: - - autoscaler.((system_domain)) - - name: autoscaler_service_broker - registration_interval: 20s - port: *brokerServerPort - tags: - component: autoscaler_service_broker - uris: - - autoscalerservicebroker.((system_domain)) - - name: autoscaler_metrics_forwarder - registration_interval: 20s - port: *metricsforwarderServerPort - tags: - component: autoscaler_metrics_forwarder - uris: - - autoscalermetrics.((system_domain)) - - name: autoscaler_metricsforwarder_health - registration_interval: 20s - port: *metricsforwarderHealthPort - tags: - component: autoscaler_metricsforwarder_health - uris: - - autoscaler-metricsforwarder.((system_domain)) - - name: loggregator_agent - release: loggregator-agent - consumes: - doppler: {from: doppler, deployment: cf} - properties: - loggregator: - tls: - ca_cert: ((loggregator_ca.certificate)) - agent: - cert: ((loggregator_tls_agent.certificate)) - key: ((loggregator_tls_agent.private_key)) - metrics: - ca_cert: ((loggregator_agent_metrics_tls.ca)) - cert: ((loggregator_agent_metrics_tls.certificate)) - key: ((loggregator_agent_metrics_tls.private_key)) - server_name: loggregator_agent_server - -variables: -- name: database_password - type: password - update_mode: converge -- name: autoscaler_service_broker_password - type: password - update_mode: converge -- name: autoscaler_scheduler_health_password - type: password - update_mode: converge -- name: autoscaler_eventgenerator_health_password - type: password - update_mode: converge -- name: autoscaler_metricsforwarder_health_password - type: password - update_mode: converge -- name: autoscaler_metricsgateway_health_password - type: password - update_mode: converge -- name: autoscaler_metricsserver_health_password - type: password - update_mode: converge -- name: autoscaler_operator_health_password - type: password - update_mode: converge -- name: autoscaler_scalingengine_health_password - type: password - update_mode: converge -- name: scalingengine_ca - type: certificate - update_mode: converge - options: - is_ca: true - common_name: scalingengineCA -- name: scalingengine_server - type: certificate - update_mode: converge - options: - ca: scalingengine_ca - common_name: scalingengine.service.cf.internal - alternative_names: - - scalingengine.service.cf.internal - extended_key_usage: - - client_auth - - server_auth -- name: scalingengine_client - type: certificate - update_mode: converge - options: - ca: scalingengine_ca - common_name: scalingengine client - extended_key_usage: - - client_auth -- name: eventgenerator_ca - type: certificate - update_mode: converge - options: - is_ca: true - common_name: eventgeneratorCA -- name: eventgenerator_server - type: certificate - update_mode: converge - options: - ca: eventgenerator_ca - common_name: eventgenerator.service.cf.internal - alternative_names: - - eventgenerator.service.cf.internal - extended_key_usage: - - client_auth - - server_auth -- name: eventgenerator_client - type: certificate - update_mode: converge - options: - ca: eventgenerator_ca - common_name: eventgenerator client - extended_key_usage: - - client_auth -- name: apiserver_ca - type: certificate - update_mode: converge - options: - is_ca: true - common_name: apiserverCA -- name: apiserver_server - type: certificate - update_mode: converge - options: - ca: apiserver_ca - common_name: apiserver.service.cf.internal - alternative_names: - - apiserver.service.cf.internal - extended_key_usage: - - client_auth - - server_auth -- name: apiserver_public_ca - type: certificate - update_mode: converge - options: - is_ca: true - common_name: apiserverpublicCA -- name: apiserver_public_server - type: certificate - update_mode: converge - options: - ca: apiserver_public_ca - common_name: autoscaler.((system_domain)) - alternative_names: - - autoscaler.((system_domain)) - extended_key_usage: - - client_auth - - server_auth -- name: apiserver_client - type: certificate - update_mode: converge - options: - ca: apiserver_ca - common_name: apiserver client - extended_key_usage: - - client_auth -- name: servicebroker_ca - type: certificate - update_mode: converge - options: - is_ca: true - common_name: servicebrokerCA -- name: servicebroker_server - type: certificate - update_mode: converge - options: - ca: servicebroker_ca - common_name: servicebroker.service.cf.internal - alternative_names: - - servicebroker.service.cf.internal - extended_key_usage: - - client_auth - - server_auth -- name: servicebroker_client - type: certificate - update_mode: converge - options: - ca: servicebroker_ca - common_name: servicebroker client - extended_key_usage: - - client_auth -- name: servicebroker_public_ca - type: certificate - update_mode: converge - options: - is_ca: true - common_name: servicebrokerCA -- name: servicebroker_public_server - type: certificate - update_mode: converge - options: - ca: servicebroker_public_ca - common_name: autoscalerservicebroker.((system_domain)) - alternative_names: - - autoscalerservicebroker.((system_domain)) - extended_key_usage: - - client_auth - - server_auth -- name: scheduler_ca - type: certificate - update_mode: converge - options: - is_ca: true - common_name: schedulerCA -- name: scheduler_server - type: certificate - update_mode: converge - options: - ca: scheduler_ca - common_name: autoscalerscheduler.service.cf.internal - alternative_names: - - autoscalerscheduler.service.cf.internal - extended_key_usage: - - client_auth - - server_auth -- name: scheduler_client - type: certificate - update_mode: converge - options: - ca: scheduler_ca - common_name: scheduler client - extended_key_usage: - - client_auth -- name: metricsserver_ca - type: certificate - update_mode: converge - options: - is_ca: true - common_name: metricsserverCA -- name: metricsserver_server - type: certificate - update_mode: converge - options: - ca: metricsserver_ca - common_name: metricsserver.service.cf.internal - alternative_names: ["metricsserver.service.cf.internal","*.asmetrics.default.app-autoscaler.bosh"] - extended_key_usage: - - client_auth - - server_auth -- name: metricsserver_client - type: certificate - update_mode: converge - options: - ca: metricsserver_ca - common_name: metricsserver client - extended_key_usage: - - client_auth -- name: postgres_ca - type: certificate - update_mode: converge - options: - is_ca: true - common_name: postgresCA -- name: postgres_server - type: certificate - update_mode: converge - options: - ca: postgres_ca - common_name: autoscalerpostgres.service.cf.internal - alternative_names: - - autoscalerpostgres.service.cf.internal - extended_key_usage: - - client_auth - - server_auth -- name: metric_scraper_ca - type: certificate - update_mode: converge - options: - common_name: metricScraperCA - is_ca: true -- name: loggregator_agent_metrics_tls - type: certificate - update_mode: converge - options: - ca: metric_scraper_ca - common_name: loggregator_agent_server - alternative_names: - - loggregator_agent_server - extended_key_usage: - - server_auth - From d80ef7d99fad58edf4ef5d9fe82620824d93149f Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Wed, 5 Jun 2024 09:10:28 +0200 Subject: [PATCH 026/186] merge configure-log-cache-and-forward-metrics-via-mtls.yml into app-autoscaler.yml manifest also: remove ops file that sets nozzle shard id --- ci/autoscaler/pipeline.yml | 3 - ci/autoscaler/scripts/deploy-autoscaler.sh | 1 - ...nt-name-to-loggregator-nozzle-shard-id.yml | 5 - ...log-cache-and-forward-metrics-via-mtls.yml | 110 ------------------ templates/app-autoscaler.yml | 95 +++++++++------ 5 files changed, 58 insertions(+), 156 deletions(-) delete mode 100644 operations/append-deployment-name-to-loggregator-nozzle-shard-id.yml delete mode 100644 operations/configure-log-cache-and-forward-metrics-via-mtls.yml diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index ac7f18da2a..74030b2428 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -9,7 +9,6 @@ anchors: operations/loggregator-certs-from-cf.yml operations/add-extra-plan.yml operations/set-release-version.yml - operations/configure-log-cache-and-forward-metrics-via-mtls.yml operations/enable-metricsforwarder-via-metron-agent.yml operations/remove-metricsserver.yml operations/remove-metricsgateway.yml @@ -24,7 +23,6 @@ anchors: operations/loggregator-certs-from-cf.yml operations/add-extra-plan.yml operations/set-release-version.yml - operations/configure-log-cache-and-forward-metrics-via-mtls.yml operations/enable-metricsforwarder-via-syslog-agent.yml operations/remove-metricsserver.yml operations/remove-metricsgateway.yml @@ -37,7 +35,6 @@ anchors: operations/add-postgres-variables.yml operations/enable-nats-tls.yml operations/loggregator-certs-from-cf.yml - operations/append-deployment-name-to-loggregator-nozzle-shard-id.yml operations/postgres-persistent-disk.yml operations/add-extra-plan.yml operations/set-release-version.yml diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index d8e0351339..01c8baf83c 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -16,7 +16,6 @@ ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\ ${autoscaler_dir}/operations/loggregator-certs-from-cf.yml\ ${autoscaler_dir}/operations/add-extra-plan.yml\ ${autoscaler_dir}/operations/set-release-version.yml\ - ${autoscaler_dir}/operations/configure-log-cache-and-forward-metrics-via-mtls.yml\ ${autoscaler_dir}/operations/remove-metricsserver.yml\ ${autoscaler_dir}/operations/remove-metricsgateway.yml\ ${autoscaler_dir}/operations/enable-log-cache-via-uaa.yml\ diff --git a/operations/append-deployment-name-to-loggregator-nozzle-shard-id.yml b/operations/append-deployment-name-to-loggregator-nozzle-shard-id.yml deleted file mode 100644 index 067bfbc5ab..0000000000 --- a/operations/append-deployment-name-to-loggregator-nozzle-shard-id.yml +++ /dev/null @@ -1,5 +0,0 @@ -# This can be used to make the Loggregator nozzle of the autoscaler unique and is primarily used in our CI where multiple deployments run at the same time in the same CF foundation. ---- -- type: replace - path: /instance_groups/name=metricsgateway/jobs/name=metricsgateway/properties/autoscaler/metricsgateway/nozzle/shard_id? - value: CF_AUTOSCALER_((deployment_name)) diff --git a/operations/configure-log-cache-and-forward-metrics-via-mtls.yml b/operations/configure-log-cache-and-forward-metrics-via-mtls.yml deleted file mode 100644 index 3316b0a527..0000000000 --- a/operations/configure-log-cache-and-forward-metrics-via-mtls.yml +++ /dev/null @@ -1,110 +0,0 @@ -# safeguard: make sure there is no loggregator agent configured -- type: remove - path: /instance_groups/name=metricsforwarder/jobs/name=loggregator_agent? - - -# configure metricscollector -- type: replace - path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/eventgenerator/metricscollector/use_log_cache? - value: true -- type: replace - path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/eventgenerator/metricscollector/host - value: logcache -- type: replace - path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/eventgenerator/metricscollector/port - value: 8080 -- type: replace - path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/eventgenerator/metricscollector/ca_cert - value: ((/bosh-autoscaler/cf/log_cache.ca)) -- type: replace - path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/eventgenerator/metricscollector/client_cert - value: ((/bosh-autoscaler/cf/log_cache.certificate)) -- type: replace - path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/eventgenerator/metricscollector/client_key - value: ((/bosh-autoscaler/cf/log_cache.private_key)) - - -# configure metricsforwarder -- type: replace - path: /instance_groups/name=metricsforwarder/jobs/name=metricsforwarder/properties/autoscaler/metricsforwarder/loggregator - value: - tls: # connection to syslog-agent - ca_cert: ((!metricsforwarder_autoscaler_metricsforwarder_loggregator_tls.ca)) - cert: ((!metricsforwarder_autoscaler_metricsforwarder_loggregator_tls.certificate)) - key: ((!metricsforwarder_autoscaler_metricsforwarder_loggregator_tls.private_key)) - - -# configure syslog-agent - -# safeguard: make sure there is no syslog-agent configured -- type: remove - path: /instance_groups/name=metricsforwarder/jobs/name=loggr-syslog-agent? - -# add configuration -- type: replace - path: /instance_groups/name=metricsforwarder/jobs/- - value: - name: loggr-syslog-agent - release: loggregator-agent - properties: - tls: # connection to syslog-agent - ca_cert: ((!loggr_syslog_agent_tls.ca)) - cert: ((!loggr_syslog_agent_tls.certificate)) - key: ((!loggr_syslog_agent_tls.private_key)) - cache: # connection to syslog-binding-cache - tls: - ca_cert: ((!loggr_syslog_agent_cache_tls.ca)) - cert: ((!loggr_syslog_agent_cache_tls.certificate)) - key: ((!loggr_syslog_agent_cache_tls.private_key)) - cn: loggr_syslog_binding_cache - # url: the value is automatically being generated if syslog-binding-cache is deployed https://github.com/cloudfoundry/loggregator-agent-release/blob/a5366d6d7c490417d12f990c1af0437a1feb067f/jobs/loggr-syslog-agent/templates/bpm.yml.erb#L60 - metrics: # connection for metric scrapers, here are dummy values configured since the /metrics endpoint can't be disabled via configuration - ca_cert: ((!loggr_syslog_agent_metrics.ca)) - cert: ((!loggr_syslog_agent_metrics.certificate)) - key: ((!loggr_syslog_agent_metrics.private_key)) - server_name: metrics.config.is.required.by.job.specification.but.not.needed.in.our.case - - -# configure syslog-binding-cache - -# safeguard: make sure there is no syslog-binding-cache configured -- type: remove - path: /instance_groups/name=metricsforwarder/jobs/name=loggr-syslog-binding-cache? - -# add configuration -- type: replace - path: /instance_groups/name=metricsforwarder/jobs/- - value: - name: loggr-syslog-binding-cache - release: loggregator-agent - consumes: - cloud_controller: { from: cloud_controller, deployment: cf } # required by job to resolve API URL https://github.com/cloudfoundry/loggregator-agent-release/blob/0e3340f17f94d06cb3d4c11d1553a9a2a5bfb891/jobs/loggr-syslog-binding-cache/templates/bpm.yml.erb#L4 - properties: - tls: # connection to syslog-binding-cache api, e.g. /v2/aggregate & /v2/bindings - ca_cert: ((!loggr_syslog_binding_cache_tls.ca)) - cert: ((!loggr_syslog_binding_cache_tls.certificate)) - key: ((!loggr_syslog_binding_cache_tls.private_key)) - cn: loggr_syslog_agent_tls - external_port: 9000 - aggregate_drains: # connection to log-cache - - url: "syslog-tls://log-cache.service.cf.internal:6067?include-metrics-deprecated=true&ssl-strict-internal=true" - # reusing these certificates here is a workaround so that we don't need to generate own ones. - # the problem is that when we generate own certificates (see variables section of app-autoscaler.yml), - # we have no possibility to reuse the CA from CF to issue new certificates. - ca: ((/bosh-autoscaler/cf/log_cache_syslog_tls.ca)) - cert: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.certificate)) - key: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.private_key)) - metrics: # connection for metric scrapers, here are dummy values configured since the /metrics endpoint can't be disabled via configuration - ca_cert: ((!loggr_syslog_binding_cache_metrics.ca)) - cert: ((!loggr_syslog_binding_cache_metrics.certificate)) - key: ((!loggr_syslog_binding_cache_metrics.private_key)) - server_name: metrics.config.is.required.by.job.specification.but.not.needed.in.our.case - api: # connection to CF cloud controller - # here are dummy values configured since there is no need to query the CC API for all bindings. - # if a customer wants to ever receive their own custom metrics in their own syslog-drain, we would need to configure this properly. - tls: - cn: api.tls.config.is.required.by.job.specification.but.not.needed.in.our.case - ca_cert: ((!loggr_syslog_binding_cache_api_tls.ca)) - cert: ((!loggr_syslog_binding_cache_api_tls.certificate)) - key: ((!loggr_syslog_binding_cache_api_tls.private_key)) - polling_interval: 876000h # 100 years, workaround to basically never poll the cloud controller API diff --git a/templates/app-autoscaler.yml b/templates/app-autoscaler.yml index 5efa2bb159..e22b399f7e 100644 --- a/templates/app-autoscaler.yml +++ b/templates/app-autoscaler.yml @@ -82,20 +82,6 @@ addons: deployment: ((deployment_name)) network: default domain: bosh - - domain: *metricsgateway_domain - targets: - - query: '*' - instance_group: metricsgateway - deployment: ((deployment_name)) - network: default - domain: bosh - - domain: *metricsserver_domain - targets: - - query: '*' - instance_group: metricsserver - deployment: ((deployment_name)) - network: default - domain: bosh # Cf internal names - domain: nats.service.cf.internal targets: @@ -264,7 +250,7 @@ instance_groups: ca_cert: ((!metricsserver_client_cert.ca)) client_cert: ((!metricsserver_client_cert.certificate)) client_key: ((!metricsserver_client_cert.private_key)) - host: *metricsserver_domain + host: logcache event_generator: ca_cert: ((!eventgenerator_client_cert.ca)) client_cert: ((!eventgenerator_client_cert.certificate)) @@ -583,11 +569,12 @@ instance_groups: client_key: ((!scalingengine_client_cert.private_key)) host: *scalingengine_domain metricscollector: - ca_cert: ((!metricsserver_client_cert.ca)) - client_cert: ((!metricsserver_client_cert.certificate)) - client_key: ((!metricsserver_client_cert.private_key)) - port: *metricsserverPort - host: *metricsserver_domain + use_log_cache: true + ca_cert: ((/bosh-autoscaler/cf/log_cache.ca)) + client_cert: ((/bosh-autoscaler/cf/log_cache.certificate)) + client_key: ((/bosh-autoscaler/cf/log_cache.private_key)) + port: 8080 + host: logcache - name: route_registrar release: routing consumes: @@ -630,28 +617,62 @@ instance_groups: port: &metricsforwarderServerPort 6201 loggregator: tls: - ca_cert: ((loggregator_tls_agent.ca)) - cert: ((loggregator_tls_agent.certificate)) - key: ((loggregator_tls_agent.private_key)) + ca_cert: ((!metricsforwarder_autoscaler_metricsforwarder_loggregator_tls.ca)) + cert: ((!metricsforwarder_autoscaler_metricsforwarder_loggregator_tls.certificate)) + key: ((!metricsforwarder_autoscaler_metricsforwarder_loggregator_tls.private_key)) storedprocedure_db: *database - - name: loggregator_agent + - name: loggr-syslog-agent + release: loggregator-agent + properties: + tls: # connection to syslog-agent + ca_cert: ((!loggr_syslog_agent_tls.ca)) + cert: ((!loggr_syslog_agent_tls.certificate)) + key: ((!loggr_syslog_agent_tls.private_key)) + cache: # connection to syslog-binding-cache + tls: + ca_cert: ((!loggr_syslog_agent_cache_tls.ca)) + cert: ((!loggr_syslog_agent_cache_tls.certificate)) + key: ((!loggr_syslog_agent_cache_tls.private_key)) + cn: loggr_syslog_binding_cache + # url: the value is automatically being generated if syslog-binding-cache is deployed https://github.com/cloudfoundry/loggregator-agent-release/blob/a5366d6d7c490417d12f990c1af0437a1feb067f/jobs/loggr-syslog-agent/templates/bpm.yml.erb#L60 + metrics: # connection for metric scrapers, here are dummy values configured since the /metrics endpoint can't be disabled via configuration + ca_cert: ((!loggr_syslog_agent_metrics.ca)) + cert: ((!loggr_syslog_agent_metrics.certificate)) + key: ((!loggr_syslog_agent_metrics.private_key)) + server_name: metrics.config.is.required.by.job.specification.but.not.needed.in.our.case + - name: loggr-syslog-binding-cache release: loggregator-agent consumes: - doppler: - from: doppler - deployment: cf + cloud_controller: { from: cloud_controller, deployment: cf } # required by job to resolve API URL https://github.com/cloudfoundry/loggregator-agent-release/blob/0e3340f17f94d06cb3d4c11d1553a9a2a5bfb891/jobs/loggr-syslog-binding-cache/templates/bpm.yml.erb#L4 properties: - loggregator: + tls: # connection to syslog-binding-cache api, e.g. /v2/aggregate & /v2/bindings + ca_cert: ((!loggr_syslog_binding_cache_tls.ca)) + cert: ((!loggr_syslog_binding_cache_tls.certificate)) + key: ((!loggr_syslog_binding_cache_tls.private_key)) + cn: loggr_syslog_agent_tls + external_port: 9000 + aggregate_drains: # connection to log-cache + - url: "syslog-tls://log-cache.service.cf.internal:6067?include-metrics-deprecated=true&ssl-strict-internal=true" + # reusing these certificates here is a workaround so that we don't need to generate own ones. + # the problem is that when we generate own certificates (see variables section of app-autoscaler.yml), + # we have no possibility to reuse the CA from CF to issue new certificates. + ca: ((/bosh-autoscaler/cf/log_cache_syslog_tls.ca)) + cert: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.certificate)) + key: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.private_key)) + metrics: # connection for metric scrapers, here are dummy values configured since the /metrics endpoint can't be disabled via configuration + ca_cert: ((!loggr_syslog_binding_cache_metrics.ca)) + cert: ((!loggr_syslog_binding_cache_metrics.certificate)) + key: ((!loggr_syslog_binding_cache_metrics.private_key)) + server_name: metrics.config.is.required.by.job.specification.but.not.needed.in.our.case + api: # connection to CF cloud controller + # here are dummy values configured since there is no need to query the CC API for all bindings. + # if a customer wants to ever receive their own custom metrics in their own syslog-drain, we would need to configure this properly. tls: - ca_cert: ((loggregator_ca.certificate)) - agent: - cert: ((loggregator_tls_agent.certificate)) - key: ((loggregator_tls_agent.private_key)) - metrics: - ca_cert: ((loggregator_agent_metrics_tls.ca)) - cert: ((loggregator_agent_metrics_tls.certificate)) - key: ((loggregator_agent_metrics_tls.private_key)) - server_name: loggregator_agent_server + cn: api.tls.config.is.required.by.job.specification.but.not.needed.in.our.case + ca_cert: ((!loggr_syslog_binding_cache_api_tls.ca)) + cert: ((!loggr_syslog_binding_cache_api_tls.certificate)) + key: ((!loggr_syslog_binding_cache_api_tls.private_key)) + polling_interval: 876000h # 100 years, workaround to basically never poll the cloud controller API - name: route_registrar release: routing consumes: From dcd8592e5f30a83df1478d38b9bf6c08b6267549 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Wed, 5 Jun 2024 10:32:29 +0200 Subject: [PATCH 027/186] disable requirement of succeeded jobs for testing --- ci/autoscaler/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index 74030b2428..8115b71ff8 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -222,7 +222,7 @@ jobs: - in_parallel: - get: bbl-state - get: app-autoscaler-release - passed: [unit-tests, integration-tests] + # passed: [unit-tests, integration-tests] disabled this here just for testing trigger: true - get: ci - task: make-prerelease @@ -275,7 +275,7 @@ jobs: - in_parallel: - get: bbl-state - get: app-autoscaler-release - passed: [unit-tests, integration-tests] + # passed: [unit-tests, integration-tests] disabled this here just for testing trigger: true - get: ci - task: make-prerelease From b7a934d54e40b6256db312230f4bf0ee641aa978 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Wed, 5 Jun 2024 10:32:40 +0200 Subject: [PATCH 028/186] align make target names --- ci/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/Makefile b/ci/Makefile index 4317bd8c3a..abdf794ea2 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -2,6 +2,6 @@ set-autoscaler-pipeline: @./autoscaler/set-pipeline.sh -.PHONY: unpause-pipeline -unpause-pipeline: +.PHONY: unpause-autoscaler-pipeline +unpause-autoscaler-pipeline: @./scripts/unpause-pipeline.sh From 4a0e805ccba99676acc3ac6b617daee92ae44625 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Wed, 5 Jun 2024 10:46:41 +0200 Subject: [PATCH 029/186] remove metricsserver and metricsgateway from deployment manifest template, related ops-files and manifest-tests --- ci/autoscaler/pipeline.yml | 4 - ci/autoscaler/scripts/deploy-autoscaler.sh | 2 - example/operation/cf-mysql-db.yml | 26 +-- .../disable-basicauth-on-health-endpoints.yml | 23 +-- .../enable-name-based-deployments.yml | 57 ------- example/operation/external-db.yml | 25 +-- .../disable-basicauth-on-health-endpoints.yml | 24 --- operations/enable-nats-tls.yml | 36 ---- operations/remove-metricsgateway.yml | 12 -- operations/remove-metricsserver.yml | 20 --- templates/app-autoscaler.yml | 160 ------------------ templates/manifest_tests.sh | 28 +-- 12 files changed, 6 insertions(+), 411 deletions(-) delete mode 100644 operations/remove-metricsgateway.yml delete mode 100644 operations/remove-metricsserver.yml diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index 8115b71ff8..973081ab5f 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -10,8 +10,6 @@ anchors: operations/add-extra-plan.yml operations/set-release-version.yml operations/enable-metricsforwarder-via-metron-agent.yml - operations/remove-metricsserver.yml - operations/remove-metricsgateway.yml operations/enable-scheduler-logging.yml app-autoscaler-ops-files-log-cache-syslog: &app-autoscaler-ops-files-log-cache-syslog @@ -24,8 +22,6 @@ anchors: operations/add-extra-plan.yml operations/set-release-version.yml operations/enable-metricsforwarder-via-syslog-agent.yml - operations/remove-metricsserver.yml - operations/remove-metricsgateway.yml operations/enable-scheduler-logging.yml app-autoscaler-ops-files-upgrade: &app-autoscaler-ops-files-upgrade diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index 01c8baf83c..f8a0f2a336 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -16,8 +16,6 @@ ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\ ${autoscaler_dir}/operations/loggregator-certs-from-cf.yml\ ${autoscaler_dir}/operations/add-extra-plan.yml\ ${autoscaler_dir}/operations/set-release-version.yml\ - ${autoscaler_dir}/operations/remove-metricsserver.yml\ - ${autoscaler_dir}/operations/remove-metricsgateway.yml\ ${autoscaler_dir}/operations/enable-log-cache-via-uaa.yml\ ${autoscaler_dir}/operations/enable-metricsforwarder-via-syslog-agent.yml\ ${autoscaler_dir}/operations/enable-scheduler-logging.yml"} diff --git a/example/operation/cf-mysql-db.yml b/example/operation/cf-mysql-db.yml index 441574e742..9cbb8e0b94 100644 --- a/example/operation/cf-mysql-db.yml +++ b/example/operation/cf-mysql-db.yml @@ -29,7 +29,7 @@ roles: - name: autoscaler password: ((autoscaler_database_password)) - tag: default + tag: default - type: replace path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/scalingengine_db_connection_config @@ -97,21 +97,6 @@ path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/lock_db_connection_config value: *databaseConnectionConfig -# asmetrics/metricsserver -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/instancemetrics_db - value: *external_database -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/instancemetrics_db_connection_config - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig - # asmetrics/eventgenerator - type: replace path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/appmetrics_db @@ -127,14 +112,6 @@ path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/policy_db_connection_config value: *databaseConnectionConfig -# asnozzle/metricsgateway -- type: replace - path: /instance_groups/name=asnozzle/jobs/name=metricsgateway/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asnozzle/jobs/name=metricsgateway/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig - # asapi/golangapiserver - type: replace path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/policy_db @@ -148,6 +125,7 @@ - type: replace path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/binding_db_connection_config value: *databaseConnectionConfig + # asapi/metricsforwarder - type: replace path: /instance_groups/name=asapi/jobs/name=metricsforwarder/properties/autoscaler/policy_db diff --git a/example/operation/disable-basicauth-on-health-endpoints.yml b/example/operation/disable-basicauth-on-health-endpoints.yml index 84d8104883..d079d28174 100644 --- a/example/operation/disable-basicauth-on-health-endpoints.yml +++ b/example/operation/disable-basicauth-on-health-endpoints.yml @@ -1,5 +1,5 @@ --- -- type: remove +- type: remove path: /variables/name=autoscaler_scheduler_health_password - type: remove @@ -8,12 +8,6 @@ - type: remove path: /variables/name=autoscaler_metricsforwarder_health_password -- type: remove - path: /variables/name=autoscaler_metricsgateway_health_password - -- type: remove - path: /variables/name=autoscaler_metricsserver_health_password - - type: remove path: /variables/name=autoscaler_operator_health_password @@ -43,25 +37,12 @@ path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/operator/health/password #asmetrics -- type: remove - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/metricsserver/health/username - -- type: remove - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/metricsserver/health/password - - type: remove path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/eventgenerator/health/username - type: remove path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/eventgenerator/health/password -# asnozzle -- type: remove - path: /instance_groups/name=asnozzle/jobs/name=metricsgateway/properties/autoscaler/metricsgateway/health/username - -- type: remove - path: /instance_groups/name=asnozzle/jobs/name=metricsgateway/properties/autoscaler/metricsgateway/health/password - # asapi - type: remove path: /instance_groups/name=asapi/jobs/name=metricsforwarder/properties/autoscaler/metricsforwarder/health/username @@ -81,4 +62,4 @@ path: /instance_groups/name=asnozzle/jobs/name=route_registrar - type: remove - path: /instance_groups/name=asapi/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_metricsforwarder_health \ No newline at end of file + path: /instance_groups/name=asapi/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_metricsforwarder_health diff --git a/example/operation/enable-name-based-deployments.yml b/example/operation/enable-name-based-deployments.yml index 92828bab0b..9b450e8f4f 100644 --- a/example/operation/enable-name-based-deployments.yml +++ b/example/operation/enable-name-based-deployments.yml @@ -16,8 +16,6 @@ servicebroker: &servicebroker_domain ((deployment_name)).servicebroker.service.cf.internal eventgenerator: &eventgenerator_domain ((deployment_name)).eventgenerator.service.cf.internal scalingengine: &scalingengine_domain ((deployment_name)).scalingengine.service.cf.internal - metricsgateway: &metricsgateway_domain ((deployment_name)).metricsgateway.service.cf.internal - metricsserver: &metricsserver_domain ((deployment_name)).metricsserver.service.cf.internal public_domains: metricsforwarder: &metricsforwarder_public_domain ((deployment_name))metrics.((system_domain)) metricsforwarder_mtls: &metricsforwarder_mtls_public_domain ((deployment_name))-metricsforwarder-mtls.((system_domain)) @@ -70,20 +68,6 @@ deployment: ((deployment_name)) network: default domain: bosh - - domain: *metricsgateway_domain - targets: - - query: '*' - instance_group: asnozzle - deployment: ((deployment_name)) - network: default - domain: bosh - - domain: *metricsserver_domain - targets: - - query: '*' - instance_group: asmetrics - deployment: ((deployment_name)) - network: default - domain: bosh # Cf internal names - domain: nats.service.cf.internal targets: @@ -186,27 +170,12 @@ value: *postgres_domain - -########### -# asmetrics - metricsserver -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/instancemetrics_db/address? - value: *postgres_domain - -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/policy_db/address? - value: *postgres_domain - ########### # asmetrics - eventgenerator - type: replace path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/eventgenerator/scaling_engine/host value: *scalingengine_domain -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/eventgenerator/metricscollector/host - value: *metricsserver_domain - - type: replace path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/appmetrics_db/address? value: *postgres_domain @@ -215,12 +184,6 @@ path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/policy_db/address? value: *postgres_domain -########### -# asnozzle - metricsgateway -- type: replace - path: /instance_groups/name=asnozzle/jobs/name=metricsgateway/properties/autoscaler/policy_db/address? - value: *postgres_domain - ########### # asapi - golangapiserver - type: replace @@ -290,16 +253,6 @@ path: /instance_groups/name=asmetrics/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_eventgenerator_health/uris value: [ ((deployment_name))-eventgenerator.((system_domain)) ] -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_metricsserver_health/uris - value: [ ((deployment_name))-metricsserver.((system_domain)) ] - -## asnozzle - -- type: replace - path: /instance_groups/name=asnozzle/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_metricsgateway_health/uris - value: [ ((deployment_name))-metricsgateway.((system_domain)) ] - ## asactors - type: replace @@ -364,16 +317,6 @@ path: /variables/name=scheduler_server/options/alternative_names value: [*scheduler_domain] -# - metricsserver_server -- type: replace - path: /variables/name=metricsserver_server/options/common_name - value: *metricsserver_domain -- type: replace - path: /variables/name=metricsserver_server/options/alternative_names - value: - - *metricsserver_domain - - "*.asmetrics.default.((deployment_name)).bosh" - # - postgres_server - type: replace path: /variables/name=postgres_server/options/common_name diff --git a/example/operation/external-db.yml b/example/operation/external-db.yml index 05be973fae..8359349097 100644 --- a/example/operation/external-db.yml +++ b/example/operation/external-db.yml @@ -19,7 +19,7 @@ roles: - name: ((database.username)) password: ((database.password)) - tag: default + tag: default - type: replace path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/scalingengine_db_connection_config @@ -87,21 +87,6 @@ path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/lock_db_connection_config value: *databaseConnectionConfig -# asmetrics/metricsserver -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/instancemetrics_db - value: *external_database -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/instancemetrics_db_connection_config - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig - # asmetrics/eventgenerator - type: replace path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/appmetrics_db @@ -117,14 +102,6 @@ path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/policy_db_connection_config value: *databaseConnectionConfig -# asnozzle/metricsgateway -- type: replace - path: /instance_groups/name=asnozzle/jobs/name=metricsgateway/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asnozzle/jobs/name=metricsgateway/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig - # asapi/golangapiserver - type: replace path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/policy_db diff --git a/operations/disable-basicauth-on-health-endpoints.yml b/operations/disable-basicauth-on-health-endpoints.yml index 8d7c5de06c..369db9bbcb 100644 --- a/operations/disable-basicauth-on-health-endpoints.yml +++ b/operations/disable-basicauth-on-health-endpoints.yml @@ -8,12 +8,6 @@ - type: remove path: /variables/name=autoscaler_metricsforwarder_health_password -- type: remove - path: /variables/name=autoscaler_metricsgateway_health_password - -- type: remove - path: /variables/name=autoscaler_metricsserver_health_password - - type: remove path: /variables/name=autoscaler_operator_health_password @@ -42,24 +36,12 @@ - type: remove path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/operator/health/password -- type: remove - path: /instance_groups/name=metricsserver/jobs/name=metricsserver/properties/autoscaler/metricsserver/health/username - -- type: remove - path: /instance_groups/name=metricsserver/jobs/name=metricsserver/properties/autoscaler/metricsserver/health/password - - type: remove path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/eventgenerator/health/username - type: remove path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/eventgenerator/health/password -- type: remove - path: /instance_groups/name=metricsgateway/jobs/name=metricsgateway/properties/autoscaler/metricsgateway/health/username - -- type: remove - path: /instance_groups/name=metricsgateway/jobs/name=metricsgateway/properties/autoscaler/metricsgateway/health/password - # asapi - type: remove path: /instance_groups/name=metricsforwarder/jobs/name=metricsforwarder/properties/autoscaler/metricsforwarder/health/username @@ -78,12 +60,6 @@ - type: remove path: /instance_groups/name=operator/jobs/name=route_registrar -- type: remove - path: /instance_groups/name=metricsserver/jobs/name=route_registrar - -- type: remove - path: /instance_groups/name=metricsgateway/jobs/name=route_registrar - - type: remove path: /instance_groups/name=eventgenerator/jobs/name=route_registrar diff --git a/operations/enable-nats-tls.yml b/operations/enable-nats-tls.yml index c92d21e328..367f2f0e71 100644 --- a/operations/enable-nats-tls.yml +++ b/operations/enable-nats-tls.yml @@ -91,42 +91,6 @@ from: nats-tls deployment: cf -# metricsgateway -- type: replace - path: /instance_groups/name=metricsgateway/jobs/name=route_registrar/properties/nats? - value: - tls: - enabled: true - client_cert: ((/bosh-autoscaler/cf/nats_client_cert.certificate)) - client_key: ((/bosh-autoscaler/cf/nats_client_cert.private_key)) - -- type: remove - path: /instance_groups/name=metricsgateway/jobs/name=route_registrar/consumes/nats - -- type: replace - path: /instance_groups/name=metricsgateway/jobs/name=route_registrar/consumes/nats-tls? - value: - from: nats-tls - deployment: cf - -# metricsserver -- type: replace - path: /instance_groups/name=metricsserver/jobs/name=route_registrar/properties/nats? - value: - tls: - enabled: true - client_cert: ((/bosh-autoscaler/cf/nats_client_cert.certificate)) - client_key: ((/bosh-autoscaler/cf/nats_client_cert.private_key)) - -- type: remove - path: /instance_groups/name=metricsserver/jobs/name=route_registrar/consumes/nats - -- type: replace - path: /instance_groups/name=metricsserver/jobs/name=route_registrar/consumes/nats-tls? - value: - from: nats-tls - deployment: cf - # eventgenerator - type: replace path: /instance_groups/name=eventgenerator/jobs/name=route_registrar/properties/nats? diff --git a/operations/remove-metricsgateway.yml b/operations/remove-metricsgateway.yml deleted file mode 100644 index b5ad557411..0000000000 --- a/operations/remove-metricsgateway.yml +++ /dev/null @@ -1,12 +0,0 @@ -- type: remove - path: /variables/name=autoscaler_metricsgateway_health_password - -- type: remove - path: /variables/name=metricsgateway_monitor_basic_auth_password - -- type: remove - path: /instance_groups/name=metricsgateway - -- type: remove - path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=((deployment_name)).metricsgateway.service.cf.internal - diff --git a/operations/remove-metricsserver.yml b/operations/remove-metricsserver.yml deleted file mode 100644 index c77baa8d3c..0000000000 --- a/operations/remove-metricsserver.yml +++ /dev/null @@ -1,20 +0,0 @@ -- type: remove - path: /variables/name=metricsserver_server_cert - -- type: remove - path: /variables/name=metricsserver_client_cert - -- type: remove - path: /variables/name=autoscaler_metricsserver_health_password - -- type: remove - path: /variables/name=metricsserver_monitor_basic_auth_password - -- type: remove - path: /instance_groups/name=metricsserver - -- type: remove - path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=((deployment_name)).metricsserver.service.cf.internal - -- type: remove - path: /instance_groups/name=apiserver/jobs/name=golangapiserver/properties/autoscaler/apiserver/metrics_collector diff --git a/templates/app-autoscaler.yml b/templates/app-autoscaler.yml index e22b399f7e..9971cbf768 100644 --- a/templates/app-autoscaler.yml +++ b/templates/app-autoscaler.yml @@ -22,8 +22,6 @@ domains: servicebroker: &servicebroker_domain ((deployment_name)).servicebroker.service.cf.internal eventgenerator: &eventgenerator_domain ((deployment_name)).eventgenerator.service.cf.internal scalingengine: &scalingengine_domain ((deployment_name)).scalingengine.service.cf.internal - metricsgateway: &metricsgateway_domain ((deployment_name)).metricsgateway.service.cf.internal - metricsserver: &metricsserver_domain ((deployment_name)).metricsserver.service.cf.internal public_domains: metricsforwarder: &metricsforwarder_public_domain ((deployment_name))metrics.((system_domain)) @@ -246,11 +244,6 @@ instance_groups: client_cert: ((!scalingengine_client_cert.certificate)) client_key: ((!scalingengine_client_cert.private_key)) host: *scalingengine_domain - metrics_collector: - ca_cert: ((!metricsserver_client_cert.ca)) - client_cert: ((!metricsserver_client_cert.certificate)) - client_key: ((!metricsserver_client_cert.private_key)) - host: logcache event_generator: ca_cert: ((!eventgenerator_client_cert.ca)) client_cert: ((!eventgenerator_client_cert.certificate)) @@ -417,122 +410,6 @@ instance_groups: uris: - ((deployment_name))-operator.((system_domain)) -# Metricsserver -- name: metricsserver - azs: - - z1 - instances: 1 - vm_type: minimal - stemcell: default - networks: - - name: default - jobs: - - name: metricsserver - release: app-autoscaler - properties: - autoscaler: - instancemetrics_db: *database - instancemetrics_db_connection_config: *databaseConnectionConfig - policy_db: *database - policy_db_connection_config: *databaseConnectionConfig - metricsserver: - logging: - level: info - server: - port: &metricsserverPort 6103 - http_client_timeout: 60s - ca_cert: ((!metricsserver_server_cert.ca)) - server_cert: ((!metricsserver_server_cert.certificate)) - server_key: ((!metricsserver_server_cert.private_key)) - collector: - keep_alive_time: 60s - refresh_interval: 60s - collect_interval: 60s - save_interval: 5s - persist_metrics: true - envelope_processor_count: 4 - metric_cache_size_per_app: 1000 - envelope_channel_size: 1000 - metric_channel_size: 1000 - health: - port: &metricsserverHealthPort 6303 - username: metricsserver - password: ((autoscaler_metricsserver_health_password)) - - name: route_registrar - release: routing - consumes: - nats: {from: nats, deployment: cf} - properties: - route_registrar: - routes: - - name: autoscaler_metricsserver_health - registration_interval: 20s - port: *metricsserverHealthPort - tags: - component: autoscaler_metricsserver_health - uris: - - ((deployment_name))-metricsserver.((system_domain)) - -# Metricsgateway -- name: metricsgateway - azs: - - z1 - instances: 1 - vm_type: minimal - stemcell: default - networks: - - name: default - jobs: - - name: metricsgateway - release: app-autoscaler - properties: - autoscaler: - policy_db: *database - policy_db_connection_config.max_open_connections: *databaseConnectionConfig - metricsgateway: - logging: - level: info - envelop_chan_size: 1000 - nozzle_count: 4 - app_manager: - app_refresh_interval: 5s - emitter: - buffer_size: 500 - keep_alive_interval: 5s - handshake_timeout: 1s - max_setup_retry_count: 3 - max_close_retry_count: 3 - retry_delay: 1s - metricsserver_client: - ca_cert: ((!metricsserver_client_cert.ca)) - cert: ((!metricsserver_client_cert.certificate)) - key: ((!metricsserver_client_cert.private_key)) - nozzle: - rlp_addr: "reverse-log-proxy.service.cf.internal:8082" - shard_id: "CF_AUTOSCALER" - loggregator_rlp_tls: - cert: ((loggregator_tls_rlp.certificate)) - key: ((loggregator_tls_rlp.private_key)) - ca_cert: ((loggregator_ca.certificate)) - health: - port: &metricsgatewayHealthPort 6503 - username: metricsgateway - password: ((autoscaler_metricsgateway_health_password)) - - name: route_registrar - release: routing - consumes: - nats: {from: nats, deployment: cf} - properties: - route_registrar: - routes: - - name: autoscaler_metricsgateway_health - registration_interval: 20s - port: *metricsgatewayHealthPort - tags: - component: autoscaler_metricsgateway_health - uris: - - ((deployment_name))-metricsgateway.((system_domain)) - # Eventgenerator Instance Group - name: eventgenerator azs: @@ -732,10 +609,6 @@ variables: length: 128 - name: autoscaler_metricsforwarder_health_password type: password -- name: autoscaler_metricsgateway_health_password - type: password -- name: autoscaler_metricsserver_health_password - type: password - name: app_autoscaler_sbss_restricted_dbuser_password type: password options: @@ -802,14 +675,6 @@ variables: length: 128 - name: autoscaler_scalingengine_health_password type: password -- name: metricsgateway_monitor_basic_auth_password - type: password - options: - length: 128 -- name: metricsserver_monitor_basic_auth_password - type: password - options: - length: 128 - name: operator_monitor_basic_auth_password type: password options: @@ -894,31 +759,6 @@ variables: - client_auth key_length: 4096 duration: 180 -- name: metricsserver_server_cert - type: certificate - update_mode: converge - options: - ca: app_autoscaler_ca_cert - common_name: *metricsserver_domain - alternative_names: - - *metricsserver_domain - - "*.metricsserver.default.((deployment_name)).bosh" - - extended_key_usage: - - client_auth - - server_auth - key_length: 4096 - duration: 180 -- name: metricsserver_client_cert - type: certificate - update_mode: converge - options: - ca: app_autoscaler_ca_cert - common_name: metricsserver_client - extended_key_usage: - - client_auth - key_length: 4096 - duration: 180 - name: scalingengine_server_cert type: certificate update_mode: converge diff --git a/templates/manifest_tests.sh b/templates/manifest_tests.sh index abdcedebfd..7533e039c2 100755 --- a/templates/manifest_tests.sh +++ b/templates/manifest_tests.sh @@ -9,8 +9,6 @@ scheduler_instance_group="${SCHEDULER_INSTANCE_GROUP:-scheduler}" metricsforwarder_instance_group="${METICSFORWARDER_INSTANCE_GROUP:-metricsforwarder}" operator_instance_group="${OPERATOR_INSTANCE_GROUP:-operator}" eventgenerator_instance_group="${EVENTGENERATOR_INSTANCE_GROUP:-eventgenerator}" -metricsserver_instance_group="${METIRCSSERVER_INSTANCE_GROUP:-metricsserver}" -metricsgateway_instance_group="${METRICSGATEWAY_INSTANCE_GROUP:-metricsgateway}" # this is a really basic check to validate that the peristent disk value is set. # FIXME we need a much better way of doing this. @@ -28,7 +26,7 @@ if [ "${ACTUAL}" != "10GB" ]; then fi -COMPONENTS="scheduler eventgenerator metricsforwarder metricsgateway metricsserver operator scalingengine" +COMPONENTS="scheduler eventgenerator metricsforwarder operator scalingengine" for COMPONENT in $COMPONENTS; do ACTUAL=$(bosh int "$manifest_path" | yq e ".variables[] | select(.name == \"autoscaler_${COMPONENT}_health_password\").type" -) if [ "${ACTUAL}" != "password" ]; then @@ -66,24 +64,12 @@ if [ "${ACTUAL}" != "((autoscaler_operator_health_password))" ]; then exit 1 fi -ACTUAL=$(bosh int "$manifest_path" | yq e ".instance_groups[] | select(.name == \"$metricsserver_instance_group\") | .jobs[] | select(.name == \"metricsserver\").properties.autoscaler.metricsserver.health.password" -) -if [ "${ACTUAL}" != "((autoscaler_metricsserver_health_password))" ]; then - echo "FAILED: $metricsserver_instance_group/metricsserver health password should be set" - exit 1 -fi - ACTUAL=$(bosh int "$manifest_path" | yq e ".instance_groups[] | select(.name == \"$eventgenerator_instance_group\") | .jobs[] | select(.name == \"eventgenerator\").properties.autoscaler.eventgenerator.health.password" -) if [ "${ACTUAL}" != "((autoscaler_eventgenerator_health_password))" ]; then echo "FAILED: asmetrics/eventgenerator health password should be set" exit 1 fi -ACTUAL=$(bosh int "$manifest_path" | yq e ".instance_groups[] | select(.name == \"$metricsgateway_instance_group\") | .jobs[] | select(.name == \"metricsgateway\").properties.autoscaler.metricsgateway.health.password" -) -if [ "${ACTUAL}" != "((autoscaler_metricsgateway_health_password))" ]; then - echo "FAILED: $metricsgateway_instance_group/metricsgateway health password should be set" - exit 1 -fi - ACTUAL=$(bosh int "$manifest_path" | yq e ".instance_groups[] | select(.name == \"$metricsforwarder_instance_group\") | .jobs[] | select(.name == \"metricsforwarder\").properties.autoscaler.metricsforwarder.health.password" -) if [ "${ACTUAL}" != "((autoscaler_metricsforwarder_health_password))" ]; then echo "FAILED: $metricsforwarder_instance_group/metricsforwarder health password should be set" @@ -114,24 +100,12 @@ if [ "${ACTUAL}" != "null" ]; then exit 1 fi -ACTUAL=$(bosh int -o "$operation_dir_path/disable-basicauth-on-health-endpoints.yml" "$manifest_path" | yq e ".instance_groups[] | select(.name == \"$metricsserver_instance_group\") | .jobs[] | select(.name == \"metricsserver\").properties.autoscaler.metricsserver.health.password" -) -if [ "${ACTUAL}" != "null" ]; then - echo "FAILED: $metricsserver_instance_group/metricsserver health password should not be set" - exit 1 -fi - ACTUAL=$(bosh int -o "$operation_dir_path/disable-basicauth-on-health-endpoints.yml" "$manifest_path" | yq e ".instance_groups[] | select(.name == \"$eventgenerator_instance_group\") | .jobs[] | select(.name == \"eventgenerator\").properties.autoscaler.eventgenerator.health.password" -) if [ "${ACTUAL}" != "null" ]; then echo "FAILED: $eventgenerator_instance_group/eventgenerator health password should not be set" exit 1 fi -ACTUAL=$(bosh int -o "$operation_dir_path/disable-basicauth-on-health-endpoints.yml" "$manifest_path" | yq e ".instance_groups[] | select(.name == \"$metricsgateway_instance_group\") | .jobs[] | select(.name == \"metricsgateway\").properties.autoscaler.metricsgateway.health.password" -) -if [ "${ACTUAL}" != "null" ]; then - echo "FAILED: $metricsgateway_instance_group/metricsgatway health password should not be set" - exit 1 -fi - ACTUAL=$(bosh int -o "$operation_dir_path/disable-basicauth-on-health-endpoints.yml" "$manifest_path" | yq e ".instance_groups[] | select(.name == \"$metricsforwarder_instance_group\") | .jobs[] | select(.name == \"metricsforwarder\").properties.autoscaler.metricsforwarder.health.password" -) if [ "${ACTUAL}" != "null" ]; then echo "FAILED: $metricsforwarder_instance_group/metricsforwarder health password should not be set" From 3012a3e3a3958ab33facff60012300eddbc1296f Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Wed, 5 Jun 2024 10:50:51 +0200 Subject: [PATCH 030/186] remove unused loggregator certificates and ops-files --- ci/autoscaler/pipeline.yml | 3 --- ci/autoscaler/scripts/deploy-autoscaler.sh | 1 - .../tasks/deploy-previous-autoscaler.yml | 1 - .../operation/loggregator-certs-from-cf.yml | 22 ------------------- operations/loggregator-certs-from-cf.yml | 22 ------------------- scripts/generate_test_certs.sh | 10 --------- templates/app-autoscaler.yml | 12 ---------- 7 files changed, 71 deletions(-) delete mode 100644 example/operation/loggregator-certs-from-cf.yml delete mode 100644 operations/loggregator-certs-from-cf.yml diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index 973081ab5f..78e788e264 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -6,7 +6,6 @@ anchors: operations/instance-identity-cert-from-cf.yml operations/add-postgres-variables.yml operations/enable-nats-tls.yml - operations/loggregator-certs-from-cf.yml operations/add-extra-plan.yml operations/set-release-version.yml operations/enable-metricsforwarder-via-metron-agent.yml @@ -18,7 +17,6 @@ anchors: operations/instance-identity-cert-from-cf.yml operations/add-postgres-variables.yml operations/enable-nats-tls.yml - operations/loggregator-certs-from-cf.yml operations/add-extra-plan.yml operations/set-release-version.yml operations/enable-metricsforwarder-via-syslog-agent.yml @@ -30,7 +28,6 @@ anchors: operations/instance-identity-cert-from-cf.yml operations/add-postgres-variables.yml operations/enable-nats-tls.yml - operations/loggregator-certs-from-cf.yml operations/postgres-persistent-disk.yml operations/add-extra-plan.yml operations/set-release-version.yml diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index f8a0f2a336..2dcfc864af 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -13,7 +13,6 @@ ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\ ${autoscaler_dir}/operations/add-postgres-variables.yml\ ${autoscaler_dir}/operations/connect_to_postgres_with_certs.yml\ ${autoscaler_dir}/operations/enable-nats-tls.yml\ - ${autoscaler_dir}/operations/loggregator-certs-from-cf.yml\ ${autoscaler_dir}/operations/add-extra-plan.yml\ ${autoscaler_dir}/operations/set-release-version.yml\ ${autoscaler_dir}/operations/enable-log-cache-via-uaa.yml\ diff --git a/ci/autoscaler/tasks/deploy-previous-autoscaler.yml b/ci/autoscaler/tasks/deploy-previous-autoscaler.yml index 910984dd8e..5f705d031d 100644 --- a/ci/autoscaler/tasks/deploy-previous-autoscaler.yml +++ b/ci/autoscaler/tasks/deploy-previous-autoscaler.yml @@ -14,7 +14,6 @@ params: # Optional: relative to the root of the release input. The files will be # checked to see if they exist, if not they will be excluded from the list OPS_FILES: | - example/operation/loggregator-certs-from-cf.yml example/operation/postgres-persistent-disk.yml diff --git a/example/operation/loggregator-certs-from-cf.yml b/example/operation/loggregator-certs-from-cf.yml deleted file mode 100644 index b636765e29..0000000000 --- a/example/operation/loggregator-certs-from-cf.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- type: replace - path: /instance_groups/name=asnozzle/jobs/name=metricsgateway/properties/autoscaler/metricsgateway/nozzle/loggregator_rlp_tls - value: - cert: ((/bosh-autoscaler/cf/loggregator_tls_rlp.certificate)) - key: ((/bosh-autoscaler/cf/loggregator_tls_rlp.private_key)) - ca_cert: ((/bosh-autoscaler/cf/loggregator_ca.certificate)) - -- type: replace - path: /instance_groups/name=asapi/jobs/name=metricsforwarder/properties/autoscaler/metricsforwarder/loggregator/tls - value: - ca_cert: ((/bosh-autoscaler/cf/loggregator_tls_agent.ca)) - cert: ((/bosh-autoscaler/cf/loggregator_tls_agent.certificate)) - key: ((/bosh-autoscaler/cf/loggregator_tls_agent.private_key)) - -- type: replace - path: /instance_groups/name=asapi/jobs/name=loggregator_agent/properties/loggregator/tls - value: - ca_cert: ((/bosh-autoscaler/cf/loggregator_ca.certificate)) - agent: - cert: ((/bosh-autoscaler/cf/loggregator_tls_agent.certificate)) - key: ((/bosh-autoscaler/cf/loggregator_tls_agent.private_key)) \ No newline at end of file diff --git a/operations/loggregator-certs-from-cf.yml b/operations/loggregator-certs-from-cf.yml deleted file mode 100644 index 3a63e88d8a..0000000000 --- a/operations/loggregator-certs-from-cf.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- type: replace - path: /instance_groups/name=metricsgateway/jobs/name=metricsgateway/properties/autoscaler/metricsgateway/nozzle/loggregator_rlp_tls - value: - cert: ((/bosh-autoscaler/cf/loggregator_tls_rlp.certificate)) - key: ((/bosh-autoscaler/cf/loggregator_tls_rlp.private_key)) - ca_cert: ((/bosh-autoscaler/cf/loggregator_ca.certificate)) - -- type: replace - path: /instance_groups/name=metricsforwarder/jobs/name=metricsforwarder/properties/autoscaler/metricsforwarder/loggregator/tls - value: - ca_cert: ((/bosh-autoscaler/cf/loggregator_tls_agent.ca)) - cert: ((/bosh-autoscaler/cf/loggregator_tls_agent.certificate)) - key: ((/bosh-autoscaler/cf/loggregator_tls_agent.private_key)) - -- type: replace - path: /instance_groups/name=metricsforwarder/jobs/name=loggregator_agent/properties/loggregator/tls - value: - ca_cert: ((/bosh-autoscaler/cf/loggregator_ca.certificate)) - agent: - cert: ((/bosh-autoscaler/cf/loggregator_tls_agent.certificate)) - key: ((/bosh-autoscaler/cf/loggregator_tls_agent.private_key)) diff --git a/scripts/generate_test_certs.sh b/scripts/generate_test_certs.sh index 029cc3cc16..d3c54015ba 100755 --- a/scripts/generate_test_certs.sh +++ b/scripts/generate_test_certs.sh @@ -16,11 +16,6 @@ ${CERTSTRAP} --depot-path "${depot_path}" init --passphrase '' --common-name aut mv -f "${depot_path}"/autoscalerCA.crt "${depot_path}"/autoscaler-ca.crt mv -f "${depot_path}"/autoscalerCA.key "${depot_path}"/autoscaler-ca.key -# CA to distribute to dummy loggregator_agent certs -${CERTSTRAP} --depot-path "${depot_path}" init --passphrase '' --common-name loggregatorCA --years "20" -mv -f "${depot_path}"/loggregatorCA.crt "${depot_path}"/loggregator-ca.crt -mv -f "${depot_path}"/loggregatorCA.key "${depot_path}"/loggregator-ca.key - # CA to distribute to dummy syslog emitter certs ${CERTSTRAP} --depot-path "${depot_path}" init --passphrase '' --common-name LogCacheSyslogServerCA --years "20" mv -f "${depot_path}"/LogCacheSyslogServerCA.crt "${depot_path}"/log-cache-syslog-server-ca.crt @@ -91,11 +86,6 @@ ${CERTSTRAP} --depot-path "${depot_path}" sign metricserver --CA autoscaler-ca - ${CERTSTRAP} --depot-path "${depot_path}" request-cert --passphrase '' --domain metricserver_client ${CERTSTRAP} --depot-path "${depot_path}" sign metricserver_client --CA autoscaler-ca --years "20" -# metricsforwarder certificate for loggregator_agent -${CERTSTRAP} --depot-path "${depot_path}" request-cert --passphrase '' --domain metron -${CERTSTRAP} --depot-path "${depot_path}" sign metron --CA loggregator-ca --years "20" - - # metricsforwarder certificate for log-cache-syslog-server ${CERTSTRAP} --depot-path "${depot_path}" request-cert --passphrase '' --domain cf-app ${CERTSTRAP} --depot-path "${depot_path}" sign cf-app --CA log-cache-syslog-server-ca --years "20" diff --git a/templates/app-autoscaler.yml b/templates/app-autoscaler.yml index 9971cbf768..e7b1148987 100644 --- a/templates/app-autoscaler.yml +++ b/templates/app-autoscaler.yml @@ -790,18 +790,6 @@ variables: is_ca: true key_length: 4096 duration: 180 -- name: loggregator_agent_metrics_tls - type: certificate - update_mode: converge - options: - ca: metric_scraper_ca - common_name: loggregator_agent_server - alternative_names: - - loggregator_agent_server - extended_key_usage: - - server_auth - key_length: 4096 - duration: 180 # metricsforwarder loggregator certificates - name: metricsforwarder_autoscaler_metricsforwarder_loggregator_tls From 69c439e0db2446031517a512e6295ad9acda4b5a Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Wed, 5 Jun 2024 13:09:56 +0200 Subject: [PATCH 031/186] fix phony of make-target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index aefd25358e..6a9b0f8ec9 100644 --- a/Makefile +++ b/Makefile @@ -252,7 +252,7 @@ spec-test: bundle install bundle exec rspec -.PHONY: release +.PHONY: bosh-release bosh-release: go-mod-tidy go-mod-vendor scheduler db build/autoscaler-test.tgz build/autoscaler-test.tgz: @echo " - building bosh release into build/autoscaler-test.tgz" From 501ea8c513b627c383bab5e0fdcb4b325a431d5a Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Wed, 5 Jun 2024 13:10:48 +0200 Subject: [PATCH 032/186] delete job and package of metricsserver and metricsgateway --- jobs/metricsgateway/monit | 5 - jobs/metricsgateway/spec | 129 ---------- .../templates/bpm-pre-start.erb | 5 - jobs/metricsgateway/templates/bpm.yml.erb | 11 - .../templates/healthendpoint.crt.erb | 3 - .../templates/healthendpoint.key.erb | 3 - .../templates/healthendpoint_ca.crt.erb | 3 - .../templates/metricsgateway.yml.erb | 88 ------- .../templates/metricsserver_client.crt.erb | 4 - .../templates/metricsserver_client.key.erb | 4 - .../templates/metricsserver_client_ca.crt.erb | 4 - .../templates/policy_db.crt.erb | 3 - .../templates/policy_db.key.erb | 3 - .../templates/policy_db_ca.crt.erb | 3 - .../templates/reverselogproxy_client.crt.erb | 4 - .../templates/reverselogproxy_client.key.erb | 3 - .../reverselogproxy_client_ca.crt.erb | 4 - jobs/metricsserver/monit | 5 - jobs/metricsserver/spec | 171 ------------- .../templates/appinstancemetrics_db.crt.erb | 3 - .../templates/appinstancemetrics_db.key.erb | 3 - .../appinstancemetrics_db_ca.crt.erb | 3 - .../metricsserver/templates/bpm-pre-start.erb | 64 ----- jobs/metricsserver/templates/bpm.yml.erb | 11 - .../templates/healthendpoint.crt.erb | 3 - .../templates/healthendpoint.key.erb | 3 - .../templates/healthendpoint_ca.crt.erb | 3 - .../templates/liquibase.properties | 1 - .../templates/metricsserver.yml.erb | 100 -------- .../templates/metricsserver_ca.crt.erb | 3 - .../templates/metricsserver_server.crt.erb | 3 - .../templates/metricsserver_server.key.erb | 3 - .../metricsserver/templates/policy_db.crt.erb | 3 - .../metricsserver/templates/policy_db.key.erb | 3 - .../templates/policy_db_ca.crt.erb | 3 - packages/metricsgateway/packaging | 12 - packages/metricsgateway/spec | 224 ----------------- packages/metricsserver/packaging | 13 - packages/metricsserver/spec | 230 ------------------ .../metricsforwarder/metricsforwarder_spec.rb | 118 --------- .../metricsgateway/metricsgateway_spec.rb | 69 ------ 41 files changed, 1333 deletions(-) delete mode 100644 jobs/metricsgateway/monit delete mode 100644 jobs/metricsgateway/spec delete mode 100644 jobs/metricsgateway/templates/bpm-pre-start.erb delete mode 100644 jobs/metricsgateway/templates/bpm.yml.erb delete mode 100644 jobs/metricsgateway/templates/healthendpoint.crt.erb delete mode 100644 jobs/metricsgateway/templates/healthendpoint.key.erb delete mode 100644 jobs/metricsgateway/templates/healthendpoint_ca.crt.erb delete mode 100644 jobs/metricsgateway/templates/metricsgateway.yml.erb delete mode 100644 jobs/metricsgateway/templates/metricsserver_client.crt.erb delete mode 100644 jobs/metricsgateway/templates/metricsserver_client.key.erb delete mode 100644 jobs/metricsgateway/templates/metricsserver_client_ca.crt.erb delete mode 100644 jobs/metricsgateway/templates/policy_db.crt.erb delete mode 100644 jobs/metricsgateway/templates/policy_db.key.erb delete mode 100644 jobs/metricsgateway/templates/policy_db_ca.crt.erb delete mode 100644 jobs/metricsgateway/templates/reverselogproxy_client.crt.erb delete mode 100644 jobs/metricsgateway/templates/reverselogproxy_client.key.erb delete mode 100644 jobs/metricsgateway/templates/reverselogproxy_client_ca.crt.erb delete mode 100644 jobs/metricsserver/monit delete mode 100644 jobs/metricsserver/spec delete mode 100644 jobs/metricsserver/templates/appinstancemetrics_db.crt.erb delete mode 100644 jobs/metricsserver/templates/appinstancemetrics_db.key.erb delete mode 100644 jobs/metricsserver/templates/appinstancemetrics_db_ca.crt.erb delete mode 100644 jobs/metricsserver/templates/bpm-pre-start.erb delete mode 100644 jobs/metricsserver/templates/bpm.yml.erb delete mode 100644 jobs/metricsserver/templates/healthendpoint.crt.erb delete mode 100644 jobs/metricsserver/templates/healthendpoint.key.erb delete mode 100644 jobs/metricsserver/templates/healthendpoint_ca.crt.erb delete mode 100644 jobs/metricsserver/templates/liquibase.properties delete mode 100644 jobs/metricsserver/templates/metricsserver.yml.erb delete mode 100644 jobs/metricsserver/templates/metricsserver_ca.crt.erb delete mode 100644 jobs/metricsserver/templates/metricsserver_server.crt.erb delete mode 100644 jobs/metricsserver/templates/metricsserver_server.key.erb delete mode 100644 jobs/metricsserver/templates/policy_db.crt.erb delete mode 100644 jobs/metricsserver/templates/policy_db.key.erb delete mode 100644 jobs/metricsserver/templates/policy_db_ca.crt.erb delete mode 100644 packages/metricsgateway/packaging delete mode 100644 packages/metricsgateway/spec delete mode 100644 packages/metricsserver/packaging delete mode 100644 packages/metricsserver/spec delete mode 100644 spec/jobs/metricsforwarder/metricsforwarder_spec.rb delete mode 100644 spec/jobs/metricsgateway/metricsgateway_spec.rb diff --git a/jobs/metricsgateway/monit b/jobs/metricsgateway/monit deleted file mode 100644 index d69ade6980..0000000000 --- a/jobs/metricsgateway/monit +++ /dev/null @@ -1,5 +0,0 @@ -check process metricsgateway - with pidfile /var/vcap/sys/run/bpm/metricsgateway/metricsgateway.pid - start program "/var/vcap/jobs/bpm/bin/bpm start metricsgateway" - stop program "/var/vcap/jobs/bpm/bin/bpm stop metricsgateway" - group vcap \ No newline at end of file diff --git a/jobs/metricsgateway/spec b/jobs/metricsgateway/spec deleted file mode 100644 index cd9a204ec9..0000000000 --- a/jobs/metricsgateway/spec +++ /dev/null @@ -1,129 +0,0 @@ ---- -name: metricsgateway -templates: - bpm-pre-start.erb: bin/bpm-pre-start - bpm.yml.erb: config/bpm.yml - metricsgateway.yml.erb: config/metricsgateway.yml - - healthendpoint_ca.crt.erb: config/certs/healthendpoint/ca.crt - healthendpoint.crt.erb: config/certs/healthendpoint/server.crt - healthendpoint.key.erb: config/certs/healthendpoint/server.key - - metricsserver_client_ca.crt.erb: config/certs/metricsserver_client/ca.crt - metricsserver_client.crt.erb: config/certs/metricsserver_client/server.crt - metricsserver_client.key.erb: config/certs/metricsserver_client/server.key - - reverselogproxy_client_ca.crt.erb: config/certs/reverselogproxy_client/ca.crt - reverselogproxy_client.crt.erb: config/certs/reverselogproxy_client/client.crt - reverselogproxy_client.key.erb: config/certs/reverselogproxy_client/client.key - - policy_db_ca.crt.erb: config/certs/policy_db/ca.crt - policy_db.crt.erb: config/certs/policy_db/crt - policy_db.key.erb: config/certs/policy_db/key - -packages: - - openjdk-17 - - db - - metricsgateway - - common - -consumes: -- name: metricsserver - type: metricsserver - -properties: - autoscaler.metricsgateway.logging.level: - description: "The level of logging" - default: "info" - autoscaler.metricsgateway.envelop_chan_size: - description: "The channel size to transport envelope from nozzles to emitters" - default: 500 - autoscaler.metricsgateway.nozzle_count: - description: "The go-loggregator client count in a metricsgateway" - default: 3 - autoscaler.metricsgateway.app_manager.app_refresh_interval: - description: "The time interval to refresh cached application from policy database" - default: 60s - - autoscaler.metricsgateway.emitter.buffer_size: - description: "The size of channel in each emitter to store envelopes before sending to metricsserver" - default: 500 - autoscaler.metricsgateway.emitter.keep_alive_interval: - description: "The time interval of keep alive for the websocket connection between emitter and metricsserver" - default: 5s - autoscaler.metricsgateway.emitter.handshake_timeout: - description: "The timeout of webscoket handshake between emitter and metricsserver" - default: 1s - autoscaler.metricsgateway.emitter.max_setup_retry_count: - description: "The maximum retry times to set up a websocket connection between emitter and metricsserver" - default: 5 - autoscaler.metricsgateway.emitter.max_close_retry_count: - description: "The maximum retry times to close a websocket connection between emitter and metricsserver" - default: 5 - autoscaler.metricsgateway.emitter.retry_delay: - description: "The duration between two websocket retrying" - default: 10s - autoscaler.metricsgateway.emitter.metricsserver_client.cert: - description: "PEM-encoded tls client certificate to connect to metricsserver" - autoscaler.metricsgateway.emitter.metricsserver_client.key: - description: "PEM-encoded tls client key to connect to metricsserver" - autoscaler.metricsgateway.emitter.metricsserver_client.ca_cert: - description: "PEM-encoded ca certificate of metricsserver" - autoscaler.metricsgateway.nozzle.rlp_addr: - description: "The address of ReverseLogProxy" - autoscaler.metricsgateway.nozzle.shard_id: - description: "The shard id for nozzle to fetch envelopes from ReverseLogProxy" - default: "CF_AUTOSCALER" - autoscaler.metricsgateway.nozzle.loggregator_rlp_tls.cert: - description: "PEM-encoded tls client certificate to connect to ReverseLogProxy" - autoscaler.metricsgateway.nozzle.loggregator_rlp_tls.key: - description: "PEM-encoded tls client key to connect to ReverseLogProxy" - autoscaler.metricsgateway.nozzle.loggregator_rlp_tls.ca_cert: - description: "PEM-encoded ca certificate of ReverseLogProxy" - - autoscaler.policy_db.address: - description: "IP address on which the policydb server will listen" - default: "autoscalerpostgres.service.cf.internal" - autoscaler.policy_db.databases: - description: "The list of databases used in policydb database including name" - autoscaler.policy_db.db_scheme: - description: "Database scheme to be used to access policydb" - autoscaler.policy_db.port: - description: "Port on which the policydb server will listen" - autoscaler.policy_db.roles: - description: "The list of database roles used in policydb database including name/password" - autoscaler.policy_db.tls.ca: - default: '' - description: 'PEM-encoded ca certificate of tls database server' - autoscaler.policy_db.tls.certificate: - default: '' - description: 'PEM-encoded certificate of tls database client' - autoscaler.policy_db.tls.private_key: - default: '' - description: 'PEM-encoded key of tls database client' - autoscaler.policy_db.sslmode: - default: disable - description: "The sslmode to connect to postgres server" - - autoscaler.policy_db_connection_config.max_open_connections: - default: 20 - autoscaler.policy_db_connection_config.max_idle_connections: - default: 10 - autoscaler.policy_db_connection_config.connection_max_lifetime: - default: 60s - - autoscaler.metricsgateway.health.port: - description: "The listening port of health endpoint" - default: 6503 - autoscaler.metricsgateway.health.ca_cert: - description: "PEM-encoded CA certificate for the health endpoint" - autoscaler.metricsgateway.health.server_cert: - description: "PEM-encoded server certificate for the health endpoint" - autoscaler.metricsgateway.health.server_key: - description: "PEM-encoded server key for the health endpoint" - autoscaler.metricsgateway.health.username: - description: "the username of health endpoint" - default: '' - autoscaler.metricsgateway.health.password: - description: "the password of health endpoint" - default: '' diff --git a/jobs/metricsgateway/templates/bpm-pre-start.erb b/jobs/metricsgateway/templates/bpm-pre-start.erb deleted file mode 100644 index ea9a571191..0000000000 --- a/jobs/metricsgateway/templates/bpm-pre-start.erb +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -source /var/vcap/packages/common/dns-helper.sh - -export PATH=/var/vcap/packages/openjdk-17/bin:$PATH diff --git a/jobs/metricsgateway/templates/bpm.yml.erb b/jobs/metricsgateway/templates/bpm.yml.erb deleted file mode 100644 index 1cd16104ce..0000000000 --- a/jobs/metricsgateway/templates/bpm.yml.erb +++ /dev/null @@ -1,11 +0,0 @@ -processes: - - name: metricsgateway - executable: /var/vcap/packages/metricsgateway/metricsgateway - args: - - -c - - /var/vcap/jobs/metricsgateway/config/metricsgateway.yml - env: - DT_MONITOR: true - DT_APPLICATIONID: metricsgateway - hooks: - pre_start: /var/vcap/jobs/metricsgateway/bin/bpm-pre-start diff --git a/jobs/metricsgateway/templates/healthendpoint.crt.erb b/jobs/metricsgateway/templates/healthendpoint.crt.erb deleted file mode 100644 index c2d7a747bc..0000000000 --- a/jobs/metricsgateway/templates/healthendpoint.crt.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.metricsgateway.health.server_cert") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsgateway/templates/healthendpoint.key.erb b/jobs/metricsgateway/templates/healthendpoint.key.erb deleted file mode 100644 index 4209051e45..0000000000 --- a/jobs/metricsgateway/templates/healthendpoint.key.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.metricsgateway.health.server_key") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsgateway/templates/healthendpoint_ca.crt.erb b/jobs/metricsgateway/templates/healthendpoint_ca.crt.erb deleted file mode 100644 index 6b4ac56d20..0000000000 --- a/jobs/metricsgateway/templates/healthendpoint_ca.crt.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.metricsgateway.health.ca_cert") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsgateway/templates/metricsgateway.yml.erb b/jobs/metricsgateway/templates/metricsgateway.yml.erb deleted file mode 100644 index 7ada15e5dc..0000000000 --- a/jobs/metricsgateway/templates/metricsgateway.yml.erb +++ /dev/null @@ -1,88 +0,0 @@ -<% - def p_arr(property) - p(property,nil) || [] - end - - def build_db_url(db_name, job_name) - db_scheme = p("autoscaler.#{db_name}.db_scheme") - db_address = p("autoscaler.#{db_name}.address") - db_port = p("autoscaler.#{db_name}.port") - db_roles = p_arr("autoscaler.#{db_name}.roles") - db_dbs = p_arr("autoscaler.#{db_name}.databases") - db_role = db_roles.find { |role| role['tag'] == db_name.tr('_', '') } || db_roles.find { |role| role['tag'] == 'default' } - db_database = db_dbs.find { |database| database['tag'] == db_name.tr('_', '') } || db_dbs.find { |database| database['tag'] == 'default' } - - if db_scheme == "postgres" - db_url = "#{db_scheme}://#{ERB::Util.url_encode(db_role['name'])}:#{ERB::Util.url_encode(db_role['password'])}@#{db_address}:#{db_port}/#{db_database['name']}?application_name=#{job_name}&sslmode=" + p("autoscaler.#{db_name}.sslmode") - else - db_url = "#{db_role['name']}:#{db_role['password']}@tcp(#{db_address}:#{db_port})/#{db_database['name']}?tls=" + p("autoscaler.#{db_name}.sslmode") - end - append_db_tls_configs(db_name, db_url, job_name) - end - - def append_db_tls_configs(db_name, db_url, job_name) - unless p("autoscaler.#{db_name}.tls.ca") == "" - db_url = "#{db_url}&sslrootcert=/var/vcap/jobs/#{job_name}/config/certs/#{db_name}/ca.crt" - end - unless p("autoscaler.#{db_name}.tls.certificate") == "" - db_url = "#{db_url}&sslcert=/var/vcap/jobs/#{job_name}/config/certs/#{db_name}/crt" - end - unless p("autoscaler.#{db_name}.tls.private_key") == "" - db_url = "#{db_url}&sslkey=/var/vcap/jobs/#{job_name}/config/certs/#{db_name}/key" - end - db_url -end - -########################################### -# Template Main # -########################################### - job_name = 'metricsgateway' - policy_db_url = build_db_url('policy_db', job_name) - - metricsserver_sorted_instances=link("metricsserver").instances.sort_by {|i|i.address} - metricsserver_addrs=metricsserver_sorted_instances.map{|i| "'wss://#{i.address}:#{link("metricsserver").p('autoscaler.metricsserver.collector.port')}'"} - metricsserver_nodeAddrs="[" + metricsserver_addrs.join(",") +"]" - -%> - - -logging: - level: <%= p("autoscaler.metricsgateway.logging.level") %> -envelop_chan_size: <%= p("autoscaler.metricsgateway.envelop_chan_size") %> -nozzle_count: <%= p("autoscaler.metricsgateway.nozzle_count") %> -metric_server_addrs: <%= metricsserver_nodeAddrs %> -app_manager: - app_refresh_interval: <%= p("autoscaler.metricsgateway.app_manager.app_refresh_interval") %> - policy_db: - url: <%= policy_db_url %> - max_open_connections: <%= p("autoscaler.policy_db_connection_config.max_open_connections") %> - max_idle_connections: <%= p("autoscaler.policy_db_connection_config.max_idle_connections") %> - connection_max_lifetime: <%= p("autoscaler.policy_db_connection_config.connection_max_lifetime") %> -emitter: - metrics_server_client_tls: - key_file: /var/vcap/jobs/metricsgateway/config/certs/metricsserver_client/server.key - cert_file: /var/vcap/jobs/metricsgateway/config/certs/metricsserver_client/server.crt - ca_file: /var/vcap/jobs/metricsgateway/config/certs/metricsserver_client/ca.crt - buffer_size: <%= p("autoscaler.metricsgateway.emitter.buffer_size") %> - keep_alive_interval: <%= p("autoscaler.metricsgateway.emitter.keep_alive_interval") %> - handshake_timeout: <%= p("autoscaler.metricsgateway.emitter.handshake_timeout") %> - max_setup_retry_count: <%= p("autoscaler.metricsgateway.emitter.max_setup_retry_count") %> - max_close_retry_count: <%= p("autoscaler.metricsgateway.emitter.max_close_retry_count") %> - retry_delay: <%= p("autoscaler.metricsgateway.emitter.retry_delay") %> -nozzle: - rlp_client_tls: - key_file: /var/vcap/jobs/metricsgateway/config/certs/reverselogproxy_client/client.key - cert_file: /var/vcap/jobs/metricsgateway/config/certs/reverselogproxy_client/client.crt - ca_file: /var/vcap/jobs/metricsgateway/config/certs/reverselogproxy_client/ca.crt - rlp_addr: <%= p("autoscaler.metricsgateway.nozzle.rlp_addr") %> - shard_id: <%= p("autoscaler.metricsgateway.nozzle.shard_id") %> -health: - port: <%= p("autoscaler.metricsgateway.health.port") %> - username: <%= p("autoscaler.metricsgateway.health.username") %> - password: <%= p("autoscaler.metricsgateway.health.password") %> - <% if_p("autoscaler.metricsgateway.health.ca_cert", "autoscaler.metricsgateway.health.server_cert", "autoscaler.metricsgateway.health.server_key") do %> - tls: - ca_file: /var/vcap/jobs/metricsgateway/config/certs/healthendpoint/ca.crt - cert_file: /var/vcap/jobs/metricsgateway/config/certs/healthendpoint/server.crt - key_file: /var/vcap/jobs/metricsgateway/config/certs/healthendpoint/server.key - <% end %> diff --git a/jobs/metricsgateway/templates/metricsserver_client.crt.erb b/jobs/metricsgateway/templates/metricsserver_client.crt.erb deleted file mode 100644 index 20490221e4..0000000000 --- a/jobs/metricsgateway/templates/metricsserver_client.crt.erb +++ /dev/null @@ -1,4 +0,0 @@ -<% if_p("autoscaler.metricsgateway.emitter.metricsserver_client.cert") do |value| %> -<%= value %> -<% end %> - diff --git a/jobs/metricsgateway/templates/metricsserver_client.key.erb b/jobs/metricsgateway/templates/metricsserver_client.key.erb deleted file mode 100644 index c33b098552..0000000000 --- a/jobs/metricsgateway/templates/metricsserver_client.key.erb +++ /dev/null @@ -1,4 +0,0 @@ -<% if_p("autoscaler.metricsgateway.emitter.metricsserver_client.key") do |value| %> -<%= value %> -<% end %> - diff --git a/jobs/metricsgateway/templates/metricsserver_client_ca.crt.erb b/jobs/metricsgateway/templates/metricsserver_client_ca.crt.erb deleted file mode 100644 index 2f3791bf1e..0000000000 --- a/jobs/metricsgateway/templates/metricsserver_client_ca.crt.erb +++ /dev/null @@ -1,4 +0,0 @@ -<% if_p("autoscaler.metricsgateway.emitter.metricsserver_client.ca_cert") do |value| %> -<%= value %> -<% end %> - diff --git a/jobs/metricsgateway/templates/policy_db.crt.erb b/jobs/metricsgateway/templates/policy_db.crt.erb deleted file mode 100644 index 3c19ec6240..0000000000 --- a/jobs/metricsgateway/templates/policy_db.crt.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.policy_db.tls.certificate") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsgateway/templates/policy_db.key.erb b/jobs/metricsgateway/templates/policy_db.key.erb deleted file mode 100644 index 6063d669f7..0000000000 --- a/jobs/metricsgateway/templates/policy_db.key.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.policy_db.tls.private_key") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsgateway/templates/policy_db_ca.crt.erb b/jobs/metricsgateway/templates/policy_db_ca.crt.erb deleted file mode 100644 index cd61f88ecd..0000000000 --- a/jobs/metricsgateway/templates/policy_db_ca.crt.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.policy_db.tls.ca") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsgateway/templates/reverselogproxy_client.crt.erb b/jobs/metricsgateway/templates/reverselogproxy_client.crt.erb deleted file mode 100644 index c0f50b7e5d..0000000000 --- a/jobs/metricsgateway/templates/reverselogproxy_client.crt.erb +++ /dev/null @@ -1,4 +0,0 @@ -<% if_p("autoscaler.metricsgateway.nozzle.loggregator_rlp_tls.cert") do |value| %> -<%= value %> -<% end %> - diff --git a/jobs/metricsgateway/templates/reverselogproxy_client.key.erb b/jobs/metricsgateway/templates/reverselogproxy_client.key.erb deleted file mode 100644 index 0188da43b6..0000000000 --- a/jobs/metricsgateway/templates/reverselogproxy_client.key.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.metricsgateway.nozzle.loggregator_rlp_tls.key") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsgateway/templates/reverselogproxy_client_ca.crt.erb b/jobs/metricsgateway/templates/reverselogproxy_client_ca.crt.erb deleted file mode 100644 index fdc37adde9..0000000000 --- a/jobs/metricsgateway/templates/reverselogproxy_client_ca.crt.erb +++ /dev/null @@ -1,4 +0,0 @@ -<% if_p("autoscaler.metricsgateway.nozzle.loggregator_rlp_tls.ca_cert") do |value| %> -<%= value %> -<% end %> - diff --git a/jobs/metricsserver/monit b/jobs/metricsserver/monit deleted file mode 100644 index be48818df7..0000000000 --- a/jobs/metricsserver/monit +++ /dev/null @@ -1,5 +0,0 @@ -check process metricsserver - with pidfile /var/vcap/sys/run/bpm/metricsserver/metricsserver.pid - start program "/var/vcap/jobs/bpm/bin/bpm start metricsserver" - stop program "/var/vcap/jobs/bpm/bin/bpm stop metricsserver" - group vcap \ No newline at end of file diff --git a/jobs/metricsserver/spec b/jobs/metricsserver/spec deleted file mode 100644 index a0dc33d4e0..0000000000 --- a/jobs/metricsserver/spec +++ /dev/null @@ -1,171 +0,0 @@ ---- -name: metricsserver -templates: - bpm-pre-start.erb: bin/bpm-pre-start - bpm.yml.erb: config/bpm.yml - metricsserver.yml.erb: config/metricsserver.yml - liquibase.properties: bin/liquibase.properties - - metricsserver_ca.crt.erb: config/certs/metricsserver/ca.crt - metricsserver_server.crt.erb: config/certs/metricsserver/server.crt - metricsserver_server.key.erb: config/certs/metricsserver/server.key - - healthendpoint_ca.crt.erb: config/certs/healthendpoint/ca.crt - healthendpoint.crt.erb: config/certs/healthendpoint/server.crt - healthendpoint.key.erb: config/certs/healthendpoint/server.key - - policy_db_ca.crt.erb: config/certs/policy_db/ca.crt - policy_db.crt.erb: config/certs/policy_db/crt - policy_db.key.erb: config/certs/policy_db/key - - appinstancemetrics_db_ca.crt.erb: config/certs/instancemetrics_db/ca.crt - appinstancemetrics_db.crt.erb: config/certs/instancemetrics_db/crt - appinstancemetrics_db.key.erb: config/certs/instancemetrics_db/key - -packages: - - openjdk-17 - - db - - metricsserver - - common - - changeloglockcleaner - -provides: -- name: metricsserver - type: metricsserver - properties: - - autoscaler.metricsserver.server.port - - autoscaler.metricsserver.collector.port - -consumes: -- name: metricsserver - type: metricsserver - -properties: - autoscaler.policy_db.address: - description: "IP address on which the policydb server will listen" - default: "autoscalerpostgres.service.cf.internal" - autoscaler.policy_db.databases: - description: "The list of databases used in policydb database including name" - autoscaler.policy_db.db_scheme: - description: "Database scheme to be used to access policydb" - autoscaler.policy_db.port: - description: "Port on which the policydb server will listen" - autoscaler.policy_db.roles: - description: "The list of database roles used in policydb database including name/password" - autoscaler.policy_db.tls.ca: - default: '' - description: 'PEM-encoded ca certificate of tls database server' - autoscaler.policy_db.tls.certificate: - default: '' - description: 'PEM-encoded certificate of tls database client' - autoscaler.policy_db.tls.private_key: - default: '' - description: 'PEM-encoded key of tls database client' - autoscaler.policy_db.sslmode: - default: disable - description: "The sslmode to connect to postgres server" - - autoscaler.instancemetrics_db.address: - description: "IP address on which the instancemetricsdb server will listen" - default: "autoscalerpostgres.service.cf.internal" - autoscaler.instancemetrics_db.databases: - description: "The list of databases used in instancemetricsdb database including name" - autoscaler.instancemetrics_db.db_scheme: - description: "Database scheme to be used to access instancemetricsdb" - autoscaler.instancemetrics_db.port: - description: "Port on which the instancemetricsdb server will listen" - autoscaler.instancemetrics_db.roles: - description: "The list of database roles used in instancemetricsdb database including name/password" - autoscaler.instancemetrics_db.tls.ca: - default: '' - description: 'PEM-encoded ca certificate for TLS database server' - autoscaler.instancemetrics_db.tls.certificate: - default: '' - description: 'PEM-encoded certificate for TLS database client' - autoscaler.instancemetrics_db.tls.private_key: - default: '' - description: 'PEM-encoded key for TLS database client' - autoscaler.instancemetrics_db.sslmode: - default: disable - description: "The sslmode to connect to postgres server" - - - autoscaler.instancemetrics_db_connection_config.max_open_connections: - default: 20 - autoscaler.instancemetrics_db_connection_config.max_idle_connections: - default: 10 - autoscaler.instancemetrics_db_connection_config.connection_max_lifetime: - default: 60s - - autoscaler.policy_db_connection_config.max_open_connections: - default: 20 - autoscaler.policy_db_connection_config.max_idle_connections: - default: 10 - autoscaler.policy_db_connection_config.connection_max_lifetime: - default: 60s - - autoscaler.metricsserver.logging.level: - description: "The level of logging" - default: "info" - - autoscaler.metricsserver.server.port: - description: "The listening port of http server" - default: 6103 - autoscaler.metricsserver.http_client_timeout: - description: "Http client timeout for metricsserver to communicate with other autoscaler components" - default: 60s - autoscaler.metricsserver.ca_cert: - description: "PEM-encoded CA certificate" - autoscaler.metricsserver.server_cert: - description: "PEM-encoded server certificate" - autoscaler.metricsserver.server_key: - description: "PEM-encoded server key" - autoscaler.metricsserver.collector.port: - description: "The listening port of websocket server" - default: 7103 - autoscaler.metricsserver.collector.keep_alive_time: - description: "Keep alive duration of the websocket between metricsserver and emitter" - default: 60s - autoscaler.metricsserver.collector.refresh_interval: - description: "The time interval to refresh app policies from policy database" - default: 60s - autoscaler.metricsserver.collector.collect_interval: - description: "The time interval to collect container metrics from loggregator" - default: 30s - autoscaler.metricsserver.collector.save_interval: - description: "The time interval to save metrics in database" - default: 5s - autoscaler.metricsserver.collector.persist_metrics: - description: "Whether to persist instancemetrics into database" - default: true - autoscaler.metricsserver.collector.envelope_processor_count: - description: "The envelope processor count to convert envelopes to app instance metrics" - default: 5 - autoscaler.metricsserver.collector.metric_cache_size_per_app: - description: "The instance metrics cache size for each application" - default: 1000 - autoscaler.metricsserver.collector.envelope_channel_size: - description: "The size of envelop channel to transport envelopes from websocket server to an envelope processor" - default: 1000 - autoscaler.metricsserver.collector.metric_channel_size: - description: "The size of channel to transport instance metric from envelope processors to cache" - default: 1000 - - autoscaler.metricsserver.health.port: - description: "The listening port of health endpoint" - default: 6303 - autoscaler.metricsserver.health.ca_cert: - description: "PEM-encoded CA certificate for the health endpoint" - autoscaler.metricsserver.health.server_cert: - description: "PEM-encoded server certificate for the health endpoint" - autoscaler.metricsserver.health.server_key: - description: "PEM-encoded server key for the health endpoint" - autoscaler.metricsserver.health.username: - description: "the username of health endpoint" - default: '' - autoscaler.metricsserver.health.password: - description: "the password of health endpoint" - default: '' - autoscaler.changeloglock_timeout_seconds: - default: 180 - description: "Liquibase changelog lock timeout duration in seconds" diff --git a/jobs/metricsserver/templates/appinstancemetrics_db.crt.erb b/jobs/metricsserver/templates/appinstancemetrics_db.crt.erb deleted file mode 100644 index bccd1b5ffe..0000000000 --- a/jobs/metricsserver/templates/appinstancemetrics_db.crt.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.instancemetrics_db.tls.certificate") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsserver/templates/appinstancemetrics_db.key.erb b/jobs/metricsserver/templates/appinstancemetrics_db.key.erb deleted file mode 100644 index 9d64259e6a..0000000000 --- a/jobs/metricsserver/templates/appinstancemetrics_db.key.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.instancemetrics_db.tls.private_key") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsserver/templates/appinstancemetrics_db_ca.crt.erb b/jobs/metricsserver/templates/appinstancemetrics_db_ca.crt.erb deleted file mode 100644 index d4e13326b9..0000000000 --- a/jobs/metricsserver/templates/appinstancemetrics_db_ca.crt.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.instancemetrics_db.tls.ca") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsserver/templates/bpm-pre-start.erb b/jobs/metricsserver/templates/bpm-pre-start.erb deleted file mode 100644 index 6b61ec4446..0000000000 --- a/jobs/metricsserver/templates/bpm-pre-start.erb +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -<% - def p_arr(property) - p(property,nil) || [] - end - - role = p_arr('autoscaler.instancemetrics_db.roles').find { |role| role['tag'] == 'instancemetricsdb' or role['tag'] == 'default' } - database = p_arr('autoscaler.instancemetrics_db.databases').find { |database| database['tag'] == 'instancemetricsdb' or database['tag'] == 'default' } -%> -source /var/vcap/packages/common/dns-helper.sh -source /var/vcap/packages/common/retry-helper.sh -source /var/vcap/packages/common/cert-helper.sh - -export PATH=/var/vcap/packages/openjdk-17/bin:$PATH - -HOST='<%= p("autoscaler.instancemetrics_db.address") %>' -DBNAME='<%= database['name'] %>' -USER='<%= role['name'] %>' -PASSWORD='<%= role['password'] %>' -PORT='<%= p("autoscaler.instancemetrics_db.port") %>' -SSLMODE='<%= p("autoscaler.instancemetrics_db.sslmode") %>' -<% if p('autoscaler.instancemetrics_db.db_scheme') == "postgres" %> -JDBCDBURL="jdbc:postgresql://$HOST:$PORT/$DBNAME?sslmode=$SSLMODE" -DBURI="postgres://$USER:$PASSWORD@$HOST:$PORT/$DBNAME?sslmode=$SSLMODE" -<% unless p('autoscaler.instancemetrics_db.tls.ca') == "" %> -JDBCDBURL=$JDBCDBURL"&sslrootcert=/var/vcap/jobs/metricsserver/config/certs/instancemetrics_db/ca.crt" -DBURI=$DBURI"&sslrootcert=/var/vcap/jobs/metricsserver/config/certs/instancemetrics_db/ca.crt" -<% end %> -<% # TODO Refactor this block -unless p("autoscaler.instancemetrics_db.tls.certificate") == "" %> -JDBCDBURL=$JDBCDBURL"&sslcert=/var/vcap/jobs/metricsserver/config/certs/instancemetrics_db/crt" -DBURI=$DBURI"&sslcert=/var/vcap/jobs/metricsserver/config/certs/instancemetrics_db/crt" -<% end %> -<% unless p("autoscaler.instancemetrics_db.tls.private_key") == "" %> - declare -r rsa_key_file=/var/vcap/jobs/metricsserver/config/certs/instancemetrics_db/key - declare -r pk8_file=/var/vcap/jobs/metricsserver/config/certs/instancemetrics_db/key.pk8 - convert_to_pk8 "$rsa_key_file" "$pk8_file" - JDBCDBURL=$JDBCDBURL"&sslkey=$pk8_file" - DBURI=$DBURI"&sslkey=$rsa_key_file" -<% end %> -# mysql case -<% else %> -JDBCDBURL="jdbc:mysql://$HOST:$PORT/$DBNAME?autoReconnect=true" -DBURI="$USER:$PASSWORD@tcp($HOST:$PORT)/$DBNAME?tls=$SSLMODE" -<% unless p('autoscaler.instancemetrics_db.tls.ca') == "" %> - declare -r instancemetrics_db_trust_store=/var/vcap/jobs/metricsserver/config/certs/instancemetrics_db/cacerts - /var/vcap/packages/common/init-db-truststore.sh appinstancemetrics_db /var/vcap/jobs/metricsserver/config/certs/instancemetrics_db/ca.crt "$instancemetrics_db_trust_store" - JDBCDBURL=$JDBCDBURL"&useSSL=true&requireSSL=true&verifyServerCertificate=true&enabledTLSProtocols=TLSv1.2&trustCertificateKeyStorePassword=123456&trustCertificateKeyStoreUrl=file:$instancemetrics_db_trust_store&trustCertificateKeyStoreType=pkcs12" - DBURI=$DBURI"&sslrootcert=/var/vcap/jobs/metricsserver/config/certs/instancemetrics_db/ca.crt" -<% end %> -<% end %> - -detect_dns $HOST $PORT - -/var/vcap/packages/changeloglockcleaner/changeloglockcleaner -d "$DBURI" -e <%= p("autoscaler.changeloglock_timeout_seconds") %> - -<% if p('autoscaler.instancemetrics_db.db_scheme') == "postgres" %> -retry java -cp "/var/vcap/packages/db/target/lib/*:/var/vcap/packages/metricsserver" liquibase.integration.commandline.Main --url "$JDBCDBURL" --username=$USER --password=$PASSWORD \ ---driver=org.postgresql.Driver --changeLogFile=metricscollector.db.changelog.yml update -<% else %> -retry java -cp "/var/vcap/packages/db/target/lib/*:/var/vcap/packages/metricsserver" liquibase.integration.commandline.Main --url "$JDBCDBURL" --username=$USER --password=$PASSWORD \ ---driver=com.mysql.cj.jdbc.Driver --changeLogFile=metricscollector.db.changelog.yml update -<% end %> diff --git a/jobs/metricsserver/templates/bpm.yml.erb b/jobs/metricsserver/templates/bpm.yml.erb deleted file mode 100644 index 09bd35b6bf..0000000000 --- a/jobs/metricsserver/templates/bpm.yml.erb +++ /dev/null @@ -1,11 +0,0 @@ -processes: - - name: metricsserver - executable: /var/vcap/packages/metricsserver/metricsserver - args: - - -c - - /var/vcap/jobs/metricsserver/config/metricsserver.yml - env: - DT_MONITOR: true - DT_APPLICATIONID: metricsserver - hooks: - pre_start: /var/vcap/jobs/metricsserver/bin/bpm-pre-start diff --git a/jobs/metricsserver/templates/healthendpoint.crt.erb b/jobs/metricsserver/templates/healthendpoint.crt.erb deleted file mode 100644 index 5ae94caaef..0000000000 --- a/jobs/metricsserver/templates/healthendpoint.crt.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.metricsserver.health.server_cert") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsserver/templates/healthendpoint.key.erb b/jobs/metricsserver/templates/healthendpoint.key.erb deleted file mode 100644 index 693519b384..0000000000 --- a/jobs/metricsserver/templates/healthendpoint.key.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.metricsserver.health.server_key") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsserver/templates/healthendpoint_ca.crt.erb b/jobs/metricsserver/templates/healthendpoint_ca.crt.erb deleted file mode 100644 index d4c40e255e..0000000000 --- a/jobs/metricsserver/templates/healthendpoint_ca.crt.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.metricsserver.health.ca_cert") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsserver/templates/liquibase.properties b/jobs/metricsserver/templates/liquibase.properties deleted file mode 100644 index 163e3b1d97..0000000000 --- a/jobs/metricsserver/templates/liquibase.properties +++ /dev/null @@ -1 +0,0 @@ -liquibase.hub.mode=off diff --git a/jobs/metricsserver/templates/metricsserver.yml.erb b/jobs/metricsserver/templates/metricsserver.yml.erb deleted file mode 100644 index 3d79c62f23..0000000000 --- a/jobs/metricsserver/templates/metricsserver.yml.erb +++ /dev/null @@ -1,100 +0,0 @@ -<% - def p_arr(property) - p(property,nil) || [] - end - - def build_db_url(db_name, job_name) - db_scheme = p("autoscaler.#{db_name}.db_scheme") - db_address = p("autoscaler.#{db_name}.address") - db_port = p("autoscaler.#{db_name}.port") - db_roles = p_arr("autoscaler.#{db_name}.roles") - db_dbs = p_arr("autoscaler.#{db_name}.databases") - db_role = db_roles.find { |role| role['tag'] == db_name.tr('_', '') } || db_roles.find { |role| role['tag'] == 'default' } - db_database = db_dbs.find { |database| database['tag'] == db_name.tr('_', '') } || db_dbs.find { |database| database['tag'] == 'default' } - - if db_scheme == "postgres" - db_url = "#{db_scheme}://#{ERB::Util.url_encode(db_role['name'])}:#{ERB::Util.url_encode(db_role['password'])}@#{db_address}:#{db_port}/#{db_database['name']}?application_name=#{job_name}&sslmode=" + p("autoscaler.#{db_name}.sslmode") - else - db_url = "#{db_role['name']}:#{db_role['password']}@tcp(#{db_address}:#{db_port})/#{db_database['name']}?tls=" + p("autoscaler.#{db_name}.sslmode") - end - append_db_tls_configs(db_name, db_url, job_name) - end - - def append_db_tls_configs(db_name, db_url, job_name) - unless p("autoscaler.#{db_name}.tls.ca") == "" - db_url = "#{db_url}&sslrootcert=/var/vcap/jobs/#{job_name}/config/certs/#{db_name}/ca.crt" - end - unless p("autoscaler.#{db_name}.tls.certificate") == "" - db_url = "#{db_url}&sslcert=/var/vcap/jobs/#{job_name}/config/certs/#{db_name}/crt" - end - unless p("autoscaler.#{db_name}.tls.private_key") == "" - db_url = "#{db_url}&sslkey=/var/vcap/jobs/#{job_name}/config/certs/#{db_name}/key" - end - db_url -end - -########################################### -# Template Main # -########################################### - job_name = 'metricsserver' - instance_metrics_db_url = build_db_url('instancemetrics_db', job_name) - policy_db_url = build_db_url('policy_db', job_name) - - sorted_instances=link("metricsserver").instances.sort_by {|i|i.address} - nodeIndex=sorted_instances.index(sorted_instances.find{|i|i.id == spec.id}) - addrs=sorted_instances.map{|i| "'#{i.address}'"} - nodeAddrs="[" + addrs.join(",") +"]" - -%> - -logging: - level: <%= p("autoscaler.metricsserver.logging.level") %> -http_client_timeout: <%= p("autoscaler.metricsserver.http_client_timeout") %> -node_addrs: <%= nodeAddrs %> -node_index: <%= nodeIndex %> -db: - policy_db: - url: <%= policy_db_url %> - max_open_connections: <%= p("autoscaler.policy_db_connection_config.max_open_connections") %> - max_idle_connections: <%= p("autoscaler.policy_db_connection_config.max_idle_connections") %> - connection_max_lifetime: <%= p("autoscaler.policy_db_connection_config.connection_max_lifetime") %> - instance_metrics_db: - url: <%= instance_metrics_db_url %> - max_open_connections: <%= p("autoscaler.instancemetrics_db_connection_config.max_open_connections") %> - max_idle_connections: <%= p("autoscaler.instancemetrics_db_connection_config.max_idle_connections") %> - connection_max_lifetime: <%= p("autoscaler.instancemetrics_db_connection_config.connection_max_lifetime") %> -collector: - port: <%= p("autoscaler.metricsserver.collector.port") %> - keep_alive_time: <%= p("autoscaler.metricsserver.collector.keep_alive_time") %> - refresh_interval: <%= p("autoscaler.metricsserver.collector.refresh_interval") %> - collect_interval: <%= p("autoscaler.metricsserver.collector.collect_interval") %> - save_interval: <%= p("autoscaler.metricsserver.collector.save_interval") %> - persist_metrics: <%= p("autoscaler.metricsserver.collector.persist_metrics") %> - envelope_processor_count: <%= p("autoscaler.metricsserver.collector.envelope_processor_count") %> - metric_cache_size_per_app: <%= p("autoscaler.metricsserver.collector.metric_cache_size_per_app") %> - envelope_channel_size: <%= p("autoscaler.metricsserver.collector.envelope_channel_size") %> - metric_channel_size: <%= p("autoscaler.metricsserver.collector.metric_channel_size") %> - tls: - key_file: /var/vcap/jobs/metricsserver/config/certs/metricsserver/server.key - cert_file: /var/vcap/jobs/metricsserver/config/certs/metricsserver/server.crt - ca_file: /var/vcap/jobs/metricsserver/config/certs/metricsserver/ca.crt - -server: - port: <%= p("autoscaler.metricsserver.server.port") %> - tls: - key_file: /var/vcap/jobs/metricsserver/config/certs/metricsserver/server.key - cert_file: /var/vcap/jobs/metricsserver/config/certs/metricsserver/server.crt - ca_file: /var/vcap/jobs/metricsserver/config/certs/metricsserver/ca.crt -health: - port: <%= p("autoscaler.metricsserver.health.port") %> - username: <%= p("autoscaler.metricsserver.health.username") %> - password: <%= p("autoscaler.metricsserver.health.password") %> - <% if_p("autoscaler.metricsserver.health.ca_cert", "autoscaler.metricsserver.health.server_cert", "autoscaler.metricsserver.health.server_key") do %> - tls: - ca_file: /var/vcap/jobs/metricsserver/config/certs/healthendpoint/ca.crt - cert_file: /var/vcap/jobs/metricsserver/config/certs/healthendpoint/server.crt - key_file: /var/vcap/jobs/metricsserver/config/certs/healthendpoint/server.key - <% end %> - - - diff --git a/jobs/metricsserver/templates/metricsserver_ca.crt.erb b/jobs/metricsserver/templates/metricsserver_ca.crt.erb deleted file mode 100644 index 3949634603..0000000000 --- a/jobs/metricsserver/templates/metricsserver_ca.crt.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.metricsserver.ca_cert") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsserver/templates/metricsserver_server.crt.erb b/jobs/metricsserver/templates/metricsserver_server.crt.erb deleted file mode 100644 index 8ea61f8eb6..0000000000 --- a/jobs/metricsserver/templates/metricsserver_server.crt.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.metricsserver.server_cert") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsserver/templates/metricsserver_server.key.erb b/jobs/metricsserver/templates/metricsserver_server.key.erb deleted file mode 100644 index c2efd20bb0..0000000000 --- a/jobs/metricsserver/templates/metricsserver_server.key.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.metricsserver.server_key") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsserver/templates/policy_db.crt.erb b/jobs/metricsserver/templates/policy_db.crt.erb deleted file mode 100644 index 3c19ec6240..0000000000 --- a/jobs/metricsserver/templates/policy_db.crt.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.policy_db.tls.certificate") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsserver/templates/policy_db.key.erb b/jobs/metricsserver/templates/policy_db.key.erb deleted file mode 100644 index 6063d669f7..0000000000 --- a/jobs/metricsserver/templates/policy_db.key.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.policy_db.tls.private_key") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/metricsserver/templates/policy_db_ca.crt.erb b/jobs/metricsserver/templates/policy_db_ca.crt.erb deleted file mode 100644 index cd61f88ecd..0000000000 --- a/jobs/metricsserver/templates/policy_db_ca.crt.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.policy_db.tls.ca") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/packages/metricsgateway/packaging b/packages/metricsgateway/packaging deleted file mode 100644 index 74291ff3c3..0000000000 --- a/packages/metricsgateway/packaging +++ /dev/null @@ -1,12 +0,0 @@ -set -e -x - -export GOROOT=$(readlink -nf /var/vcap/packages/golang-1-linux) -export PATH=$GOROOT/bin:$PATH -export GOPATH=${BOSH_COMPILE_TARGET} -export GOCACHE=/tmp/gocache - -pushd ${BOSH_COMPILE_TARGET}/autoscaler - GOPROXY=off make build-metricsgateway -popd - -cp -a ${BOSH_COMPILE_TARGET}/autoscaler/build/metricsgateway ${BOSH_INSTALL_TARGET} diff --git a/packages/metricsgateway/spec b/packages/metricsgateway/spec deleted file mode 100644 index 31f59047ec..0000000000 --- a/packages/metricsgateway/spec +++ /dev/null @@ -1,224 +0,0 @@ ---- -name: metricsgateway - -dependencies: -- golang-1-linux - -files: -- autoscaler/* -- autoscaler/vendor/* -- autoscaler/cf/* # gosub -- autoscaler/db/* # gosub -- autoscaler/db/sqldb/* # gosub -- autoscaler/healthendpoint/* # gosub -- autoscaler/helpers/* # gosub -- autoscaler/metricsforwarder/server/common/* # gosub -- autoscaler/metricsgateway/* # gosub -- autoscaler/metricsgateway/cmd/metricsgateway/* # gosub -- autoscaler/metricsgateway/config/* # gosub -- autoscaler/metricsgateway/helpers/* # gosub -- autoscaler/models/* # gosub -- autoscaler/routes/* # gosub -- autoscaler/vendor/code.cloudfoundry.org/cfhttp/v2/* # gosub -- autoscaler/vendor/code.cloudfoundry.org/clock/* # gosub -- autoscaler/vendor/code.cloudfoundry.org/go-diodes/* # gosub -- autoscaler/vendor/code.cloudfoundry.org/go-loggregator/v9/* # gosub -- autoscaler/vendor/code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2/* # gosub -- autoscaler/vendor/code.cloudfoundry.org/lager/v3/* # gosub -- autoscaler/vendor/code.cloudfoundry.org/lager/v3/internal/truncate/* # gosub -- autoscaler/vendor/code.cloudfoundry.org/tlsconfig/* # gosub -- autoscaler/vendor/filippo.io/edwards25519/* # gosub -- autoscaler/vendor/filippo.io/edwards25519/field/* # gosub -- autoscaler/vendor/github.com/beorn7/perks/quantile/* # gosub -- autoscaler/vendor/github.com/cespare/xxhash/v2/* # gosub -- autoscaler/vendor/github.com/go-logr/logr/* # gosub -- autoscaler/vendor/github.com/go-logr/logr/funcr/* # gosub -- autoscaler/vendor/github.com/go-logr/stdr/* # gosub -- autoscaler/vendor/github.com/go-sql-driver/mysql/* # gosub -- autoscaler/vendor/github.com/golang/protobuf/proto/* # gosub -- autoscaler/vendor/github.com/gorilla/mux/* # gosub -- autoscaler/vendor/github.com/gorilla/websocket/* # gosub -- autoscaler/vendor/github.com/hashicorp/go-cleanhttp/* # gosub -- autoscaler/vendor/github.com/hashicorp/go-retryablehttp/* # gosub -- autoscaler/vendor/github.com/jackc/pgpassfile/* # gosub -- autoscaler/vendor/github.com/jackc/pgservicefile/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/internal/iobufpool/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/internal/pgio/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/internal/sanitize/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/internal/stmtcache/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/pgconn/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/pgconn/ctxwatch/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/pgconn/internal/bgreader/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/pgproto3/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/pgtype/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/pgxpool/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/stdlib/* # gosub -- autoscaler/vendor/github.com/jackc/puddle/v2/* # gosub -- autoscaler/vendor/github.com/jackc/puddle/v2/internal/genstack/* # gosub -- autoscaler/vendor/github.com/jmoiron/sqlx/* # gosub -- autoscaler/vendor/github.com/jmoiron/sqlx/reflectx/* # gosub -- autoscaler/vendor/github.com/nu7hatch/gouuid/* # gosub -- autoscaler/vendor/github.com/openzipkin/zipkin-go/idgenerator/* # gosub -- autoscaler/vendor/github.com/openzipkin/zipkin-go/model/* # gosub -- autoscaler/vendor/github.com/prometheus/client_golang/prometheus/* # gosub -- autoscaler/vendor/github.com/prometheus/client_golang/prometheus/collectors/* # gosub -- autoscaler/vendor/github.com/prometheus/client_golang/prometheus/internal/* # gosub -- autoscaler/vendor/github.com/prometheus/client_golang/prometheus/promhttp/* # gosub -- autoscaler/vendor/github.com/prometheus/client_model/go/* # gosub -- autoscaler/vendor/github.com/prometheus/common/expfmt/* # gosub -- autoscaler/vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/* # gosub -- autoscaler/vendor/github.com/prometheus/common/model/* # gosub -- autoscaler/vendor/github.com/prometheus/procfs/* # gosub -- autoscaler/vendor/github.com/prometheus/procfs/internal/fs/* # gosub -- autoscaler/vendor/github.com/prometheus/procfs/internal/util/* # gosub -- autoscaler/vendor/github.com/tedsuo/ifrit/* # gosub -- autoscaler/vendor/github.com/tedsuo/ifrit/grouper/* # gosub -- autoscaler/vendor/github.com/tedsuo/ifrit/http_server/* # gosub -- autoscaler/vendor/github.com/tedsuo/ifrit/sigmon/* # gosub -- autoscaler/vendor/github.com/uptrace/opentelemetry-go-extra/otelsql/* # gosub -- autoscaler/vendor/github.com/uptrace/opentelemetry-go-extra/otelsqlx/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/attribute/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/baggage/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/codes/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/internal/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/internal/attribute/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/internal/baggage/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/internal/global/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/metric/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/metric/embedded/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/propagation/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/sdk/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/sdk/instrumentation/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/sdk/internal/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/sdk/internal/env/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/sdk/resource/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/sdk/trace/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/semconv/internal/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/semconv/v1.10.0/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/semconv/v1.21.0/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/semconv/v1.24.0/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/trace/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/trace/embedded/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/trace/noop/* # gosub -- autoscaler/vendor/golang.org/x/crypto/bcrypt/* # gosub -- autoscaler/vendor/golang.org/x/crypto/blowfish/* # gosub -- autoscaler/vendor/golang.org/x/crypto/pbkdf2/* # gosub -- autoscaler/vendor/golang.org/x/exp/maps/* # gosub -- autoscaler/vendor/golang.org/x/net/context/* # gosub -- autoscaler/vendor/golang.org/x/net/http/httpguts/* # gosub -- autoscaler/vendor/golang.org/x/net/http2/* # gosub -- autoscaler/vendor/golang.org/x/net/http2/hpack/* # gosub -- autoscaler/vendor/golang.org/x/net/idna/* # gosub -- autoscaler/vendor/golang.org/x/net/internal/socks/* # gosub -- autoscaler/vendor/golang.org/x/net/internal/timeseries/* # gosub -- autoscaler/vendor/golang.org/x/net/proxy/* # gosub -- autoscaler/vendor/golang.org/x/net/trace/* # gosub -- autoscaler/vendor/golang.org/x/sync/semaphore/* # gosub -- autoscaler/vendor/golang.org/x/sys/unix/* # gosub -- autoscaler/vendor/golang.org/x/sys/windows/* # gosub -- autoscaler/vendor/golang.org/x/sys/windows/registry/* # gosub -- autoscaler/vendor/golang.org/x/text/cases/* # gosub -- autoscaler/vendor/golang.org/x/text/internal/* # gosub -- autoscaler/vendor/golang.org/x/text/internal/language/* # gosub -- autoscaler/vendor/golang.org/x/text/internal/language/compact/* # gosub -- autoscaler/vendor/golang.org/x/text/internal/tag/* # gosub -- autoscaler/vendor/golang.org/x/text/language/* # gosub -- autoscaler/vendor/golang.org/x/text/runes/* # gosub -- autoscaler/vendor/golang.org/x/text/secure/bidirule/* # gosub -- autoscaler/vendor/golang.org/x/text/secure/precis/* # gosub -- autoscaler/vendor/golang.org/x/text/transform/* # gosub -- autoscaler/vendor/golang.org/x/text/unicode/bidi/* # gosub -- autoscaler/vendor/golang.org/x/text/unicode/norm/* # gosub -- autoscaler/vendor/golang.org/x/text/width/* # gosub -- autoscaler/vendor/google.golang.org/genproto/googleapis/rpc/status/* # gosub -- autoscaler/vendor/google.golang.org/grpc/* # gosub -- autoscaler/vendor/google.golang.org/grpc/attributes/* # gosub -- autoscaler/vendor/google.golang.org/grpc/backoff/* # gosub -- autoscaler/vendor/google.golang.org/grpc/balancer/* # gosub -- autoscaler/vendor/google.golang.org/grpc/balancer/base/* # gosub -- autoscaler/vendor/google.golang.org/grpc/balancer/grpclb/state/* # gosub -- autoscaler/vendor/google.golang.org/grpc/balancer/roundrobin/* # gosub -- autoscaler/vendor/google.golang.org/grpc/binarylog/grpc_binarylog_v1/* # gosub -- autoscaler/vendor/google.golang.org/grpc/channelz/* # gosub -- autoscaler/vendor/google.golang.org/grpc/codes/* # gosub -- autoscaler/vendor/google.golang.org/grpc/connectivity/* # gosub -- autoscaler/vendor/google.golang.org/grpc/credentials/* # gosub -- autoscaler/vendor/google.golang.org/grpc/credentials/insecure/* # gosub -- autoscaler/vendor/google.golang.org/grpc/encoding/* # gosub -- autoscaler/vendor/google.golang.org/grpc/encoding/proto/* # gosub -- autoscaler/vendor/google.golang.org/grpc/grpclog/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/backoff/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/balancer/gracefulswitch/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/balancerload/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/binarylog/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/buffer/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/channelz/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/credentials/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/envconfig/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/grpclog/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/grpcrand/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/grpcsync/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/grpcutil/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/idle/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/metadata/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/pretty/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/resolver/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/resolver/dns/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/resolver/dns/internal/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/resolver/passthrough/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/resolver/unix/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/serviceconfig/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/status/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/syscall/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/transport/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/transport/networktype/* # gosub -- autoscaler/vendor/google.golang.org/grpc/keepalive/* # gosub -- autoscaler/vendor/google.golang.org/grpc/metadata/* # gosub -- autoscaler/vendor/google.golang.org/grpc/peer/* # gosub -- autoscaler/vendor/google.golang.org/grpc/resolver/* # gosub -- autoscaler/vendor/google.golang.org/grpc/resolver/dns/* # gosub -- autoscaler/vendor/google.golang.org/grpc/serviceconfig/* # gosub -- autoscaler/vendor/google.golang.org/grpc/stats/* # gosub -- autoscaler/vendor/google.golang.org/grpc/status/* # gosub -- autoscaler/vendor/google.golang.org/grpc/tap/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/encoding/protodelim/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/encoding/protojson/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/encoding/prototext/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/encoding/protowire/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/descfmt/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/descopts/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/detrand/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/editiondefaults/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/editionssupport/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/encoding/defval/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/encoding/json/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/encoding/messageset/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/encoding/tag/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/encoding/text/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/errors/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/filedesc/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/filetype/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/flags/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/genid/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/impl/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/order/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/pragma/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/set/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/strs/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/version/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/proto/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/protoadapt/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/reflect/protodesc/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/reflect/protoreflect/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/reflect/protoregistry/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/runtime/protoiface/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/runtime/protoimpl/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/types/descriptorpb/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/types/gofeaturespb/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/types/known/anypb/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/types/known/durationpb/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/types/known/timestamppb/* # gosub -- autoscaler/vendor/gopkg.in/yaml.v3/* # gosub diff --git a/packages/metricsserver/packaging b/packages/metricsserver/packaging deleted file mode 100644 index aa8ada9230..0000000000 --- a/packages/metricsserver/packaging +++ /dev/null @@ -1,13 +0,0 @@ -set -e -x - -export GOROOT=$(readlink -nf /var/vcap/packages/golang-1-linux) -export PATH=$GOROOT/bin:$PATH -export GOPATH=${BOSH_COMPILE_TARGET} -export GOCACHE=/tmp/gocache - -pushd ${BOSH_COMPILE_TARGET}/autoscaler - GOPROXY=off make build-metricsserver -popd - -cp -a ${BOSH_COMPILE_TARGET}/autoscaler/build/metricsserver ${BOSH_INSTALL_TARGET} -cp -a ${BOSH_COMPILE_TARGET}/autoscaler/metricsserver/db/metricscollector.db.changelog.yml ${BOSH_INSTALL_TARGET} diff --git a/packages/metricsserver/spec b/packages/metricsserver/spec deleted file mode 100644 index e5f8b6d283..0000000000 --- a/packages/metricsserver/spec +++ /dev/null @@ -1,230 +0,0 @@ ---- -name: metricsserver - -dependencies: -- golang-1-linux - -files: -- autoscaler/* -- autoscaler/vendor/* -- autoscaler/metricsserver/db/* -- autoscaler/cf/* # gosub -- autoscaler/collection/* # gosub -- autoscaler/db/* # gosub -- autoscaler/db/sqldb/* # gosub -- autoscaler/envelopeprocessor/* # gosub -- autoscaler/healthendpoint/* # gosub -- autoscaler/helpers/* # gosub -- autoscaler/helpers/handlers/* # gosub -- autoscaler/metricsforwarder/server/common/* # gosub -- autoscaler/metricsserver/cmd/metricsserver/* # gosub -- autoscaler/metricsserver/collector/* # gosub -- autoscaler/metricsserver/config/* # gosub -- autoscaler/models/* # gosub -- autoscaler/routes/* # gosub -- autoscaler/vendor/code.cloudfoundry.org/cfhttp/v2/* # gosub -- autoscaler/vendor/code.cloudfoundry.org/clock/* # gosub -- autoscaler/vendor/code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2/* # gosub -- autoscaler/vendor/code.cloudfoundry.org/lager/v3/* # gosub -- autoscaler/vendor/code.cloudfoundry.org/lager/v3/internal/truncate/* # gosub -- autoscaler/vendor/code.cloudfoundry.org/tlsconfig/* # gosub -- autoscaler/vendor/dario.cat/mergo/* # gosub -- autoscaler/vendor/filippo.io/edwards25519/* # gosub -- autoscaler/vendor/filippo.io/edwards25519/field/* # gosub -- autoscaler/vendor/github.com/beorn7/perks/quantile/* # gosub -- autoscaler/vendor/github.com/cespare/xxhash/v2/* # gosub -- autoscaler/vendor/github.com/felixge/httpsnoop/* # gosub -- autoscaler/vendor/github.com/go-logr/logr/* # gosub -- autoscaler/vendor/github.com/go-logr/logr/funcr/* # gosub -- autoscaler/vendor/github.com/go-logr/stdr/* # gosub -- autoscaler/vendor/github.com/go-sql-driver/mysql/* # gosub -- autoscaler/vendor/github.com/golang/protobuf/proto/* # gosub -- autoscaler/vendor/github.com/gorilla/mux/* # gosub -- autoscaler/vendor/github.com/gorilla/websocket/* # gosub -- autoscaler/vendor/github.com/hashicorp/go-cleanhttp/* # gosub -- autoscaler/vendor/github.com/hashicorp/go-retryablehttp/* # gosub -- autoscaler/vendor/github.com/jackc/pgpassfile/* # gosub -- autoscaler/vendor/github.com/jackc/pgservicefile/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/internal/iobufpool/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/internal/pgio/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/internal/sanitize/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/internal/stmtcache/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/pgconn/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/pgconn/ctxwatch/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/pgconn/internal/bgreader/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/pgproto3/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/pgtype/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/pgxpool/* # gosub -- autoscaler/vendor/github.com/jackc/pgx/v5/stdlib/* # gosub -- autoscaler/vendor/github.com/jackc/puddle/v2/* # gosub -- autoscaler/vendor/github.com/jackc/puddle/v2/internal/genstack/* # gosub -- autoscaler/vendor/github.com/jmoiron/sqlx/* # gosub -- autoscaler/vendor/github.com/jmoiron/sqlx/reflectx/* # gosub -- autoscaler/vendor/github.com/nu7hatch/gouuid/* # gosub -- autoscaler/vendor/github.com/openzipkin/zipkin-go/idgenerator/* # gosub -- autoscaler/vendor/github.com/openzipkin/zipkin-go/model/* # gosub -- autoscaler/vendor/github.com/prometheus/client_golang/prometheus/* # gosub -- autoscaler/vendor/github.com/prometheus/client_golang/prometheus/collectors/* # gosub -- autoscaler/vendor/github.com/prometheus/client_golang/prometheus/internal/* # gosub -- autoscaler/vendor/github.com/prometheus/client_golang/prometheus/promhttp/* # gosub -- autoscaler/vendor/github.com/prometheus/client_model/go/* # gosub -- autoscaler/vendor/github.com/prometheus/common/expfmt/* # gosub -- autoscaler/vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/* # gosub -- autoscaler/vendor/github.com/prometheus/common/model/* # gosub -- autoscaler/vendor/github.com/prometheus/procfs/* # gosub -- autoscaler/vendor/github.com/prometheus/procfs/internal/fs/* # gosub -- autoscaler/vendor/github.com/prometheus/procfs/internal/util/* # gosub -- autoscaler/vendor/github.com/tedsuo/ifrit/* # gosub -- autoscaler/vendor/github.com/tedsuo/ifrit/grouper/* # gosub -- autoscaler/vendor/github.com/tedsuo/ifrit/http_server/* # gosub -- autoscaler/vendor/github.com/tedsuo/ifrit/sigmon/* # gosub -- autoscaler/vendor/github.com/uptrace/opentelemetry-go-extra/otelsql/* # gosub -- autoscaler/vendor/github.com/uptrace/opentelemetry-go-extra/otelsqlx/* # gosub -- autoscaler/vendor/go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux/* # gosub -- autoscaler/vendor/go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux/internal/semconvutil/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/attribute/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/baggage/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/codes/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/internal/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/internal/attribute/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/internal/baggage/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/internal/global/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/metric/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/metric/embedded/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/propagation/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/sdk/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/sdk/instrumentation/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/sdk/internal/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/sdk/internal/env/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/sdk/resource/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/sdk/trace/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/semconv/internal/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/semconv/v1.10.0/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/semconv/v1.20.0/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/semconv/v1.21.0/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/semconv/v1.24.0/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/trace/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/trace/embedded/* # gosub -- autoscaler/vendor/go.opentelemetry.io/otel/trace/noop/* # gosub -- autoscaler/vendor/golang.org/x/crypto/bcrypt/* # gosub -- autoscaler/vendor/golang.org/x/crypto/blowfish/* # gosub -- autoscaler/vendor/golang.org/x/crypto/pbkdf2/* # gosub -- autoscaler/vendor/golang.org/x/exp/maps/* # gosub -- autoscaler/vendor/golang.org/x/net/context/* # gosub -- autoscaler/vendor/golang.org/x/net/http/httpguts/* # gosub -- autoscaler/vendor/golang.org/x/net/http2/* # gosub -- autoscaler/vendor/golang.org/x/net/http2/hpack/* # gosub -- autoscaler/vendor/golang.org/x/net/idna/* # gosub -- autoscaler/vendor/golang.org/x/net/internal/socks/* # gosub -- autoscaler/vendor/golang.org/x/net/internal/timeseries/* # gosub -- autoscaler/vendor/golang.org/x/net/proxy/* # gosub -- autoscaler/vendor/golang.org/x/net/trace/* # gosub -- autoscaler/vendor/golang.org/x/sync/semaphore/* # gosub -- autoscaler/vendor/golang.org/x/sys/unix/* # gosub -- autoscaler/vendor/golang.org/x/sys/windows/* # gosub -- autoscaler/vendor/golang.org/x/sys/windows/registry/* # gosub -- autoscaler/vendor/golang.org/x/text/cases/* # gosub -- autoscaler/vendor/golang.org/x/text/internal/* # gosub -- autoscaler/vendor/golang.org/x/text/internal/language/* # gosub -- autoscaler/vendor/golang.org/x/text/internal/language/compact/* # gosub -- autoscaler/vendor/golang.org/x/text/internal/tag/* # gosub -- autoscaler/vendor/golang.org/x/text/language/* # gosub -- autoscaler/vendor/golang.org/x/text/runes/* # gosub -- autoscaler/vendor/golang.org/x/text/secure/bidirule/* # gosub -- autoscaler/vendor/golang.org/x/text/secure/precis/* # gosub -- autoscaler/vendor/golang.org/x/text/transform/* # gosub -- autoscaler/vendor/golang.org/x/text/unicode/bidi/* # gosub -- autoscaler/vendor/golang.org/x/text/unicode/norm/* # gosub -- autoscaler/vendor/golang.org/x/text/width/* # gosub -- autoscaler/vendor/google.golang.org/genproto/googleapis/rpc/status/* # gosub -- autoscaler/vendor/google.golang.org/grpc/* # gosub -- autoscaler/vendor/google.golang.org/grpc/attributes/* # gosub -- autoscaler/vendor/google.golang.org/grpc/backoff/* # gosub -- autoscaler/vendor/google.golang.org/grpc/balancer/* # gosub -- autoscaler/vendor/google.golang.org/grpc/balancer/base/* # gosub -- autoscaler/vendor/google.golang.org/grpc/balancer/grpclb/state/* # gosub -- autoscaler/vendor/google.golang.org/grpc/balancer/roundrobin/* # gosub -- autoscaler/vendor/google.golang.org/grpc/binarylog/grpc_binarylog_v1/* # gosub -- autoscaler/vendor/google.golang.org/grpc/channelz/* # gosub -- autoscaler/vendor/google.golang.org/grpc/codes/* # gosub -- autoscaler/vendor/google.golang.org/grpc/connectivity/* # gosub -- autoscaler/vendor/google.golang.org/grpc/credentials/* # gosub -- autoscaler/vendor/google.golang.org/grpc/credentials/insecure/* # gosub -- autoscaler/vendor/google.golang.org/grpc/encoding/* # gosub -- autoscaler/vendor/google.golang.org/grpc/encoding/proto/* # gosub -- autoscaler/vendor/google.golang.org/grpc/grpclog/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/backoff/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/balancer/gracefulswitch/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/balancerload/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/binarylog/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/buffer/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/channelz/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/credentials/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/envconfig/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/grpclog/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/grpcrand/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/grpcsync/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/grpcutil/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/idle/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/metadata/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/pretty/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/resolver/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/resolver/dns/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/resolver/dns/internal/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/resolver/passthrough/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/resolver/unix/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/serviceconfig/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/status/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/syscall/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/transport/* # gosub -- autoscaler/vendor/google.golang.org/grpc/internal/transport/networktype/* # gosub -- autoscaler/vendor/google.golang.org/grpc/keepalive/* # gosub -- autoscaler/vendor/google.golang.org/grpc/metadata/* # gosub -- autoscaler/vendor/google.golang.org/grpc/peer/* # gosub -- autoscaler/vendor/google.golang.org/grpc/resolver/* # gosub -- autoscaler/vendor/google.golang.org/grpc/resolver/dns/* # gosub -- autoscaler/vendor/google.golang.org/grpc/serviceconfig/* # gosub -- autoscaler/vendor/google.golang.org/grpc/stats/* # gosub -- autoscaler/vendor/google.golang.org/grpc/status/* # gosub -- autoscaler/vendor/google.golang.org/grpc/tap/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/encoding/protodelim/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/encoding/protojson/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/encoding/prototext/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/encoding/protowire/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/descfmt/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/descopts/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/detrand/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/editiondefaults/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/editionssupport/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/encoding/defval/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/encoding/json/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/encoding/messageset/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/encoding/tag/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/encoding/text/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/errors/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/filedesc/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/filetype/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/flags/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/genid/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/impl/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/order/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/pragma/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/set/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/strs/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/internal/version/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/proto/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/protoadapt/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/reflect/protodesc/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/reflect/protoreflect/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/reflect/protoregistry/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/runtime/protoiface/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/runtime/protoimpl/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/types/descriptorpb/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/types/gofeaturespb/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/types/known/anypb/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/types/known/durationpb/* # gosub -- autoscaler/vendor/google.golang.org/protobuf/types/known/timestamppb/* # gosub -- autoscaler/vendor/gopkg.in/yaml.v3/* # gosub diff --git a/spec/jobs/metricsforwarder/metricsforwarder_spec.rb b/spec/jobs/metricsforwarder/metricsforwarder_spec.rb deleted file mode 100644 index 4c64b52f61..0000000000 --- a/spec/jobs/metricsforwarder/metricsforwarder_spec.rb +++ /dev/null @@ -1,118 +0,0 @@ -require "rspec" -require "json" -require "bosh/template/test" -require "rspec/file_fixtures" -require "yaml" - -describe "metricsforwarder" do - let(:release) { Bosh::Template::Test::ReleaseDir.new(File.join(File.dirname(__FILE__), "../../..")) } - let(:job) { release.job("metricsforwarder") } - let(:template) { job.template("config/metricsforwarder.yml") } - let(:properties) { YAML.safe_load(fixture("metricsforwarder.yml").read) } - let(:rendered_template) { YAML.safe_load(template.render(properties)) } - - context "config/metricsforwarder.yml" do - it "supports syslog forwarding" do - properties["autoscaler"]["metricsforwarder"] = { - "syslog" => { - "server_address" => "syslog-server" - } - } - - expect(rendered_template).to include( - { - "syslog" => { - "server_address" => "syslog-server", - "port" => 6067, - "tls" => { - "key_file" => "/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.key", - "cert_file" => "/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.crt", - "ca_file" => "/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/ca.crt" - } - } - } - ) - end - - it "does not set username nor password if not configured" do - properties["autoscaler"]["metricsforwarder"] = { - "health" => { - "port" => 1234 - } - } - - expect(rendered_template["health"]) - .to include( - {"port" => 1234} - ) - end - - it "check metricsforwarder basic auth username and password" do - properties["autoscaler"]["metricsforwarder"] = { - "health" => { - "port" => 1234, - "username" => "test-user", - "password" => "test-user-password" - } - } - - expect(rendered_template["health"]) - .to include( - {"port" => 1234, - "username" => "test-user", - "password" => "test-user-password"} - ) - end - - it "has a cred helper impl by default" do - expect(rendered_template).to include({ - "cred_helper_impl" => "default" - }) - end - - it "has a cred helper impl configured for stored procedures" do - properties["autoscaler"]["metricsforwarder"] = { - "cred_helper" => { - "impl" => "stored_procedure", - "stored_procedure_config" => { - "schema_name" => "SCHEMA", - "create_binding_credential_procedure_name" => "CREATE_BINDING_CREDENTIAL", - "drop_binding_credential_procedure_name" => "DROP_BINDING_CREDENTIAL", - "drop_all_binding_credential_procedure_name" => "DROP_ALL_BINDING_CREDENTIALS", - "validate_binding_credential_procedure_name" => "VALIDATE_BINDING_CREDENTIALS" - } - } - } - - expect(rendered_template).to include( - { - "cred_helper_impl" => "stored_procedure", - "stored_procedure_binding_credential_config" => { - "schema_name" => "SCHEMA", - "create_binding_credential_procedure_name" => "CREATE_BINDING_CREDENTIAL", - "drop_binding_credential_procedure_name" => "DROP_BINDING_CREDENTIAL", - "drop_all_binding_credential_procedure_name" => "DROP_ALL_BINDING_CREDENTIALS", - "validate_binding_credential_procedure_name" => "VALIDATE_BINDING_CREDENTIALS" - } - } - ) - end - - context "uses tls" do - context "policy_db" do - it "includes the ca, cert and key in url when configured" do - rendered_template["db"]["policy_db"]["url"].tap do |url| - check_if_certs_in_url(url, "policy_db") - end - end - - it "does not include the ca, cert and key in url when not configured" do - properties["autoscaler"]["policy_db"]["tls"] = nil - rendered_template["db"]["policy_db"]["url"].tap do |url| - check_if_certs_not_in_url(url, "policy_db") - end - end - end - end - end -end diff --git a/spec/jobs/metricsgateway/metricsgateway_spec.rb b/spec/jobs/metricsgateway/metricsgateway_spec.rb deleted file mode 100644 index 924942d083..0000000000 --- a/spec/jobs/metricsgateway/metricsgateway_spec.rb +++ /dev/null @@ -1,69 +0,0 @@ -require "rspec" -require "json" -require "bosh/template/test" -require "yaml" -require "rspec/file_fixtures" - -describe "metricsgateway" do - let(:release) { Bosh::Template::Test::ReleaseDir.new(File.join(File.dirname(__FILE__), "../../..")) } - let(:job) { release.job("metricsgateway") } - let(:template) { job.template("config/metricsgateway.yml") } - let(:properties) { YAML.safe_load(fixture("metricsgateway.yml").read) } - let(:links) { [Bosh::Template::Test::Link.new(name: "metricsserver")] } - let(:rendered_template) { YAML.safe_load(template.render(properties, consumes: links)) } - - context "config/metricsgateway.yml" do - it "does not set username nor password if not configured" do - properties["autoscaler"]["metricsgateway"] = { - "health" => { - "port" => 1234 - }, - "nozzle" => { - "rlp_addr" => "localhost" - } - } - - expect(rendered_template["health"]) - .to include( - {"port" => 1234} - ) - end - - it "check metricsgateway basic auth username and password" do - properties["autoscaler"]["metricsgateway"] = { - "health" => { - "port" => 1234, - "username" => "test-user", - "password" => "test-user-password" - }, - "nozzle" => { - "rlp_addr" => "localhost" - } - } - - expect(rendered_template["health"]) - .to include( - {"port" => 1234, - "username" => "test-user", - "password" => "test-user-password"} - ) - end - - context "apiserver uses tls" do - context "policy_db" do - it "includes the ca, cert and key in url when configured" do - rendered_template["app_manager"]["policy_db"]["url"].tap do |url| - check_if_certs_in_url(url, "policy_db") - end - end - - it "does not include the ca, cert and key in url when not configured" do - properties["autoscaler"]["policy_db"]["tls"] = nil - rendered_template["app_manager"]["policy_db"]["url"].tap do |url| - check_if_certs_not_in_url(url, "policy_db") - end - end - end - end - end -end From fc67f0e57cbda055b78b4f4cf78a48de1d599360 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 7 Jun 2024 16:49:11 +0200 Subject: [PATCH 033/186] remove metricsserver and metricsgateway code, adjust integration tests to work with a brand new logcache mock, get rid of appinstancemetrics database table --- jobs/operator/spec | 11 - .../templates/appinstancemetrics_db.crt.erb | 3 - .../templates/appinstancemetrics_db.key.erb | 3 - .../appinstancemetrics_db_ca.crt.erb | 3 - jobs/operator/templates/operator.yml.erb | 9 - scripts/generate_test_certs.sh | 14 +- scripts/sync-package-specs | 4 +- spec/fixtures/metricsgateway.yml | 27 - spec/fixtures/metricsserver.yml | 39 - spec/jobs/common/health_endpoint_spec.rb | 4 - spec/jobs/metricsserver/metricsserver_spec.rb | 78 -- spec/jobs/operator/operator_spec.rb | 15 - .../db/sqldb/instancemetrics_sqldb.go | 190 --- .../db/sqldb/instancemetrics_sqldb_test.go | 697 ----------- src/autoscaler/db/sqldb/sqldb_suite_test.go | 28 +- .../aggregator/metric_poller.go | 2 +- src/autoscaler/generate-fakes.go | 2 - src/autoscaler/go.mod | 1 - src/autoscaler/go.sum | 2 - .../healthendpoint/counter_collector.go | 64 - .../healthendpoint/counter_collector_test.go | 113 -- src/autoscaler/integration/components_test.go | 157 +-- src/autoscaler/integration/helpers_test.go | 9 - ...tegration_golangapi_eventgenerator_test.go | 1 - ...ntegration_golangapi_scalingengine_test.go | 1 - .../integration_golangapi_scheduler_test.go | 1 - ...ache_eventgenerator_scalingengine_test.go} | 125 +- .../integration_operator_others_test.go | 17 +- .../integration/integration_suite_test.go | 145 +-- .../integration/mock/log_cache_mock.go | 153 +++ src/autoscaler/metricsgateway/appManager.go | 79 -- .../metricsgateway/appManager_test.go | 92 -- .../metricsgateway/cmd/metricsgateway/main.go | 169 --- .../metricsgateway_suite_test.go | 259 ---- .../cmd/metricsgateway/metricsgateway_test.go | 196 --- .../metricsgateway/config/config.go | 174 --- .../config/config_suite_test.go | 13 - .../metricsgateway/config/config_test.go | 1050 ----------------- src/autoscaler/metricsgateway/dispatcher.go | 49 - .../metricsgateway/dispatcher_test.go | 148 --- src/autoscaler/metricsgateway/emitter.go | 89 -- src/autoscaler/metricsgateway/emitter_test.go | 127 -- .../metricsgateway/exampleconfig/example.yaml | 34 - .../helpers/helpers_suite_test.go | 13 - .../metricsgateway/helpers/ws_helper.go | 187 --- .../metricsgateway/helpers/ws_helper_test.go | 216 ---- .../metricsgateway_suite_test.go | 19 - src/autoscaler/metricsgateway/nozzle.go | 140 --- src/autoscaler/metricsgateway/nozzle_test.go | 345 ------ .../metricsserver/cmd/metricsserver/main.go | 167 --- .../metricsserver/metricsserver_suite_test.go | 183 --- .../cmd/metricsserver/metricsserver_test.go | 212 ---- .../metricsserver/collector/collector.go | 244 ---- .../collector/collector_suite_test.go | 72 -- .../metricsserver/collector/collector_test.go | 339 ------ .../metricsserver/collector/config.go | 34 - .../collector/envelope_processor.go | 136 --- .../collector/envelope_processor_test.go | 361 ------ .../metricsserver/collector/metric_handler.go | 164 --- .../collector/metric_handler_test.go | 426 ------- .../metricsserver/collector/server.go | 38 - .../metricsserver/collector/server_test.go | 66 -- .../metricsserver/collector/ws_handler.go | 142 --- .../metricsserver/collector/ws_server.go | 24 - src/autoscaler/metricsserver/config/config.go | 157 --- .../metricsserver/config/config_suite_test.go | 13 - .../metricsserver/config/config_test.go | 352 ------ .../db/metricscollector.db.changelog.yml | 10 + .../metricsserver/exampleconfig/example.yml | 31 - src/autoscaler/operator/cmd/operator/main.go | 16 +- .../cmd/operator/operator_suite_test.go | 9 - .../operator/cmd/operator/operator_test.go | 33 +- src/autoscaler/operator/config/config.go | 17 - src/autoscaler/operator/config/config_test.go | 79 -- .../operator/config/testdata/invalid.yml | 8 - .../operator/config/testdata/partial.yml | 4 - .../operator/config/testdata/valid.yml | 8 - .../operator/exampleconfig/example.yml | 8 - src/autoscaler/operator/instancemetricsdb.go | 40 - .../operator/instancemetricsdb_test.go | 65 - src/autoscaler/testhelpers/fakeLoggregator.go | 226 ---- 81 files changed, 307 insertions(+), 8694 deletions(-) delete mode 100644 jobs/operator/templates/appinstancemetrics_db.crt.erb delete mode 100644 jobs/operator/templates/appinstancemetrics_db.key.erb delete mode 100644 jobs/operator/templates/appinstancemetrics_db_ca.crt.erb delete mode 100644 spec/fixtures/metricsgateway.yml delete mode 100644 spec/fixtures/metricsserver.yml delete mode 100644 spec/jobs/metricsserver/metricsserver_spec.rb delete mode 100644 src/autoscaler/db/sqldb/instancemetrics_sqldb.go delete mode 100644 src/autoscaler/db/sqldb/instancemetrics_sqldb_test.go delete mode 100644 src/autoscaler/healthendpoint/counter_collector.go delete mode 100644 src/autoscaler/healthendpoint/counter_collector_test.go rename src/autoscaler/integration/{integration_metricsgateway_metricserver_eventgenerator_scalingengine_test.go => integration_logcache_eventgenerator_scalingengine_test.go} (86%) create mode 100644 src/autoscaler/integration/mock/log_cache_mock.go delete mode 100644 src/autoscaler/metricsgateway/appManager.go delete mode 100644 src/autoscaler/metricsgateway/appManager_test.go delete mode 100644 src/autoscaler/metricsgateway/cmd/metricsgateway/main.go delete mode 100644 src/autoscaler/metricsgateway/cmd/metricsgateway/metricsgateway_suite_test.go delete mode 100644 src/autoscaler/metricsgateway/cmd/metricsgateway/metricsgateway_test.go delete mode 100644 src/autoscaler/metricsgateway/config/config.go delete mode 100644 src/autoscaler/metricsgateway/config/config_suite_test.go delete mode 100644 src/autoscaler/metricsgateway/config/config_test.go delete mode 100644 src/autoscaler/metricsgateway/dispatcher.go delete mode 100644 src/autoscaler/metricsgateway/dispatcher_test.go delete mode 100644 src/autoscaler/metricsgateway/emitter.go delete mode 100644 src/autoscaler/metricsgateway/emitter_test.go delete mode 100644 src/autoscaler/metricsgateway/exampleconfig/example.yaml delete mode 100644 src/autoscaler/metricsgateway/helpers/helpers_suite_test.go delete mode 100644 src/autoscaler/metricsgateway/helpers/ws_helper.go delete mode 100644 src/autoscaler/metricsgateway/helpers/ws_helper_test.go delete mode 100644 src/autoscaler/metricsgateway/metricsgateway_suite_test.go delete mode 100644 src/autoscaler/metricsgateway/nozzle.go delete mode 100644 src/autoscaler/metricsgateway/nozzle_test.go delete mode 100644 src/autoscaler/metricsserver/cmd/metricsserver/main.go delete mode 100644 src/autoscaler/metricsserver/cmd/metricsserver/metricsserver_suite_test.go delete mode 100644 src/autoscaler/metricsserver/cmd/metricsserver/metricsserver_test.go delete mode 100644 src/autoscaler/metricsserver/collector/collector.go delete mode 100644 src/autoscaler/metricsserver/collector/collector_suite_test.go delete mode 100644 src/autoscaler/metricsserver/collector/collector_test.go delete mode 100644 src/autoscaler/metricsserver/collector/config.go delete mode 100644 src/autoscaler/metricsserver/collector/envelope_processor.go delete mode 100644 src/autoscaler/metricsserver/collector/envelope_processor_test.go delete mode 100644 src/autoscaler/metricsserver/collector/metric_handler.go delete mode 100644 src/autoscaler/metricsserver/collector/metric_handler_test.go delete mode 100644 src/autoscaler/metricsserver/collector/server.go delete mode 100644 src/autoscaler/metricsserver/collector/server_test.go delete mode 100644 src/autoscaler/metricsserver/collector/ws_handler.go delete mode 100644 src/autoscaler/metricsserver/collector/ws_server.go delete mode 100644 src/autoscaler/metricsserver/config/config.go delete mode 100644 src/autoscaler/metricsserver/config/config_suite_test.go delete mode 100644 src/autoscaler/metricsserver/config/config_test.go delete mode 100644 src/autoscaler/metricsserver/exampleconfig/example.yml delete mode 100644 src/autoscaler/operator/instancemetricsdb.go delete mode 100644 src/autoscaler/operator/instancemetricsdb_test.go delete mode 100644 src/autoscaler/testhelpers/fakeLoggregator.go diff --git a/jobs/operator/spec b/jobs/operator/spec index 9da22934f4..ba529df29d 100644 --- a/jobs/operator/spec +++ b/jobs/operator/spec @@ -26,10 +26,6 @@ templates: appmetrics_db.crt.erb: config/certs/appmetrics_db/crt appmetrics_db.key.erb: config/certs/appmetrics_db/key - appinstancemetrics_db_ca.crt.erb: config/certs/instancemetrics_db/ca.crt - appinstancemetrics_db.crt.erb: config/certs/instancemetrics_db/crt - appinstancemetrics_db.key.erb: config/certs/instancemetrics_db/key - lock_db_ca.crt.erb: config/certs/lock_db/ca.crt lock_db.crt.erb: config/certs/lock_db/crt lock_db.key.erb: config/certs/lock_db/key @@ -209,13 +205,6 @@ properties: description: "the time interval to sync applications" default: 24h - autoscaler.operator.instance_metrics_db.refresh_interval: - description: "the time interval to prune the instancemetrics database" - default: 1h - autoscaler.operator.instance_metrics_db.cutoff_duration: - description: "the cutoff days when pruning instancemetrics database" - default: 24h - autoscaler.operator.app_metrics_db.refresh_interval: description: "the time interval to prune the appmetrics database" default: 1h diff --git a/jobs/operator/templates/appinstancemetrics_db.crt.erb b/jobs/operator/templates/appinstancemetrics_db.crt.erb deleted file mode 100644 index bccd1b5ffe..0000000000 --- a/jobs/operator/templates/appinstancemetrics_db.crt.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.instancemetrics_db.tls.certificate") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/operator/templates/appinstancemetrics_db.key.erb b/jobs/operator/templates/appinstancemetrics_db.key.erb deleted file mode 100644 index 9d64259e6a..0000000000 --- a/jobs/operator/templates/appinstancemetrics_db.key.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.instancemetrics_db.tls.private_key") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/operator/templates/appinstancemetrics_db_ca.crt.erb b/jobs/operator/templates/appinstancemetrics_db_ca.crt.erb deleted file mode 100644 index d4e13326b9..0000000000 --- a/jobs/operator/templates/appinstancemetrics_db_ca.crt.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% if_p("autoscaler.instancemetrics_db.tls.ca") do |value| %> -<%= value %> -<% end %> \ No newline at end of file diff --git a/jobs/operator/templates/operator.yml.erb b/jobs/operator/templates/operator.yml.erb index 599a407074..c96a6890a0 100644 --- a/jobs/operator/templates/operator.yml.erb +++ b/jobs/operator/templates/operator.yml.erb @@ -38,7 +38,6 @@ end ########################################### job_name = 'operator' policy_db_url = build_db_url('policy_db', job_name) - instance_metrics_db_url = build_db_url('instancemetrics_db', job_name) app_metrics_db_url = build_db_url('appmetrics_db', job_name) scaling_engine_db_url = build_db_url('scalingengine_db', job_name) lock_db_url = build_db_url('lock_db', job_name) @@ -70,14 +69,6 @@ health: <% end %> http_client_timeout: <%= p("autoscaler.operator.http_client_timeout") %> -instance_metrics_db: - db: - url: <%= instance_metrics_db_url %> - max_open_connections: <%= p("autoscaler.instancemetrics_db_connection_config.max_open_connections") %> - max_idle_connections: <%= p("autoscaler.instancemetrics_db_connection_config.max_idle_connections") %> - connection_max_lifetime: <%= p("autoscaler.instancemetrics_db_connection_config.connection_max_lifetime") %> - refresh_interval: <%= p("autoscaler.operator.instance_metrics_db.refresh_interval") %> - cutoff_duration: <%= p("autoscaler.operator.instance_metrics_db.cutoff_duration") %> app_metrics_db: db: diff --git a/scripts/generate_test_certs.sh b/scripts/generate_test_certs.sh index d3c54015ba..5a119d30a7 100755 --- a/scripts/generate_test_certs.sh +++ b/scripts/generate_test_certs.sh @@ -16,6 +16,11 @@ ${CERTSTRAP} --depot-path "${depot_path}" init --passphrase '' --common-name aut mv -f "${depot_path}"/autoscalerCA.crt "${depot_path}"/autoscaler-ca.crt mv -f "${depot_path}"/autoscalerCA.key "${depot_path}"/autoscaler-ca.key +# CA to distribute to dummy loggregator_agent certs +${CERTSTRAP} --depot-path "${depot_path}" init --passphrase '' --common-name loggregatorCA --years "20" +mv -f "${depot_path}"/loggregatorCA.crt "${depot_path}"/loggregator-ca.crt +mv -f "${depot_path}"/loggregatorCA.key "${depot_path}"/loggregator-ca.key + # CA to distribute to dummy syslog emitter certs ${CERTSTRAP} --depot-path "${depot_path}" init --passphrase '' --common-name LogCacheSyslogServerCA --years "20" mv -f "${depot_path}"/LogCacheSyslogServerCA.crt "${depot_path}"/log-cache-syslog-server-ca.crt @@ -69,7 +74,6 @@ ${CERTSTRAP} --depot-path "${depot_path}" sign scheduler --CA autoscaler-ca --ye openssl pkcs12 -export -in "${depot_path}"/scheduler.crt -inkey "${depot_path}"/scheduler.key -out "${depot_path}"/scheduler.p12 -name scheduler -password pass:123456 keytool -importcert -alias autoscaler -file "${depot_path}"/autoscaler-ca.crt -keystore "${depot_path}"/autoscaler.truststore -storeType pkcs12 -storepass 123456 -noprompt - # # loggregator test server certificate ${CERTSTRAP} --depot-path "${depot_path}" request-cert --passphrase '' --domain reverselogproxy ${CERTSTRAP} --depot-path "${depot_path}" sign reverselogproxy --CA autoscaler-ca --years "20" @@ -86,10 +90,18 @@ ${CERTSTRAP} --depot-path "${depot_path}" sign metricserver --CA autoscaler-ca - ${CERTSTRAP} --depot-path "${depot_path}" request-cert --passphrase '' --domain metricserver_client ${CERTSTRAP} --depot-path "${depot_path}" sign metricserver_client --CA autoscaler-ca --years "20" +# metricsforwarder certificate for loggregator_agent +${CERTSTRAP} --depot-path "${depot_path}" request-cert --passphrase '' --domain metron +${CERTSTRAP} --depot-path "${depot_path}" sign metron --CA loggregator-ca --years "20" + # metricsforwarder certificate for log-cache-syslog-server ${CERTSTRAP} --depot-path "${depot_path}" request-cert --passphrase '' --domain cf-app ${CERTSTRAP} --depot-path "${depot_path}" sign cf-app --CA log-cache-syslog-server-ca --years "20" +# log-cache certificate +${CERTSTRAP} --depot-path "${depot_path}" request-cert --passphrase '' --domain log-cache --ip 127.0.0.1 +${CERTSTRAP} --depot-path "${depot_path}" sign log-cache --CA autoscaler-ca --years "20" + # mTLS client certificate for local testing ## certstrap with multiple OU not working at the moment. Pull request is created in the upstream. Therefore, using openssl at the moment ## https://github.com/square/certstrap/pull/120 diff --git a/scripts/sync-package-specs b/scripts/sync-package-specs index 349ad13d98..8f7558178a 100755 --- a/scripts/sync-package-specs +++ b/scripts/sync-package-specs @@ -29,7 +29,5 @@ function sync_package() { sync_package eventgenerator eventgenerator -app ./... sync_package golangapiserver api -app ./... sync_package metricsforwarder metricsforwarder -app ./... -sync_package metricsgateway metricsgateway -app ./... -sync_package metricsserver metricsserver -app ./... sync_package operator operator -app ./... -sync_package scalingengine scalingengine -app ./... \ No newline at end of file +sync_package scalingengine scalingengine -app ./... diff --git a/spec/fixtures/metricsgateway.yml b/spec/fixtures/metricsgateway.yml deleted file mode 100644 index 3ea7c70188..0000000000 --- a/spec/fixtures/metricsgateway.yml +++ /dev/null @@ -1,27 +0,0 @@ -autoscaler: - policy_db: - address: 10.11.137.101 - databases: - - name: foo - tag: default - db_scheme: postgres - port: 5432 - roles: - - name: foo - password: default - tag: default - tls: - ca: BEGIN---CA---END - certificate: BEGIN---CERT---END - private_key: BEGIN---KEY---END - cf: - api: https://api.cf.domain - auth_endpoint: https://login.cf.domain - client_id: client_id - secret: uaa_secret - uaa_api: https://login.cf.domain/uaa - grant_type: ALLOW_ALL - metricsgateway: - nozzle: - rlp_addr: - some_addr \ No newline at end of file diff --git a/spec/fixtures/metricsserver.yml b/spec/fixtures/metricsserver.yml deleted file mode 100644 index bcb0079d59..0000000000 --- a/spec/fixtures/metricsserver.yml +++ /dev/null @@ -1,39 +0,0 @@ -autoscaler: - policy_db: - address: 10.11.137.101 - databases: - - name: foo - tag: default - db_scheme: postgres - port: 5432 - roles: - - name: foo - password: default - tag: default - tls: - ca: BEGIN---CA---END - certificate: BEGIN---CERT---END - private_key: BEGIN---KEY---END - instancemetrics_db: - address: 10.11.137.101 - databases: - - name: foo - tag: default - db_scheme: postgres - port: 5432 - tls.ca: aa - roles: - - name: foo - password: default - tag: default - tls: - ca: BEGIN---CA---END - certificate: BEGIN---CERT---END - private_key: BEGIN---KEY---END - cf: - api: https://api.cf.domain - auth_endpoint: https://login.cf.domain - client_id: client_id - secret: uaa_secret - uaa_api: https://login.cf.domain/uaa - grant_type: ALLOW_ALL \ No newline at end of file diff --git a/spec/jobs/common/health_endpoint_spec.rb b/spec/jobs/common/health_endpoint_spec.rb index 0a47a9c150..98a96b2898 100644 --- a/spec/jobs/common/health_endpoint_spec.rb +++ b/spec/jobs/common/health_endpoint_spec.rb @@ -10,8 +10,6 @@ %w[apiserver golangapiserver config/apiserver.yml apiserver.yml], %w[eventgenerator eventgenerator config/eventgenerator.yml eventgenerator.yml], %w[metricsforwarder metricsforwarder config/metricsforwarder.yml metricsforwarder.yml], - %w[metricsgateway metricsgateway config/metricsgateway.yml metricsgateway.yml], - %w[metricsserver metricsserver config/metricsserver.yml metricsserver.yml], %w[operator operator config/operator.yml operator.yml], %w[scalingengine scalingengine config/scalingengine.yml scalingengine.yml] ].each do |service, release_job, config_file, properties_file| @@ -23,8 +21,6 @@ @links = case service when "eventgenerator" [Bosh::Template::Test::Link.new(name: "eventgenerator")] - when "metricsgateway", "metricsserver" - [Bosh::Template::Test::Link.new(name: "metricsserver")] else [] end diff --git a/spec/jobs/metricsserver/metricsserver_spec.rb b/spec/jobs/metricsserver/metricsserver_spec.rb deleted file mode 100644 index 6841bfb8e0..0000000000 --- a/spec/jobs/metricsserver/metricsserver_spec.rb +++ /dev/null @@ -1,78 +0,0 @@ -require "rspec" -require "json" -require "bosh/template/test" -require "yaml" -require "rspec/file_fixtures" - -describe "metricsserver" do - let(:release) { Bosh::Template::Test::ReleaseDir.new(File.join(File.dirname(__FILE__), "../../..")) } - let(:job) { release.job("metricsserver") } - let(:template) { job.template("config/metricsserver.yml") } - let(:properties) { YAML.safe_load(fixture("metricsserver.yml").read) } - let(:links) { [Bosh::Template::Test::Link.new(name: "metricsserver")] } - let(:rendered_template) { YAML.safe_load(template.render(properties, consumes: links)) } - - context "config/metricsserver.yml" do - it "does not set username nor password if not configured" do - properties["autoscaler"]["metricsserver"] = { - "health" => { - "port" => 1234 - } - } - - expect(rendered_template["health"]) - .to include( - {"port" => 1234} - ) - end - - it "check metricsserver basic auth username and password" do - properties["autoscaler"]["metricsserver"] = { - "health" => { - "port" => 1234, - "username" => "test-user", - "password" => "test-user-password" - } - } - - expect(rendered_template["health"]) - .to include( - {"port" => 1234, - "username" => "test-user", - "password" => "test-user-password"} - ) - end - - context "uses tls" do - context "policy_db" do - it "includes the ca, cert and key in url when configured" do - rendered_template["db"]["policy_db"]["url"].tap do |url| - check_if_certs_in_url(url, "policy_db") - end - end - - it "does not include the ca, cert and key in url when not configured" do - properties["autoscaler"]["policy_db"]["tls"] = nil - rendered_template["db"]["policy_db"]["url"].tap do |url| - check_if_certs_not_in_url(url, "policy_db") - end - end - end - - context "instance_metrics_db" do - it "includes the ca, cert and key in url when configured" do - rendered_template["db"]["instance_metrics_db"]["url"].tap do |url| - check_if_certs_in_url(url, "instancemetrics_db") - end - end - - it "does not include the ca, cert and key in url when not configured" do - properties["autoscaler"]["instancemetrics_db"]["tls"] = nil - rendered_template["db"]["instance_metrics_db"]["url"].tap do |url| - check_if_certs_not_in_url(url, "instancemetrics_db") - end - end - end - end - end -end diff --git a/spec/jobs/operator/operator_spec.rb b/spec/jobs/operator/operator_spec.rb index bd2f966447..c7319a71b2 100644 --- a/spec/jobs/operator/operator_spec.rb +++ b/spec/jobs/operator/operator_spec.rb @@ -58,21 +58,6 @@ end end - context "instance_metrics_db " do - it "includes the ca, cert and key in url when configured" do - rendered_template["instance_metrics_db"]["db"]["url"].tap do |url| - check_if_certs_in_url(url, "instancemetrics_db") - end - end - - it "does not include the ca, cert and key in url when not configured" do - properties["autoscaler"]["instancemetrics_db"]["tls"] = nil - rendered_template["instance_metrics_db"]["db"]["url"].tap do |url| - check_if_certs_not_in_url(url, "instancemetrics_db ") - end - end - end - context "app_metrics_db" do it "includes the ca, cert and key in url when configured" do rendered_template["app_metrics_db"]["db"]["url"].tap do |url| diff --git a/src/autoscaler/db/sqldb/instancemetrics_sqldb.go b/src/autoscaler/db/sqldb/instancemetrics_sqldb.go deleted file mode 100644 index cfa127efb7..0000000000 --- a/src/autoscaler/db/sqldb/instancemetrics_sqldb.go +++ /dev/null @@ -1,190 +0,0 @@ -package sqldb - -import ( - "context" - "database/sql" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - "code.cloudfoundry.org/lager/v3" - "github.com/jmoiron/sqlx" - "github.com/uptrace/opentelemetry-go-extra/otelsql" - "github.com/uptrace/opentelemetry-go-extra/otelsqlx" -) - -type InstanceMetricsSQLDB struct { - logger lager.Logger - dbConfig db.DatabaseConfig - sqldb *sqlx.DB -} - -func NewInstanceMetricsSQLDB(dbConfig db.DatabaseConfig, logger lager.Logger) (*InstanceMetricsSQLDB, error) { - database, err := db.GetConnection(dbConfig.URL) - if err != nil { - return nil, err - } - - sqldb, err := otelsqlx.Open(database.DriverName, database.DSN, otelsql.WithAttributes(database.OTELAttribute)) - if err != nil { - logger.Error("failed-open-instancemetrics-db", err, lager.Data{"dbConfig": dbConfig}) - return nil, err - } - - err = sqldb.Ping() - if err != nil { - sqldb.Close() - logger.Error("failed-ping-instancemetrics-db", err, lager.Data{"dbConfig": dbConfig}) - return nil, err - } - - sqldb.SetConnMaxLifetime(dbConfig.ConnectionMaxLifetime) - sqldb.SetMaxIdleConns(int(dbConfig.MaxIdleConnections)) - sqldb.SetMaxOpenConns(int(dbConfig.MaxOpenConnections)) - sqldb.SetConnMaxIdleTime(dbConfig.ConnectionMaxIdleTime) - - return &InstanceMetricsSQLDB{ - sqldb: sqldb, - logger: logger, - dbConfig: dbConfig, - }, nil -} - -func (idb *InstanceMetricsSQLDB) Close() error { - err := idb.sqldb.Close() - if err != nil { - idb.logger.Error("failed-close-instancemetrics-db", err, lager.Data{"dbConfig": idb.dbConfig}) - return err - } - return nil -} - -func (idb *InstanceMetricsSQLDB) SaveMetric(metric *models.AppInstanceMetric) error { - query := idb.sqldb.Rebind("INSERT INTO appinstancemetrics(appid, instanceindex, collectedat, name, unit, value, timestamp) values(?, ?, ?, ?, ?, ?, ?)") - _, err := idb.sqldb.Exec(query, metric.AppId, metric.InstanceIndex, metric.CollectedAt, metric.Name, metric.Unit, metric.Value, metric.Timestamp) - - if err != nil { - idb.logger.Error("failed-insert-instancemetric-into-appinstancemetrics-table", err, lager.Data{"query": query, "metric": metric}) - } - return err -} - -func (idb *InstanceMetricsSQLDB) SaveMetricsInBulk(metrics []*models.AppInstanceMetric) error { - if len(metrics) == 0 { - return nil - } - - ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second) - defer cancelFunc() - - txn, err := idb.sqldb.BeginTxx(ctx, nil) - if err != nil { - idb.logger.Error("failed-to-start-transaction", err) - return err - } - - sqlStr := "INSERT INTO appinstancemetrics(appid, instanceindex, collectedat, name, unit, value, timestamp) VALUES (:app_id, :instance_index, :collected_at, :name, :unit, :value, :timestamp)" - - _, err = txn.NamedExec(sqlStr, metrics) - if err != nil { - idb.logger.Error("failed-to-execute-statement", err) - _ = txn.Rollback() - return err - } - - err = txn.Commit() - if err != nil { - idb.logger.Error("failed-to-commit-transaction", err) - _ = txn.Rollback() - return err - } - - return nil -} - -func (idb *InstanceMetricsSQLDB) RetrieveInstanceMetrics(appid string, instanceIndex int, name string, start int64, end int64, orderType db.OrderType) ([]*models.AppInstanceMetric, error) { - var orderStr string - if orderType == db.ASC { - orderStr = db.ASCSTR - } else { - orderStr = db.DESCSTR - } - query := idb.sqldb.Rebind("SELECT instanceindex, collectedat, unit, value, timestamp FROM appinstancemetrics WHERE " + - " appid = ? " + - " AND name = ? " + - " AND timestamp >= ?" + - " AND timestamp <= ?" + - " ORDER BY timestamp " + orderStr + ", instanceindex") - - queryByInstanceIndex := idb.sqldb.Rebind("SELECT instanceindex, collectedat, unit, value, timestamp FROM appinstancemetrics WHERE " + - " appid = ? " + - " AND instanceindex = ?" + - " AND name = ? " + - " AND timestamp >= ?" + - " AND timestamp <= ?" + - " ORDER BY timestamp " + orderStr) - - if end < 0 { - end = time.Now().UnixNano() - } - var rows *sql.Rows - var err error - if instanceIndex >= 0 { - rows, err = idb.sqldb.Query(queryByInstanceIndex, appid, instanceIndex, name, start, end) - if err != nil { - idb.logger.Error("failed-retrieve-instancemetrics-from-appinstancemetrics-table", err, - lager.Data{"query": query, "appid": appid, "instanceindex": instanceIndex, "metricName": name, "start": start, "end": end, "orderType": orderType}) - return nil, err - } - } else { - rows, err = idb.sqldb.Query(query, appid, name, start, end) - if err != nil { - idb.logger.Error("failed-retrieve-instancemetrics-from-appinstancemetrics-table", err, - lager.Data{"query": query, "appid": appid, "metricName": name, "start": start, "end": end, "orderType": orderType}) - return nil, err - } - } - - defer func() { _ = rows.Close() }() - - mtrcs := []*models.AppInstanceMetric{} - var index uint32 - var collectedAt, timestamp int64 - var unit, value string - - for rows.Next() { - if err := rows.Scan(&index, &collectedAt, &unit, &value, ×tamp); err != nil { - idb.logger.Error("failed-scan-instancemetric-from-search-result", err) - return nil, err - } - - length := len(mtrcs) - if (length > 0) && (timestamp == mtrcs[length-1].Timestamp) && (index == mtrcs[length-1].InstanceIndex) { - continue - } - - metric := models.AppInstanceMetric{ - AppId: appid, - InstanceIndex: index, - CollectedAt: collectedAt, - Name: name, - Unit: unit, - Value: value, - Timestamp: timestamp, - } - mtrcs = append(mtrcs, &metric) - } - return mtrcs, rows.Err() -} -func (idb *InstanceMetricsSQLDB) PruneInstanceMetrics(ctx context.Context, before int64) error { - query := idb.sqldb.Rebind("DELETE FROM appinstancemetrics WHERE timestamp <= ?") - _, err := idb.sqldb.ExecContext(ctx, query, before) - if err != nil { - idb.logger.Error("failed-prune-instancemetric-from-appinstancemetrics-table", err, lager.Data{"query": query, "before": before}) - } - - return err -} -func (idb *InstanceMetricsSQLDB) GetDBStatus() sql.DBStats { - return idb.sqldb.Stats() -} diff --git a/src/autoscaler/db/sqldb/instancemetrics_sqldb_test.go b/src/autoscaler/db/sqldb/instancemetrics_sqldb_test.go deleted file mode 100644 index e334baa3cb..0000000000 --- a/src/autoscaler/db/sqldb/instancemetrics_sqldb_test.go +++ /dev/null @@ -1,697 +0,0 @@ -package sqldb_test - -import ( - "context" - "os" - "strings" - "sync" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db/sqldb" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - - "code.cloudfoundry.org/lager/v3" - "github.com/go-sql-driver/mysql" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/onsi/gomega/gstruct" -) - -var _ = Describe("InstancemetricsSqldb", func() { - var ( - dbConfig db.DatabaseConfig - dbHost = os.Getenv("DB_HOST") - idb *InstanceMetricsSQLDB - logger lager.Logger - err error - metric *models.AppInstanceMetric - mtrcs []*models.AppInstanceMetric - start int64 - end int64 - before int64 - orderType db.OrderType - - appId string - instanceIndex int - metricName string - testMetricName string - testMetricUnit string - ) - - dbUrl := testhelpers.GetDbUrl() - BeforeEach(func() { - logger = lager.NewLogger("instance-metrics-sqldb-test") - dbConfig = db.DatabaseConfig{ - URL: dbUrl, - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 5 * time.Second, - ConnectionMaxIdleTime: 10 * time.Millisecond, - } - instanceIndex = -1 - testMetricName = addProcessIdTo("TestMetricType") - appId = addProcessIdTo("Test-App-ID") - testMetricUnit = addProcessIdTo("TestMetricUnit") - idb, err = NewInstanceMetricsSQLDB(dbConfig, logger) - DeferCleanup(func() error { - if idb != nil { - return idb.Close() - } - return nil - }) - cleanInstanceMetricsTableForApp(appId) - DeferCleanup(func() { cleanInstanceMetricsTableForApp(appId) }) - }) - - Describe("NewInstanceMetricsSQLDB", func() { - JustBeforeEach(func() { - if idb != nil { - _ = idb.Close() - } - idb, err = NewInstanceMetricsSQLDB(dbConfig, logger) - }) - - Context("when db url is not correct", func() { - BeforeEach(func() { - if !strings.Contains(dbUrl, "postgres") { - Skip("Not configured for postgres") - } - dbConfig.URL = "postgres://not-exist-user:not-exist-password@localhost/autoscaler?sslmode=disable" - }) - It("should throw an error", func() { - Expect(err).To(HaveOccurred()) - }) - }) - - Context("when mysql db url is not correct", func() { - BeforeEach(func() { - if strings.Contains(dbUrl, "postgres") { - Skip("Mysql test") - } - dbConfig.URL = "not-exist-user:not-exist-password@tcp(" + dbHost + ")/autoscaler?tls=false" - }) - It("should throw an error", func() { - Expect(err).To(BeAssignableToTypeOf(&mysql.MySQLError{})) - }) - }) - - Context("when url is correct", func() { - It("should not error", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(idb).NotTo(BeNil()) - }) - }) - }) - - Describe("SaveMetric", func() { - BeforeEach(func() { - Expect(err).NotTo(HaveOccurred()) - }) - - Context("When inserting a metric", func() { - BeforeEach(func() { - metric = &models.AppInstanceMetric{ - AppId: appId, - InstanceIndex: 0, - CollectedAt: 111111, - Name: testMetricName, - Unit: testMetricUnit, - Value: "123", - Timestamp: 110000, - } - err = idb.SaveMetric(metric) - }) - - It("has the metric in database", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(hasInstanceMetric(appId, 0, testMetricName, 110000)).To(BeTrue()) - }) - }) - - Context("When inserting multiple metrics", func() { - BeforeEach(func() { - metric = &models.AppInstanceMetric{ - AppId: appId, - Name: testMetricName, - Unit: testMetricUnit, - } - metric.InstanceIndex = 0 - metric.CollectedAt = 111111 - metric.Value = "123" - metric.Timestamp = 111100 - err = idb.SaveMetric(metric) - Expect(err).NotTo(HaveOccurred()) - - metric.InstanceIndex = 1 - metric.CollectedAt = 111111 - metric.Value = "214365" - metric.Timestamp = 110000 - err = idb.SaveMetric(metric) - Expect(err).NotTo(HaveOccurred()) - - metric.InstanceIndex = 0 - metric.CollectedAt = 222222 - metric.Value = "654321" - metric.Timestamp = 220000 - err = idb.SaveMetric(metric) - Expect(err).NotTo(HaveOccurred()) - }) - - It("has all the metrics in database", func() { - Expect(hasInstanceMetric(appId, 0, testMetricName, 111100)).To(BeTrue()) - Expect(hasInstanceMetric(appId, 1, testMetricName, 110000)).To(BeTrue()) - Expect(hasInstanceMetric(appId, 0, testMetricName, 220000)).To(BeTrue()) - }) - }) - }) - - Describe("SaveMetricsInBulk", func() { - Context("When inserting an empty array of metrics", func() { - BeforeEach(func() { - metrics := []*models.AppInstanceMetric{} - err = idb.SaveMetricsInBulk(metrics) - }) - It("Should return nil", func() { - Expect(err).To(BeNil()) - }) - }) - - Context("When inserting an array of metrics", func() { - BeforeEach(func() { - metric1 := models.AppInstanceMetric{ - AppId: appId, - InstanceIndex: 0, - CollectedAt: 111111, - Name: testMetricName, - Unit: testMetricUnit, - Value: "123", - Timestamp: 110000, - } - metric2 := models.AppInstanceMetric{ - AppId: appId, - InstanceIndex: 1, - CollectedAt: 222222, - Name: testMetricName, - Unit: testMetricUnit, - Value: "234", - Timestamp: 220000, - } - err = idb.SaveMetricsInBulk([]*models.AppInstanceMetric{&metric1, &metric2}) - }) - - It("has the metrics in database", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(hasInstanceMetric(appId, 0, testMetricName, 110000)).To(BeTrue()) - Expect(hasInstanceMetric(appId, 1, testMetricName, 220000)).To(BeTrue()) - }) - }) - - Context("When there are errors in transaction", func() { - - BeforeEach(func() { - testMetricName = "This-is-a-too-long-meitrc-name-too-loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong" - metric1 := models.AppInstanceMetric{ - AppId: appId, - InstanceIndex: 0, - CollectedAt: 111111, - Name: testMetricName, - Unit: testMetricUnit, - Value: "123", - Timestamp: 110000, - } - metric2 := models.AppInstanceMetric{ - AppId: appId, - InstanceIndex: 1, - CollectedAt: 222222, - Name: testMetricName, - Unit: testMetricUnit, - Value: "234", - Timestamp: 220000, - } - wg := sync.WaitGroup{} - wg.Add(100) - for i := 0; i < 100; i++ { - go func() { - err := idb.SaveMetricsInBulk([]*models.AppInstanceMetric{&metric1, &metric2}) - Expect(err).To(HaveOccurred()) - defer wg.Done() - }() - } - wg.Wait() - }) - - It("all connections should be released after transactions' rolling back", func() { - - Eventually(func() int { return idb.GetDBStatus().OpenConnections }, 120*time.Second, 10*time.Millisecond).Should(BeZero()) - }) - }) - }) - - Describe("RetrieveInstanceMetrics", func() { - BeforeEach(func() { - metric = &models.AppInstanceMetric{ - AppId: appId, - Name: testMetricName, - Unit: testMetricUnit, - } - - metric.InstanceIndex = 0 - metric.CollectedAt = 111111 - metric.Value = "654321" - metric.Timestamp = 111100 - err = idb.SaveMetric(metric) - Expect(err).NotTo(HaveOccurred()) - - metric.InstanceIndex = 1 - metric.CollectedAt = 111111 - metric.Value = "214365" - metric.Timestamp = 110000 - err = idb.SaveMetric(metric) - Expect(err).NotTo(HaveOccurred()) - - metric.InstanceIndex = 1 - metric.CollectedAt = 222222 - metric.Value = "321765" - metric.Timestamp = 222200 - err = idb.SaveMetric(metric) - Expect(err).NotTo(HaveOccurred()) - - metric.InstanceIndex = 0 - metric.CollectedAt = 333333 - metric.Value = "879654" - metric.Timestamp = 110000 - err = idb.SaveMetric(metric) - Expect(err).NotTo(HaveOccurred()) - - start = 0 - end = -1 - metricName = testMetricName - orderType = db.DESC - - }) - - JustBeforeEach(func() { - mtrcs, err = idb.RetrieveInstanceMetrics(appId, instanceIndex, metricName, start, end, orderType) - }) - - Context("The app has no instance metrics", func() { - BeforeEach(func() { - appId = "app-id-no-metric" - }) - - It("returns empty metrics", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(mtrcs).To(BeEmpty()) - }) - - }) - - Context("The app has no instance metrics for a given instance index", func() { - BeforeEach(func() { - instanceIndex = 2 - }) - - It("returns empty metrics", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(mtrcs).To(BeEmpty()) - }) - - }) - - Context("when the app has no instance metrics with the given metric name", func() { - BeforeEach(func() { - metricName = "metric-name-no-metrics" - }) - - It("returns empty metrics", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(mtrcs).To(BeEmpty()) - }) - }) - - Context("when retrieving all the metrics of all the instances when start = 0, end = -1 and orderType = ASC ", func() { - BeforeEach(func() { - orderType = db.ASC - instanceIndex = -1 - }) - It("returns all the instance metrics of the app ordered by timestamp asc, instanceindex asc", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(mtrcs).To(HaveLen(4)) - Expect(*mtrcs[0]).To(gstruct.MatchAllFields(gstruct.Fields{ - "AppId": Equal(appId), - "InstanceIndex": BeEquivalentTo(0), - "CollectedAt": BeEquivalentTo(333333), - "Name": Equal(testMetricName), - "Unit": Equal(testMetricUnit), - "Value": Equal("879654"), - "Timestamp": BeEquivalentTo(110000), - })) - - Expect(*mtrcs[1]).To(gstruct.MatchAllFields(gstruct.Fields{ - "AppId": Equal(appId), - "InstanceIndex": BeEquivalentTo(1), - "CollectedAt": BeEquivalentTo(111111), - "Name": Equal(testMetricName), - "Unit": Equal(testMetricUnit), - "Value": Equal("214365"), - "Timestamp": BeEquivalentTo(110000), - })) - - Expect(*mtrcs[2]).To(gstruct.MatchAllFields(gstruct.Fields{ - "AppId": Equal(appId), - "InstanceIndex": BeEquivalentTo(0), - "CollectedAt": BeNumerically(">=", 111111), - "Name": Equal(testMetricName), - "Unit": Equal(testMetricUnit), - "Value": Equal("654321"), - "Timestamp": BeEquivalentTo(111100), - })) - - Expect(*mtrcs[3]).To(gstruct.MatchAllFields(gstruct.Fields{ - "AppId": Equal(appId), - "InstanceIndex": BeEquivalentTo(1), - "CollectedAt": BeEquivalentTo(222222), - "Name": Equal(testMetricName), - "Unit": Equal(testMetricUnit), - "Value": Equal("321765"), - "Timestamp": BeEquivalentTo(222200), - })) - }) - }) - - Context("when retrieving all the metrics of all the instances when start = 0, end = -1 and orderType = DESC ", func() { - BeforeEach(func() { - orderType = db.DESC - instanceIndex = -1 - }) - It("returns all the instance metrics of the app ordered by timestamp desc, instanceindex asc", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(mtrcs).To(HaveLen(4)) - - Expect(*mtrcs[3]).To(gstruct.MatchAllFields(gstruct.Fields{ - "AppId": Equal(appId), - "InstanceIndex": BeEquivalentTo(1), - "CollectedAt": BeEquivalentTo(111111), - "Name": Equal(testMetricName), - "Unit": Equal(testMetricUnit), - "Value": Equal("214365"), - "Timestamp": BeEquivalentTo(110000), - })) - - Expect(*mtrcs[2]).To(gstruct.MatchAllFields(gstruct.Fields{ - "AppId": Equal(appId), - "InstanceIndex": BeEquivalentTo(0), - "CollectedAt": BeEquivalentTo(333333), - "Name": Equal(testMetricName), - "Unit": Equal(testMetricUnit), - "Value": Equal("879654"), - "Timestamp": BeEquivalentTo(110000), - })) - - Expect(*mtrcs[1]).To(gstruct.MatchAllFields(gstruct.Fields{ - "AppId": Equal(appId), - "InstanceIndex": BeEquivalentTo(0), - "CollectedAt": BeNumerically(">=", 111111), - "Name": Equal(testMetricName), - "Unit": Equal(testMetricUnit), - "Value": Equal("654321"), - "Timestamp": BeEquivalentTo(111100), - })) - - Expect(*mtrcs[0]).To(gstruct.MatchAllFields(gstruct.Fields{ - "AppId": Equal(appId), - "InstanceIndex": BeEquivalentTo(1), - "CollectedAt": BeEquivalentTo(222222), - "Name": Equal(testMetricName), - "Unit": Equal(testMetricUnit), - "Value": Equal("321765"), - "Timestamp": BeEquivalentTo(222200), - })) - - }) - }) - - Context("when retrieving all the metrics of a given instance index when start = 0, end = -1 and orderType = ASC ", func() { - BeforeEach(func() { - orderType = db.ASC - instanceIndex = 1 - }) - It("removes duplicates and returns all the instance metrics of the app ordered by timestamp asc", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(mtrcs).To(HaveLen(2)) - Expect(*mtrcs[0]).To(gstruct.MatchAllFields(gstruct.Fields{ - "AppId": Equal(appId), - "InstanceIndex": BeEquivalentTo(1), - "CollectedAt": BeEquivalentTo(111111), - "Name": Equal(testMetricName), - "Unit": Equal(testMetricUnit), - "Value": Equal("214365"), - "Timestamp": BeEquivalentTo(110000), - })) - - Expect(*mtrcs[1]).To(gstruct.MatchAllFields(gstruct.Fields{ - "AppId": Equal(appId), - "InstanceIndex": BeEquivalentTo(1), - "CollectedAt": BeEquivalentTo(222222), - "Name": Equal(testMetricName), - "Unit": Equal(testMetricUnit), - "Value": Equal("321765"), - "Timestamp": BeEquivalentTo(222200), - })) - }) - }) - - Context("when retrieving all the metrics of a given instance index when start = 0, end = -1 and orderType = DESC ", func() { - BeforeEach(func() { - instanceIndex = 1 - }) - It("returns all the instance metrics of the app ordered by timestamp desc", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(mtrcs).To(HaveLen(2)) - Expect(*mtrcs[1]).To(gstruct.MatchAllFields(gstruct.Fields{ - "AppId": Equal(appId), - "InstanceIndex": BeEquivalentTo(1), - "CollectedAt": BeEquivalentTo(111111), - "Name": Equal(testMetricName), - "Unit": Equal(testMetricUnit), - "Value": Equal("214365"), - "Timestamp": BeEquivalentTo(110000), - })) - - Expect(*mtrcs[0]).To(gstruct.MatchAllFields(gstruct.Fields{ - "AppId": Equal(appId), - "InstanceIndex": BeEquivalentTo(1), - "CollectedAt": BeEquivalentTo(222222), - "Name": Equal(testMetricName), - "Unit": Equal(testMetricUnit), - "Value": Equal("321765"), - "Timestamp": BeEquivalentTo(222200), - })) - }) - }) - - Context("when end time = -1)", func() { - BeforeEach(func() { - start = 111111 - end = -1 - }) - - It("returns metrics from start time to now", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(mtrcs).To(HaveLen(1)) - }) - - }) - - Context("when end time is before all the metrics timestamps", func() { - BeforeEach(func() { - start = 22222 - end = 100000 - }) - - It("returns empty metrics", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(mtrcs).To(BeEmpty()) - }) - - }) - - Context("when start time is after all the metrics timestamps", func() { - BeforeEach(func() { - start = 222222 - end = 888888 - }) - - It("returns empty metrics", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(mtrcs).To(BeEmpty()) - }) - - }) - - Context("when start time > end time", func() { - BeforeEach(func() { - start = 200000 - end = 111111 - }) - - It("returns empty metrics", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(mtrcs).To(BeEmpty()) - }) - }) - - Context("when retrieving part of all the instances's metrics", func() { - BeforeEach(func() { - start = 111100 - end = 222222 - }) - - It("returns correct metrics", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(mtrcs).To(HaveLen(2)) - Expect(*mtrcs[1]).To(gstruct.MatchAllFields(gstruct.Fields{ - "AppId": Equal(appId), - "InstanceIndex": BeEquivalentTo(0), - "CollectedAt": BeNumerically(">=", 111111), - "Name": Equal(testMetricName), - "Unit": Equal(testMetricUnit), - "Value": Equal("654321"), - "Timestamp": BeEquivalentTo(111100), - })) - Expect(*mtrcs[0]).To(gstruct.MatchAllFields(gstruct.Fields{ - "AppId": Equal(appId), - "InstanceIndex": BeEquivalentTo(1), - "CollectedAt": BeEquivalentTo(222222), - "Name": Equal(testMetricName), - "Unit": Equal(testMetricUnit), - "Value": Equal("321765"), - "Timestamp": BeEquivalentTo(222200), - })) - }) - }) - - Context("when retrieving part of a given instance's metrics", func() { - BeforeEach(func() { - start = 111100 - end = 222222 - instanceIndex = 1 - }) - - It("returns correct metrics", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(mtrcs).To(HaveLen(1)) - Expect(*mtrcs[0]).To(gstruct.MatchAllFields(gstruct.Fields{ - "AppId": Equal(appId), - "InstanceIndex": BeEquivalentTo(1), - "CollectedAt": BeEquivalentTo(222222), - "Name": Equal(testMetricName), - "Unit": Equal(testMetricUnit), - "Value": Equal("321765"), - "Timestamp": BeEquivalentTo(222200), - })) - }) - }) - - Context("when db fails", func() { - BeforeEach(func() { - idb.Close() - }) - - It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("sql: .*"))) - }) - - }) - }) - - Describe("PruneMetrics", Serial, func() { - BeforeEach(func() { - metric = &models.AppInstanceMetric{ - AppId: appId, - Name: testMetricName, - Unit: testMetricUnit, - } - - metric.InstanceIndex = 0 - metric.CollectedAt = 111111 - metric.Value = "654321" - metric.Timestamp = 111100 - err = idb.SaveMetric(metric) - Expect(err).NotTo(HaveOccurred()) - - metric.InstanceIndex = 1 - metric.CollectedAt = 111111 - metric.Value = "214365" - metric.Timestamp = 110000 - err = idb.SaveMetric(metric) - Expect(err).NotTo(HaveOccurred()) - - metric.InstanceIndex = 1 - metric.CollectedAt = 222222 - metric.Value = "321765" - metric.Timestamp = 222200 - err = idb.SaveMetric(metric) - Expect(err).NotTo(HaveOccurred()) - - metric.InstanceIndex = 0 - metric.CollectedAt = 222222 - metric.Value = "654321" - metric.Timestamp = 111110 - err = idb.SaveMetric(metric) - Expect(err).NotTo(HaveOccurred()) - }) - - JustBeforeEach(func() { - err = idb.PruneInstanceMetrics(context.TODO(), before) - }) - - Context("when pruning metrics before all the timestamps of metrics", func() { - BeforeEach(func() { - before = 1000 - }) - - It("does not remove any metrics", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(getNumberOfInstanceMetricsForApp(appId)).To(Equal(4)) - }) - }) - - Context("when pruning all the metrics", func() { - BeforeEach(func() { - before = time.Now().UnixNano() - }) - - It("empties the metrics table", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(getNumberOfInstanceMetricsForApp(appId)).To(Equal(0)) - }) - }) - - Context("when pruning part of the metrics", func() { - BeforeEach(func() { - before = 111000 - }) - - It("removes metrics before the time specified", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(getNumberOfInstanceMetricsForApp(appId)).To(Equal(3)) - }) - }) - - Context("when db fails", func() { - BeforeEach(func() { - _ = idb.Close() - }) - - It("should error", func() { - Expect(err).To(MatchError(MatchRegexp("sql: .*"))) - }) - - }) - - }) -}) diff --git a/src/autoscaler/db/sqldb/sqldb_suite_test.go b/src/autoscaler/db/sqldb/sqldb_suite_test.go index 9a118ecf5e..b7facb5eca 100644 --- a/src/autoscaler/db/sqldb/sqldb_suite_test.go +++ b/src/autoscaler/db/sqldb/sqldb_suite_test.go @@ -87,37 +87,13 @@ var _ = SynchronizedAfterSuite(func() { } }) -func cleanInstanceMetricsTableForApp(appId string) { - _, e := dbHelper.Exec(dbHelper.Rebind("DELETE FROM appinstancemetrics where appid=?"), appId) - FailOnError("can not clean table appinstancemetrics", e) -} - -func hasInstanceMetric(appId string, index int, name string, timestamp int64) bool { - query := dbHelper.Rebind("SELECT * FROM appinstancemetrics WHERE appid = ? AND instanceindex = ? AND name = ? AND timestamp = ?") - rows, e := dbHelper.Query(query, appId, index, name, timestamp) - FailOnError("can not query table appinstancemetrics", e) - defer func() { _ = rows.Close() }() - item := rows.Next() - FailOnError("can not query table appinstancemetrics", rows.Err()) - return item -} - -func getNumberOfInstanceMetricsForApp(appId string) int { - var num int - e := dbHelper.QueryRow(dbHelper.Rebind("SELECT COUNT(*) FROM appinstancemetrics where appid = ?"), appId).Scan(&num) - if e != nil { - Fail("can not count the number of records in table appinstancemetrics: " + e.Error()) - } - return num -} - func hasServiceInstance(serviceInstanceId string) bool { query := dbHelper.Rebind("SELECT * FROM service_instance WHERE service_instance_id = ?") rows, e := dbHelper.Query(query, serviceInstanceId) FailOnError("can not query table service_instance", e) defer func() { _ = rows.Close() }() item := rows.Next() - FailOnError("can not query table appinstancemetrics", rows.Err()) + FailOnError("can not query table service_instance", rows.Err()) return item } @@ -127,7 +103,7 @@ func hasServiceInstanceWithNullDefaultPolicy(serviceInstanceId string) bool { FailOnError("can not query table service_instance", e) defer func() { _ = rows.Close() }() item := rows.Next() - FailOnError("can not query table appinstancemetrics", rows.Err()) + FailOnError("can not query table service_instance", rows.Err()) return item } diff --git a/src/autoscaler/eventgenerator/aggregator/metric_poller.go b/src/autoscaler/eventgenerator/aggregator/metric_poller.go index cf5970b7ae..b1f5140dc1 100644 --- a/src/autoscaler/eventgenerator/aggregator/metric_poller.go +++ b/src/autoscaler/eventgenerator/aggregator/metric_poller.go @@ -63,10 +63,10 @@ func (m *MetricPoller) retrieveMetric(appMonitor *models.AppMonitor) error { startTime := endTime.Add(0 - statWindow) metrics, err := m.metricClient.GetMetrics(appId, metricType, startTime, endTime) - m.logger.Debug("received metrics from metricClient", lager.Data{"retrievedMetrics": metrics}) if err != nil { return fmt.Errorf("retrieveMetric Failed: %w", err) } + m.logger.Debug("received metrics from metricClient", lager.Data{"retrievedMetrics": metrics}) avgMetric := m.aggregate(appId, metricType, metrics) m.logger.Debug("save-aggregated-appmetric", lager.Data{"appMetric": avgMetric}) m.appMetricChan <- avgMetric diff --git a/src/autoscaler/generate-fakes.go b/src/autoscaler/generate-fakes.go index d34207eec6..d6f7f4f4fa 100644 --- a/src/autoscaler/generate-fakes.go +++ b/src/autoscaler/generate-fakes.go @@ -1,7 +1,6 @@ package fakes // Multiple go:generate directives instead of counterfeiter:generate due to https://github.com/maxbrunsfeld/counterfeiter/issues/254 -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_wshelper.go ./metricsgateway/helpers WSHelper //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_cf_client.go ./cf CFClient //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_cf_context_client.go ./cf ContextClient //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_policy_db.go ./db PolicyDB @@ -26,6 +25,5 @@ package fakes //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_ratelimiter.go ./ratelimiter Limiter //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_httpstatus_collector.go ./healthendpoint HTTPStatusCollector //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_database_status.go ./healthendpoint DatabaseStatus -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_emitter.go ./metricsgateway Emitter //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_operator.go ./operator Operator //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_sychronizer.go ./scalingengine/schedule ActiveScheduleSychronizer diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 37cb412f17..bb301da425 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -17,7 +17,6 @@ require ( github.com/go-faster/jx v1.1.0 github.com/go-logr/logr v1.4.2 github.com/go-sql-driver/mysql v1.8.1 - github.com/golang/protobuf v1.5.4 github.com/gorilla/mux v1.8.1 github.com/gorilla/websocket v1.5.1 github.com/hashicorp/go-retryablehttp v0.7.7 diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 32667f28bf..5ea19cd648 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -776,8 +776,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= diff --git a/src/autoscaler/healthendpoint/counter_collector.go b/src/autoscaler/healthendpoint/counter_collector.go deleted file mode 100644 index 46e86c82c6..0000000000 --- a/src/autoscaler/healthendpoint/counter_collector.go +++ /dev/null @@ -1,64 +0,0 @@ -package healthendpoint - -import ( - "sync" - - "github.com/prometheus/client_golang/prometheus" -) - -type CounterCollector interface { - prometheus.Collector - AddCounters(counterOps ...prometheus.CounterOpts) - Add(counterOps prometheus.CounterOpts, count int64) -} - -func NewCounterCollector() CounterCollector { - return &counterCollector{ - counterMap: map[string]prometheus.Counter{}, - } -} - -type counterCollector struct { - counterMap map[string]prometheus.Counter - sync.RWMutex -} - -func (c *counterCollector) AddCounters(counterOps ...prometheus.CounterOpts) { - c.Lock() - defer c.Unlock() - for _, ops := range counterOps { - counterFullName := getCounterFullName(ops) - if _, exists := c.counterMap[counterFullName]; exists { - continue - } - c.counterMap[counterFullName] = prometheus.NewCounter(ops) - } -} - -func (c *counterCollector) Describe(ch chan<- *prometheus.Desc) { - c.RLock() - defer c.RUnlock() - for _, counter := range c.counterMap { - ch <- counter.Desc() - } -} - -func (c *counterCollector) Collect(ch chan<- prometheus.Metric) { - c.RLock() - defer c.RUnlock() - for _, counter := range c.counterMap { - ch <- counter - } -} - -func (c *counterCollector) Add(counterOps prometheus.CounterOpts, count int64) { - c.RLock() - defer c.RUnlock() - if counter, exists := c.counterMap[getCounterFullName(counterOps)]; exists { - counter.Add(float64(count)) - } -} - -func getCounterFullName(counterOps prometheus.CounterOpts) string { - return counterOps.Namespace + "_" + counterOps.Subsystem + "_" + counterOps.Name -} diff --git a/src/autoscaler/healthendpoint/counter_collector_test.go b/src/autoscaler/healthendpoint/counter_collector_test.go deleted file mode 100644 index 2a7078f496..0000000000 --- a/src/autoscaler/healthendpoint/counter_collector_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package healthendpoint_test - -import ( - "strings" - - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/healthendpoint" - - "github.com/prometheus/client_golang/prometheus/testutil" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/prometheus/client_golang/prometheus" -) - -var _ = Describe("CounterCollector", func() { - var ( - namespace1 = "test_name_space1" - subSystem1 = "test_sub_system1" - name1 = "test_name1" - help1 = "test_help1" - - namespace2 = "test_name_space2" - subSystem2 = "test_sub_system2" - name2 = "test_name2" - help2 = "test_help2" - - counterOpt1 = prometheus.CounterOpts{ - Namespace: namespace1, - Subsystem: subSystem1, - Name: name1, - Help: help1, - } - counterOpt2 = prometheus.CounterOpts{ - Namespace: namespace2, - Subsystem: subSystem2, - Name: name2, - Help: help2, - } - - descChan chan *prometheus.Desc - metricChan chan prometheus.Metric - counterCollector CounterCollector - counterDesc1 = prometheus.NewDesc( - prometheus.BuildFQName(namespace1, subSystem1, name1), - help1, - nil, - nil, - ) - counterDesc2 = prometheus.NewDesc( - prometheus.BuildFQName(namespace2, subSystem2, name2), - help2, - nil, - nil, - ) - counterMetric1 = prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: namespace1, - Subsystem: subSystem1, - Name: name1, - Help: help1, - }) - counterMetric2 = prometheus.NewCounter( - prometheus.CounterOpts{ - Namespace: namespace2, - Subsystem: subSystem2, - Name: name2, - Help: help2, - }) - ) - BeforeEach(func() { - descChan = make(chan *prometheus.Desc, 10) - metricChan = make(chan prometheus.Metric, 10) - counterCollector = NewCounterCollector() - counterCollector.AddCounters(counterOpt1, counterOpt2) - }) - Context("Describe", func() { - BeforeEach(func() { - counterCollector.Describe(descChan) - }) - It("receive descriptions", func() { - var desc1, desc2 *prometheus.Desc - Expect(descChan).To(Receive(&desc1)) - Expect(descChan).To(Receive(&desc2)) - Expect([]prometheus.Desc{*desc1, *desc2}).To(ContainElement(*counterDesc1)) - Expect([]prometheus.Desc{*desc1, *desc2}).To(ContainElement(*counterDesc2)) - }) - }) - Context("Collect", func() { - BeforeEach(func() { - counterCollector.Add(counterOpt1, 10) - counterCollector.Add(counterOpt2, 100) - counterCollector.Collect(metricChan) - counterMetric1.Add(float64(10)) - counterMetric2.Add(float64(100)) - }) - It("Receive metrics", func() { - numberReceived := testutil.CollectAndCount(counterCollector, "test_name_space1_test_sub_system1_test_name1", "test_name_space2_test_sub_system2_test_name2") - Expect(numberReceived).Should(Equal(2)) - - expected := ` - # HELP test_name_space1_test_sub_system1_test_name1 test_help1 - # TYPE test_name_space1_test_sub_system1_test_name1 counter - test_name_space1_test_sub_system1_test_name1 10 - # HELP test_name_space2_test_sub_system2_test_name2 test_help2 - # TYPE test_name_space2_test_sub_system2_test_name2 counter - test_name_space2_test_sub_system2_test_name2 100 -` - err := testutil.CollectAndCompare(counterCollector, strings.NewReader(expected)) - Expect(err).NotTo(HaveOccurred()) - }) - }) - -}) diff --git a/src/autoscaler/integration/components_test.go b/src/autoscaler/integration/components_test.go index bce423519d..5ff75f2a1f 100644 --- a/src/autoscaler/integration/components_test.go +++ b/src/autoscaler/integration/components_test.go @@ -4,25 +4,23 @@ import ( _ "embed" "text/template" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + apiConfig "code.cloudfoundry.org/app-autoscaler/src/autoscaler/api/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cf" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" egConfig "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - mgConfig "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsgateway/config" - msConfig "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsserver/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" opConfig "code.cloudfoundry.org/app-autoscaler/src/autoscaler/operator/config" seConfig "code.cloudfoundry.org/app-autoscaler/src/autoscaler/scalingengine/config" - "fmt" - "net/url" - "os" - "os/exec" - "path/filepath" - "strings" - "time" - "github.com/go-sql-driver/mysql" . "github.com/onsi/gomega" "github.com/tedsuo/ifrit/ginkgomon_v2" @@ -37,9 +35,6 @@ const ( EventGenerator = "eventGenerator" ScalingEngine = "scalingEngine" Operator = "operator" - MetricsGateway = "metricsGateway" - MetricsServerHTTP = "metricsServerHTTP" - MetricsServerWS = "metricsServerWS" ) var golangAPIInfoFilePath = "../api/exampleconfig/catalog-example.json" @@ -119,22 +114,6 @@ func (components *Components) Scheduler(confPath string, argv ...string) *ginkgo }) } -func (components *Components) MetricsServer(confPath string, argv ...string) *ginkgomon_v2.Runner { - return ginkgomon_v2.New(ginkgomon_v2.Config{ - Name: MetricsServerHTTP, - AnsiColorCode: "33m", - StartCheck: `"metricsserver.started"`, - StartCheckTimeout: 20 * time.Second, - // #nosec G204 - Command: exec.Command( - components.Executables[MetricsServerHTTP], - append([]string{ - "-c", confPath, - }, argv...)..., - ), - }) -} - func (components *Components) EventGenerator(confPath string, argv ...string) *ginkgomon_v2.Runner { return ginkgomon_v2.New(ginkgomon_v2.Config{ Name: EventGenerator, @@ -183,23 +162,7 @@ func (components *Components) Operator(confPath string, argv ...string) *ginkgom }) } -func (components *Components) MetricsGateway(confPath string, argv ...string) *ginkgomon_v2.Runner { - return ginkgomon_v2.New(ginkgomon_v2.Config{ - Name: MetricsGateway, - AnsiColorCode: "32m", - StartCheck: `"metricsgateway.started"`, - StartCheckTimeout: 20 * time.Second, - // #nosec G204 - Command: exec.Command( - components.Executables[MetricsGateway], - append([]string{ - "-c", confPath, - }, argv...)..., - ), - }) -} - -func (components *Components) PrepareGolangApiServerConfig(dbURI string, publicApiPort int, brokerPort int, cfApi string, schedulerUri string, scalingEngineUri string, metricsCollectorUri string, eventGeneratorUri string, metricsForwarderUri string, tmpDir string) string { +func (components *Components) PrepareGolangApiServerConfig(dbURI string, publicApiPort int, brokerPort int, cfApi string, schedulerUri string, scalingEngineUri string, eventGeneratorUri string, metricsForwarderUri string, tmpDir string) string { brokerCred1 := apiConfig.BrokerCredentialsConfig{ BrokerUsername: "broker_username", //BrokerUsernameHash: []byte("$2a$10$WNO1cPko4iDAT6MkhaDojeJMU8ZdNH6gt.SapsFOsC0OF4cQ9qQwu"), // ruby -r bcrypt -e 'puts BCrypt::Password.create("broker_username")' @@ -405,6 +368,7 @@ func (components *Components) PrepareEventGeneratorConfig(dbUri string, port int }, }, MetricCollector: egConfig.MetricCollectorConfig{ + UseLogCache: true, MetricCollectorURL: metricsCollectorURL, TLSClientCerts: models.TLSCerts{ KeyFile: filepath.Join(testCertDir, "eventgenerator.key"), @@ -466,13 +430,6 @@ func (components *Components) PrepareOperatorConfig(dbURI string, ccUAAURL strin ClientID: "admin", Secret: "admin", }, - InstanceMetricsDB: opConfig.DbPrunerConfig{ - RefreshInterval: 2 * time.Minute, - CutoffDuration: cutoffDuration, - DB: db.DatabaseConfig{ - URL: dbURI, - }, - }, AppMetricsDB: opConfig.DbPrunerConfig{ RefreshInterval: 2 * time.Minute, CutoffDuration: cutoffDuration, @@ -523,100 +480,6 @@ func (components *Components) PrepareOperatorConfig(dbURI string, ccUAAURL strin return writeYmlConfig(tmpDir, Operator, &conf) } -func (components *Components) PrepareMetricsGatewayConfig(dbURI string, metricServerAddresses []string, rlpAddr string, tmpDir string) string { - cfg := mgConfig.Config{ - Logging: helpers.LoggingConfig{ - Level: LOGLEVEL, - }, - EnvelopChanSize: 500, - NozzleCount: 1, - MetricServerAddrs: metricServerAddresses, - AppManager: mgConfig.AppManagerConfig{ - AppRefreshInterval: 10 * time.Second, - PolicyDB: db.DatabaseConfig{ - URL: dbURI, - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - }, - }, - Emitter: mgConfig.EmitterConfig{ - BufferSize: 500, - KeepAliveInterval: 1 * time.Second, - HandshakeTimeout: 1 * time.Second, - MaxSetupRetryCount: 3, - MaxCloseRetryCount: 3, - RetryDelay: 500 * time.Millisecond, - MetricsServerClientTLS: &models.TLSCerts{ - KeyFile: filepath.Join(testCertDir, "metricserver_client.key"), - CertFile: filepath.Join(testCertDir, "metricserver_client.crt"), - CACertFile: filepath.Join(testCertDir, "autoscaler-ca.crt"), - }, - }, - Nozzle: mgConfig.NozzleConfig{ - RLPAddr: rlpAddr, - ShardID: "autoscaler", - RLPClientTLS: &models.TLSCerts{ - KeyFile: filepath.Join(testCertDir, "reverselogproxy_client.key"), - CertFile: filepath.Join(testCertDir, "reverselogproxy_client.crt"), - CACertFile: filepath.Join(testCertDir, "autoscaler-ca.crt"), - }, - }, - } - return writeYmlConfig(tmpDir, MetricsGateway, &cfg) -} - -func (components *Components) PrepareMetricsServerConfig(dbURI string, httpClientTimeout time.Duration, httpServerPort int, wsServerPort int, tmpDir string) string { - cfg := msConfig.Config{ - Logging: helpers.LoggingConfig{ - Level: LOGLEVEL, - }, - HttpClientTimeout: httpClientTimeout, - NodeAddrs: []string{"localhost"}, - NodeIndex: 0, - DB: msConfig.DBConfig{ - PolicyDB: db.DatabaseConfig{ - URL: dbURI, - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - }, - InstanceMetricsDB: db.DatabaseConfig{ - URL: dbURI, - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - }, - }, - Collector: msConfig.CollectorConfig{ - WSPort: wsServerPort, - WSKeepAliveTime: 5 * time.Second, - TLS: models.TLSCerts{ - KeyFile: filepath.Join(testCertDir, "metricserver.key"), - CertFile: filepath.Join(testCertDir, "metricserver.crt"), - CACertFile: filepath.Join(testCertDir, "autoscaler-ca.crt"), - }, - RefreshInterval: 5 * time.Second, - CollectInterval: 1 * time.Second, - SaveInterval: 2 * time.Second, - MetricCacheSizePerApp: 100, - PersistMetrics: true, - EnvelopeProcessorCount: 2, - EnvelopeChannelSize: 100, - MetricChannelSize: 100, - }, - Server: helpers.ServerConfig{ - Port: httpServerPort, - TLS: models.TLSCerts{ - KeyFile: filepath.Join(testCertDir, "metricserver.key"), - CertFile: filepath.Join(testCertDir, "metricserver.crt"), - CACertFile: filepath.Join(testCertDir, "autoscaler-ca.crt"), - }, - }, - } - return writeYmlConfig(tmpDir, MetricsServerHTTP, &cfg) -} - func writeYmlConfig(dir string, componentName string, c interface{}) string { cfgFile, err := os.CreateTemp(dir, componentName) Expect(err).NotTo(HaveOccurred()) diff --git a/src/autoscaler/integration/helpers_test.go b/src/autoscaler/integration/helpers_test.go index c98a67f34e..d2a88d2174 100644 --- a/src/autoscaler/integration/helpers_test.go +++ b/src/autoscaler/integration/helpers_test.go @@ -15,15 +15,6 @@ import ( . "github.com/onsi/gomega" ) -type AppInstanceMetricResult struct { - TotalResults int `json:"total_results"` - TotalPages int `json:"total_pages"` - Page int `json:"page"` - PrevUrl string `json:"prev_url"` - NextUrl string `json:"next_url"` - Resources []models.AppInstanceMetric `json:"resources"` -} - type AppAggregatedMetricResult struct { TotalResults int `json:"total_results"` TotalPages int `json:"total_pages"` diff --git a/src/autoscaler/integration/integration_golangapi_eventgenerator_test.go b/src/autoscaler/integration/integration_golangapi_eventgenerator_test.go index 12597a206a..8c3af3a3ba 100644 --- a/src/autoscaler/integration/integration_golangapi_eventgenerator_test.go +++ b/src/autoscaler/integration/integration_golangapi_eventgenerator_test.go @@ -34,7 +34,6 @@ var _ = Describe("Integration_GolangApi_EventGenerator", func() { fakeCCNOAAUAA.URL(), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[Scheduler]), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[ScalingEngine]), - fmt.Sprintf("https://127.0.0.1:%d", components.Ports[MetricsCollector]), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[EventGenerator]), "https://127.0.0.1:8888", tmpDir) diff --git a/src/autoscaler/integration/integration_golangapi_scalingengine_test.go b/src/autoscaler/integration/integration_golangapi_scalingengine_test.go index 0550eccbe7..ac81660419 100644 --- a/src/autoscaler/integration/integration_golangapi_scalingengine_test.go +++ b/src/autoscaler/integration/integration_golangapi_scalingengine_test.go @@ -32,7 +32,6 @@ var _ = Describe("Integration_GolangApi_ScalingEngine", func() { fakeCCNOAAUAA.URL(), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[Scheduler]), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[ScalingEngine]), - fmt.Sprintf("https://127.0.0.1:%d", components.Ports[MetricsCollector]), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[EventGenerator]), "https://127.0.0.1:8888", tmpDir) diff --git a/src/autoscaler/integration/integration_golangapi_scheduler_test.go b/src/autoscaler/integration/integration_golangapi_scheduler_test.go index a0c46a4450..b54b46e088 100644 --- a/src/autoscaler/integration/integration_golangapi_scheduler_test.go +++ b/src/autoscaler/integration/integration_golangapi_scheduler_test.go @@ -60,7 +60,6 @@ var _ = Describe("Integration_GolangApi_Scheduler", func() { fakeCCNOAAUAA.URL(), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[Scheduler]), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[ScalingEngine]), - fmt.Sprintf("https://127.0.0.1:%d", components.Ports[MetricsCollector]), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[EventGenerator]), "https://127.0.0.1:8888", tmpDir) diff --git a/src/autoscaler/integration/integration_metricsgateway_metricserver_eventgenerator_scalingengine_test.go b/src/autoscaler/integration/integration_logcache_eventgenerator_scalingengine_test.go similarity index 86% rename from src/autoscaler/integration/integration_metricsgateway_metricserver_eventgenerator_scalingengine_test.go rename to src/autoscaler/integration/integration_logcache_eventgenerator_scalingengine_test.go index 8318bdd106..93c2117cf7 100644 --- a/src/autoscaler/integration/integration_metricsgateway_metricserver_eventgenerator_scalingengine_test.go +++ b/src/autoscaler/integration/integration_logcache_eventgenerator_scalingengine_test.go @@ -6,52 +6,40 @@ import ( "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - as_testhelpers "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" - + rpc "code.cloudfoundry.org/go-log-cache/v2/rpc/logcache_v1" "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) -var _ = Describe("Integration_Metricsgateway_Metricserver_Eventgenerator_Scalingengine", func() { +var _ = Describe("Integration_Eventgenerator_Scalingengine", func() { var ( - testAppId string - timeout = 2 * time.Duration(breachDurationSecs) * time.Second - initInstanceCount = 2 - fakeRLPServer *as_testhelpers.FakeEventProducer - envelopes []*loggregator_v2.Envelope - fakeRLPEmitInterval = 500 * time.Millisecond + testAppId string + timeout = 2 * time.Duration(breachDurationSecs) * time.Second + initInstanceCount = 2 ) BeforeEach(func() { testAppId = getRandomIdRef("testAppId") startFakeCCNOAAUAA(initInstanceCount) + startMockLogCache() }) JustBeforeEach(func() { - fakeRLPServer = startFakeRLPServer(testAppId, envelopes, fakeRLPEmitInterval) - metricsServerConfPath = components.PrepareMetricsServerConfig(dbUrl, defaultHttpClientTimeout, components.Ports[MetricsServerHTTP], components.Ports[MetricsServerWS], tmpDir) - metricsGatewayConfPath = components.PrepareMetricsGatewayConfig(dbUrl, []string{fmt.Sprintf("wss://127.0.0.1:%d", components.Ports[MetricsServerWS])}, fakeRLPServer.GetAddr(), tmpDir) - eventGeneratorConfPath = components.PrepareEventGeneratorConfig(dbUrl, components.Ports[EventGenerator], fmt.Sprintf("https://127.0.0.1:%d", components.Ports[MetricsServerHTTP]), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[ScalingEngine]), aggregatorExecuteInterval, policyPollerInterval, saveInterval, evaluationManagerInterval, defaultHttpClientTimeout, tmpDir) + eventGeneratorConfPath = components.PrepareEventGeneratorConfig(dbUrl, components.Ports[EventGenerator], fmt.Sprintf("127.0.0.1:%d", mockLogCachePort), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[ScalingEngine]), aggregatorExecuteInterval, policyPollerInterval, saveInterval, evaluationManagerInterval, defaultHttpClientTimeout, tmpDir) scalingEngineConfPath = components.PrepareScalingEngineConfig(dbUrl, components.Ports[ScalingEngine], fakeCCNOAAUAA.URL(), defaultHttpClientTimeout, tmpDir) - startMetricsServer() - startMetricsGateway() startEventGenerator() startScalingEngine() - }) AfterEach(func() { - stopFakeRLPServer(fakeRLPServer) - stopMetricsGateway() - stopMetricsServer() stopEventGenerator() stopScalingEngine() + stopMockLogCache() }) Context("MemoryUtil", func() { BeforeEach(func() { - envelopes = createContainerEnvelope(testAppId, 1, 4.0, float64(50), float64(2048000000), float64(100)) - + mockLogCacheReturnsReadEnvelopes(testAppId, createContainerEnvelope(testAppId, 1, 4.0, float64(50), float64(2048000000), float64(100))) }) Context("Scale out", func() { Context("application's responsetime break the scaling out rule for more than breach duration", func() { @@ -172,8 +160,7 @@ var _ = Describe("Integration_Metricsgateway_Metricserver_Eventgenerator_Scaling Context("MemoryUsed", func() { BeforeEach(func() { - envelopes = createContainerEnvelope(testAppId, 1, 4.0, float64(50*1024*1024), float64(2048000000), float64(100*1024*1024)) - + mockLogCacheReturnsReadEnvelopes(testAppId, createContainerEnvelope(testAppId, 1, 4.0, float64(50*1024*1024), float64(2048000000), float64(100*1024*1024))) }) Context("Scale out", func() { Context("application's memory break the scaling out rule for more than breach duration", func() { @@ -294,8 +281,7 @@ var _ = Describe("Integration_Metricsgateway_Metricserver_Eventgenerator_Scaling Context("ResponseTime", func() { BeforeEach(func() { - envelopes = createHTTPTimerEnvelope(testAppId, 1542325492000000000, 1542325492050000000) - + mockLogCacheReturnsQueryResult(testAppId, 40) }) Context("Scale out", func() { Context("application's responsetime break the scaling out rule for more than breach duration", func() { @@ -416,9 +402,7 @@ var _ = Describe("Integration_Metricsgateway_Metricserver_Eventgenerator_Scaling Context("Throughput", func() { BeforeEach(func() { - envelopes = createHTTPTimerEnvelope(testAppId, 1542325492000000000, 1542325492050000000) - fakeRLPEmitInterval = 100 * time.Millisecond - + mockLogCacheReturnsQueryResult(testAppId, 40) }) Context("Scale out", func() { Context("application's throughput break the scaling out rule for more than breach duration", func() { @@ -539,8 +523,7 @@ var _ = Describe("Integration_Metricsgateway_Metricserver_Eventgenerator_Scaling Context("CustomMetric", func() { BeforeEach(func() { - envelopes = createCustomEnvelope(testAppId, "queuelength", "number", 50) - + mockLogCacheReturnsReadEnvelopes(testAppId, createCustomEnvelope(testAppId, "queuelength", "number", 50)) }) Context("Scale out", func() { Context("application's queuelength break the scaling out rule for more than breach duration", func() { @@ -660,3 +643,85 @@ var _ = Describe("Integration_Metricsgateway_Metricserver_Eventgenerator_Scaling }) }) + +func mockLogCacheReturnsReadEnvelopes(sourceId string, envelopes []*loggregator_v2.Envelope) { + mockLogCache.ReadReturns(sourceId, &rpc.ReadResponse{ + Envelopes: &loggregator_v2.EnvelopeBatch{ + Batch: envelopes, + }, + }, nil) +} + +func mockLogCacheReturnsQueryResult(sourceId string, value float64) { + mockLogCache.InstantQueryReturns(sourceId, &rpc.PromQL_InstantQueryResult{ + Result: &rpc.PromQL_InstantQueryResult_Vector{ + Vector: &rpc.PromQL_Vector{ + Samples: []*rpc.PromQL_Sample{ + { + Metric: map[string]string{ + "instance_id": "0", + }, + Point: &rpc.PromQL_Point{ + Value: value, + }, + }, + }, + }, + }, + }, nil) +} + +func createContainerEnvelope(appId string, instanceIndex int32, cpuPercentage float64, memoryBytes float64, diskByte float64, memQuota float64) []*loggregator_v2.Envelope { + return []*loggregator_v2.Envelope{ + { + SourceId: appId, + Message: &loggregator_v2.Envelope_Gauge{ + Gauge: &loggregator_v2.Gauge{ + Metrics: map[string]*loggregator_v2.GaugeValue{ + "cpu": { + Unit: "percentage", + Value: cpuPercentage, + }, + "disk": { + Unit: "bytes", + Value: diskByte, + }, + "memory": { + Unit: "bytes", + Value: memoryBytes, + }, + "memory_quota": { + Unit: "bytes", + Value: memQuota, + }, + }, + }, + }, + }, + } +} + +func createCustomEnvelope(appId string, name string, unit string, value float64) []*loggregator_v2.Envelope { + return []*loggregator_v2.Envelope{ + { + SourceId: appId, + DeprecatedTags: map[string]*loggregator_v2.Value{ + "origin": { + Data: &loggregator_v2.Value_Text{ + Text: "autoscaler_metrics_forwarder", + }, + }, + }, + Message: &loggregator_v2.Envelope_Gauge{ + Gauge: &loggregator_v2.Gauge{ + Metrics: map[string]*loggregator_v2.GaugeValue{ + name: { + Unit: unit, + Value: value, + }, + }, + }, + }, + }, + } +} diff --git a/src/autoscaler/integration/integration_operator_others_test.go b/src/autoscaler/integration/integration_operator_others_test.go index 69427abdb2..45c3081a5f 100644 --- a/src/autoscaler/integration/integration_operator_others_test.go +++ b/src/autoscaler/integration/integration_operator_others_test.go @@ -45,7 +45,6 @@ var _ = Describe("Integration_Operator_Others", func() { fakeCCNOAAUAA.URL(), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[Scheduler]), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[ScalingEngine]), - fmt.Sprintf("https://127.0.0.1:%d", components.Ports[MetricsServerHTTP]), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[EventGenerator]), "https://127.0.0.1:8888", tmpDir) @@ -219,16 +218,6 @@ var _ = Describe("Integration_Operator_Others", func() { Describe("Pruner", func() { BeforeEach(func() { - metric := &models.AppInstanceMetric{ - AppId: testAppId, - CollectedAt: time.Now().Add(-24 * time.Hour).UnixNano(), - Name: models.MetricNameMemoryUsed, - Unit: models.UnitMegaBytes, - Value: "123456", - } - insertAppInstanceMetric(metric) - Expect(getAppInstanceMetricTotalCount(testAppId)).To(Equal(1)) - appmetric := &models.AppMetric{ AppId: testAppId, MetricType: models.MetricNameMemoryUsed, @@ -255,12 +244,10 @@ var _ = Describe("Integration_Operator_Others", func() { }) - It("opeator should remove the staled records ", func() { + It("operator should remove the stale records ", func() { Eventually(func() bool { - return getAppInstanceMetricTotalCount(testAppId) == 0 && - getScalingHistoryTotalCount(testAppId) == 0 && getScalingHistoryTotalCount(testAppId) == 0 + return getScalingHistoryTotalCount(testAppId) == 0 }, 2*time.Minute, 5*time.Second).Should(BeTrue()) - }) }) }) diff --git a/src/autoscaler/integration/integration_suite_test.go b/src/autoscaler/integration/integration_suite_test.go index 8e685c09e8..212a422b10 100644 --- a/src/autoscaler/integration/integration_suite_test.go +++ b/src/autoscaler/integration/integration_suite_test.go @@ -14,14 +14,13 @@ import ( "testing" "time" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cf/mocks" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cf" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cf/mocks" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/integration/mock" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" "code.cloudfoundry.org/lager/v3" _ "github.com/go-sql-driver/mysql" _ "github.com/jackc/pgx/v5/stdlib" @@ -49,8 +48,6 @@ var ( eventGeneratorConfPath string scalingEngineConfPath string operatorConfPath string - metricsGatewayConfPath string - metricsServerConfPath string brokerAuth string dbUrl string LOGLEVEL string @@ -58,6 +55,8 @@ var ( fakeCCNOAAUAA *mocks.Server testUserScope = []string{"cloud_controller.read", "cloud_controller.write", "password.write", "openid", "network.admin", "network.write", "uaa.user"} processMap = map[string]ifrit.Process{} + mockLogCache = &mock.LogCache{} + mockLogCachePort = 20000 + GinkgoParallelProcess() defaultHttpClientTimeout = 10 * time.Second @@ -148,8 +147,6 @@ func CompileTestedExecutables() Executables { builtExecutables[EventGenerator] = path.Join(rootDir, "src", "autoscaler", "build", "eventgenerator") builtExecutables[ScalingEngine] = path.Join(rootDir, "src", "autoscaler", "build", "scalingengine") builtExecutables[Operator] = path.Join(rootDir, "src", "autoscaler", "build", "operator") - builtExecutables[MetricsGateway] = path.Join(rootDir, "src", "autoscaler", "build", "metricsgateway") - builtExecutables[MetricsServerHTTP] = path.Join(rootDir, "src", "autoscaler", "build", "metricsserver") builtExecutables[GolangAPIServer] = path.Join(rootDir, "src", "autoscaler", "build", "api") return builtExecutables @@ -161,8 +158,6 @@ func PreparePorts() Ports { GolangServiceBroker: 23000 + GinkgoParallelProcess(), Scheduler: 15000 + GinkgoParallelProcess(), MetricsCollector: 16000 + GinkgoParallelProcess(), - MetricsServerHTTP: 20000 + GinkgoParallelProcess(), - MetricsServerWS: 21000 + GinkgoParallelProcess(), EventGenerator: 17000 + GinkgoParallelProcess(), ScalingEngine: 18000 + GinkgoParallelProcess(), } @@ -198,16 +193,17 @@ func startOperator() { })) } -func startMetricsGateway() { - processMap[MetricsGateway] = ginkgomon_v2.Invoke(grouper.NewOrdered(os.Interrupt, grouper.Members{ - {MetricsGateway, components.MetricsGateway(metricsGatewayConfPath)}, - })) -} +func startMockLogCache() { + tlsConfig, err := mock.NewTLSConfig( + filepath.Join(testCertDir, "autoscaler-ca.crt"), + filepath.Join(testCertDir, "log-cache.crt"), + filepath.Join(testCertDir, "log-cache.key"), + "log-cache", + ) + Expect(err).ToNot(HaveOccurred()) -func startMetricsServer() { - processMap[MetricsServerHTTP] = ginkgomon_v2.Invoke(grouper.NewOrdered(os.Interrupt, grouper.Members{ - {MetricsServerHTTP, components.MetricsServer(metricsServerConfPath)}, - })) + mockLogCache = mock.NewLogCache(tlsConfig) + mockLogCache.Start(mockLogCachePort) } func stopGolangApiServer() { @@ -225,11 +221,9 @@ func stopEventGenerator() { func stopOperator() { ginkgomon_v2.Kill(processMap[Operator], 5*time.Second) } -func stopMetricsGateway() { - ginkgomon_v2.Kill(processMap[MetricsGateway], 5*time.Second) -} -func stopMetricsServer() { - ginkgomon_v2.Kill(processMap[MetricsServerHTTP], 5*time.Second) + +func stopMockLogCache() { + mockLogCache.Stop() } func getRandomIdRef(ref string) string { @@ -579,9 +573,6 @@ func clearDatabase() { _, err = dbHelper.Exec("DELETE FROM app_metric") Expect(err).NotTo(HaveOccurred()) - - _, err = dbHelper.Exec("DELETE FROM appinstancemetrics") - Expect(err).NotTo(HaveOccurred()) } func insertPolicy(appId string, policyStr string, guid string) { @@ -650,14 +641,6 @@ func getScalingHistoryTotalCount(appId string) int { return count } -func insertAppInstanceMetric(appInstanceMetric *models.AppInstanceMetric) { - query := dbHelper.Rebind("INSERT INTO appinstancemetrics" + - "(appid, instanceindex, collectedat, name, unit, value, timestamp) " + - "VALUES(?, ?, ?, ?, ?, ?, ?)") - _, err := dbHelper.Exec(query, appInstanceMetric.AppId, appInstanceMetric.InstanceIndex, appInstanceMetric.CollectedAt, appInstanceMetric.Name, appInstanceMetric.Unit, appInstanceMetric.Value, appInstanceMetric.Timestamp) - Expect(err).NotTo(HaveOccurred()) -} - func insertAppMetric(appMetrics *models.AppMetric) { query := dbHelper.Rebind("INSERT INTO app_metric" + "(app_id, metric_type, unit, value, timestamp) " + @@ -666,14 +649,6 @@ func insertAppMetric(appMetrics *models.AppMetric) { Expect(err).NotTo(HaveOccurred()) } -func getAppInstanceMetricTotalCount(appId string) int { - var count int - query := dbHelper.Rebind("SELECT COUNT(*) FROM appinstancemetrics WHERE appid=?") - err := dbHelper.QueryRow(query, appId).Scan(&count) - Expect(err).NotTo(HaveOccurred()) - return count -} - func getAppMetricTotalCount(appId string) int { var count int query := dbHelper.Rebind("SELECT COUNT(*) FROM app_metric WHERE app_id=?") @@ -761,89 +736,3 @@ func startFakeCCNOAAUAA(instanceCount int) { CheckToken(testUserScope). UserInfo(http.StatusOK, testUserId) } - -func startFakeRLPServer(appId string, envelopes []*loggregator_v2.Envelope, emitInterval time.Duration) *FakeEventProducer { - fakeRLPServer, err := NewFakeEventProducer(filepath.Join(testCertDir, "reverselogproxy.crt"), filepath.Join(testCertDir, "reverselogproxy.key"), filepath.Join(testCertDir, "autoscaler-ca.crt"), emitInterval) - Expect(err).NotTo(HaveOccurred()) - fakeRLPServer.SetEnvelops(envelopes) - fakeRLPServer.Start() - return fakeRLPServer -} - -func stopFakeRLPServer(fakeRLPServer *FakeEventProducer) { - stopped := fakeRLPServer.Stop() - Expect(stopped).To(Equal(true)) -} - -func createContainerEnvelope(appId string, instanceIndex int32, cpuPercentage float64, memoryBytes float64, diskByte float64, memQuota float64) []*loggregator_v2.Envelope { - return []*loggregator_v2.Envelope{ - { - SourceId: appId, - Message: &loggregator_v2.Envelope_Gauge{ - Gauge: &loggregator_v2.Gauge{ - Metrics: map[string]*loggregator_v2.GaugeValue{ - "cpu": { - Unit: "percentage", - Value: cpuPercentage, - }, - "disk": { - Unit: "bytes", - Value: diskByte, - }, - "memory": { - Unit: "bytes", - Value: memoryBytes, - }, - "memory_quota": { - Unit: "bytes", - Value: memQuota, - }, - }, - }, - }, - }, - } -} - -func createHTTPTimerEnvelope(appId string, start int64, end int64) []*loggregator_v2.Envelope { - return []*loggregator_v2.Envelope{ - { - SourceId: appId, - Message: &loggregator_v2.Envelope_Timer{ - Timer: &loggregator_v2.Timer{ - Name: "http", - Start: start, - Stop: end, - }, - }, - DeprecatedTags: map[string]*loggregator_v2.Value{ - "peer_type": {Data: &loggregator_v2.Value_Text{Text: "Client"}}, - }, - }, - } -} - -func createCustomEnvelope(appId string, name string, unit string, value float64) []*loggregator_v2.Envelope { - return []*loggregator_v2.Envelope{ - { - SourceId: appId, - DeprecatedTags: map[string]*loggregator_v2.Value{ - "origin": { - Data: &loggregator_v2.Value_Text{ - Text: "autoscaler_metrics_forwarder", - }, - }, - }, - Message: &loggregator_v2.Envelope_Gauge{ - Gauge: &loggregator_v2.Gauge{ - Metrics: map[string]*loggregator_v2.GaugeValue{ - name: { - Unit: unit, - Value: value, - }, - }, - }, - }, - }, - } -} diff --git a/src/autoscaler/integration/mock/log_cache_mock.go b/src/autoscaler/integration/mock/log_cache_mock.go new file mode 100644 index 0000000000..e6c9bdd990 --- /dev/null +++ b/src/autoscaler/integration/mock/log_cache_mock.go @@ -0,0 +1,153 @@ +package mock + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "log" + "net" + "os" + "strings" + "sync" + + rpc "code.cloudfoundry.org/go-log-cache/v2/rpc/logcache_v1" + "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" + "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +type LogCache struct { + mutex sync.Mutex + + lis net.Listener + + readResponses map[string]*rpc.ReadResponse + readErrors map[string]error + + instantQueryResults map[string]*rpc.PromQL_InstantQueryResult + instantQueryErrors map[string]error + + tlsConfig *tls.Config + + rpc.UnimplementedEgressServer + rpc.UnimplementedIngressServer + rpc.UnimplementedPromQLQuerierServer +} + +func NewLogCache(tlsConfig *tls.Config) *LogCache { + return &LogCache{ + tlsConfig: tlsConfig, + + readResponses: map[string]*rpc.ReadResponse{}, + readErrors: map[string]error{}, + + instantQueryResults: map[string]*rpc.PromQL_InstantQueryResult{}, + instantQueryErrors: map[string]error{}, + } +} + +func (s *LogCache) Start(port int) string { + var err error + s.lis, err = net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + + var srv *grpc.Server + if s.tlsConfig == nil { + srv = grpc.NewServer() + } else { + srv = grpc.NewServer(grpc.Creds(credentials.NewTLS(s.tlsConfig))) + } + rpc.RegisterIngressServer(srv, s) + rpc.RegisterEgressServer(srv, s) + rpc.RegisterPromQLQuerierServer(srv, s) + + //nolint:errcheck + go srv.Serve(s.lis) + + return s.lis.Addr().String() +} + +func (s *LogCache) Stop() { + s.lis.Close() +} + +func (s *LogCache) Read(ctx context.Context, r *rpc.ReadRequest) (*rpc.ReadResponse, error) { + s.mutex.Lock() + defer s.mutex.Unlock() + + response := &rpc.ReadResponse{Envelopes: &loggregator_v2.EnvelopeBatch{Batch: []*loggregator_v2.Envelope{}}} + if storedResponse, ok := s.readResponses[r.SourceId]; ok { + response = storedResponse + } + + err := errors.New("default-error") + if storedError, ok := s.readErrors[r.SourceId]; ok { + err = storedError + } + + return response, err +} + +func (s *LogCache) ReadReturns(sourceId string, response *rpc.ReadResponse, err error) { + s.mutex.Lock() + defer s.mutex.Unlock() + + s.readResponses[sourceId] = response + s.readErrors[sourceId] = err +} + +func (s *LogCache) InstantQuery(ctx context.Context, r *rpc.PromQL_InstantQueryRequest) (*rpc.PromQL_InstantQueryResult, error) { + s.mutex.Lock() + defer s.mutex.Unlock() + + // every query carries the source_id, example "http{source_id='foo'}" + // find stored mock-result or mock-error by checking if any of the stored source ids is contained in the query + query := r.Query + for sourceId := range s.instantQueryResults { + if strings.Contains(query, sourceId) { + return s.instantQueryResults[sourceId], s.instantQueryErrors[sourceId] + } + } + + return nil, fmt.Errorf("default-error(query: %s)", query) +} + +func (s *LogCache) InstantQueryReturns(sourceId string, result *rpc.PromQL_InstantQueryResult, err error) { + s.mutex.Lock() + defer s.mutex.Unlock() + + s.instantQueryResults[sourceId] = result + s.instantQueryErrors[sourceId] = err +} + +func NewTLSConfig(caPath, certPath, keyPath, cn string) (*tls.Config, error) { + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return nil, err + } + + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + ServerName: cn, + Certificates: []tls.Certificate{cert}, + InsecureSkipVerify: false, + } + + caCertBytes, err := os.ReadFile(caPath) + if err != nil { + return nil, err + } + + caCertPool := x509.NewCertPool() + if ok := caCertPool.AppendCertsFromPEM(caCertBytes); !ok { + return nil, errors.New("cannot parse ca cert") + } + + tlsConfig.RootCAs = caCertPool + + return tlsConfig, nil +} diff --git a/src/autoscaler/metricsgateway/appManager.go b/src/autoscaler/metricsgateway/appManager.go deleted file mode 100644 index cd7af474d2..0000000000 --- a/src/autoscaler/metricsgateway/appManager.go +++ /dev/null @@ -1,79 +0,0 @@ -package metricsgateway - -import ( - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "golang.org/x/exp/maps" - "golang.org/x/net/context" - - "code.cloudfoundry.org/clock" - "code.cloudfoundry.org/lager/v3" - - "sync" - "time" -) - -type GetAppIDsFunc func() map[string]bool - -type AppManager struct { - logger lager.Logger - interval time.Duration - policyDB db.PolicyDB - clock clock.Clock - doneChan chan bool - appIDMap map[string]bool - pLock sync.RWMutex -} - -func NewAppManager(logger lager.Logger, clock clock.Clock, interval time.Duration, policyDB db.PolicyDB) *AppManager { - return &AppManager{ - logger: logger.Session("AppManager"), - clock: clock, - interval: interval, - policyDB: policyDB, - doneChan: make(chan bool), - appIDMap: make(map[string]bool), - } -} -func (am *AppManager) GetAppIDs() map[string]bool { - am.pLock.RLock() - defer am.pLock.RUnlock() - return maps.Clone(am.appIDMap) -} -func (am *AppManager) Start() { - go am.startAppIDsRetrieve() - am.logger.Info("started", lager.Data{"interval": am.interval}) -} - -func (am *AppManager) Stop() { - close(am.doneChan) - am.logger.Info("stopped") -} - -func (am *AppManager) startAppIDsRetrieve() { - tick := am.clock.NewTicker(am.interval) - defer tick.Stop() - for { - appIDMap, err := am.retrieveAppIDs() - if err != nil { - continue - } - am.pLock.Lock() - am.appIDMap = appIDMap - am.pLock.Unlock() - select { - case <-am.doneChan: - return - case <-tick.C(): - } - } -} - -func (am *AppManager) retrieveAppIDs() (map[string]bool, error) { - appIDMap, err := am.policyDB.GetAppIds(context.Background()) - if err != nil { - am.logger.Error("retrieve-app-ids", err) - return nil, err - } - am.logger.Debug("retrieve-app-ids", lager.Data{"count": len(appIDMap)}) - return appIDMap, nil -} diff --git a/src/autoscaler/metricsgateway/appManager_test.go b/src/autoscaler/metricsgateway/appManager_test.go deleted file mode 100644 index 6456e11264..0000000000 --- a/src/autoscaler/metricsgateway/appManager_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package metricsgateway_test - -import ( - "context" - "errors" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsgateway" - - "code.cloudfoundry.org/clock/fakeclock" - "code.cloudfoundry.org/lager/v3" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("AppManager", func() { - var ( - policyDB *fakes.FakePolicyDB - clock *fakeclock.FakeClock - appManager *AppManager - logger lager.Logger - testAppIDRetrieveInterval = 5 * time.Second - ) - - BeforeEach(func() { - policyDB = &fakes.FakePolicyDB{} - clock = fakeclock.NewFakeClock(time.Now()) - logger = lager.NewLogger("AppManager-test") - }) - Context("Start", func() { - JustBeforeEach(func() { - appManager = NewAppManager(logger, clock, testAppIDRetrieveInterval, policyDB) - appManager.Start() - - }) - - AfterEach(func() { - appManager.Stop() - }) - - Context("when the AppManager is started", func() { - BeforeEach(func() { - i := 0 - policyDB.GetAppIdsStub = func(_ context.Context) (map[string]bool, error) { - if i == 0 { - i++ - return map[string]bool{"testAppId-1": true, "testAppId-2": true}, nil - } else { - return map[string]bool{"testAppId-3": true, "testAppId-4": true}, nil - } - } - }) - It("should retrieve app ids for every interval", func() { - Eventually(policyDB.GetAppIdsCallCount).Should(Equal(1)) - Eventually(appManager.GetAppIDs).Should(Equal(map[string]bool{"testAppId-1": true, "testAppId-2": true})) - By("app ids in policy changes") - clock.Increment(1 * testAppIDRetrieveInterval) - Eventually(policyDB.GetAppIdsCallCount).Should(Equal(2)) - Eventually(appManager.GetAppIDs).Should(Equal(map[string]bool{"testAppId-3": true, "testAppId-4": true})) - }) - - Context("when retrieving policies from policyDB fails", func() { - BeforeEach(func() { - policyDB.GetAppIdsStub = func(_ context.Context) (map[string]bool, error) { - return nil, errors.New("error when retrieve app ids from policyDB") - } - }) - It("should return an empty app id map", func() { - clock.Increment(2 * testAppIDRetrieveInterval) - appIDMap := appManager.GetAppIDs() - Expect(len(appIDMap)).To(Equal(0)) - }) - }) - }) - }) - - Context("Stop", func() { - BeforeEach(func() { - appManager = NewAppManager(logger, clock, testAppIDRetrieveInterval, policyDB) - appManager.Start() - Eventually(policyDB.GetAppIdsCallCount).Should(Equal(1)) - - appManager.Stop() - }) - - It("stops the polling", func() { - clock.Increment(5 * testAppIDRetrieveInterval) - Consistently(policyDB.GetAppIdsCallCount).Should(Or(Equal(1), Equal(2))) - }) - }) -}) diff --git a/src/autoscaler/metricsgateway/cmd/metricsgateway/main.go b/src/autoscaler/metricsgateway/cmd/metricsgateway/main.go deleted file mode 100644 index 359e26927e..0000000000 --- a/src/autoscaler/metricsgateway/cmd/metricsgateway/main.go +++ /dev/null @@ -1,169 +0,0 @@ -package main - -import ( - "crypto/tls" - "flag" - "fmt" - "io" - "os" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db/sqldb" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/healthendpoint" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsgateway" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsgateway/config" - mgHelpers "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsgateway/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/routes" - "code.cloudfoundry.org/go-loggregator/v9" - - "code.cloudfoundry.org/clock" - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "code.cloudfoundry.org/lager/v3" - "github.com/prometheus/client_golang/prometheus" - "github.com/tedsuo/ifrit" - "github.com/tedsuo/ifrit/grouper" - "github.com/tedsuo/ifrit/sigmon" -) - -func main() { - var path string - flag.StringVar(&path, "c", "", "config file") - flag.Parse() - if path == "" { - _, _ = fmt.Fprintln(os.Stderr, "missing config file") - os.Exit(1) - } - - conf, err := loadConfig(path) - if err != nil { - _, _ = fmt.Fprintf(os.Stdout, "%s\n", err.Error()) - os.Exit(1) - } - - helpers.SetupOpenTelemetry() - - logger := helpers.InitLoggerFromConfig(&conf.Logging, "metricsgateway") - gatewayClock := clock.NewClock() - loggregatorClientTLSConfig, err := loggregator.NewEgressTLSConfig(conf.Nozzle.RLPClientTLS.CACertFile, conf.Nozzle.RLPClientTLS.CertFile, conf.Nozzle.RLPClientTLS.KeyFile) - if err != nil { - _, _ = fmt.Fprintf(os.Stdout, "%s\n", err.Error()) - os.Exit(1) - } - - metricServerClientTLSConfig, err := conf.Emitter.MetricsServerClientTLS.CreateClientConfig() - if err != nil { - _, _ = fmt.Fprintf(os.Stdout, "%s\n", err.Error()) - os.Exit(1) - } - - policyDb := sqldb.CreatePolicyDb(conf.AppManager.PolicyDB, logger) - defer func() { _ = policyDb.Close() }() - - envelopeCounterCollector := healthendpoint.NewCounterCollector() - - envelopChan := make(chan *loggregator_v2.Envelope, conf.EnvelopChanSize) - emitters := createEmitters(logger, conf.Emitter.BufferSize, gatewayClock, conf.Emitter.KeepAliveInterval, conf.MetricServerAddrs, metricServerClientTLSConfig, conf.Emitter.HandshakeTimeout, conf.Emitter.MaxSetupRetryCount, conf.Emitter.MaxCloseRetryCount, conf.Emitter.RetryDelay) - appManager := metricsgateway.NewAppManager(logger, gatewayClock, conf.AppManager.AppRefreshInterval, policyDb) - dispatcher := metricsgateway.NewDispatcher(logger, envelopChan, emitters) - nozzles := createNozzles(logger, conf.NozzleCount, conf.Nozzle.ShardID, conf.Nozzle.RLPAddr, loggregatorClientTLSConfig, envelopChan, appManager.GetAppIDs, envelopeCounterCollector) - promRegistry := prometheus.NewRegistry() - healthendpoint.RegisterCollectors(promRegistry, []prometheus.Collector{ - healthendpoint.NewDatabaseStatusCollector("autoscaler", "metricsgateway", "policyDB", policyDb), - healthendpoint.NewHTTPStatusCollector("autoscaler", "metricsgateway"), - envelopeCounterCollector, - }, true, logger.Session("metricsgateway-prometheus")) - - gatewayServer := ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error { - logger.Info("starting metricsgateway") - appManager.Start() - dispatcher.Start() - for _, emitter := range emitters { - err := emitter.Start() - if err != nil { - logger.Error("failed to start emitter", err) - os.Exit(1) - } - } - for _, nozzle := range nozzles { - nozzle.Start() - } - - close(ready) - <-signals - for _, nozzle := range nozzles { - nozzle.Stop() - } - for _, emitter := range emitters { - emitter.Stop() - } - dispatcher.Stop() - appManager.Stop() - - return nil - }) - - healthServer, err := healthendpoint.NewServerWithBasicAuth(conf.Health, []healthendpoint.Checker{}, logger.Session("health-server"), promRegistry, time.Now) - - if err != nil { - logger.Error("failed to create health server", err) - os.Exit(1) - } - members := grouper.Members{ - {"metricsgateway", gatewayServer}, - {"health_server", healthServer}, - } - monitor := ifrit.Invoke(sigmon.New(grouper.NewOrdered(os.Interrupt, members))) - - logger.Info("started") - - err = <-monitor.Wait() - if err != nil { - logger.Error("exited-with-failure", err) - os.Exit(1) - } - logger.Info("exited") -} - -func loadConfig(path string) (*config.Config, error) { - configFile, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("failed to open config file %q: %w", path, err) - } - - configFileBytes, err := io.ReadAll(configFile) - _ = configFile.Close() - if err != nil { - return nil, fmt.Errorf("failed to read data from config file %q: %w", path, err) - } - - conf, err := config.LoadConfig(configFileBytes) - if err != nil { - return nil, fmt.Errorf("failed to parse config file %q: %w", path, err) - } - - err = conf.Validate() - if err != nil { - return nil, fmt.Errorf("failed to validate configuration: %w", err) - } - return conf, nil -} - -func createNozzles(logger lager.Logger, nozzleCount int, shardID string, rlpAddr string, loggregatorClientTLSConfig *tls.Config, envelopChan chan *loggregator_v2.Envelope, getAppIDsFunc metricsgateway.GetAppIDsFunc, envelopeCounterCollector healthendpoint.CounterCollector) []*metricsgateway.Nozzle { - nozzles := make([]*metricsgateway.Nozzle, nozzleCount) - for i := 0; i < nozzleCount; i++ { - nozzles[i] = metricsgateway.NewNozzle(logger, i, shardID, rlpAddr, loggregatorClientTLSConfig, envelopChan, getAppIDsFunc, envelopeCounterCollector) - } - return nozzles -} - -func createEmitters(logger lager.Logger, bufferSize int, eclock clock.Clock, keepAliveInterval time.Duration, metricsServerAddrs []string, metricServerClientTLSConfig *tls.Config, handshakeTimeout time.Duration, maxSetupRetryCount int, maxCloseRetryCount int, retryDelay time.Duration) []metricsgateway.Emitter { - emitters := make([]metricsgateway.Emitter, len(metricsServerAddrs)) - for i := 0; i < len(metricsServerAddrs); i++ { - //TODO mgHelpers.NewWSHelper should be injected via a provider function because metricServerClientTLSConfig has nothing to do with the emitter only the WebSocket. - emitter := metricsgateway.NewEnvelopeEmitter(logger, bufferSize, eclock, keepAliveInterval, mgHelpers.NewWSHelper(metricsServerAddrs[i]+routes.EnvelopePath, metricServerClientTLSConfig, handshakeTimeout, logger, maxSetupRetryCount, maxCloseRetryCount, retryDelay)) - emitters[i] = emitter - } - return emitters -} diff --git a/src/autoscaler/metricsgateway/cmd/metricsgateway/metricsgateway_suite_test.go b/src/autoscaler/metricsgateway/cmd/metricsgateway/metricsgateway_suite_test.go deleted file mode 100644 index de8c8607b7..0000000000 --- a/src/autoscaler/metricsgateway/cmd/metricsgateway/metricsgateway_suite_test.go +++ /dev/null @@ -1,259 +0,0 @@ -package main_test - -import ( - "io" - "log" - "net/http" - "os" - "os/exec" - "path/filepath" - "strings" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "github.com/jmoiron/sqlx" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/onsi/gomega/gbytes" - "github.com/onsi/gomega/gexec" - "github.com/onsi/gomega/ghttp" - "google.golang.org/grpc/grpclog" - "gopkg.in/yaml.v3" - - "testing" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsgateway/config" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" -) - -var ( - conf config.Config - configFile *os.File - mgPath string - healthHttpClient *http.Client - healthport int - - testCertDir = "../../../../../test-certs" - loggregatorServerCrtPath = filepath.Join(testCertDir, "reverselogproxy.crt") - loggregatorServerKeyPath = filepath.Join(testCertDir, "reverselogproxy.key") - loggregatorClientCrtPath = filepath.Join(testCertDir, "reverselogproxy_client.crt") - loggregatorClientKeyPath = filepath.Join(testCertDir, "reverselogproxy_client.key") - - metricServerClientCrtPath = filepath.Join(testCertDir, "metricserver_client.crt") - metricServerClientKeyPath = filepath.Join(testCertDir, "metricserver_client.key") - - caPath = filepath.Join(testCertDir, "autoscaler-ca.crt") - - fakeLoggregator testhelpers.FakeEventProducer - rlpAddr string - - fakeMetricServer *ghttp.Server - metricServerAddress string - - testAppId = "test-app-id" - envelopes = []*loggregator_v2.Envelope{ - { - SourceId: testAppId, - DeprecatedTags: map[string]*loggregator_v2.Value{ - "peer_type": {Data: &loggregator_v2.Value_Text{Text: "Client"}}, - }, - Message: &loggregator_v2.Envelope_Timer{ - Timer: &loggregator_v2.Timer{ - Name: "http", - Start: 1542325492043447110, - Stop: 1542325492045491009, - }, - }, - }, - } - messageChan chan []byte - pingPongChan chan int -) - -func TestMetricsgateway(t *testing.T) { - grpclog.SetLoggerV2(grpclog.NewLoggerV2(GinkgoWriter, io.Discard, io.Discard)) - log.SetOutput(GinkgoWriter) - RegisterFailHandler(Fail) - RunSpecs(t, "Metricsgateway Suite") -} - -var _ = SynchronizedBeforeSuite(func() []byte { - eg, err := gexec.Build("code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsgateway/cmd/metricsgateway", "-race") - Expect(err).NotTo(HaveOccurred()) - initDB() - return []byte(eg) -}, func(pathByte []byte) { - mgPath = string(pathByte) - initFakeServers() - initConfig() - healthHttpClient = &http.Client{} -}) - -var _ = SynchronizedAfterSuite(func() { - os.Remove(configFile.Name()) - fakeLoggregator.Stop() - fakeMetricServer.Close() -}, func() { - gexec.CleanupBuildArtifacts() -}) - -func initDB() { - dbUrl := testhelpers.GetDbUrl() - database, err := db.GetConnection(dbUrl) - Expect(err).NotTo(HaveOccurred()) - - mgDB, err := sqlx.Open(database.DriverName, database.DSN) - Expect(err).NotTo(HaveOccurred()) - - _, err = mgDB.Exec("DELETE from policy_json") - Expect(err).NotTo(HaveOccurred()) - - policy := ` - { - "instance_min_count":1, - "instance_max_count":5, - "scaling_rules":[ - { - "metric_type":"a-metric-type", - "breach_duration_secs":120, - "threshold":300, - "operator":">", - "cool_down_secs":300, - "adjustment":"+1" - } - ] - }` - query := mgDB.Rebind("INSERT INTO policy_json(app_id, policy_json, guid) values(?, ?, ?)") - _, err = mgDB.Exec(query, testAppId, policy, "1234") - Expect(err).NotTo(HaveOccurred()) - - err = mgDB.Close() - Expect(err).NotTo(HaveOccurred()) -} - -func initConfig() { - healthport = 8000 + GinkgoParallelProcess() - dbUrl := testhelpers.GetDbUrl() - conf = config.Config{ - Logging: helpers.LoggingConfig{ - Level: "info", - }, - EnvelopChanSize: 500, - NozzleCount: 1, - MetricServerAddrs: []string{metricServerAddress}, - AppManager: config.AppManagerConfig{ - AppRefreshInterval: 10 * time.Second, - PolicyDB: db.DatabaseConfig{ - URL: dbUrl, - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - }, - }, - Emitter: config.EmitterConfig{ - BufferSize: 500, - KeepAliveInterval: 1 * time.Second, - HandshakeTimeout: 1 * time.Second, - MaxSetupRetryCount: 3, - MaxCloseRetryCount: 3, - RetryDelay: 1 * time.Second, - MetricsServerClientTLS: &models.TLSCerts{ - KeyFile: metricServerClientKeyPath, - CertFile: metricServerClientCrtPath, - CACertFile: caPath, - }, - }, - Nozzle: config.NozzleConfig{ - RLPAddr: rlpAddr, - ShardID: "autoscaler", - RLPClientTLS: &models.TLSCerts{ - KeyFile: loggregatorClientKeyPath, - CertFile: loggregatorClientCrtPath, - CACertFile: caPath, - }, - }, - Health: helpers.HealthConfig{ - ServerConfig: helpers.ServerConfig{ - Port: healthport, - }, - HealthCheckUsername: "metricsgatewayhealthcheckuser", - HealthCheckPassword: "metricsgatewayhealthcheckpassword", - }, - } - configFile = writeConfig(&conf) -} - -func initFakeServers() { - fakeLoggregator, err := testhelpers.NewFakeEventProducer(loggregatorServerCrtPath, loggregatorServerKeyPath, caPath, 500*time.Millisecond) - Expect(err).NotTo(HaveOccurred()) - fakeLoggregator.Start() - rlpAddr = fakeLoggregator.GetAddr() - fakeLoggregator.SetEnvelops(envelopes) - - fakeMetricServer = ghttp.NewServer() - metricServerAddress = strings.Replace(fakeMetricServer.URL(), "http", "ws", 1) - - messageChan = make(chan []byte, 10) - pingPongChan = make(chan int, 10) - wsh := testhelpers.NewWebsocketHandler(messageChan, pingPongChan, 5*time.Second) - fakeMetricServer.RouteToHandler("GET", "/v1/envelopes", wsh.ServeWebsocket) -} - -func writeConfig(c *config.Config) *os.File { - cfg, err := os.CreateTemp("", "mg") - Expect(err).NotTo(HaveOccurred()) - defer cfg.Close() - configBytes, err := yaml.Marshal(c) - Expect(err).NotTo(HaveOccurred()) - err = os.WriteFile(cfg.Name(), configBytes, 0600) - Expect(err).NotTo(HaveOccurred()) - return cfg -} - -type MetricsGatewayRunner struct { - configPath string - startCheck string - Session *gexec.Session -} - -func NewMetricsGatewayRunner() *MetricsGatewayRunner { - return &MetricsGatewayRunner{ - configPath: configFile.Name(), - startCheck: "metricsgateway.started", - } -} - -func (mg *MetricsGatewayRunner) Start() { - // #nosec G204 - mgSession, err := gexec.Start(exec.Command( - mgPath, - "-c", - mg.configPath, - ), - gexec.NewPrefixedWriter("\x1b[32m[o]\x1b[32m[mg]\x1b[0m ", GinkgoWriter), - gexec.NewPrefixedWriter("\x1b[91m[e]\x1b[32m[mg]\x1b[0m ", GinkgoWriter), - ) - Expect(err).NotTo(HaveOccurred()) - - if mg.startCheck != "" { - Eventually(mgSession.Buffer, 2).Should(gbytes.Say(mg.startCheck)) - } - - mg.Session = mgSession -} - -func (mg *MetricsGatewayRunner) Interrupt() { - if mg.Session != nil { - mg.Session.Interrupt().Wait(5 * time.Second) - } -} - -func (mg *MetricsGatewayRunner) KillWithFire() { - if mg.Session != nil { - mg.Session.Kill().Wait(5 * time.Second) - } -} diff --git a/src/autoscaler/metricsgateway/cmd/metricsgateway/metricsgateway_test.go b/src/autoscaler/metricsgateway/cmd/metricsgateway/metricsgateway_test.go deleted file mode 100644 index 00ca8d4fba..0000000000 --- a/src/autoscaler/metricsgateway/cmd/metricsgateway/metricsgateway_test.go +++ /dev/null @@ -1,196 +0,0 @@ -package main_test - -import ( - "fmt" - "io" - "net/http" - "os" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gbytes" - . "github.com/onsi/gomega/gexec" -) - -var _ = Describe("Metricsgateway", func() { - var ( - runner *MetricsGatewayRunner - ) - - BeforeEach(func() { - runner = NewMetricsGatewayRunner() - }) - JustBeforeEach(func() { - runner.Start() - }) - AfterEach(func() { - runner.KillWithFire() - }) - - Describe("Metricscollector configuration check", func() { - Context("with a valid config file", func() { - - It("Starts successfully, retrives envelopes and emit envelopes", func() { - Consistently(runner.Session, 5*time.Second).ShouldNot(Exit()) - Eventually(func() bool { return len(fakeMetricServer.ReceivedRequests()) >= 1 }, 5*time.Second).Should(BeTrue()) - Eventually(messageChan, 5*time.Second).Should(Receive()) - }) - }) - Context("with a invalid config file", func() { - BeforeEach(func() { - runner.startCheck = "" - badfile, err := os.CreateTemp("", "bad-mc-config") - Expect(err).NotTo(HaveOccurred()) - runner.configPath = badfile.Name() - err = os.WriteFile(runner.configPath, []byte("bogus"), os.ModePerm) - Expect(err).NotTo(HaveOccurred()) - - }) - - AfterEach(func() { - os.Remove(runner.configPath) - }) - - It("fails with an error", func() { - Eventually(runner.Session).Should(Exit(1)) - Expect(runner.Session.Buffer()).To(Say("failed to parse config file")) - }) - }) - Context("with a missing config file", func() { - BeforeEach(func() { - runner.startCheck = "" - runner.configPath = "bogus" - }) - - It("fails with an error", func() { - Eventually(runner.Session).Should(Exit(1)) - Expect(runner.Session.Buffer()).To(Say("failed to open config file")) - }) - }) - Context("with missing configuration", func() { - BeforeEach(func() { - runner.startCheck = "" - missingConfig := conf - missingConfig.AppManager.PolicyDB.URL = "" - runner.configPath = writeConfig(&missingConfig).Name() - }) - - AfterEach(func() { - os.Remove(runner.configPath) - }) - - It("should fail validation", func() { - Eventually(runner.Session).Should(Exit(1)) - Expect(runner.Session.Buffer()).To(Say("failed to validate configuration")) - }) - }) - }) - Describe("when it fails to connect to metricsserver when starting", func() { - BeforeEach(func() { - wrongConfig := conf - wrongConfig.MetricServerAddrs = []string{"wss://localhost:9999"} - runner.configPath = writeConfig(&wrongConfig).Name() - runner.startCheck = "" - - }) - It("fails to start", func() { - Eventually(runner.Session, time.Duration(2*conf.Emitter.MaxSetupRetryCount)*conf.Emitter.RetryDelay).Should(Exit(1)) - Expect(runner.Session.Buffer()).To(Say("failed to start emitter")) - }) - }) - Describe("when an interrupt is sent", func() { - - It("should stop", func() { - runner.Session.Interrupt() - Eventually(runner.Session, 5).Should(Exit(0)) - }) - }) - - Describe("when Health server is ready to serve RESTful API without basic Auth", func() { - BeforeEach(func() { - basicAuthConfig := conf - basicAuthConfig.Health.HealthCheckUsername = "" - basicAuthConfig.Health.HealthCheckPassword = "" - runner.configPath = writeConfig(&basicAuthConfig).Name() - }) - - Context("when a request to query health comes", func() { - It("returns with a 200", func() { - rsp, err := healthHttpClient.Get(fmt.Sprintf("http://127.0.0.1:%d/health", healthport)) - Expect(err).NotTo(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) - raw, _ := io.ReadAll(rsp.Body) - healthData := string(raw) - Expect(healthData).To(ContainSubstring("autoscaler_metricsgateway_concurrent_http_request")) - Expect(healthData).To(ContainSubstring("autoscaler_metricsgateway_policyDB")) - Expect(healthData).To(ContainSubstring("go_goroutines")) - Expect(healthData).To(ContainSubstring("go_memstats_alloc_bytes")) - Expect(healthData).To(ContainSubstring("autoscaler_metricsgateway_envelope_number_from_rlp")) - rsp.Body.Close() - - }) - }) - }) - - Describe("when Health server is ready to serve RESTful API with basic Auth", func() { - Context("when username and password are incorrect for basic authentication during health check", func() { - It("should return 401", func() { - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) - - req.SetBasicAuth("wrongusername", "wrongpassword") - - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusUnauthorized)) - }) - }) - - Context("when username and password are correct for basic authentication during health check", func() { - It("should return 200", func() { - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) - - req.SetBasicAuth(conf.Health.HealthCheckUsername, conf.Health.HealthCheckPassword) - - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) - }) - }) - - }) - - Describe("when Health server is ready to serve RESTful API with basic Auth", func() { - Context("when username and password are incorrect for basic authentication during health check", func() { - It("should return 401", func() { - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) - - req.SetBasicAuth("wrongusername", "wrongpassword") - - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusUnauthorized)) - }) - }) - - Context("when username and password are correct for basic authentication during health check", func() { - It("should return 200", func() { - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) - - req.SetBasicAuth(conf.Health.HealthCheckUsername, conf.Health.HealthCheckPassword) - - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) - }) - }) - }) -}) diff --git a/src/autoscaler/metricsgateway/config/config.go b/src/autoscaler/metricsgateway/config/config.go deleted file mode 100644 index 232f1a8b9c..0000000000 --- a/src/autoscaler/metricsgateway/config/config.go +++ /dev/null @@ -1,174 +0,0 @@ -package config - -import ( - "bytes" - "fmt" - "strings" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - - "time" - - "gopkg.in/yaml.v3" -) - -const ( - DefaultShardID = "CF_AUTOSCALER" - DefaultLoggingLevel = "info" - DefaultAppRefreshInterval = 60 * time.Second - DefaultHandshakeTimeout = 500 * time.Millisecond - DefaultKeepAliveInterval = 5 * time.Second - DefaultNozzleCount = 3 - DefaultEnvelopChanSize = 500 - DefaultEmitterBufferSize = 500 - DefaultMaxSetupRetryCount = 10 - DefaultMaxCloseRetryCount = 10 - DefaultRetryDelay = 10 * time.Second -) - -type AppManagerConfig struct { - AppRefreshInterval time.Duration `yaml:"app_refresh_interval"` - PolicyDB db.DatabaseConfig `yaml:"policy_db"` -} - -type NozzleConfig struct { - RLPClientTLS *models.TLSCerts `yaml:"rlp_client_tls"` - RLPAddr string `yaml:"rlp_addr"` - ShardID string `yaml:"shard_id"` -} - -type EmitterConfig struct { - MetricsServerClientTLS *models.TLSCerts `yaml:"metrics_server_client_tls"` - BufferSize int `yaml:"buffer_size"` - KeepAliveInterval time.Duration `yaml:"keep_alive_interval"` - HandshakeTimeout time.Duration `yaml:"handshake_timeout"` - - MaxSetupRetryCount int `yaml:"max_setup_retry_count"` - MaxCloseRetryCount int `yaml:"max_close_retry_count"` - RetryDelay time.Duration `yaml:"retry_delay"` -} - -type Config struct { - Logging helpers.LoggingConfig `yaml:"logging"` - EnvelopChanSize int `yaml:"envelop_chan_size"` - NozzleCount int `yaml:"nozzle_count"` - MetricServerAddrs []string `yaml:"metric_server_addrs"` - AppManager AppManagerConfig `yaml:"app_manager"` - Emitter EmitterConfig `yaml:"emitter"` - Nozzle NozzleConfig `yaml:"nozzle"` - Health helpers.HealthConfig `yaml:"health"` -} - -func LoadConfig(config []byte) (*Config, error) { - conf := &Config{ - Logging: helpers.LoggingConfig{ - Level: DefaultLoggingLevel, - }, - EnvelopChanSize: DefaultEnvelopChanSize, - NozzleCount: DefaultNozzleCount, - Emitter: EmitterConfig{ - BufferSize: DefaultEmitterBufferSize, - KeepAliveInterval: DefaultKeepAliveInterval, - HandshakeTimeout: DefaultHandshakeTimeout, - MaxSetupRetryCount: DefaultMaxSetupRetryCount, - MaxCloseRetryCount: DefaultMaxCloseRetryCount, - RetryDelay: DefaultRetryDelay, - }, - Nozzle: NozzleConfig{ - ShardID: DefaultShardID, - }, - AppManager: AppManagerConfig{ - AppRefreshInterval: DefaultAppRefreshInterval, - }, - } - - dec := yaml.NewDecoder(bytes.NewBuffer(config)) - dec.KnownFields(true) - err := dec.Decode(conf) - if err != nil { - return nil, err - } - - conf.Logging.Level = strings.ToLower(conf.Logging.Level) - return conf, nil -} - -func (c *Config) Validate() error { - if c.NozzleCount <= 0 { - return fmt.Errorf("Configuration error: nozzle_count is less-equal than 0") - } - if c.EnvelopChanSize <= 0 { - return fmt.Errorf("Configuration error: envelope_chan_size is less-equal than 0") - } - - if len(c.MetricServerAddrs) <= 0 { - return fmt.Errorf("Configuration error: metrics_server_addrs is empty") - } - - if c.AppManager.PolicyDB.URL == "" { - return fmt.Errorf("Configuration error: app_manager.policy_db.url is empty") - } - if c.AppManager.PolicyDB.MaxOpenConnections <= 0 { - return fmt.Errorf("Configuration error: app_manager.policy_db.max_open_connections is less-equal than 0") - } - if c.AppManager.PolicyDB.MaxIdleConnections <= 0 { - return fmt.Errorf("Configuration error: app_manager.policy_db.max_idle_connections is less-equal than 0") - } - if c.AppManager.PolicyDB.ConnectionMaxLifetime == time.Duration(0) { - return fmt.Errorf("Configuration error: app_manager.policy_db.connection_max_lifetime is 0") - } - if c.AppManager.AppRefreshInterval == time.Duration(0) { - return fmt.Errorf("Configuration error: app_manager.app_refresh_interval is 0") - } - if c.Emitter.BufferSize <= 0 { - return fmt.Errorf("Configuration error: emitter.buffer_size is less-equal than 0") - } - if c.Emitter.HandshakeTimeout == time.Duration(0) { - return fmt.Errorf("Configuration error: emitter.handshake_timeout is 0") - } - if c.Emitter.KeepAliveInterval == time.Duration(0) { - return fmt.Errorf("Configuration error: emitter.keep_alive_interval is 0") - } - if c.Emitter.MaxSetupRetryCount <= 0 { - return fmt.Errorf("Configuration error: emitter.max_setup_retry_count is less-equal than 0") - } - if c.Emitter.MaxCloseRetryCount <= 0 { - return fmt.Errorf("Configuration error: emitter.max_close_retry_count is less-equal than 0") - } - if c.Emitter.RetryDelay == time.Duration(0) { - return fmt.Errorf("Configuration error: emitter.retry_delay is 0") - } - if c.Emitter.MetricsServerClientTLS.CertFile == "" { - return fmt.Errorf("Configuration error: emitter.metrics_server_client_tls.cert_file is empty") - } - if c.Emitter.MetricsServerClientTLS.KeyFile == "" { - return fmt.Errorf("Configuration error: emitter.metrics_server_client_tls.key_file is empty") - } - if c.Emitter.MetricsServerClientTLS.CACertFile == "" { - return fmt.Errorf("Configuration error: emitter.metrics_server_client_tls.ca_file is empty") - } - - if err := c.Health.Validate(); err != nil { - return err - } - - if c.Nozzle.RLPAddr == "" { - return fmt.Errorf("Configuration error: nozzle.rlp_addr is empty") - } - if c.Nozzle.ShardID == "" { - return fmt.Errorf("Configuration error: nozzle.shard_id is empty") - } - if c.Nozzle.RLPClientTLS.CertFile == "" { - return fmt.Errorf("Configuration error: nozzle.rlp_client_tls.cert_file is empty") - } - if c.Nozzle.RLPClientTLS.KeyFile == "" { - return fmt.Errorf("Configuration error: nozzle.rlp_client_tls.key_file is empty") - } - if c.Nozzle.RLPClientTLS.CACertFile == "" { - return fmt.Errorf("Configuration error: nozzle.rlp_client_tls.ca_file is empty") - } - - return nil -} diff --git a/src/autoscaler/metricsgateway/config/config_suite_test.go b/src/autoscaler/metricsgateway/config/config_suite_test.go deleted file mode 100644 index 4d4d240f67..0000000000 --- a/src/autoscaler/metricsgateway/config/config_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package config_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "testing" -) - -func TestConfig(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Config Suite") -} diff --git a/src/autoscaler/metricsgateway/config/config_test.go b/src/autoscaler/metricsgateway/config/config_test.go deleted file mode 100644 index 20e3c7fd87..0000000000 --- a/src/autoscaler/metricsgateway/config/config_test.go +++ /dev/null @@ -1,1050 +0,0 @@ -package config_test - -import ( - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsgateway/config" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "gopkg.in/yaml.v3" -) - -var _ = Describe("Config", func() { - - var ( - conf *Config - err error - configBytes []byte - ) - - Context("Load Config", func() { - JustBeforeEach(func() { - conf, err = LoadConfig(configBytes) - }) - - Context("with invalid yaml", func() { - BeforeEach(func() { - configBytes = []byte(` - logging: - level: "debug" -envelop_chan_size: 500 -nozzle_count: 3 -`) - }) - - It("returns an error", func() { - Expect(err).To(MatchError(MatchRegexp("yaml: .*"))) - }) - }) - - Context("valid config yaml", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: "debug" -envelop_chan_size: 800 -nozzle_count: 10 -metric_server_addrs: - - wss://localhost:8080 - - wss://localhost:9080 -app_manager: - app_refresh_interval: 10s - policy_db: - url: postgres://postgres:password@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: 800 - keep_alive_interval: 10s - handshake_timeout: 100ms - max_setup_retry_count: 10 - max_close_retry_count: 10 - retry_delay: 1s -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: wss://localhost:9999 - shard_id: autoscaler -health: - port: 8081 -`) - }) - It("returns the config", func() { - Expect(err).ShouldNot(HaveOccurred()) - - Expect(conf.Logging.Level).To(Equal("debug")) - Expect(conf.EnvelopChanSize).To(Equal(800)) - Expect(conf.NozzleCount).To(Equal(10)) - Expect(conf.MetricServerAddrs).To(Equal([]string{"wss://localhost:8080", "wss://localhost:9080"})) - Expect(conf.AppManager.AppRefreshInterval).To(Equal(10 * time.Second)) - Expect(conf.AppManager.PolicyDB).To(Equal(db.DatabaseConfig{ - URL: "postgres://postgres:password@localhost/autoscaler?sslmode=disable", - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - })) - Expect(conf.Emitter.BufferSize).To(Equal(800)) - Expect(conf.Emitter.HandshakeTimeout).To(Equal(100 * time.Millisecond)) - Expect(conf.Emitter.KeepAliveInterval).To(Equal(10 * time.Second)) - Expect(conf.Emitter.MetricsServerClientTLS).To(Equal(&models.TLSCerts{ - KeyFile: "metrc_server_client.cert", - CertFile: "metrc_server_client.key", - CACertFile: "autoscaler_ca.cert", - })) - Expect(conf.Nozzle.RLPAddr).To(Equal("wss://localhost:9999")) - Expect(conf.Nozzle.ShardID).To(Equal("autoscaler")) - Expect(conf.Nozzle.RLPClientTLS).To(Equal(&models.TLSCerts{ - KeyFile: "loggregator_client.cert", - CertFile: "loggregator_client.key", - CACertFile: "autoscaler_ca.cert", - })) - Expect(conf.Health.Port).To(Equal(8081)) - - }) - }) - - Context("valid partial yaml", func() { - BeforeEach(func() { - configBytes = []byte(` -metric_server_addrs: - - localhost:8080 - - localhost:9080 -app_manager: - policy_db: - url: postgres://postgres:postgres@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: localhost:9999 -health: - port: 8081 -`) - }) - It("returns the default values", func() { - Expect(err).To(BeNil()) - - Expect(conf.Logging.Level).To(Equal("info")) - Expect(conf.EnvelopChanSize).To(Equal(500)) - Expect(conf.NozzleCount).To(Equal(3)) - Expect(conf.AppManager.AppRefreshInterval).To(Equal(60 * time.Second)) - - Expect(conf.Emitter.BufferSize).To(Equal(500)) - Expect(conf.Emitter.HandshakeTimeout).To(Equal(500 * time.Millisecond)) - Expect(conf.Emitter.KeepAliveInterval).To(Equal(5 * time.Second)) - - Expect(conf.Emitter.MaxSetupRetryCount).To(Equal(10)) - Expect(conf.Emitter.MaxCloseRetryCount).To(Equal(10)) - Expect(conf.Emitter.RetryDelay).To(Equal(10 * time.Second)) - - Expect(conf.Nozzle.ShardID).To(Equal("CF_AUTOSCALER")) - }) - }) - - Context("when it gives a non integer envelop_chan_size", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: "debug" -envelop_chan_size: NOT-INTEGER-VALUE -nozzle_count: 10 -metric_server_addrs: - - wss://localhost:8080 - - wss://localhost:9080 -app_manager: - app_refresh_interval: 10s - policy_db: - url: postgres://postgres:password@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: 800 - keep_alive_interval: 10s - handshake_timeout: 100ms - max_setup_retry_count: 10 - max_close_retry_count: 10 - retry_delay: 1s -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: wss://localhost:9999 - shard_id: autoscaler -health: - port: 8081 -`) - }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) - }) - }) - - Context("when it gives a non integer nozzle_count", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: "debug" -envelop_chan_size: 800 -nozzle_count: NOT-INTEGER-VALUE -metric_server_addrs: - - wss://localhost:8080 - - wss://localhost:9080 -app_manager: - app_refresh_interval: 10s - policy_db: - url: postgres://postgres:password@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: 800 - keep_alive_interval: 10s - handshake_timeout: 100ms - max_setup_retry_count: 10 - max_close_retry_count: 10 - retry_delay: 1s -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: wss://localhost:9999 - shard_id: autoscaler -health: - port: 8081 -`) - }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) - }) - - }) - - Context("when metric_server_addrs is not an array", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: "debug" -envelop_chan_size: 800 -nozzle_count: 3 -metric_server_addrs: wss://localhost:8080 -app_manager: - app_refresh_interval: 10s - policy_db: - url: postgres://postgres:password@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: 800 - keep_alive_interval: 10s - handshake_timeout: 100ms - max_setup_retry_count: 10 - max_close_retry_count: 10 - retry_delay: 1s -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: wss://localhost:9999 - shard_id: autoscaler -health: - port: 8081 -`) - }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into \\[\\]string"))) - }) - - }) - Context("when app_manager.app_refresh_interval is a time.Duration", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: "debug" -envelop_chan_size: 800 -nozzle_count: 10 -metric_server_addrs: - - wss://localhost:8080 - - wss://localhost:9080 -app_manager: - app_refresh_interval: 10k - policy_db: - url: postgres://postgres:password@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: 800 - keep_alive_interval: 10s - handshake_timeout: 100ms - max_setup_retry_count: 10 - max_close_retry_count: 10 - retry_delay: 1s -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: wss://localhost:9999 - shard_id: autoscaler -health: - port: 8081 -`) - }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal .* into time.Duration"))) - }) - - }) - Context("when it gives a non integer max_open_connections of policydb", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: "debug" -envelop_chan_size: 800 -nozzle_count: 10 -metric_server_addrs: - - wss://localhost:8080 - - wss://localhost:9080 -app_manager: - app_refresh_interval: 10s - policy_db: - url: postgres://postgres:password@localhost/autoscaler?sslmode=disable - max_open_connections: NOT-INTEGAER-VALUE - max_idle_connections: 5 - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: 800 - keep_alive_interval: 10s - handshake_timeout: 100ms - max_setup_retry_count: 10 - max_close_retry_count: 10 - retry_delay: 1s -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: wss://localhost:9999 - shard_id: autoscaler -health: - port: 8081 -`) - }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) - }) - - }) - - Context("when it gives a non integer max_idle_connections of policydb", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: "debug" -envelop_chan_size: 800 -nozzle_count: 10 -metric_server_addrs: - - wss://localhost:8080 - - wss://localhost:9080 -app_manager: - app_refresh_interval: 10s - policy_db: - url: postgres://postgres:password@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: NOT-INTEGER-VALUE - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: 800 - keep_alive_interval: 10s - handshake_timeout: 100ms - max_setup_retry_count: 10 - max_close_retry_count: 10 - retry_delay: 1s -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: wss://localhost:9999 - shard_id: autoscaler -health: - port: 8081 -`) - }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) - }) - - }) - - Context("when policy_db.connection_max_lifetime is not a time.Duration", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: "debug" -envelop_chan_size: 800 -nozzle_count: 10 -metric_server_addrs: - - wss://localhost:8080 - - wss://localhost:9080 -app_manager: - app_refresh_interval: 10s - policy_db: - url: postgres://postgres:password@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60k -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: 800 - keep_alive_interval: 10s - handshake_timeout: 100ms - max_setup_retry_count: 10 - max_close_retry_count: 10 - retry_delay: 1s -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: wss://localhost:9999 - shard_id: autoscaler -health: - port: 8081 -`) - }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal .* into time.Duration"))) - }) - - }) - - Context("when it gives a non-integer emitter.buffer_size", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: "debug" -envelop_chan_size: 800 -nozzle_count: 10 -metric_server_addrs: - - wss://localhost:8080 - - wss://localhost:9080 -app_manager: - app_refresh_interval: 10s - policy_db: - url: postgres://postgres:password@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: NOT-INTEGER-VALUE - keep_alive_interval: 10s - handshake_timeout: 100ms - max_setup_retry_count: 10 - max_close_retry_count: 10 - retry_delay: 1s -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: wss://localhost:9999 - shard_id: autoscaler -health: - port: 8081 -`) - }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) - }) - - }) - - Context("when emitter.keep_alive_interval is not a time.Duration", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: "debug" -envelop_chan_size: 800 -nozzle_count: 10 -metric_server_addrs: - - wss://localhost:8080 - - wss://localhost:9080 -app_manager: - app_refresh_interval: 10s - policy_db: - url: postgres://postgres:password@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: 800 - keep_alive_interval: 10kk - handshake_timeout: 100ms - max_setup_retry_count: 10 - max_close_retry_count: 10 - retry_delay: 1s -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: wss://localhost:9999 - shard_id: autoscaler -health: - port: 8081 -`) - }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal .* into time.Duration"))) - }) - - }) - - Context("when emitter.handshake_timeout is not a time.Duration", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: "debug" -envelop_chan_size: 800 -nozzle_count: 10 -metric_server_addrs: - - wss://localhost:8080 - - wss://localhost:9080 -app_manager: - app_refresh_interval: 10s - policy_db: - url: postgres://postgres:password@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: 800 - keep_alive_interval: 10s - handshake_timeout: 100kk - max_setup_retry_count: 10 - max_close_retry_count: 10 - retry_delay: 1s -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: wss://localhost:9999 - shard_id: autoscaler -health: - port: 8081 -`) - }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into time.Duration"))) - }) - - }) - - Context("when it gives a non-integer emitter.max_setup_retry_count", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: "debug" -envelop_chan_size: 800 -nozzle_count: 10 -metric_server_addrs: - - wss://localhost:8080 - - wss://localhost:9080 -app_manager: - app_refresh_interval: 10s - policy_db: - url: postgres://postgres:password@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: 800 - keep_alive_interval: 10s - handshake_timeout: 100ms - max_setup_retry_count: NOT-INTEGER-VALUE - max_close_retry_count: 10 - retry_delay: 1s -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: wss://localhost:9999 - shard_id: autoscaler -health: - port: 8081 -`) - }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) - }) - - }) - - Context("when it gives a non-integer emitter.max_close_retry_count", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: "debug" -envelop_chan_size: 800 -nozzle_count: 10 -metric_server_addrs: - - wss://localhost:8080 - - wss://localhost:9080 -app_manager: - app_refresh_interval: 10s - policy_db: - url: postgres://postgres:password@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: 800 - keep_alive_interval: 10s - handshake_timeout: 100ms - max_setup_retry_count: 10 - max_close_retry_count: NOT-INTEGER-VALUE - retry_delay: 1s -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: wss://localhost:9999 - shard_id: autoscaler -health: - port: 8081 -`) - }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) - }) - - }) - - Context("when emitter.retry_delay is not a time.Duration", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: "debug" -envelop_chan_size: 800 -nozzle_count: 10 -metric_server_addrs: - - wss://localhost:8080 - - wss://localhost:9080 -app_manager: - app_refresh_interval: 10s - policy_db: - url: postgres://postgres:password@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: 800 - keep_alive_interval: 10s - handshake_timeout: 100kk - max_setup_retry_count: 10 - max_close_retry_count: 10 - retry_delay: 1kk -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: wss://localhost:9999 - shard_id: autoscaler -health: - port: 8081 -`) - }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into time.Duration"))) - }) - - }) - - Context("when it gives a non-integer health.port", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: "debug" -envelop_chan_size: 800 -nozzle_count: 10 -metric_server_addrs: - - wss://localhost:8080 - - wss://localhost:9080 -app_manager: - app_refresh_interval: 10s - policy_db: - url: postgres://postgres:password@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: 800 - keep_alive_interval: 10s - handshake_timeout: 100ms - max_setup_retry_count: 10 - max_close_retry_count: 10 - retry_delay: 1s -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: wss://localhost:9999 - shard_id: autoscaler -health: - port: NOT-INTEGER-VALUE -`) - }) - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - Expect(err).To(MatchError(MatchRegexp("cannot unmarshal.*into int"))) - }) - - }) - - }) - - Context("Validate config", func() { - BeforeEach(func() { - conf = &Config{ - Logging: helpers.LoggingConfig{ - Level: "info", - }, - EnvelopChanSize: 500, - NozzleCount: 3, - MetricServerAddrs: []string{"wss://localhost:8080", "wss://localhost:9080"}, - AppManager: AppManagerConfig{ - AppRefreshInterval: 10 * time.Second, - PolicyDB: db.DatabaseConfig{ - URL: "postgres://postgres:password@localhost/autoscaler?sslmode=disable", - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - }, - }, - Emitter: EmitterConfig{ - BufferSize: 500, - KeepAliveInterval: 1 * time.Second, - HandshakeTimeout: 1 * time.Second, - MaxSetupRetryCount: 10, - MaxCloseRetryCount: 10, - RetryDelay: 1 * time.Second, - MetricsServerClientTLS: &models.TLSCerts{ - KeyFile: "metrc_server_client.cert", - CertFile: "metrc_server_client.key", - CACertFile: "autoscaler_ca.cert", - }, - }, - Nozzle: NozzleConfig{ - RLPAddr: "wss://localhost:9999", - ShardID: DefaultShardID, - RLPClientTLS: &models.TLSCerts{ - KeyFile: "loggregator_client.cert", - CertFile: "loggregator_client.key", - CACertFile: "autoscaler_ca.cert", - }, - }, - Health: helpers.HealthConfig{ - ServerConfig: helpers.ServerConfig{ - Port: 8081, - }, - }, - } - }) - JustBeforeEach(func() { - err = conf.Validate() - }) - - Context("when nozzle_count <= 0", func() { - BeforeEach(func() { - conf.NozzleCount = -1 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: nozzle_count is less-equal than 0")) - }) - }) - - Context("when envelope_chan_size <= 0", func() { - BeforeEach(func() { - conf.EnvelopChanSize = -1 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: envelope_chan_size is less-equal than 0")) - }) - }) - - Context("when metrics_server_addrs is empty", func() { - BeforeEach(func() { - conf.MetricServerAddrs = []string{} - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: metrics_server_addrs is empty")) - }) - }) - Context("when app_manager.policy_db.url is empty", func() { - BeforeEach(func() { - conf.AppManager.PolicyDB.URL = "" - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: app_manager.policy_db.url is empty")) - }) - }) - - Context("when app_manager.policy_db.max_open_connections <= 0", func() { - BeforeEach(func() { - conf.AppManager.PolicyDB.MaxOpenConnections = -1 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: app_manager.policy_db.max_open_connections is less-equal than 0")) - }) - }) - - Context("when app_manager.policy_db.max_idle_connections <= 0", func() { - BeforeEach(func() { - conf.AppManager.PolicyDB.MaxIdleConnections = -1 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: app_manager.policy_db.max_idle_connections is less-equal than 0")) - }) - }) - - Context("when app_manager.policy_db.connection_max_lifetime is 0", func() { - BeforeEach(func() { - conf.AppManager.PolicyDB.ConnectionMaxLifetime = 0 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: app_manager.policy_db.connection_max_lifetime is 0")) - }) - }) - - Context("when app_manager.app_refresh_interval is 0", func() { - BeforeEach(func() { - conf.AppManager.AppRefreshInterval = 0 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: app_manager.app_refresh_interval is 0")) - }) - }) - - Context("when emitter.buffer_size <= 0", func() { - BeforeEach(func() { - conf.Emitter.BufferSize = -1 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: emitter.buffer_size is less-equal than 0")) - }) - }) - - Context("when emitter.handshake_timeout is 0", func() { - BeforeEach(func() { - conf.Emitter.HandshakeTimeout = 0 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: emitter.handshake_timeout is 0")) - }) - }) - - Context("when emitter.keep_alive_interval is 0", func() { - BeforeEach(func() { - conf.Emitter.KeepAliveInterval = 0 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: emitter.keep_alive_interval is 0")) - }) - }) - - Context("when emitter.max_setup_retry_count <= 0", func() { - BeforeEach(func() { - conf.Emitter.MaxSetupRetryCount = -1 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: emitter.max_setup_retry_count is less-equal than 0")) - }) - }) - - Context("when emitter.max_close_retry_count <= 0", func() { - BeforeEach(func() { - conf.Emitter.MaxCloseRetryCount = -1 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: emitter.max_close_retry_count is less-equal than 0")) - }) - }) - - Context("when emitter.retry_delay is 0", func() { - BeforeEach(func() { - conf.Emitter.RetryDelay = 0 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: emitter.retry_delay is 0")) - }) - }) - - Context("when emitter.metrics_server_client_tls.cert_file is empty", func() { - BeforeEach(func() { - conf.Emitter.MetricsServerClientTLS.CertFile = "" - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: emitter.metrics_server_client_tls.cert_file is empty")) - }) - }) - Context("when emitter.metrics_server_client_tls.key_file is empty", func() { - BeforeEach(func() { - conf.Emitter.MetricsServerClientTLS.KeyFile = "" - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: emitter.metrics_server_client_tls.key_file is empty")) - }) - }) - Context("when emitter.metrics_server_client_tls.ca_file is empty", func() { - BeforeEach(func() { - conf.Emitter.MetricsServerClientTLS.CACertFile = "" - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: emitter.metrics_server_client_tls.ca_file is empty")) - }) - }) - - Context("when nozzle.rlp_addr is empty", func() { - BeforeEach(func() { - conf.Nozzle.RLPAddr = "" - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: nozzle.rlp_addr is empty")) - }) - }) - - Context("when nozzle.shard_id is empty", func() { - BeforeEach(func() { - conf.Nozzle.ShardID = "" - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: nozzle.shard_id is empty")) - }) - }) - - Context("when nozzle.rlp_client_tls.cert_file is empty", func() { - BeforeEach(func() { - conf.Nozzle.RLPClientTLS.CertFile = "" - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: nozzle.rlp_client_tls.cert_file is empty")) - }) - }) - Context("when nozzle.rlp_client_tls.key_file is empty", func() { - BeforeEach(func() { - conf.Nozzle.RLPClientTLS.KeyFile = "" - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: nozzle.rlp_client_tls.key_file is empty")) - }) - }) - Context("when nozzle.rlp_client_tls.ca_file is empty", func() { - BeforeEach(func() { - conf.Nozzle.RLPClientTLS.CACertFile = "" - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: nozzle.rlp_client_tls.ca_file is empty")) - }) - }) - - }) -}) diff --git a/src/autoscaler/metricsgateway/dispatcher.go b/src/autoscaler/metricsgateway/dispatcher.go deleted file mode 100644 index 47da6d6354..0000000000 --- a/src/autoscaler/metricsgateway/dispatcher.go +++ /dev/null @@ -1,49 +0,0 @@ -package metricsgateway - -import ( - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "code.cloudfoundry.org/lager/v3" -) - -type Dispatcher struct { - logger lager.Logger - envelopChan chan *loggregator_v2.Envelope - doneChan chan bool - emitters []Emitter -} - -func NewDispatcher(logger lager.Logger, envelopChan chan *loggregator_v2.Envelope, emitters []Emitter) *Dispatcher { - return &Dispatcher{ - logger: logger.Session("Dispather"), - envelopChan: envelopChan, - emitters: emitters, - doneChan: make(chan bool), - } -} -func (d *Dispatcher) Start() { - go d.dispatch() - d.logger.Info("dispatcher-started") -} - -func (d *Dispatcher) Stop() { - d.doneChan <- true -} -func (d *Dispatcher) dispatch() { - for { - select { - case <-d.doneChan: - d.logger.Info("dispatcher-stopped") - return - case e := <-d.envelopChan: - appID := e.SourceId - emmitter := d.getEmitter(appID) - emmitter.Accept(e) - } - } -} - -func (d *Dispatcher) getEmitter(appID string) Emitter { - return d.emitters[helpers.FNVHash(appID)%(uint32)(len(d.emitters))] -} diff --git a/src/autoscaler/metricsgateway/dispatcher_test.go b/src/autoscaler/metricsgateway/dispatcher_test.go deleted file mode 100644 index 1a86cd5f42..0000000000 --- a/src/autoscaler/metricsgateway/dispatcher_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package metricsgateway_test - -import ( - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsgateway" - - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "code.cloudfoundry.org/lager/v3/lagertest" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Dispatcher", func() { - var ( - logger *lagertest.TestLogger - envelopChan chan *loggregator_v2.Envelope - resultChans []chan *loggregator_v2.Envelope - emitters []Emitter - dispatcher *Dispatcher - testAppId1 = "the-test-appid1" - testAppId2 = "the-test-appid2" - testAppId3 = "the-test-appid3" - ) - BeforeEach(func() { - envelopChan = make(chan *loggregator_v2.Envelope, 10) - resultChans = []chan *loggregator_v2.Envelope{} - logger = lagertest.NewTestLogger("Dispatcher") - - }) - Context("Start", func() { - AfterEach(func() { - dispatcher.Stop() - }) - JustBeforeEach(func() { - dispatcher.Start() - envelopChan <- &loggregator_v2.Envelope{ - SourceId: testAppId1, - } - envelopChan <- &loggregator_v2.Envelope{ - SourceId: testAppId2, - } - envelopChan <- &loggregator_v2.Envelope{ - SourceId: testAppId3, - } - }) - Context("when there is only one emitter", func() { - BeforeEach(func() { - resultChans = []chan *loggregator_v2.Envelope{ - make(chan *loggregator_v2.Envelope, 100), - } - emitters = []Emitter{ - &fakes.FakeEmitter{ - AcceptStub: func(envelope *loggregator_v2.Envelope) { - resultChans[0] <- envelope - }, - }, - } - dispatcher = NewDispatcher(logger, envelopChan, emitters) - }) - It("dispatch envelope to emitter", func() { - Eventually(resultChans[0]).Should(Receive(Equal(&loggregator_v2.Envelope{ - SourceId: testAppId1, - }))) - Eventually(resultChans[0]).Should(Receive(Equal(&loggregator_v2.Envelope{ - SourceId: testAppId2, - }))) - Eventually(resultChans[0]).Should(Receive(Equal(&loggregator_v2.Envelope{ - SourceId: testAppId3, - }))) - }) - }) - - Context("when there 4 emitters", func() { - BeforeEach(func() { - resultChans = []chan *loggregator_v2.Envelope{ - make(chan *loggregator_v2.Envelope, 5), - make(chan *loggregator_v2.Envelope, 5), - make(chan *loggregator_v2.Envelope, 5), - make(chan *loggregator_v2.Envelope, 5), - } - emitters = []Emitter{ - &fakes.FakeEmitter{ - AcceptStub: func(envelope *loggregator_v2.Envelope) { - resultChans[0] <- envelope - }, - }, - &fakes.FakeEmitter{ - AcceptStub: func(envelope *loggregator_v2.Envelope) { - resultChans[1] <- envelope - }, - }, - &fakes.FakeEmitter{ - AcceptStub: func(envelope *loggregator_v2.Envelope) { - resultChans[2] <- envelope - }, - }, - &fakes.FakeEmitter{ - AcceptStub: func(envelope *loggregator_v2.Envelope) { - resultChans[3] <- envelope - }, - }, - } - dispatcher = NewDispatcher(logger, envelopChan, emitters) - }) - It("dispatch envelope to emitters", func() { - Eventually(resultChans[3]).Should(Receive(Equal(&loggregator_v2.Envelope{ - SourceId: testAppId1, - }))) - Eventually(resultChans[2]).Should(Receive(Equal(&loggregator_v2.Envelope{ - SourceId: testAppId2, - }))) - Eventually(resultChans[1]).Should(Receive(Equal(&loggregator_v2.Envelope{ - SourceId: testAppId3, - }))) - Consistently(resultChans[0]).ShouldNot(Receive()) - }) - }) - - }) - - Context("Stop", func() { - BeforeEach(func() { - resultChans = []chan *loggregator_v2.Envelope{ - make(chan *loggregator_v2.Envelope, 100), - } - emitters = []Emitter{ - &fakes.FakeEmitter{ - AcceptStub: func(envelope *loggregator_v2.Envelope) { - resultChans[0] <- envelope - }, - }, - } - dispatcher = NewDispatcher(logger, envelopChan, emitters) - dispatcher.Start() - envelopChan <- &loggregator_v2.Envelope{ - SourceId: testAppId1, - } - Eventually(resultChans[0]).Should(Receive()) - dispatcher.Stop() - envelopChan <- &loggregator_v2.Envelope{ - SourceId: testAppId1, - } - }) - It("stop dispathing", func() { - Consistently(resultChans[0]).ShouldNot(Receive()) - }) - }) -}) diff --git a/src/autoscaler/metricsgateway/emitter.go b/src/autoscaler/metricsgateway/emitter.go deleted file mode 100644 index 67e8cf9f2d..0000000000 --- a/src/autoscaler/metricsgateway/emitter.go +++ /dev/null @@ -1,89 +0,0 @@ -package metricsgateway - -import ( - "time" - - "code.cloudfoundry.org/clock" - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "code.cloudfoundry.org/lager/v3" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsgateway/helpers" -) - -type Emitter interface { - Accept(envelope *loggregator_v2.Envelope) - Emit(envelope *loggregator_v2.Envelope) error - Start() error - Stop() -} - -type EnvelopeEmitter struct { - logger lager.Logger - envelopChan chan *loggregator_v2.Envelope - doneChan chan bool - keepAliveInterval time.Duration - eclock clock.Clock - ticker clock.Ticker - wsHelper helpers.WSHelper -} - -func NewEnvelopeEmitter(logger lager.Logger, bufferSize int, eclock clock.Clock, keepAliveInterval time.Duration, wsHelper helpers.WSHelper) Emitter { - return &EnvelopeEmitter{ - logger: logger.Session("EnvelopeEmitter"), - envelopChan: make(chan *loggregator_v2.Envelope, bufferSize), - doneChan: make(chan bool), - eclock: eclock, - keepAliveInterval: keepAliveInterval, - wsHelper: wsHelper, - } -} - -func (e *EnvelopeEmitter) Start() error { - err := e.wsHelper.SetupConn() - if err != nil { - e.logger.Error("failed-to-start-emimtter", err) - return err - } - go e.startEmitEnvelope() - e.logger.Info("started") - return nil -} - -func (e *EnvelopeEmitter) startEmitEnvelope() { - e.ticker = e.eclock.NewTicker(e.keepAliveInterval) - for { - select { - case <-e.doneChan: - e.logger.Info("stopped") - return - case envelope := <-e.envelopChan: - err := e.Emit(envelope) - if err != nil { - e.logger.Error("failed-to-emit-envelope", err, lager.Data{"message": envelope}) - } - case <-e.ticker.C(): - err := e.wsHelper.Ping() - if err != nil { - e.logger.Error("failed-to-ping-metricserver", err) - } - } - } -} - -func (e *EnvelopeEmitter) Stop() { - err := e.wsHelper.CloseConn() - if err != nil { - e.logger.Error("failed-to-stop", err) - } - e.doneChan <- true -} - -func (e *EnvelopeEmitter) Accept(envelope *loggregator_v2.Envelope) { - e.logger.Debug("accept-envelope", lager.Data{"envelope": envelope}) - e.envelopChan <- envelope -} -func (e *EnvelopeEmitter) Emit(envelope *loggregator_v2.Envelope) error { - e.logger.Debug("emit-envelope", lager.Data{"envelope": envelope}) - err := e.wsHelper.Write(envelope) - return err -} diff --git a/src/autoscaler/metricsgateway/emitter_test.go b/src/autoscaler/metricsgateway/emitter_test.go deleted file mode 100644 index 1be6757498..0000000000 --- a/src/autoscaler/metricsgateway/emitter_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package metricsgateway_test - -import ( - "errors" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsgateway" - - "code.cloudfoundry.org/clock/fakeclock" - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "code.cloudfoundry.org/lager/v3/lagertest" - "github.com/gorilla/websocket" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gbytes" -) - -var _ = Describe("Emitter", func() { - var ( - logger *lagertest.TestLogger - envelopChan = make(chan *loggregator_v2.Envelope, 10) - wsMessageChan = make(chan int, 10) - - bufferSize = 500 - fakeWSHelper *fakes.FakeWSHelper - emitter Emitter - testAppId = "test-app-id" - fclock *fakeclock.FakeClock - verifyWSConnectionInterval = 5 * time.Second - - testEnvelope = loggregator_v2.Envelope{ - SourceId: testAppId, - Message: &loggregator_v2.Envelope_Gauge{ - Gauge: &loggregator_v2.Gauge{ - Metrics: map[string]*loggregator_v2.GaugeValue{ - "cpu": { - Unit: "percentage", - Value: 20.5, - }, - "disk": { - Unit: "bytes", - Value: 3000000000, - }, - "memory": { - Unit: "bytes", - Value: 1000000000, - }, - "memory_quota": { - Unit: "bytes", - Value: 2000000000, - }, - }, - }, - }, - } - ) - BeforeEach(func() { - logger = lagertest.NewTestLogger("emitter") - fclock = fakeclock.NewFakeClock(time.Now()) - fakeWSHelper = &fakes.FakeWSHelper{} - fakeWSHelper.WriteStub = func(envelope *loggregator_v2.Envelope) error { - envelopChan <- envelope - return nil - } - fakeWSHelper.PingStub = func() error { - wsMessageChan <- websocket.PingMessage - return nil - } - fakeWSHelper.CloseConnStub = func() error { - wsMessageChan <- websocket.CloseMessage - return nil - } - - }) - Context("Start", func() { - var startError error - JustBeforeEach(func() { - emitter = NewEnvelopeEmitter(logger, bufferSize, fclock, verifyWSConnectionInterval, fakeWSHelper) - startError = emitter.Start() - }) - It("should emit envelops to metricServer", func() { - Expect(startError).NotTo(HaveOccurred()) - emitter.Accept(&testEnvelope) - Eventually(envelopChan).Should(Receive()) - }) - - It("should send ping message to metricServer periodically", func() { - Expect(startError).NotTo(HaveOccurred()) - fclock.WaitForWatcherAndIncrement(1 * verifyWSConnectionInterval) - Eventually(wsMessageChan).Should(Receive(Equal(websocket.PingMessage))) - fclock.WaitForWatcherAndIncrement(1 * verifyWSConnectionInterval) - Eventually(wsMessageChan).Should(Receive(Equal(websocket.PingMessage))) - }) - Context("when it fails to connect metricsserver", func() { - BeforeEach(func() { - fakeWSHelper.SetupConnStub = func() error { - return errors.New("connection-error") - } - }) - It("failed to start", func() { - Expect(startError).To(HaveOccurred()) - emitter.Accept(&testEnvelope) - Consistently(envelopChan).ShouldNot(Receive()) - }) - }) - }) - - Context("Stop", func() { - BeforeEach(func() { - emitter = NewEnvelopeEmitter(logger, bufferSize, fclock, verifyWSConnectionInterval, fakeWSHelper) - err := emitter.Start() - Expect(err).NotTo(HaveOccurred()) - Eventually(logger.Buffer).Should(Say("started")) - emitter.Accept(&testEnvelope) - Eventually(envelopChan).Should(Receive()) - emitter.Stop() - By("should close ws connection") - Eventually(wsMessageChan).Should(Receive(Equal(websocket.CloseMessage))) - Eventually(envelopChan).ShouldNot(Receive()) - }) - It("should stop emitting envelope", func() { - emitter.Accept(&testEnvelope) - Consistently(envelopChan).ShouldNot(Receive()) - }) - }) -}) diff --git a/src/autoscaler/metricsgateway/exampleconfig/example.yaml b/src/autoscaler/metricsgateway/exampleconfig/example.yaml deleted file mode 100644 index 44e35db105..0000000000 --- a/src/autoscaler/metricsgateway/exampleconfig/example.yaml +++ /dev/null @@ -1,34 +0,0 @@ -logging: - level: "debug" -envelop_chan_size: 500 -Nozzle_count: 3 -metric_server_addrs: - - localhost:8080 - - localhost:9080 -app_manager: - app_refresh_interval: 5s - policy_db: - url: postgres://postgres:postgres@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -emitter: - metrics_server_client_tls: - key_file: "metrc_server_client.cert" - cert_file: "metrc_server_client.key" - ca_file: "autoscaler_ca.cert" - buffer_size: 500 - keep_alive_interval: 5s - handshake_timeout: 500ms - max_setup_retry_count: 10 - max_close_retry_count: 10 - retry_delay: 1s -nozzle: - rlp_client_tls: - key_file: "loggregator_client.cert" - cert_file: "loggregator_client.key" - ca_file: "autoscaler_ca.cert" - rlp_addr: localhost:9999 - shard_id: autoscaler -health: - port: 8081 \ No newline at end of file diff --git a/src/autoscaler/metricsgateway/helpers/helpers_suite_test.go b/src/autoscaler/metricsgateway/helpers/helpers_suite_test.go deleted file mode 100644 index 96cf17caab..0000000000 --- a/src/autoscaler/metricsgateway/helpers/helpers_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package helpers_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "testing" -) - -func TestHelpers(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Helpers Suite") -} diff --git a/src/autoscaler/metricsgateway/helpers/ws_helper.go b/src/autoscaler/metricsgateway/helpers/ws_helper.go deleted file mode 100644 index 9cdea46702..0000000000 --- a/src/autoscaler/metricsgateway/helpers/ws_helper.go +++ /dev/null @@ -1,187 +0,0 @@ -package helpers - -import ( - "crypto/tls" - "fmt" - "net/url" - - "net/http" - "sync" - "time" - - "code.cloudfoundry.org/lager/v3" - - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - //TODO remove static check and use non deprecated version https://github.com/cloudfoundry/app-autoscaler-release/issues/978 - //nolint:staticcheck - "github.com/golang/protobuf/proto" - "github.com/gorilla/websocket" -) - -type WSHelper interface { - SetupConn() error - CloseConn() error - IsClosed() bool - Write(envelope *loggregator_v2.Envelope) error - Read() error - Ping() error -} - -type connection struct { - rwMu sync.RWMutex - conn *websocket.Conn -} - -func (c *connection) getConnection() *websocket.Conn { - c.rwMu.RLock() - defer c.rwMu.RUnlock() - return c.conn -} -func (c *connection) setConnection(conn *websocket.Conn) { - c.rwMu.Lock() - defer c.rwMu.Unlock() - c.conn = conn -} - -func (c *connection) Close() error { - c.rwMu.Lock() - defer c.rwMu.Unlock() - err := c.conn.Close() - c.conn = nil - return err -} - -var _ WSHelper = &WsHelper{} - -type WsHelper struct { - dialer websocket.Dialer - maxSetupRetryCount int - maxCloseRetryCount int - retryDelay time.Duration - logger lager.Logger - metricServerURL string - connection connection - CloseWaitTime time.Duration -} - -func NewWSHelper(metricServerURL string, tlsConfig *tls.Config, handshakeTimeout time.Duration, logger lager.Logger, maxSetupRetryCount int, maxCloseRetryCount int, retryDelay time.Duration) *WsHelper { - return &WsHelper{ - metricServerURL: metricServerURL, - dialer: websocket.Dialer{ - TLSClientConfig: tlsConfig, - Proxy: http.ProxyFromEnvironment, - HandshakeTimeout: handshakeTimeout, - }, - logger: logger.Session("WSHelper"), - maxSetupRetryCount: maxSetupRetryCount, - maxCloseRetryCount: maxCloseRetryCount, - retryDelay: retryDelay, - CloseWaitTime: 5 * time.Second, - } -} - -func (wh *WsHelper) SetupConn() error { - wh.logger.Info("setup-new-ws-connection") - URL, err := url.Parse(wh.metricServerURL) - if err != nil { - return err - } - - if URL.Scheme != "wss" && URL.Scheme != "ws" { - return fmt.Errorf("Invalid scheme '%s'", URL.Scheme) - } - retryCount := 0 - for { - // dial docs says not to close the response body by the application - //nolint:bodyclose - con, _, err := wh.dialer.Dial(wh.metricServerURL, nil) - if err != nil { - wh.logger.Error("failed-to-create-websocket-connection-to-metricserver", err, lager.Data{"metricServerURL": wh.metricServerURL}) - if retryCount < wh.maxSetupRetryCount { - retryCount++ - time.Sleep(wh.retryDelay) - } else { - return fmt.Errorf("failed after %d retries: %w", retryCount, err) - } - } else { - go func() { - for { - _, _, err := con.ReadMessage() - if err != nil { - wh.logger.Error("failed-to-read-message", err) - return - } - } - }() - wh.connection.setConnection(con) - return nil - } - } -} - -func (wh *WsHelper) IsClosed() bool { - return wh.connection.getConnection() == nil -} - -func (wh *WsHelper) CloseConn() error { - retryCount := 0 - for { - wh.logger.Info("close-ws-connection") - err := wh.connection.getConnection().WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Time{}) - if err != nil { - if retryCount < wh.maxCloseRetryCount { - retryCount++ - time.Sleep(wh.retryDelay) - } else { - return fmt.Errorf("failed to close correctly after %d retries: %w", retryCount, err) - } - } else { - go func() { - time.AfterFunc(wh.CloseWaitTime, func() { - _ = wh.connection.Close() - }) - }() - return nil - } - } -} - -func (wh *WsHelper) Write(envelope *loggregator_v2.Envelope) error { - bytes, err := proto.Marshal(envelope) - if err != nil { - wh.logger.Error("failed-to-marshal-envelope", err, lager.Data{"envelope": envelope}) - return err - } - wh.logger.Debug("writing-envelope-to-server", lager.Data{"envelope": envelope}) - err = wh.connection.getConnection().WriteMessage(websocket.BinaryMessage, bytes) - //TODO should retry sending a message. - if err != nil { - wh.logger.Error("failed-to-write-envelope", err) - return wh.reconnect() - } - return nil -} -func (wh *WsHelper) Read() error { - return nil -} -func (wh *WsHelper) Ping() error { - err := wh.connection.getConnection().WriteControl(websocket.PingMessage, nil, time.Now().Add(1*time.Second)) - if err != nil { - wh.logger.Error("failed-to-send-ping", err) - return wh.reconnect() - } - return nil -} - -func (wh *WsHelper) reconnect() error { - err := wh.CloseConn() - if err != nil { - wh.logger.Error("failed-to-close-websocket-connection", err) - } - - err = wh.SetupConn() - if err != nil { - return err - } - return nil -} diff --git a/src/autoscaler/metricsgateway/helpers/ws_helper_test.go b/src/autoscaler/metricsgateway/helpers/ws_helper_test.go deleted file mode 100644 index 6aed382339..0000000000 --- a/src/autoscaler/metricsgateway/helpers/ws_helper_test.go +++ /dev/null @@ -1,216 +0,0 @@ -package helpers_test - -import ( - "fmt" - "strings" - "time" - - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsgateway/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/routes" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" - - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "code.cloudfoundry.org/lager/v3/lagertest" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/onsi/gomega/ghttp" -) - -var _ = Describe("WsHelper", func() { - var ( - fakeMetricServer *ghttp.Server - metricServerAddress string - testHandshakeTimeout = 5 * time.Millisecond - testMaxSetupRetryCount = 3 - testMaxCloseRetryCount = 3 - testRetryDelay = 1 * time.Millisecond - messageChan chan []byte - pingPongChan chan int - wsHelper *WsHelper - logger *lagertest.TestLogger - wsh *testhelpers.WebsocketHandler - testAppId = "test-app-id" - testEnvelope = loggregator_v2.Envelope{ - SourceId: testAppId, - Message: &loggregator_v2.Envelope_Gauge{ - Gauge: &loggregator_v2.Gauge{ - Metrics: map[string]*loggregator_v2.GaugeValue{ - "cpu": { - Unit: "percentage", - Value: 20.5, - }, - "disk": { - Unit: "bytes", - Value: 3000000000, - }, - "memory": { - Unit: "bytes", - Value: 1000000000, - }, - "memory_quota": { - Unit: "bytes", - Value: 2000000000, - }, - }, - }, - }, - } - ) - BeforeEach(func() { - fakeMetricServer = ghttp.NewServer() - metricServerAddress = strings.Replace(fakeMetricServer.URL(), "http", "ws", 1) - messageChan = make(chan []byte, 10) - pingPongChan = make(chan int, 10) - wsh = testhelpers.NewWebsocketHandler(messageChan, pingPongChan, 5*time.Second) - fakeMetricServer.RouteToHandler("GET", "/v1/envelopes", wsh.ServeWebsocket) - logger = lagertest.NewTestLogger("ws_helper") - }) - AfterEach(func() { - fakeMetricServer.Close() - }) - Describe("SetupConn", func() { - var err error - JustBeforeEach(func() { - wsHelper = NewWSHelper(metricServerAddress+routes.EnvelopePath, nil, testHandshakeTimeout, logger, testMaxSetupRetryCount, testMaxCloseRetryCount, testRetryDelay) - err = wsHelper.SetupConn() - - }) - It("set up websocket connection", func() { - Expect(err).NotTo(HaveOccurred()) - }) - Context("when scheme of url is not ws/wss", func() { - BeforeEach(func() { - metricServerAddress = strings.Replace(metricServerAddress, "ws", "http", 1) - }) - It("fails to setup websocket connection", func() { - Expect(err).To(Equal(fmt.Errorf("Invalid scheme '%s'", "http"))) - }) - }) - Context("when maximum number of setup retries reached", func() { - BeforeEach(func() { - fakeMetricServer.RouteToHandler("GET", "/v1/envelopes", - testhelpers.RespondWithMultiple( - ghttp.RespondWith(500, ""), - ghttp.RespondWith(500, ""), - ghttp.RespondWith(500, ""), - ghttp.RespondWith(500, ""), - ), - ) - }) - It("fails to setup websocket connection", func() { - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(MatchRegexp(fmt.Sprintf(".*failed after %d retries:.*", testMaxCloseRetryCount)))) - Expect(err).To(MatchError(MatchRegexp(".*bad handshake.*"))) - Expect(len(fakeMetricServer.ReceivedRequests())).To(Equal(4)) - }) - }) - Context("when retries then connects correctly", func() { - BeforeEach(func() { - fakeMetricServer.RouteToHandler("GET", "/v1/envelopes", - testhelpers.RespondWithMultiple( - ghttp.RespondWith(500, ""), - ghttp.RespondWith(500, ""), - ghttp.RespondWith(500, ""), - wsh.ServeWebsocket, - ), - ) - }) - It("successfully connects", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(len(fakeMetricServer.ReceivedRequests())).To(Equal(4)) - }) - }) - }) - Describe("CloseConn", func() { - var err error - BeforeEach(func() { - wsHelper = NewWSHelper(metricServerAddress+routes.EnvelopePath, nil, testHandshakeTimeout, logger, testMaxSetupRetryCount, testMaxCloseRetryCount, testRetryDelay) - err = wsHelper.SetupConn() - Expect(err).NotTo(HaveOccurred()) - - err = wsHelper.Ping() - Expect(err).ShouldNot(HaveOccurred()) - Eventually(pingPongChan, 5*time.Second, 1*time.Second).Should(Receive(Equal(1))) - }) - It("close the websocket connection", func() { - wsHelper.CloseWaitTime = 100 * time.Millisecond - err = wsHelper.CloseConn() - Expect(err).NotTo(HaveOccurred()) - Eventually(wsHelper.IsClosed, 200*time.Millisecond, 50*time.Millisecond).Should(BeTrue()) - }) - Context("when maximum number of close retries reached", func() { - BeforeEach(func() { - fakeMetricServer.Close() - wsh.CloseWSConnection() - }) - It("fails to close websocket connection", func() { - var err error - Eventually(func() error { - err = wsHelper.CloseConn() - return err - }).Should(HaveOccurred()) - - Expect(err).To(MatchError(MatchRegexp(fmt.Sprintf("failed to close correctly after %d retries:.*", testMaxCloseRetryCount)))) - Expect(err).To(MatchError(MatchRegexp(".*close sent.*"))) - }) - }) - }) - Describe("Ping", func() { - var err error - BeforeEach(func() { - wsHelper = NewWSHelper(metricServerAddress+routes.EnvelopePath, nil, testHandshakeTimeout, logger, testMaxSetupRetryCount, testMaxCloseRetryCount, testRetryDelay) - err = wsHelper.SetupConn() - Expect(err).NotTo(HaveOccurred()) - - }) - It("send ping message to server", func() { - err = wsHelper.Ping() - Expect(err).ShouldNot(HaveOccurred()) - Eventually(pingPongChan, 5*time.Second, 100*time.Millisecond).Should(Receive(Equal(1))) - }) - Context("when server is down", func() { - BeforeEach(func() { - fakeMetricServer.Close() - wsh.CloseWSConnection() - - }) - It("fails to ping and reconnect", func() { - var err error - Eventually(func() error { - err = wsHelper.Ping() - return err - }, "1s").Should(HaveOccurred()) - Expect(err).To(MatchError(MatchRegexp(fmt.Sprintf(".*failed after %d retries:.*", testMaxCloseRetryCount)))) - }) - }) - }) - Describe("Write", func() { - var err error - BeforeEach(func() { - wsHelper = NewWSHelper(metricServerAddress+routes.EnvelopePath, nil, testHandshakeTimeout, logger, testMaxSetupRetryCount, testMaxCloseRetryCount, testRetryDelay) - err = wsHelper.SetupConn() - Expect(err).NotTo(HaveOccurred()) - - }) - It("write envelops to server", func() { - Consistently(messageChan).ShouldNot(Receive()) - err = wsHelper.Write(&testEnvelope) - Expect(err).NotTo(HaveOccurred()) - Eventually(messageChan).Should(Receive()) - }) - Context("when server is down", Pending, func() { - //TODO disabling due to flakeynes https://github.com/cloudfoundry/app-autoscaler-release/issues/1013 - BeforeEach(func() { - fakeMetricServer.Close() - wsh.CloseWSConnection() - }) - It("fails to write envelops", func() { - Eventually(func() error { - err = wsHelper.Write(&testEnvelope) - return err - }).Should(HaveOccurred()) - Expect(err).To(MatchError(MatchRegexp(fmt.Sprintf(".*failed after %d retries:.*", testMaxCloseRetryCount)))) - }) - }) - }) -}) diff --git a/src/autoscaler/metricsgateway/metricsgateway_suite_test.go b/src/autoscaler/metricsgateway/metricsgateway_suite_test.go deleted file mode 100644 index 73c3cfa947..0000000000 --- a/src/autoscaler/metricsgateway/metricsgateway_suite_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package metricsgateway_test - -import ( - "io" - "log" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "google.golang.org/grpc/grpclog" - - "testing" -) - -func TestMetricsgateway(t *testing.T) { - grpclog.SetLoggerV2(grpclog.NewLoggerV2(GinkgoWriter, io.Discard, io.Discard)) - log.SetOutput(GinkgoWriter) - RegisterFailHandler(Fail) - RunSpecs(t, "Metricsgateway Suite") -} diff --git a/src/autoscaler/metricsgateway/nozzle.go b/src/autoscaler/metricsgateway/nozzle.go deleted file mode 100644 index c489b7e046..0000000000 --- a/src/autoscaler/metricsgateway/nozzle.go +++ /dev/null @@ -1,140 +0,0 @@ -package metricsgateway - -import ( - "context" - "crypto/tls" - "time" - - "code.cloudfoundry.org/go-loggregator/v9" - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "code.cloudfoundry.org/lager/v3" - "github.com/prometheus/client_golang/prometheus" - "google.golang.org/grpc" - "google.golang.org/grpc/keepalive" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/healthendpoint" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" -) - -const METRICS_FORWARDER_ORIGIN = "autoscaler_metrics_forwarder" - -var selectors = []*loggregator_v2.Selector{ - { - Message: &loggregator_v2.Selector_Gauge{ - Gauge: &loggregator_v2.GaugeSelector{}, - }, - }, - { - Message: &loggregator_v2.Selector_Timer{ - Timer: &loggregator_v2.TimerSelector{}, - }, - }, -} -var envelopeCounter = prometheus.CounterOpts{ - Namespace: "autoscaler", - Subsystem: "metricsgateway", - Name: "envelope_number_from_rlp", - Help: "the total envelopes number got from rlp", -} - -type Nozzle struct { - logger lager.Logger - rlpAddr string - tls *tls.Config - envelopChan chan *loggregator_v2.Envelope - index int - shardID string - getAppIDsFunc GetAppIDsFunc - context context.Context - cancelFunc context.CancelFunc - envelopeCounterCollector healthendpoint.CounterCollector -} - -func NewNozzle(logger lager.Logger, index int, shardID string, rlpAddr string, tls *tls.Config, envelopChan chan *loggregator_v2.Envelope, getAppIDsFunc GetAppIDsFunc, envelopeCounterCollector healthendpoint.CounterCollector) *Nozzle { - ctx, cancelFunc := context.WithCancel(context.Background()) - envelopeCounterCollector.AddCounters(envelopeCounter) - return &Nozzle{ - logger: logger.Session("Nozzle"), - index: index, - shardID: shardID, - rlpAddr: rlpAddr, - tls: tls, - envelopChan: envelopChan, - getAppIDsFunc: getAppIDsFunc, - context: ctx, - cancelFunc: cancelFunc, - envelopeCounterCollector: envelopeCounterCollector, - } -} - -func (n *Nozzle) Start() { - n.envelopeCounterCollector.AddCounters(envelopeCounter) - go n.streamMetrics() - n.logger.Info("started", lager.Data{"index": n.index}) -} - -func (n *Nozzle) Stop() { - n.cancelFunc() -} - -func (n *Nozzle) streamMetrics() { - streamConnector := loggregator.NewEnvelopeStreamConnector(n.rlpAddr, n.tls, - loggregator.WithEnvelopeStreamLogger(helpers.NewLoggregatorGRPCLogger(n.logger.Session("envelope_streamer"))), - loggregator.WithEnvelopeStreamConnectorDialOptions(grpc.WithKeepaliveParams(keepalive.ClientParameters{ - Time: 10 * time.Second, - Timeout: 30 * time.Second, - PermitWithoutStream: true, - })), - ) - rx := streamConnector.Stream(n.context, &loggregator_v2.EgressBatchRequest{ - ShardId: n.shardID, - Selectors: selectors, - }) - for { - select { - case <-n.context.Done(): - n.logger.Info("nozzle-stopped", lager.Data{"index": n.index}) - return - default: - } - envelops := rx() - if envelops != nil { - n.envelopeCounterCollector.Add(envelopeCounter, int64(len(envelops))) - n.filterEnvelopes(envelops) - } - } -} - -func (n *Nozzle) filterEnvelopes(envelops []*loggregator_v2.Envelope) { - for _, e := range envelops { - _, exist := n.getAppIDsFunc()[e.SourceId] - if exist { - switch e.GetMessage().(type) { - case *loggregator_v2.Envelope_Gauge: - if isContainerMetricEnvelope(e) { - n.logger.Debug("filter-envelopes-get-container-metrics", lager.Data{"index": n.index, "appID": e.SourceId, "message": e.Message}) - n.envelopChan <- e - } else if e.GetDeprecatedTags()["origin"].GetText() == METRICS_FORWARDER_ORIGIN { - n.logger.Debug("filter-envelopes-get-custom-metrics", lager.Data{"index": n.index, "appID": e.SourceId, "message": e.Message}) - n.envelopChan <- e - } - case *loggregator_v2.Envelope_Timer: - peerType := e.GetTags()["peer_type"] - if peerTypeFromDeprecatedTags := e.GetDeprecatedTags()["peer_type"]; peerType == "" && peerTypeFromDeprecatedTags != nil { - peerType = peerTypeFromDeprecatedTags.GetText() - } - - if e.GetTimer().GetName() == "http" && (peerType == "" || peerType == "Client") { - n.logger.Debug("filter-envelopes-get-httpstartstop", lager.Data{"index": n.index, "appID": e.SourceId, "message": e.Message}) - n.envelopChan <- e - } - } - } - } -} - -func isContainerMetricEnvelope(e *loggregator_v2.Envelope) bool { - _, memoryQuotaExists := e.GetGauge().GetMetrics()["memory_quota"] - _, cpuEntitlementExists := e.GetGauge().GetMetrics()["cpu_entitlement"] - return memoryQuotaExists || cpuEntitlementExists -} diff --git a/src/autoscaler/metricsgateway/nozzle_test.go b/src/autoscaler/metricsgateway/nozzle_test.go deleted file mode 100644 index 615b7317a6..0000000000 --- a/src/autoscaler/metricsgateway/nozzle_test.go +++ /dev/null @@ -1,345 +0,0 @@ -package metricsgateway_test - -import ( - "crypto/tls" - "path/filepath" - "time" - - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "code.cloudfoundry.org/lager/v3/lagertest" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gbytes" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/healthendpoint" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsgateway" - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" -) - -var _ = Describe("Nozzle", func() { - - var ( - testCertDir = "../../../test-certs" - serverCrtPath = filepath.Join(testCertDir, "reverselogproxy.crt") - serverKeyPath = filepath.Join(testCertDir, "reverselogproxy.key") - clientCrtPath = filepath.Join(testCertDir, "reverselogproxy_client.crt") - clientKeyPath = filepath.Join(testCertDir, "reverselogproxy_client.key") - caPath = filepath.Join(testCertDir, "autoscaler-ca.crt") - fakeLoggregator FakeEventProducer - testAppId = "test-app-id" - envelopes []*loggregator_v2.Envelope - nonContainerMetricGaugeEnvelope = loggregator_v2.Envelope{ - SourceId: "uaa", - Message: &loggregator_v2.Envelope_Gauge{ - Gauge: &loggregator_v2.Gauge{ - Metrics: map[string]*loggregator_v2.GaugeValue{ - "cpu": { - Unit: "percentage", - Value: 33.2, - }, - "memory": { - Unit: "bytes", - Value: 1000000000, - }, - }, - }, - }, - } - - containerMetricEnvelope1 = loggregator_v2.Envelope{ - SourceId: testAppId, - Message: &loggregator_v2.Envelope_Gauge{ - Gauge: &loggregator_v2.Gauge{ - Metrics: map[string]*loggregator_v2.GaugeValue{ - "cpu": { - Unit: "percentage", - Value: 20.5, - }, - "disk": { - Unit: "bytes", - Value: 3000000000, - }, - "memory": { - Unit: "bytes", - Value: 1000000000, - }, - "memory_quota": { - Unit: "bytes", - Value: 2000000000, - }, - }, - }, - }, - } - - // cpu_entitlement is emitted in a separate envelope https://docs.cloudfoundry.org/loggregator/container-metrics.html - containerMetricEnvelope2 = loggregator_v2.Envelope{ - SourceId: testAppId, - Message: &loggregator_v2.Envelope_Gauge{ - Gauge: &loggregator_v2.Gauge{ - Metrics: map[string]*loggregator_v2.GaugeValue{ - "cpu_entitlement": { - Unit: "percentage", - Value: 20.5, - }, - }, - }, - }, - } - - customMetricEnvelope = loggregator_v2.Envelope{ - SourceId: testAppId, - DeprecatedTags: map[string]*loggregator_v2.Value{ - "origin": { - Data: &loggregator_v2.Value_Text{ - Text: "autoscaler_metrics_forwarder", - }, - }, - }, - Message: &loggregator_v2.Envelope_Gauge{ - Gauge: &loggregator_v2.Gauge{ - Metrics: map[string]*loggregator_v2.GaugeValue{ - "queue_length": { - Unit: "number", - Value: 100, - }, - }, - }, - }, - } - - nonCustomMetricEnvelope = loggregator_v2.Envelope{ - SourceId: testAppId, - Tags: map[string]string{"origin": "other-origin"}, - Message: &loggregator_v2.Envelope_Gauge{ - Gauge: &loggregator_v2.Gauge{ - Metrics: map[string]*loggregator_v2.GaugeValue{ - "queue_length": { - Unit: "number", - Value: 100, - }, - }, - }, - }, - } - - httpStartStopEnvelope = loggregator_v2.Envelope{ - SourceId: testAppId, - DeprecatedTags: map[string]*loggregator_v2.Value{ - "peer_type": {Data: &loggregator_v2.Value_Text{Text: "Client"}}, - }, - Message: &loggregator_v2.Envelope_Timer{ - Timer: &loggregator_v2.Timer{ - Name: "http", - Start: 1542325492043447110, - Stop: 1542325492045491009, - }, - }, - } - serverHttpStartStopEnvelope = loggregator_v2.Envelope{ - SourceId: testAppId, - DeprecatedTags: map[string]*loggregator_v2.Value{ - "peer_type": {Data: &loggregator_v2.Value_Text{Text: "Server"}}, - }, - Message: &loggregator_v2.Envelope_Timer{ - Timer: &loggregator_v2.Timer{ - Name: "http", - Start: 1542325492043447110, - Stop: 1542325492045491009, - }, - }, - } - nonHttpStartStopTimerEnvelope = loggregator_v2.Envelope{ - SourceId: testAppId, - Message: &loggregator_v2.Envelope_Timer{ - Timer: &loggregator_v2.Timer{ - Name: "other_timer", - Start: 1542325492043307300, - Stop: 1542325492045818196, - }, - }, - } - - logger *lagertest.TestLogger - index = 0 - shardID = "autoscaler" - envelopChan chan *loggregator_v2.Envelope - getAppIDs metricsgateway.GetAppIDsFunc - nozzle *metricsgateway.Nozzle - rlpAddr string - tlsConf *tls.Config - appIDs map[string]bool - LogServerName string - envelopeCounterCollector = healthendpoint.NewCounterCollector() - ) - BeforeEach(func() { - envelopChan = make(chan *loggregator_v2.Envelope, 1000) - logger = lagertest.NewTestLogger("AppManager-test") - getAppIDs = func() map[string]bool { - return appIDs - } - envelopes = []*loggregator_v2.Envelope{ - &containerMetricEnvelope1, - } - LogServerName = "reverselogproxy" - - }) - AfterEach(func() { - fakeLoggregator.Stop() - - }) - Context("Start", func() { - JustBeforeEach(func() { - fakeLoggregator, err := NewFakeEventProducer(serverCrtPath, serverKeyPath, caPath, 500*time.Millisecond) - Expect(err).NotTo(HaveOccurred()) - fakeLoggregator.Start() - tlsConf, err = NewClientMutualTLSConfig(clientCrtPath, clientKeyPath, caPath, LogServerName) - Expect(err).NotTo(HaveOccurred()) - rlpAddr = fakeLoggregator.GetAddr() - fakeLoggregator.SetEnvelops(envelopes) - nozzle = metricsgateway.NewNozzle(logger, index, shardID, rlpAddr, tlsConf, envelopChan, getAppIDs, envelopeCounterCollector) - nozzle.Start() - }) - BeforeEach(func() { - appIDs = map[string]bool{testAppId: true} - }) - AfterEach(func() { - nozzle.Stop() - }) - Context("when there is err when connect to loggregator", func() { - BeforeEach(func() { - LogServerName = "wrong-server-name" - }) - It("should output the err", func() { - Eventually(logger.Buffer).Should(Say("Error connecting to Logs Provider")) - }) - }) - Context("when there is no app", func() { - BeforeEach(func() { - appIDs = map[string]bool{} - }) - It("should not accept any envelop", func() { - Consistently(envelopChan).ShouldNot(Receive()) - }) - }) - Context("when the app ID of the received envelope is not in policy database", func() { - BeforeEach(func() { - appIDs = map[string]bool{"another-test-app-id": true} - }) - It("should not accept the envelope", func() { - Consistently(envelopChan).ShouldNot(Receive()) - }) - }) - Context("when the gauge envelope is not a container metric", func() { - BeforeEach(func() { - envelopes = []*loggregator_v2.Envelope{ - &nonContainerMetricGaugeEnvelope, - } - }) - It("should not accept the envelope", func() { - Consistently(envelopChan).ShouldNot(Receive()) - }) - }) - Context("there is container metric envelope 1", func() { - BeforeEach(func() { - envelopes = []*loggregator_v2.Envelope{ - &containerMetricEnvelope1, - } - }) - It("should accept the envelope", func() { - Eventually(envelopChan).Should(Receive()) - - }) - }) - - Context("there is container metric envelope 2", func() { - BeforeEach(func() { - envelopes = []*loggregator_v2.Envelope{ - &containerMetricEnvelope2, - } - }) - It("should accept the envelope", func() { - Eventually(envelopChan).Should(Receive()) - - }) - }) - - Context("there is custom metric envelope", func() { - BeforeEach(func() { - envelopes = []*loggregator_v2.Envelope{ - &customMetricEnvelope, - } - }) - It("should accept the envelope", func() { - Eventually(envelopChan).Should(Receive()) - }) - }) - - Context("when the gauge envelope is not a custom metric", func() { - BeforeEach(func() { - envelopes = []*loggregator_v2.Envelope{ - &nonCustomMetricEnvelope, - } - }) - It("should not accept the envelope", func() { - Consistently(envelopChan).ShouldNot(Receive()) - }) - }) - - Context("when there is httpstartstop envelope", func() { - BeforeEach(func() { - envelopes = []*loggregator_v2.Envelope{ - &httpStartStopEnvelope, - } - }) - It("should accept the envelope", func() { - Eventually(envelopChan).Should(Receive()) - }) - }) - - Context("when there is a server httpstartstop timer envelope", func() { - BeforeEach(func() { - envelopes = []*loggregator_v2.Envelope{ - &serverHttpStartStopEnvelope, - } - }) - It("should not accept the envelope", func() { - Eventually(envelopChan).ShouldNot(Receive()) - }) - }) - - Context("when there is non httpstartstop timer envelope", func() { - BeforeEach(func() { - envelopes = []*loggregator_v2.Envelope{ - &nonHttpStartStopTimerEnvelope, - } - }) - It("should not accept the envelope", func() { - Eventually(envelopChan).ShouldNot(Receive()) - }) - }) - - }) - - Context("Stop", func() { - BeforeEach(func() { - appIDs = map[string]bool{testAppId: true} - fakeLoggregator, err := NewFakeEventProducer(serverCrtPath, serverKeyPath, caPath, 500*time.Millisecond) - Expect(err).NotTo(HaveOccurred()) - fakeLoggregator.Start() - tlsConf, err = NewClientMutualTLSConfig(clientCrtPath, clientKeyPath, caPath, "reverselogproxy") - Expect(err).NotTo(HaveOccurred()) - rlpAddr = fakeLoggregator.GetAddr() - fakeLoggregator.SetEnvelops(envelopes) - nozzle = metricsgateway.NewNozzle(logger, index, shardID, rlpAddr, tlsConf, envelopChan, getAppIDs, envelopeCounterCollector) - nozzle.Start() - Eventually(envelopChan).Should(Receive()) - nozzle.Stop() - Eventually(logger.Buffer).Should(Say("nozzle-stopped")) - }) - It("should not accept any envelop", func() { - Eventually(envelopChan).ShouldNot(Receive()) - }) - - }) -}) diff --git a/src/autoscaler/metricsserver/cmd/metricsserver/main.go b/src/autoscaler/metricsserver/cmd/metricsserver/main.go deleted file mode 100644 index 223ad424f8..0000000000 --- a/src/autoscaler/metricsserver/cmd/metricsserver/main.go +++ /dev/null @@ -1,167 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "os" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db/sqldb" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/healthendpoint" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsserver/collector" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsserver/config" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - - "code.cloudfoundry.org/clock" - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "code.cloudfoundry.org/lager/v3" - "github.com/prometheus/client_golang/prometheus" - "github.com/tedsuo/ifrit" - "github.com/tedsuo/ifrit/grouper" - "github.com/tedsuo/ifrit/sigmon" -) - -func main() { - var path string - flag.StringVar(&path, "c", "", "config file") - flag.Parse() - if path == "" { - _, _ = fmt.Fprintln(os.Stderr, "missing config file") - os.Exit(1) - } - - configFile, err := os.Open(path) - if err != nil { - _, _ = fmt.Fprintf(os.Stdout, "failed to open config file '%s' : %s\n", path, err.Error()) - os.Exit(1) - } - - var conf *config.Config - conf, err = config.LoadConfig(configFile) - if err != nil { - _, _ = fmt.Fprintf(os.Stdout, "failed to read config file '%s' : %s\n", path, err.Error()) - os.Exit(1) - } - _ = configFile.Close() - - err = conf.Validate() - if err != nil { - _, _ = fmt.Fprintf(os.Stdout, "failed to validate configuration : %s\n", err.Error()) - os.Exit(1) - } - - helpers.SetupOpenTelemetry() - - logger := helpers.InitLoggerFromConfig(&conf.Logging, "metricsserver") - msClock := clock.NewClock() - - var instanceMetricsDB db.InstanceMetricsDB - instanceMetricsDB, err = sqldb.NewInstanceMetricsSQLDB(conf.DB.InstanceMetricsDB, logger.Session("instancemetrics-db")) - if err != nil { - logger.Error("failed to connect instancemetrics database", err) - os.Exit(1) - } - defer func() { _ = instanceMetricsDB.Close() }() - - policyDb := sqldb.CreatePolicyDb(conf.DB.PolicyDB, logger) - defer func() { _ = policyDb.Close() }() - - metricsChan := make(chan *models.AppInstanceMetric, conf.Collector.MetricChannelSize) - - httpStatusCollector := healthendpoint.NewHTTPStatusCollector("autoscaler", "metricsserver") - promRegistry := prometheus.NewRegistry() - healthendpoint.RegisterCollectors(promRegistry, []prometheus.Collector{ - healthendpoint.NewDatabaseStatusCollector("autoscaler", "metricsserver", "instanceMetricsDB", instanceMetricsDB), - healthendpoint.NewDatabaseStatusCollector("autoscaler", "metricsserver", "policyDB", policyDb), - httpStatusCollector, - }, true, logger.Session("metricsserver-prometheus")) - - coll := collector.NewCollector(logger.Session("metricsserver-collector"), conf.Collector.RefreshInterval, conf.Collector.CollectInterval, conf.Collector.PersistMetrics, - conf.Collector.SaveInterval, conf.NodeIndex, len(conf.NodeAddrs), conf.Collector.MetricCacheSizePerApp, policyDb, instanceMetricsDB, msClock, metricsChan) - - envelopeChannels := make([]chan *loggregator_v2.Envelope, conf.Collector.EnvelopeProcessorCount) - for i := 0; i < conf.Collector.EnvelopeProcessorCount; i++ { - envelopeChannels[i] = make(chan *loggregator_v2.Envelope, conf.Collector.EnvelopeChannelSize) - } - - envelopeProcessors, err := createEnvelopeProcessors(logger, msClock, conf, envelopeChannels, metricsChan, coll.GetAppIDs) - if err != nil { - logger.Error("failed to create Envelope Processors", err) - os.Exit(1) - } - - metricsServer := ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error { - logger.Info("starting collector", lager.Data{"NodeIndex": conf.NodeIndex, "NodeAddrs": conf.NodeAddrs}) - - coll.Start() - for _, envelopeProcessor := range envelopeProcessors { - envelopeProcessor.Start() - } - - close(ready) - - <-signals - for _, envelopeProcessor := range envelopeProcessors { - envelopeProcessor.Stop() - } - coll.Stop() - return nil - }) - - wsServer, err := collector.NewWSServer(logger.Session("ws_server"), conf.Collector.TLS, conf.Collector.WSPort, - conf.Collector.WSKeepAliveTime, envelopeChannels, httpStatusCollector) - if err != nil { - logger.Error("failed to create web socket server", err) - os.Exit(1) - } - - healthServer, err := healthendpoint.NewServerWithBasicAuth(conf.Health, []healthendpoint.Checker{}, logger.Session("health-server"), promRegistry, time.Now) - if err != nil { - logger.Error("failed to create health server", err) - os.Exit(1) - } - - serverConfig := collector.FromConfig(conf) - - httpServer, err := collector.NewServer(logger.Session("http_server"), &serverConfig, coll.QueryMetrics, httpStatusCollector) - if err != nil { - logger.Error("failed to create http server", err) - os.Exit(1) - } - - members := grouper.Members{ - {"metric_server", metricsServer}, - {"ws_server", wsServer}, - {"http_server", httpServer}, - {"health_server", healthServer}, - } - monitor := ifrit.Invoke(sigmon.New(grouper.NewOrdered(os.Interrupt, members))) - - logger.Info("started") - - err = <-monitor.Wait() - if err != nil { - logger.Error("exit-with-failure", err) - os.Exit(1) - } - logger.Info("exit") -} - -func createEnvelopeProcessors( - logger lager.Logger, - clock clock.Clock, - conf *config.Config, - envelopeChan []chan *loggregator_v2.Envelope, - metricChan chan<- *models.AppInstanceMetric, - getAppIDs func() map[string]bool, -) ([]collector.EnvelopeProcessor, error) { - count := conf.Collector.EnvelopeProcessorCount - envelopeProcessors := make([]collector.EnvelopeProcessor, count) - - for i := 0; i < count; i++ { - envelopeProcessors[i] = collector.NewEnvelopeProcessor(logger, conf.Collector.CollectInterval, clock, i, count, envelopeChan[i], metricChan, getAppIDs) - } - return envelopeProcessors, nil -} diff --git a/src/autoscaler/metricsserver/cmd/metricsserver/metricsserver_suite_test.go b/src/autoscaler/metricsserver/cmd/metricsserver/metricsserver_suite_test.go deleted file mode 100644 index 0a37ed997d..0000000000 --- a/src/autoscaler/metricsserver/cmd/metricsserver/metricsserver_suite_test.go +++ /dev/null @@ -1,183 +0,0 @@ -package main_test - -import ( - "net/http" - "os" - "os/exec" - "path/filepath" - "testing" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" - - _ "github.com/go-sql-driver/mysql" - "github.com/jmoiron/sqlx" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/onsi/gomega/gbytes" - "github.com/onsi/gomega/gexec" - yaml "gopkg.in/yaml.v3" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsserver/config" -) - -var ( - msPath string - cfg config.Config - msPort int - healthport int - configFile *os.File - httpClient *http.Client - healthHttpClient *http.Client -) - -func TestMetricsServer(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "MetricsServer Suite") -} - -var _ = SynchronizedBeforeSuite(func() []byte { - ms, err := gexec.Build("code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsserver/cmd/metricsserver", "-race") - Expect(err).NotTo(HaveOccurred()) - - dbUrl := testhelpers.GetDbUrl() - database, err := db.GetConnection(dbUrl) - Expect(err).NotTo(HaveOccurred()) - - msDB, err := sqlx.Open(database.DriverName, database.DSN) - Expect(err).NotTo(HaveOccurred()) - - _, err = msDB.Exec("DELETE FROM appinstancemetrics") - Expect(err).NotTo(HaveOccurred()) - - _, err = msDB.Exec("DELETE from policy_json") - Expect(err).NotTo(HaveOccurred()) - - policy := ` - { - "instance_min_count": 1, - "instance_max_count": 5 - }` - query := msDB.Rebind("INSERT INTO policy_json(app_id, policy_json, guid) values(?, ?, ?)") - _, err = msDB.Exec(query, "an-app-id", policy, "1234") - Expect(err).NotTo(HaveOccurred()) - - err = msDB.Close() - Expect(err).NotTo(HaveOccurred()) - - return []byte(ms) -}, func(pathsByte []byte) { - msPath = string(pathsByte) - - testCertDir := "../../../../../test-certs" - msPort = 7000 + GinkgoParallelProcess() - healthport = 8000 + GinkgoParallelProcess() - cfg.Server.Port = msPort - - cfg.Health.Port = healthport - - cfg.Logging.Level = "info" - - dbUrl := testhelpers.GetDbUrl() - cfg.DB.InstanceMetricsDB = db.DatabaseConfig{ - URL: dbUrl, - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 10 * time.Second, - } - cfg.DB.PolicyDB = db.DatabaseConfig{ - URL: dbUrl, - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 10 * time.Second, - } - - cfg.Collector.CollectInterval = 10 * time.Second - cfg.Collector.RefreshInterval = 30 * time.Second - cfg.Collector.SaveInterval = 5 * time.Second - cfg.Collector.MetricCacheSizePerApp = 100 - cfg.Collector.TLS.KeyFile = filepath.Join(testCertDir, "metricserver.key") - cfg.Collector.TLS.CertFile = filepath.Join(testCertDir, "metricserver.crt") - cfg.Collector.TLS.CACertFile = filepath.Join(testCertDir, "autoscaler-ca.crt") - cfg.HttpClientTimeout = 10 * time.Second - cfg.NodeAddrs = []string{"localhost"} - cfg.NodeIndex = 0 - cfg.Collector.WSKeepAliveTime = 1 * time.Minute - - cfg.Collector.PersistMetrics = true - cfg.Collector.EnvelopeProcessorCount = 5 - cfg.Collector.EnvelopeChannelSize = 1000 - cfg.Collector.MetricChannelSize = 1000 - - cfg.Health.HealthCheckUsername = "metricsserverhealthcheckuser" - cfg.Health.HealthCheckPassword = "metricsserverhealthcheckpassword" - - configFile = writeConfig(&cfg) - httpClient = testhelpers.NewMetricServerClient() - healthHttpClient = &http.Client{} -}) - -var _ = SynchronizedAfterSuite(func() { - os.Remove(configFile.Name()) -}, func() { - gexec.CleanupBuildArtifacts() -}) - -func writeConfig(c *config.Config) *os.File { - cfg, err := os.CreateTemp("", "ms") - Expect(err).NotTo(HaveOccurred()) - defer cfg.Close() - - bytes, err := yaml.Marshal(c) - Expect(err).NotTo(HaveOccurred()) - - _, err = cfg.Write(bytes) - Expect(err).NotTo(HaveOccurred()) - - return cfg -} - -type MetricsServerRunner struct { - configPath string - startCheck string - Session *gexec.Session -} - -func NewMetricsServerRunner() *MetricsServerRunner { - return &MetricsServerRunner{ - configPath: configFile.Name(), - startCheck: "metricsserver.started", - } -} - -func (ms *MetricsServerRunner) Start() { - // #nosec G204 - msSession, err := gexec.Start(exec.Command( - msPath, - "-c", - ms.configPath, - ), - gexec.NewPrefixedWriter("\x1b[32m[o]\x1b[32m[mc]\x1b[0m ", GinkgoWriter), - gexec.NewPrefixedWriter("\x1b[91m[e]\x1b[32m[mc]\x1b[0m ", GinkgoWriter), - ) - Expect(err).NotTo(HaveOccurred()) - - if ms.startCheck != "" { - Eventually(msSession.Buffer, 2).Should(gbytes.Say(ms.startCheck)) - } - - ms.Session = msSession -} - -func (ms *MetricsServerRunner) Interrupt() { - if ms.Session != nil { - ms.Session.Interrupt().Wait(5 * time.Second) - } -} - -func (ms *MetricsServerRunner) KillWithFire() { - if ms.Session != nil { - ms.Session.Kill().Wait(5 * time.Second) - } -} diff --git a/src/autoscaler/metricsserver/cmd/metricsserver/metricsserver_test.go b/src/autoscaler/metricsserver/cmd/metricsserver/metricsserver_test.go deleted file mode 100644 index eab57b378a..0000000000 --- a/src/autoscaler/metricsserver/cmd/metricsserver/metricsserver_test.go +++ /dev/null @@ -1,212 +0,0 @@ -package main_test - -import ( - "fmt" - "io" - "net/http" - "os" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - . "github.com/onsi/gomega/gbytes" - . "github.com/onsi/gomega/gexec" -) - -var _ = Describe("MetricsServer", func() { - var ( - runner *MetricsServerRunner - ) - - BeforeEach(func() { - runner = NewMetricsServerRunner() - }) - - AfterEach(func() { - runner.KillWithFire() - }) - - Describe("MetricsServer configuration check", func() { - - Context("with a missing config file", func() { - BeforeEach(func() { - runner.startCheck = "" - runner.configPath = "bogus" - runner.Start() - }) - - It("fails with an error", func() { - Eventually(runner.Session).Should(Exit(1)) - Expect(runner.Session.Buffer()).To(Say("failed to open config file")) - }) - }) - - Context("with an invalid config file", func() { - BeforeEach(func() { - runner.startCheck = "" - badfile, err := os.CreateTemp("", "bad-ms-config") - Expect(err).NotTo(HaveOccurred()) - runner.configPath = badfile.Name() - err = os.WriteFile(runner.configPath, []byte("bogus"), os.ModePerm) - Expect(err).NotTo(HaveOccurred()) - runner.Start() - }) - - AfterEach(func() { - os.Remove(runner.configPath) - }) - - It("fails with an error", func() { - Eventually(runner.Session).Should(Exit(1)) - Expect(runner.Session.Buffer()).To(Say("failed to read config file")) - }) - }) - - Context("with missing configuration", func() { - BeforeEach(func() { - runner.startCheck = "" - missingConfig := cfg - missingConfig.Server.Port = 7000 + GinkgoParallelProcess() - missingConfig.Logging.Level = "debug" - missingConfig.Collector.EnvelopeChannelSize = 0 - runner.configPath = writeConfig(&missingConfig).Name() - runner.Start() - }) - - AfterEach(func() { - os.Remove(runner.configPath) - }) - - It("should fail validation", func() { - Eventually(runner.Session).Should(Exit(1)) - Expect(runner.Session.Buffer()).To(Say("failed to validate configuration")) - }) - }) - }) - - Describe("when interrupt is sent", func() { - BeforeEach(func() { - runner.Start() - }) - - It("should stop", func() { - runner.Session.Interrupt() - Eventually(runner.Session, 5).Should(Exit(0)) - }) - - }) - - Describe("MetricsServer REST API", func() { - Context("when a request for metrics history comes", func() { - BeforeEach(func() { - runner.Start() - }) - - It("returns with a 200", func() { - rsp, err := httpClient.Get(fmt.Sprintf("http://127.0.0.1:%d/v1/apps/an-app-id/metric_histories/a-metric-type", msPort)) - Expect(err).NotTo(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) - rsp.Body.Close() - }) - }) - - }) - - Describe("when Health server is ready to serve RESTful API", func() { - BeforeEach(func() { - - basicAuthConfig := cfg - basicAuthConfig.Health.HealthCheckUsername = "" - basicAuthConfig.Health.HealthCheckPassword = "" - runner.configPath = writeConfig(&basicAuthConfig).Name() - runner.Start() - - }) - Context("when a request to query health comes", func() { - It("returns with a 200", func() { - rsp, err := healthHttpClient.Get(fmt.Sprintf("http://127.0.0.1:%d", healthport)) - Expect(err).NotTo(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) - raw, _ := io.ReadAll(rsp.Body) - healthData := string(raw) - Expect(healthData).To(ContainSubstring("autoscaler_metricsserver_concurrent_http_request")) - Expect(healthData).To(ContainSubstring("autoscaler_metricsserver_policyDB")) - Expect(healthData).To(ContainSubstring("autoscaler_metricsserver_instanceMetricsDB")) - Expect(healthData).To(ContainSubstring("go_goroutines")) - Expect(healthData).To(ContainSubstring("go_memstats_alloc_bytes")) - rsp.Body.Close() - - }) - }) - }) - - Describe("when Health server is ready to serve RESTful API with basic Auth", func() { - BeforeEach(func() { - runner.Start() - }) - - Context("when username and password are incorrect for basic authentication during health check", func() { - It("should return 401", func() { - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) - - req.SetBasicAuth("wrongusername", "wrongpassword") - - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusUnauthorized)) - }) - }) - - Context("when username and password are correct for basic authentication during health check", func() { - It("should return 200", func() { - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) - - req.SetBasicAuth(cfg.Health.HealthCheckUsername, cfg.Health.HealthCheckPassword) - - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) - }) - }) - }) - - Describe("when Health server is ready to serve RESTful API with basic Auth", func() { - BeforeEach(func() { - runner.Start() - }) - - Context("when username and password are incorrect for basic authentication during health check", func() { - It("should return 401", func() { - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) - - req.SetBasicAuth("wrongusername", "wrongpassword") - - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusUnauthorized)) - }) - }) - - Context("when username and password are correct for basic authentication during health check", func() { - It("should return 200", func() { - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) - - req.SetBasicAuth(cfg.Health.HealthCheckUsername, cfg.Health.HealthCheckPassword) - - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) - }) - }) - }) - - //TODO : Add test cases for testing WebServer endpoints -}) diff --git a/src/autoscaler/metricsserver/collector/collector.go b/src/autoscaler/metricsserver/collector/collector.go deleted file mode 100644 index 3254612e2a..0000000000 --- a/src/autoscaler/metricsserver/collector/collector.go +++ /dev/null @@ -1,244 +0,0 @@ -package collector - -import ( - "fmt" - "sync" - "time" - - "golang.org/x/exp/maps" - "golang.org/x/net/context" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/collection" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - - "code.cloudfoundry.org/clock" - "code.cloudfoundry.org/lager/v3" -) - -type MetricQueryFunc func(appID string, instanceIndex int, name string, - start, end int64, order db.OrderType) ([]*models.AppInstanceMetric, error) - -type MetricCollector interface { - Start() - Stop() - GetAppIDs() map[string]bool - QueryMetrics(string, int, string, int64, int64, db.OrderType) ([]*models.AppInstanceMetric, error) - QueryMetricsWithLabels(string, int64, int64, db.OrderType, map[string]string) ([]*models.AppInstanceMetric, bool) -} - -type metricCollector struct { - logger lager.Logger - refreshInterval time.Duration - collectInterval time.Duration - PersistMetrics bool - saveInterval time.Duration - nodeNum int - nodeIndex int - metricCacheSizePerApp int - policyDb db.PolicyDB - instancemetricsDb db.InstanceMetricsDB - cclock clock.Clock - doneChan chan bool - doneSaveChan chan bool - ticker clock.Ticker - lock *sync.RWMutex // AppsIDsLock - metricsChan <-chan *models.AppInstanceMetric - appIDs map[string]bool // Node specific apps ids - metricCache map[string]*collection.TSDCache - mLock *sync.RWMutex // MetricsLock -} - -func NewCollector(logger lager.Logger, refreshInterval time.Duration, collectInterval time.Duration, PersistMetrics bool, saveInterval time.Duration, - nodeIndex, nodeNum int, metricCacheSizePerApp int, policyDb db.PolicyDB, instancemetricsDb db.InstanceMetricsDB, - cclock clock.Clock, metricsChan <-chan *models.AppInstanceMetric) *metricCollector { - return &metricCollector{ - refreshInterval: refreshInterval, - collectInterval: collectInterval, - saveInterval: saveInterval, - PersistMetrics: PersistMetrics, - nodeIndex: nodeIndex, - nodeNum: nodeNum, - metricCacheSizePerApp: metricCacheSizePerApp, - logger: logger, - policyDb: policyDb, - instancemetricsDb: instancemetricsDb, - cclock: cclock, - doneChan: make(chan bool), - doneSaveChan: make(chan bool), - lock: &sync.RWMutex{}, - metricsChan: metricsChan, - appIDs: map[string]bool{}, - metricCache: make(map[string]*collection.TSDCache), - mLock: &sync.RWMutex{}, - } -} - -func (c *metricCollector) Start() { - c.ticker = c.cclock.NewTicker(c.refreshInterval) - go c.startAppRefresh() - go c.saveMetrics() - - c.logger.Info("collector-started") -} - -func (c *metricCollector) startAppRefresh() { - for { - c.refreshApps() - select { - case <-c.doneChan: - return - case <-c.ticker.C(): - } - } -} - -// Remove metrics from cache that it should not track anymore, when a policy changes or the node does -// not take case or that app anymore -func (c *metricCollector) refreshApps() { - apps, err := c.policyDb.GetAppIds(context.Background()) - if err != nil { - c.logger.Error("refresh-apps", err) - return - } - - appIDs := map[string]bool{} - for id := range apps { - if helpers.FNVHash(id)%uint32(c.nodeNum) == uint32(c.nodeIndex) { - appIDs[id] = true - } - } - - c.lock.Lock() - c.appIDs = appIDs - c.lock.Unlock() - - c.mLock.Lock() - for id := range c.metricCache { - if _, exist := appIDs[id]; !exist { - delete(c.metricCache, id) - } - } - for id := range appIDs { - if _, exist := c.metricCache[id]; !exist { - c.metricCache[id] = collection.NewTSDCache(c.metricCacheSizePerApp) - } - } - c.mLock.Unlock() -} - -func (c *metricCollector) Stop() { - if c.ticker != nil { - c.ticker.Stop() - c.doneChan <- true - c.doneSaveChan <- true - } - c.logger.Info("collector-stopped") -} - -func (c *metricCollector) GetAppIDs() map[string]bool { - c.lock.RLock() - defer c.lock.RUnlock() - return maps.Clone(c.appIDs) -} - -func (c *metricCollector) QueryMetrics(appID string, instanceIndex int, name string, start, end int64, order db.OrderType) ([]*models.AppInstanceMetric, error) { - if end == -1 { - end = c.cclock.Now().UnixNano() - } - - c.mLock.RLock() - appCache, exist := c.metricCache[appID] - c.mLock.RUnlock() - - if exist { - labels := map[string]string{models.MetricLabelName: name} - if instanceIndex >= 0 { - labels[models.MetricLabelInstanceIndex] = fmt.Sprintf("%d", instanceIndex) - } - result, hit := appCache.Query(start, end+1, labels) - if hit || !c.PersistMetrics { - metrics := make([]*models.AppInstanceMetric, len(result)) - for index, tsd := range result { - if order == db.ASC { - metrics[index] = tsd.(*models.AppInstanceMetric) - } else { - metrics[len(result)-1-index] = tsd.(*models.AppInstanceMetric) - } - } - return metrics, nil - } - } - - if c.PersistMetrics { - return c.instancemetricsDb.RetrieveInstanceMetrics(appID, instanceIndex, name, start, end, order) - } - return []*models.AppInstanceMetric{}, nil -} - -func (c *metricCollector) QueryMetricsWithLabels(appID string, start, end int64, order db.OrderType, labels map[string]string) ([]*models.AppInstanceMetric, bool) { - if end == -1 { - end = c.cclock.Now().UnixNano() - } - - c.mLock.RLock() - appCache, exist := c.metricCache[appID] - c.mLock.RUnlock() - - if exist { - result, hit := appCache.Query(start, end+1, labels) - if hit || !c.PersistMetrics { - metrics := make([]*models.AppInstanceMetric, len(result)) - if order == db.ASC { - for index, tsd := range result { - metrics[index] = tsd.(*models.AppInstanceMetric) - } - } else { - for index, tsd := range result { - metrics[len(result)-1-index] = tsd.(*models.AppInstanceMetric) - } - } - return metrics, true - } - } - return nil, false -} - -func (c *metricCollector) saveMetrics() { - ticker := c.cclock.NewTicker(c.saveInterval) - metrics := []*models.AppInstanceMetric{} - for { - select { - case m := <-c.metricsChan: - c.saveMetricToCache(m) - if c.PersistMetrics { - metrics = append(metrics, m) - } - case <-ticker.C(): - if c.PersistMetrics && len(metrics) > 0 { - go func(db db.InstanceMetricsDB, metrics []*models.AppInstanceMetric) { - err := db.SaveMetricsInBulk(metrics) - if err != nil { - c.logger.Error("Failed to save metrics", err) - } - }(c.instancemetricsDb, metrics) - metrics = nil - } - case <-c.doneSaveChan: - return - } - } -} - -func (c *metricCollector) saveMetricToCache(m *models.AppInstanceMetric) bool { - c.mLock.RLock() - appCache := c.metricCache[m.AppId] - c.mLock.RUnlock() - - if appCache != nil { - appCache.Put(m) - return true - } - return false -} diff --git a/src/autoscaler/metricsserver/collector/collector_suite_test.go b/src/autoscaler/metricsserver/collector/collector_suite_test.go deleted file mode 100644 index b6fbb059e5..0000000000 --- a/src/autoscaler/metricsserver/collector/collector_suite_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package collector_test - -import ( - "fmt" - "net/url" - "strconv" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "github.com/tedsuo/ifrit/ginkgomon_v2" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsserver/collector" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - - "testing" - - "code.cloudfoundry.org/lager/v3" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/tedsuo/ifrit" -) - -var ( - serverProcess ifrit.Process - serverUrl *url.URL -) - -const ( - TestCollectInterval = 1 * time.Second - TestRefreshInterval = 2 * time.Second - TestSaveInterval = 2 * time.Second -) - -func TestCollector(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Collector Suite") -} - -var _ = BeforeSuite(func() { - - port := 1111 + GinkgoParallelProcess() - serverConf := &collector.ServerConfig{ - ServerConfig: helpers.ServerConfig{ - Port: port, - }, - NodeAddrs: []string{fmt.Sprintf("%s:%d", "localhost", port)}, - NodeIndex: 0, - } - - queryFunc := func(appID string, instanceIndex int, name string, start, end int64, order db.OrderType) ([]*models.AppInstanceMetric, error) { - return nil, nil - } - - httpStatusCollector := &fakes.FakeHTTPStatusCollector{} - - logger := lager.NewLogger("collector_suite_test") - logger.RegisterSink(lager.NewWriterSink(GinkgoWriter, lager.DEBUG)) - - httpServer, err := collector.NewServer(logger, serverConf, queryFunc, httpStatusCollector) - Expect(err).NotTo(HaveOccurred()) - - serverUrl, err = url.Parse("http://127.0.0.1:" + strconv.Itoa(port)) - Expect(err).ToNot(HaveOccurred()) - - serverProcess = ginkgomon_v2.Invoke(httpServer) -}) - -var _ = AfterSuite(func() { - ginkgomon_v2.Interrupt(serverProcess) -}) diff --git a/src/autoscaler/metricsserver/collector/collector_test.go b/src/autoscaler/metricsserver/collector/collector_test.go deleted file mode 100644 index b56ced877d..0000000000 --- a/src/autoscaler/metricsserver/collector/collector_test.go +++ /dev/null @@ -1,339 +0,0 @@ -package collector_test - -import ( - "context" - "errors" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsserver/collector" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - - "code.cloudfoundry.org/clock/fakeclock" - "code.cloudfoundry.org/lager/v3/lagertest" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/onsi/gomega/gbytes" -) - -var _ = Describe("Collector", func() { - - var ( - policyDb *fakes.FakePolicyDB - instanceMetricsDb *fakes.FakeInstanceMetricsDB - fclock *fakeclock.FakeClock - buffer *gbytes.Buffer - logger *lagertest.TestLogger - nodeNum int - nodeIndex int - metric1, metric2, metric3 *models.AppInstanceMetric - isMetricPersistencySuppported bool - metricCacheSizePerApp int - mc MetricCollector - metricsChan chan *models.AppInstanceMetric - ) - - BeforeEach(func() { - nodeNum = 1 - nodeIndex = 0 - policyDb = &fakes.FakePolicyDB{} - instanceMetricsDb = &fakes.FakeInstanceMetricsDB{} - logger = lagertest.NewTestLogger("collector-test") - buffer = logger.Buffer() - fclock = fakeclock.NewFakeClock(time.Now()) - isMetricPersistencySuppported = false - metricCacheSizePerApp = 3 - metricsChan = make(chan *models.AppInstanceMetric, 10) - metric1 = &models.AppInstanceMetric{ - AppId: "an-app-id", - InstanceIndex: 0, - CollectedAt: 2222, - Name: models.MetricNameThroughput, - Unit: models.UnitRPS, - Value: "3", - Timestamp: 2222, - } - metric2 = &models.AppInstanceMetric{ - AppId: "an-app-id", - InstanceIndex: 0, - CollectedAt: 3333, - Name: models.MetricNameThroughput, - Unit: models.UnitRPS, - Value: "5", - Timestamp: 3333, - } - - metric3 = &models.AppInstanceMetric{ - AppId: "an-app-id", - InstanceIndex: 0, - CollectedAt: 6666, - Name: models.MetricNameThroughput, - Unit: models.UnitRPS, - Value: "5", - Timestamp: 6666, - } - - }) - - Describe("Start", func() { - JustBeforeEach(func() { - mc = NewCollector(logger, TestRefreshInterval, TestCollectInterval, isMetricPersistencySuppported, TestSaveInterval, - nodeIndex, nodeNum, metricCacheSizePerApp, policyDb, instanceMetricsDb, fclock, metricsChan) - mc.Start() - }) - - AfterEach(func() { - mc.Stop() - }) - - It("refreshes the apps with given interval", func() { - Eventually(policyDb.GetAppIdsCallCount).Should(Equal(1), "policyDb.GetAppIds called in poll loop") - - fclock.Increment(TestRefreshInterval) - Eventually(policyDb.GetAppIdsCallCount).Should(Equal(2), "policyDb.GetAppIds called in poll loop") - - fclock.Increment(TestRefreshInterval) - Eventually(policyDb.GetAppIdsCallCount).Should(Equal(3), "policyDb.GetAppIds called in poll loop") - - }) - - Context("when getting apps from policy db succeeds", func() { - Context("when no apps in policy db", func() { - BeforeEach(func() { - policyDb.GetAppIdsReturns(make(map[string]bool), nil) - }) - - It("does nothing", func() { - Consistently(mc.GetAppIDs).Should(BeEmpty()) - - fclock.WaitForWatcherAndIncrement(TestRefreshInterval) - Consistently(mc.GetAppIDs).Should(BeEmpty()) - }) - }) - - Context("when there are apps in policy db", func() { - BeforeEach(func() { - policyDb.GetAppIdsStub = func(_ context.Context) (map[string]bool, error) { - switch policyDb.GetAppIdsCallCount() { - case 1: - return map[string]bool{"app-id-1": true, "app-id-3": true}, nil - case 2: - return map[string]bool{"app-id-2": true, "app-id-3": true}, nil - default: - return map[string]bool{"app-id-1": true, "app-id-2": true}, nil - } - } - }) - - It("retrieves the app IDs", func() { - Eventually(mc.GetAppIDs).Should(Equal(map[string]bool{"app-id-1": true, "app-id-3": true}), "mc.GetAppIds match after poll loop %d", policyDb.GetAppIdsCallCount()) - - fclock.WaitForWatcherAndIncrement(TestRefreshInterval) - Eventually(mc.GetAppIDs).Should(Equal(map[string]bool{"app-id-2": true, "app-id-3": true}), "mc.GetAppIds match after poll loop %d", policyDb.GetAppIdsCallCount()) - - fclock.Increment(TestRefreshInterval) - Eventually(mc.GetAppIDs).Should(Equal(map[string]bool{"app-id-1": true, "app-id-2": true}), "mc.GetAppIds match after poll loop %d", policyDb.GetAppIdsCallCount()) - }) - - It("The app list is cloned", func() { - Eventually(mc.GetAppIDs).Should(Equal(map[string]bool{"app-id-1": true, "app-id-3": true}), "mc.GetAppIds match after poll loop %d", policyDb.GetAppIdsCallCount()) - apps := mc.GetAppIDs() - delete(apps, "app-id-1") - Expect(mc.GetAppIDs()).To(Equal(map[string]bool{"app-id-1": true, "app-id-3": true})) - }) - - }) - - Context("when running with 3 nodes", func() { - BeforeEach(func() { - nodeNum = 3 - policyDb.GetAppIdsStub = func(_ context.Context) (map[string]bool, error) { - switch policyDb.GetAppIdsCallCount() { - case 1: - return map[string]bool{"app-id-1": true, "app-id-2": true, "app-id-3": true, "app-id-4": true}, nil - case 2: - return map[string]bool{"app-id-3": true, "app-id-4": true, "app-id-5": true, "app-id-6": true}, nil - default: - return map[string]bool{"app-id-5": true, "app-id-6": true, "app-id-7": true, "app-id-8": true}, nil - } - } - }) - Context("when current index is 0", func() { - BeforeEach(func() { - nodeIndex = 0 - }) - It("retrieves app shard 0", func() { - Eventually(mc.GetAppIDs).Should(Equal(map[string]bool{"app-id-3": true, "app-id-4": true})) - - fclock.WaitForWatcherAndIncrement(TestRefreshInterval) - Eventually(mc.GetAppIDs).Should(Equal(map[string]bool{"app-id-3": true, "app-id-4": true})) - - fclock.Increment(TestRefreshInterval) - Eventually(mc.GetAppIDs).Should(Equal(map[string]bool{"app-id-8": true})) - }) - }) - Context("when current index is 1", func() { - BeforeEach(func() { - nodeIndex = 1 - }) - It("retrieves app shard 1", func() { - Consistently(mc.GetAppIDs).Should(BeEmpty()) - - fclock.WaitForWatcherAndIncrement(TestRefreshInterval) - Eventually(mc.GetAppIDs).Should(Equal(map[string]bool{"app-id-5": true, "app-id-6": true})) - - fclock.Increment(TestRefreshInterval) - Consistently(mc.GetAppIDs).Should(Equal(map[string]bool{"app-id-5": true, "app-id-6": true})) - - }) - }) - Context("when current index is 2", func() { - BeforeEach(func() { - nodeIndex = 2 - }) - It("retrieves app shard 2", func() { - Eventually(mc.GetAppIDs).Should(Equal(map[string]bool{"app-id-1": true, "app-id-2": true})) - - fclock.WaitForWatcherAndIncrement(TestRefreshInterval) - Eventually(mc.GetAppIDs).Should(BeEmpty()) - - fclock.Increment(TestRefreshInterval) - Eventually(mc.GetAppIDs).Should(Equal(map[string]bool{"app-id-7": true})) - - }) - }) - - }) - - }) - - Context("when getting apps from policy db fails", func() { - BeforeEach(func() { - policyDb.GetAppIdsReturns(nil, errors.New("test collector error")) - }) - - It("logs the error", func() { - Eventually(buffer).Should(gbytes.Say("test collector error")) - Consistently(mc.GetAppIDs).Should(BeEmpty()) - - fclock.WaitForWatcherAndIncrement(TestRefreshInterval) - Eventually(buffer).Should(gbytes.Say("test collector error")) - Consistently(mc.GetAppIDs).Should(BeEmpty()) - }) - }) - - Context("when there are incoming metrics", func() { - BeforeEach(func() { - policyDb.GetAppIdsReturns(map[string]bool{"an-app-id": true}, nil) - Expect(metricsChan).Should(BeSent(metric1)) - Expect(metricsChan).Should(BeSent(metric2)) - Expect(metricsChan).Should(BeSent(metric3)) - - }) - Context("when metric persistency is not supported", func() { - It("does not save metrics to db", func() { - Consistently(instanceMetricsDb.SaveMetricCallCount).Should(BeZero()) - fclock.WaitForWatcherAndIncrement(TestSaveInterval) - Consistently(instanceMetricsDb.SaveMetricCallCount).Should(BeZero()) - }) - }) - Context("when metric persistency is supported", func() { - BeforeEach(func() { - isMetricPersistencySuppported = true - }) - It("save metrics to db", func() { - Consistently(instanceMetricsDb.SaveMetricCallCount).Should(BeZero()) - - fclock.WaitForWatcherAndIncrement(TestSaveInterval) - Eventually(instanceMetricsDb.SaveMetricsInBulkCallCount).Should(Equal(1)) - }) - }) - }) - Context("when there is no metrics", func() { - BeforeEach(func() { - policyDb.GetAppIdsReturns(map[string]bool{"an-app-id": true}, nil) - - }) - Context("when metric persistency is supported", func() { - BeforeEach(func() { - isMetricPersistencySuppported = true - }) - It("does not save metrics to db", func() { - Consistently(instanceMetricsDb.SaveMetricCallCount).Should(BeZero()) - - fclock.WaitForWatcherAndIncrement(TestSaveInterval) - Consistently(instanceMetricsDb.SaveMetricsInBulkCallCount).Should(BeZero()) - }) - }) - }) - }) - - Describe("QueryMetrics", func() { - JustBeforeEach(func() { - policyDb.GetAppIdsReturns(map[string]bool{"an-app-id": true}, nil) - mc = NewCollector(logger, TestRefreshInterval, TestCollectInterval, isMetricPersistencySuppported, TestSaveInterval, - nodeIndex, nodeNum, metricCacheSizePerApp, policyDb, instanceMetricsDb, fclock, metricsChan) - mc.Start() - Eventually(mc.GetAppIDs).Should(Equal(map[string]bool{"an-app-id": true})) - - Expect(metricsChan).To(BeSent(metric1)) - Expect(metricsChan).To(BeSent(metric2)) - Expect(metricsChan).To(BeSent(metric3)) - - }) - - AfterEach(func() { - mc.Stop() - }) - - Context("when cache hits", func() { - It("returns the metrics in cache", func() { - time.Sleep(100 * time.Millisecond) - result, err := mc.QueryMetrics("an-app-id", 0, models.MetricNameThroughput, 2222, 5555, db.ASC) - Expect(err).NotTo(HaveOccurred()) - Expect(result).To(Equal([]*models.AppInstanceMetric{metric1, metric2})) - Expect(instanceMetricsDb.RetrieveInstanceMetricsCallCount()).To(BeZero()) - }) - }) - - Context("when cache misses", func() { - BeforeEach(func() { - metricCacheSizePerApp = 2 - }) - Context("when metrics persistency is not supported", func() { - It("return empty result", func() { - time.Sleep(100 * time.Millisecond) - result, err := mc.QueryMetrics("an-app-id", 0, models.MetricNameThroughput, 0, 3331, db.ASC) - Expect(err).NotTo(HaveOccurred()) - Expect(result).To(BeEmpty()) - }) - }) - Context("when metrics persistency is supported", func() { - BeforeEach(func() { - isMetricPersistencySuppported = true - }) - It("retrieves metrics from database", func() { - time.Sleep(100 * time.Millisecond) - _, err := mc.QueryMetrics("an-app-id", 0, models.MetricNameThroughput, 0, 3331, db.ASC) - Expect(err).ShouldNot(HaveOccurred()) - Expect(instanceMetricsDb.RetrieveInstanceMetricsCallCount()).Should(Equal(1)) - }) - - Context("when retrieving metrics from db fails", func() { - BeforeEach(func() { - instanceMetricsDb.RetrieveInstanceMetricsReturns(nil, errors.New("an error")) - }) - It("returns the error", func() { - time.Sleep(100 * time.Millisecond) - result, err := mc.QueryMetrics("an-app-id", 0, models.MetricNameThroughput, 2222, 5555, db.ASC) - Expect(instanceMetricsDb.RetrieveInstanceMetricsCallCount()).Should(Equal(1)) - Expect(err).Should(HaveOccurred()) - Eventually(result).Should(BeNil()) - }) - }) - }) - }) - }) - -}) diff --git a/src/autoscaler/metricsserver/collector/config.go b/src/autoscaler/metricsserver/collector/config.go deleted file mode 100644 index aa30608133..0000000000 --- a/src/autoscaler/metricsserver/collector/config.go +++ /dev/null @@ -1,34 +0,0 @@ -package collector - -import ( - "fmt" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsserver/config" -) - -// ServerConfig a type to wrap both the config.ServerConfig and config.Config data. -type ServerConfig struct { - helpers.ServerConfig `yaml:",inline"` - NodeAddrs []string `yaml:"node_addrs"` - NodeIndex int `yaml:"node_index"` -} - -func FromConfig(config *config.Config) ServerConfig { - return ServerConfig{ - ServerConfig: helpers.ServerConfig{ - Port: config.Server.Port, - TLS: config.Server.TLS, - }, - NodeAddrs: withPort(config.NodeAddrs, config.Server.Port), - NodeIndex: config.NodeIndex, - } -} - -func withPort(addrs []string, port int) []string { - serverNodeAddrs := make([]string, len(addrs)) - for i, n := range addrs { - serverNodeAddrs[i] = fmt.Sprintf("%s:%d", n, port) - } - return serverNodeAddrs -} diff --git a/src/autoscaler/metricsserver/collector/envelope_processor.go b/src/autoscaler/metricsserver/collector/envelope_processor.go deleted file mode 100644 index 20d92351dd..0000000000 --- a/src/autoscaler/metricsserver/collector/envelope_processor.go +++ /dev/null @@ -1,136 +0,0 @@ -package collector - -import ( - "sync" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/envelopeprocessor" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - - "code.cloudfoundry.org/clock" - loggregator_v2 "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "code.cloudfoundry.org/lager/v3" -) - -type EnvelopeProcessor interface { - Start() - Stop() - IsCacheEmpty() bool -} - -type envelopeProcessor struct { - logger lager.Logger - collectInterval time.Duration - doneChan chan bool - clock clock.Clock - processorIndex int - numProcessors int - httpStartStopEnvelopes map[string][]*loggregator_v2.Envelope // appID map of envelopes - envelopeChan <-chan *loggregator_v2.Envelope - metricChan chan<- *models.AppInstanceMetric - getAppIDs func() map[string]bool - mu sync.RWMutex -} - -func NewEnvelopeProcessor( - logger lager.Logger, - collectInterval time.Duration, - clock clock.Clock, - processorIndex, - numProcessors int, - envelopeChan <-chan *loggregator_v2.Envelope, - metricChan chan<- *models.AppInstanceMetric, - getAppIDs func() map[string]bool, -) *envelopeProcessor { - return &envelopeProcessor{ - logger: logger, - collectInterval: collectInterval, - doneChan: make(chan bool), - clock: clock, - httpStartStopEnvelopes: map[string][]*loggregator_v2.Envelope{}, - processorIndex: processorIndex, - numProcessors: numProcessors, - envelopeChan: envelopeChan, - metricChan: metricChan, - getAppIDs: getAppIDs, - } -} - -func (ep *envelopeProcessor) Start() { - go ep.processEvents() - ep.logger.Info("envelop-processor-started", lager.Data{"processor-index": ep.processorIndex, "processor-num": ep.numProcessors}) -} - -func (ep *envelopeProcessor) Stop() { - ep.doneChan <- true -} - -func (ep *envelopeProcessor) processEvents() { - ticker := ep.clock.NewTicker(ep.collectInterval) - for { - select { - case <-ep.doneChan: - ticker.Stop() - return - - case e := <-ep.envelopeChan: - metrics := ep.getAppInstanceMetrics(e) - for i := range metrics { - ep.metricChan <- &metrics[i] - } - - case <-ticker.C(): - ep.processHttpStartStopMetrics() - } - } -} - -func (ep *envelopeProcessor) IsCacheEmpty() bool { - ep.mu.RLock() - defer ep.mu.RUnlock() - return len(ep.httpStartStopEnvelopes) == 0 -} - -func (ep *envelopeProcessor) getAppInstanceMetrics(e *loggregator_v2.Envelope) []models.AppInstanceMetric { - switch e.GetMessage().(type) { - case *loggregator_v2.Envelope_Gauge: - appInstanceMetrics, _ := envelopeprocessor.GetGaugeInstanceMetrics([]*loggregator_v2.Envelope{e}, ep.clock.Now().UnixNano()) - return appInstanceMetrics - case *loggregator_v2.Envelope_Timer: - ep.cacheHttpStartStopEnvelop(e) - return []models.AppInstanceMetric{} - default: - return []models.AppInstanceMetric{} - } -} -func (ep *envelopeProcessor) isMetrissrvRespForApp(appID string) bool { - return helpers.FNVHash(appID)%uint32(ep.numProcessors) == uint32(ep.processorIndex) -} - -func (ep *envelopeProcessor) processHttpStartStopMetrics() { - ep.mu.Lock() - defer ep.mu.Unlock() - ep.logger.Debug("compute-and-save-metrics", lager.Data{"message": "start to compute and save metrics"}) - - for appID := range ep.getAppIDs() { - if !ep.isMetrissrvRespForApp(appID) { // skip apps we are not responsible for - continue - } - - metrics := envelopeprocessor.GetHttpStartStopInstanceMetrics(ep.httpStartStopEnvelopes[appID], appID, ep.clock.Now().UnixNano(), ep.collectInterval) - for i := range metrics { - ep.metricChan <- &metrics[i] - } - } - - //TODO we should grab the old and replace with and empty one in the beginning of the function to relieve lock contention. - //i.e. save the old one and replease with empty then release the lock and process the envelopes. - ep.httpStartStopEnvelopes = map[string][]*loggregator_v2.Envelope{} -} - -func (ep *envelopeProcessor) cacheHttpStartStopEnvelop(e *loggregator_v2.Envelope) { - ep.mu.Lock() - defer ep.mu.Unlock() - ep.httpStartStopEnvelopes[e.SourceId] = append(ep.httpStartStopEnvelopes[e.SourceId], e) -} diff --git a/src/autoscaler/metricsserver/collector/envelope_processor_test.go b/src/autoscaler/metricsserver/collector/envelope_processor_test.go deleted file mode 100644 index 587e4f26ba..0000000000 --- a/src/autoscaler/metricsserver/collector/envelope_processor_test.go +++ /dev/null @@ -1,361 +0,0 @@ -package collector_test - -import ( - "time" - - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsserver/collector" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - - "code.cloudfoundry.org/clock/fakeclock" - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "code.cloudfoundry.org/lager/v3/lagertest" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("EnvelopeProcessor", func() { - - var ( - logger *lagertest.TestLogger - fclock *fakeclock.FakeClock - processorIndex int - numProcessors int - envelopeChan chan *loggregator_v2.Envelope - metricChan chan *models.AppInstanceMetric - getAppIDs func() map[string]bool - processor EnvelopeProcessor - ) - - BeforeEach(func() { - numProcessors = 1 - processorIndex = 0 - logger = lagertest.NewTestLogger("collector-test") - fclock = fakeclock.NewFakeClock(time.Now()) - envelopeChan = make(chan *loggregator_v2.Envelope, 10) - metricChan = make(chan *models.AppInstanceMetric, 10) - getAppIDs = func() map[string]bool { - return map[string]bool{} - } - }) - - Describe("Start", func() { - JustBeforeEach(func() { - processor = NewEnvelopeProcessor(logger, TestCollectInterval, fclock, - processorIndex, numProcessors, envelopeChan, metricChan, getAppIDs) - processor.Start() - // make sure the envelopes have been processed - time.Sleep(2 * time.Second) - }) - AfterEach(func() { - processor.Stop() - }) - - Context("processing container metrics", func() { - BeforeEach(func() { - Expect(envelopeChan).Should(BeSent(GenerateContainerMetrics("test-app-id", "0", 10.2, 10*1024*1024, 20*1024*1024, 1111))) - Expect(envelopeChan).Should(BeSent(GenerateContainerMetrics("test-app-id", "1", 10.6, 10.2*1024*1024, 20*1024*1024, 1111))) - - }) - It("sends standard app instance metrics to channel", func() { - Eventually(metricChan).Should(Receive(Equal(&models.AppInstanceMetric{ - AppId: "test-app-id", - InstanceIndex: 0, - CollectedAt: fclock.Now().UnixNano(), - Name: models.MetricNameMemoryUsed, - Unit: models.UnitMegaBytes, - Value: "10", - Timestamp: 1111, - }))) - - Eventually(metricChan).Should(Receive(Equal(&models.AppInstanceMetric{ - AppId: "test-app-id", - InstanceIndex: 0, - CollectedAt: fclock.Now().UnixNano(), - Name: models.MetricNameMemoryUtil, - Unit: models.UnitPercentage, - Value: "50", - Timestamp: 1111, - }))) - - Eventually(metricChan).Should(Receive(Equal(&models.AppInstanceMetric{ - AppId: "test-app-id", - InstanceIndex: 0, - CollectedAt: fclock.Now().UnixNano(), - Name: models.MetricNameCPU, - Unit: models.UnitPercentage, - Value: "11", - Timestamp: 1111, - }))) - - Eventually(metricChan).Should(Receive(Equal(&models.AppInstanceMetric{ - AppId: "test-app-id", - InstanceIndex: 1, - CollectedAt: fclock.Now().UnixNano(), - Name: models.MetricNameMemoryUsed, - Unit: models.UnitMegaBytes, - Value: "11", - Timestamp: 1111, - }))) - - Eventually(metricChan).Should(Receive(Equal(&models.AppInstanceMetric{ - AppId: "test-app-id", - InstanceIndex: 1, - CollectedAt: fclock.Now().UnixNano(), - Name: models.MetricNameMemoryUtil, - Unit: models.UnitPercentage, - Value: "51", - Timestamp: 1111, - }))) - - Eventually(metricChan).Should(Receive(Equal(&models.AppInstanceMetric{ - AppId: "test-app-id", - InstanceIndex: 1, - CollectedAt: fclock.Now().UnixNano(), - Name: models.MetricNameCPU, - Unit: models.UnitPercentage, - Value: "11", - Timestamp: 1111, - }))) - - }) - }) - - Context("processing custom metrics", func() { - BeforeEach(func() { - Expect(envelopeChan).Should(BeSent(GenerateCustomMetrics("test-app-id", "1", "custom_name", "custom_unit", 11.88, 1111))) - Expect(envelopeChan).Should(BeSent(GenerateCustomMetrics("test-app-id", "0", "custom_name", "custom_unit", 11.08, 1111))) - }) - It("sends standard app instance metrics to channel", func() { - Eventually(metricChan).Should(Receive(Equal(&models.AppInstanceMetric{ - AppId: "test-app-id", - InstanceIndex: 1, - CollectedAt: fclock.Now().UnixNano(), - Name: "custom_name", - Unit: "custom_unit", - Value: "12", - Timestamp: 1111, - }))) - - Eventually(metricChan).Should(Receive(Equal(&models.AppInstanceMetric{ - AppId: "test-app-id", - InstanceIndex: 0, - CollectedAt: fclock.Now().UnixNano(), - Name: "custom_name", - Unit: "custom_unit", - Value: "12", - Timestamp: 1111, - }))) - - }) - }) - - Context("Computing throughput and responsetime", func() { - BeforeEach(func() { - getAppIDs = func() map[string]bool { - return map[string]bool{"test-app-id": true} - } - Expect(envelopeChan).Should(BeSent(GenerateHttpStartStopEnvelope("test-app-id", "0", 10*1000*1000, 25*1000*1000, 1111))) - Expect(envelopeChan).Should(BeSent(GenerateHttpStartStopEnvelope("test-app-id", "1", 10*1000*1000, 30*1000*1000, 1111))) - Expect(envelopeChan).Should(BeSent(GenerateHttpStartStopEnvelope("test-app-id", "0", 20*1000*1000, 30*1000*1000, 1111))) - Expect(envelopeChan).Should(BeSent(GenerateHttpStartStopEnvelope("test-app-id", "1", 20*1000*1000, 50*1000*1000, 1111))) - Expect(envelopeChan).Should(BeSent(GenerateHttpStartStopEnvelope("test-app-id", "1", 20*1000*1000, 30*1000*1000, 1111))) - }) - - It("sends throughput and responsetime metric to channel", func() { - Consistently(metricChan).ShouldNot(Receive()) - - fclock.WaitForWatcherAndIncrement(TestCollectInterval) - metrics := []*models.AppInstanceMetric{} - - Eventually(func() int { - metrics = append(metrics, <-metricChan) - return len(metrics) - }).Should(Equal(4)) - Expect(metrics).To(ContainElement(&models.AppInstanceMetric{ - AppId: "test-app-id", - InstanceIndex: 0, - CollectedAt: fclock.Now().UnixNano(), - Name: models.MetricNameThroughput, - Unit: models.UnitRPS, - Value: "2", - Timestamp: fclock.Now().UnixNano(), - })) - - Expect(metrics).To(ContainElement(&models.AppInstanceMetric{ - AppId: "test-app-id", - InstanceIndex: 0, - CollectedAt: fclock.Now().UnixNano(), - Name: models.MetricNameResponseTime, - Unit: models.UnitMilliseconds, - Value: "13", - Timestamp: fclock.Now().UnixNano(), - })) - - Expect(metrics).To(ContainElement(&models.AppInstanceMetric{ - AppId: "test-app-id", - InstanceIndex: 1, - CollectedAt: fclock.Now().UnixNano(), - Name: models.MetricNameThroughput, - Unit: models.UnitRPS, - Value: "3", - Timestamp: fclock.Now().UnixNano(), - })) - - Expect(metrics).To(ContainElement(&models.AppInstanceMetric{ - AppId: "test-app-id", - InstanceIndex: 1, - CollectedAt: fclock.Now().UnixNano(), - Name: models.MetricNameResponseTime, - Unit: models.UnitMilliseconds, - Value: "20", - Timestamp: fclock.Now().UnixNano(), - })) - - Expect(processor.IsCacheEmpty()).To(BeTrue()) - }) - - Context("when the app does not have http requests", func() { - BeforeEach(func() { - getAppIDs = func() map[string]bool { - return map[string]bool{"another-test-app-id": true} - } - }) - - Context("when the current processor is responsible for the app", func() { - BeforeEach(func() { - numProcessors = 3 - processorIndex = 0 - }) - - It("sends send 0 throughput and responsetime metric", func() { - Consistently(metricChan).ShouldNot(Receive()) - fclock.WaitForWatcherAndIncrement(TestCollectInterval) - Eventually(metricChan).Should(Receive(Equal(&models.AppInstanceMetric{ - AppId: "another-test-app-id", - InstanceIndex: 0, - CollectedAt: fclock.Now().UnixNano(), - Name: models.MetricNameThroughput, - Unit: models.UnitRPS, - Value: "0", - Timestamp: fclock.Now().UnixNano(), - }))) - Eventually(metricChan).Should(Receive(Equal(&models.AppInstanceMetric{ - AppId: "another-test-app-id", - InstanceIndex: 0, - CollectedAt: fclock.Now().UnixNano(), - Name: models.MetricNameResponseTime, - Unit: models.UnitMilliseconds, - Value: "0", - Timestamp: fclock.Now().UnixNano(), - }))) - - }) - }) - - Context("when the current processor is not responsible for the app", func() { - BeforeEach(func() { - numProcessors = 3 - processorIndex = 1 - }) - - It("sends nothing", func() { - Consistently(metricChan).ShouldNot(Receive()) - fclock.WaitForWatcherAndIncrement(TestCollectInterval) - Consistently(metricChan).ShouldNot(Receive()) - }) - }) - - }) - - }) - - }) - Describe("Stop", func() { - BeforeEach(func() { - getAppIDs = func() map[string]bool { - return map[string]bool{"test-app-id": true} - } - processor = NewEnvelopeProcessor(logger, TestCollectInterval, fclock, processorIndex, numProcessors, envelopeChan, metricChan, getAppIDs) - processor.Start() - }) - It("Stops processing the envelops", func() { - Expect(envelopeChan).Should(BeSent(GenerateContainerMetrics("test-app-id", "0", 10.2, 10*1024*1024, 20*1024*1024, 1111))) - Eventually(metricChan).Should(Receive()) - Eventually(metricChan).Should(Receive()) - Eventually(metricChan).Should(Receive()) - - Expect(envelopeChan).Should(BeSent(GenerateHttpStartStopEnvelope("test-app-id", "0", 10*1000*1000, 20*1000*1000, 1111))) - fclock.WaitForWatcherAndIncrement(TestCollectInterval) - Eventually(metricChan).Should(Receive()) - Eventually(metricChan).Should(Receive()) - - processor.Stop() - fclock.Increment(TestCollectInterval) - Consistently(metricChan).ShouldNot(Receive()) - Consistently(metricChan).ShouldNot(Receive()) - }) - }) -}) - -func GenerateContainerMetrics(sourceID, instance string, cpu, memory, memoryQuota float64, timestamp int64) *loggregator_v2.Envelope { - e := &loggregator_v2.Envelope{ - SourceId: sourceID, - InstanceId: instance, - Message: &loggregator_v2.Envelope_Gauge{ - Gauge: &loggregator_v2.Gauge{ - Metrics: map[string]*loggregator_v2.GaugeValue{ - "cpu": { - Unit: "percentage", - Value: cpu, - }, - "memory": { - Unit: "bytes", - Value: memory, - }, - "memory_quota": { - Unit: "bytes", - Value: memoryQuota, - }, - }, - }, - }, - Timestamp: timestamp, - } - return e -} - -func GenerateCustomMetrics(sourceID, instance, name, unit string, value float64, timestamp int64) *loggregator_v2.Envelope { - e := &loggregator_v2.Envelope{ - SourceId: sourceID, - InstanceId: instance, - Message: &loggregator_v2.Envelope_Gauge{ - Gauge: &loggregator_v2.Gauge{ - Metrics: map[string]*loggregator_v2.GaugeValue{ - name: { - Unit: unit, - Value: value, - }, - }, - }, - }, - Timestamp: timestamp, - } - return e -} - -func GenerateHttpStartStopEnvelope(sourceID, instance string, start, stop, timestamp int64) *loggregator_v2.Envelope { - e := &loggregator_v2.Envelope{ - SourceId: sourceID, - InstanceId: instance, - Message: &loggregator_v2.Envelope_Timer{ - Timer: &loggregator_v2.Timer{ - Name: "http", - Start: start, - Stop: stop, - }, - }, - Timestamp: timestamp, - } - return e -} diff --git a/src/autoscaler/metricsserver/collector/metric_handler.go b/src/autoscaler/metricsserver/collector/metric_handler.go deleted file mode 100644 index 4aa559b4c4..0000000000 --- a/src/autoscaler/metricsserver/collector/metric_handler.go +++ /dev/null @@ -1,164 +0,0 @@ -package collector - -import ( - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers/handlers" - "code.cloudfoundry.org/lager/v3" - - "encoding/json" - "fmt" - "net/http" - "strconv" - "strings" -) - -type MetricHandler struct { - nodeIndex int - nodeAdds []string - logger lager.Logger - queryFunc MetricQueryFunc -} - -func NewMetricHandler(logger lager.Logger, nodeIndex int, nodeAdds []string, query MetricQueryFunc) *MetricHandler { - return &MetricHandler{ - nodeIndex: nodeIndex, - nodeAdds: nodeAdds, - logger: logger, - queryFunc: query, - } -} - -func (h *MetricHandler) GetMetricHistories(w http.ResponseWriter, r *http.Request, vars map[string]string) { - appId := vars["appid"] - if r.URL.Query()["referer"] == nil { - shardID := helpers.FNVHash(appId) % uint32(len(h.nodeAdds)) - if shardID != uint32(h.nodeIndex) { - params := r.URL.Query() - params.Add("referer", h.nodeAdds[h.nodeIndex]) - newURL := "https://" + h.nodeAdds[shardID] + r.URL.Path + "?" + params.Encode() - h.logger.Debug("get-metric-histories-redirect", lager.Data{"appId": appId, "newURL": newURL}) - http.Redirect(w, r, newURL, http.StatusFound) - return - } - } - metricType := vars["metrictype"] - instanceIndexParam := r.URL.Query()["instanceindex"] - startParam := r.URL.Query()["start"] - endParam := r.URL.Query()["end"] - orderParam := r.URL.Query()["order"] - - h.logger.Debug("get-metric-histories", lager.Data{"appId": appId, "metrictype": metricType, "instanceIndex": instanceIndexParam, "start": startParam, "end": endParam, "order": orderParam}) - - var err error - start := int64(0) - end := int64(-1) - order := db.ASC - instanceIndex := -1 - - if len(instanceIndexParam) == 1 { - instanceIndex, err = strconv.Atoi(instanceIndexParam[0]) - if err != nil { - h.logger.Error("get-metric-histories-parse-instance-index", err, lager.Data{"instanceIndex": instanceIndexParam}) - handlers.WriteJSONResponse(w, http.StatusBadRequest, models.ErrorResponse{ - Code: "Bad-Request", - Message: "Error parsing instanceIndex"}) - return - } - if instanceIndex < 0 { - h.logger.Error("get-metric-histories-parse-instance-index", err, lager.Data{"instanceIndex": instanceIndexParam}) - handlers.WriteJSONResponse(w, http.StatusBadRequest, models.ErrorResponse{ - Code: "Bad-Request", - Message: "InstanceIndex must be greater than or equal to 0"}) - return - } - } else if len(instanceIndexParam) > 1 { - h.logger.Error("get-metric-histories-get-instance-index", err, lager.Data{"instanceIndex": instanceIndexParam}) - handlers.WriteJSONResponse(w, http.StatusBadRequest, models.ErrorResponse{ - Code: "Bad-Request", - Message: "Incorrect instanceIndex parameter in query string"}) - return - } - - if len(startParam) == 1 { - start, err = strconv.ParseInt(startParam[0], 10, 64) - if err != nil { - h.logger.Error("get-metric-histories-parse-start-time", err, lager.Data{"start": startParam}) - handlers.WriteJSONResponse(w, http.StatusBadRequest, models.ErrorResponse{ - Code: "Bad-Request", - Message: "Error parsing start time"}) - return - } - } else if len(startParam) > 1 { - h.logger.Error("get-metric-histories-get-start-time", err, lager.Data{"start": startParam}) - handlers.WriteJSONResponse(w, http.StatusBadRequest, models.ErrorResponse{ - Code: "Bad-Request", - Message: "Incorrect start parameter in query string"}) - return - } - - if len(endParam) == 1 { - end, err = strconv.ParseInt(endParam[0], 10, 64) - if err != nil { - h.logger.Error("get-metric-histories-parse-end-time", err, lager.Data{"end": endParam}) - handlers.WriteJSONResponse(w, http.StatusBadRequest, models.ErrorResponse{ - Code: "Bad-Request", - Message: "Error parsing end time"}) - return - } - } else if len(endParam) > 1 { - h.logger.Error("get-metric-histories-get-end-time", err, lager.Data{"end": endParam}) - handlers.WriteJSONResponse(w, http.StatusBadRequest, models.ErrorResponse{ - Code: "Bad-Request", - Message: "Incorrect end parameter in query string"}) - return - } - - if len(orderParam) == 1 { - orderStr := strings.ToUpper(orderParam[0]) - if orderStr == db.DESCSTR { - order = db.DESC - } else if orderStr == db.ASCSTR { - order = db.ASC - } else { - h.logger.Error("get-metric-histories-parse-order", err, lager.Data{"order": orderParam}) - handlers.WriteJSONResponse(w, http.StatusBadRequest, models.ErrorResponse{ - Code: "Bad-Request", - Message: fmt.Sprintf("Incorrect order parameter in query string, the value can only be %s or %s", db.ASCSTR, db.DESCSTR), - }) - return - } - } else if len(orderParam) > 1 { - h.logger.Error("get-metric-histories-parse-order", err, lager.Data{"order": orderParam}) - handlers.WriteJSONResponse(w, http.StatusBadRequest, models.ErrorResponse{ - Code: "Bad-Request", - Message: "Incorrect order parameter in query string"}) - return - } - - mtrcs, err := h.queryFunc(appId, instanceIndex, metricType, start, end, order) - if err != nil { - h.logger.Error("get-metric-histories-retrieve-metrics", err, lager.Data{"appId": appId, "metrictype": metricType, "instanceIndex": instanceIndex, "start": start, "end": end, "order": order}) - handlers.WriteJSONResponse(w, http.StatusInternalServerError, models.ErrorResponse{ - Code: "Internal-Server-Error", - Message: "Error getting instance metric histories"}) - return - } - - var body []byte - body, err = json.Marshal(mtrcs) - if err != nil { - h.logger.Error("get-metric-histories-marshal", err, lager.Data{"appId": appId, "metrictype": metricType, "metrics": mtrcs}) - - handlers.WriteJSONResponse(w, http.StatusInternalServerError, models.ErrorResponse{ - Code: "Internal-Server-Error", - Message: "Error marshal metric histories"}) - return - } - _, err = w.Write(body) - if err != nil { - h.logger.Error("unable-to-write-body", err, lager.Data{"appId": appId, "metrictype": metricType, "metrics": mtrcs}) - } -} diff --git a/src/autoscaler/metricsserver/collector/metric_handler_test.go b/src/autoscaler/metricsserver/collector/metric_handler_test.go deleted file mode 100644 index 2d699c6ab4..0000000000 --- a/src/autoscaler/metricsserver/collector/metric_handler_test.go +++ /dev/null @@ -1,426 +0,0 @@ -package collector_test - -import ( - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsserver/collector" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - - "code.cloudfoundry.org/lager/v3" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "encoding/json" - "errors" - "fmt" - "net/http" - "net/http/httptest" - "net/url" -) - -var testUrlMetricHistories = "http://metrics_collector_hostname/v1/apps/an-app-id/metric_histories/a-metric-type" - -var _ = Describe("MetricHandler", func() { - - var ( - handler *MetricHandler - nodeIndex int - nodeAddrs []string - resp *httptest.ResponseRecorder - req *http.Request - err, queryErr error - metric1 models.AppInstanceMetric - metric2 models.AppInstanceMetric - metrics []*models.AppInstanceMetric - appID string - instanceIndex int - metricName string - startTime, endTime int64 - queryOrder db.OrderType - ) - - BeforeEach(func() { - nodeIndex = 0 - nodeAddrs = []string{"localhost:8080"} - resp = httptest.NewRecorder() - queryErr = nil - }) - - Describe("GetMetricHistory", func() { - JustBeforeEach(func() { - logger := lager.NewLogger("handler-test") - queryFunc := func(id string, index int, name string, - start, end int64, order db.OrderType) ([]*models.AppInstanceMetric, error) { - appID = id - instanceIndex = index - metricName = name - startTime = start - endTime = end - queryOrder = order - return metrics, queryErr - } - handler = NewMetricHandler(logger, nodeIndex, nodeAddrs, queryFunc) - handler.GetMetricHistories(resp, req, map[string]string{"appid": "an-app-id", "metrictype": "a-metric-type"}) - }) - - Context("when request query string is invalid", func() { - - Context("when there are multiple instanceindex pararmeters in query string", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?instanceindex=123&instanceindex=231", nil) - Expect(err).ToNot(HaveOccurred()) - - }) - - It("returns 400", func() { - Expect(resp.Code).To(Equal(http.StatusBadRequest)) - - errJson := &models.ErrorResponse{} - err = json.Unmarshal(resp.Body.Bytes(), errJson) - - Expect(err).ToNot(HaveOccurred()) - Expect(errJson).To(Equal(&models.ErrorResponse{ - Code: "Bad-Request", - Message: "Incorrect instanceIndex parameter in query string", - })) - }) - }) - - Context("when instanceindex is not a number", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?instanceindex=abc", nil) - Expect(err).ToNot(HaveOccurred()) - - }) - - It("returns 400", func() { - Expect(resp.Code).To(Equal(http.StatusBadRequest)) - - errJson := &models.ErrorResponse{} - err = json.Unmarshal(resp.Body.Bytes(), errJson) - - Expect(err).ToNot(HaveOccurred()) - Expect(errJson).To(Equal(&models.ErrorResponse{ - Code: "Bad-Request", - Message: "Error parsing instanceIndex", - })) - }) - }) - - Context("when instanceindex is smaller than 0", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?instanceindex=-1", nil) - Expect(err).ToNot(HaveOccurred()) - - }) - - It("returns 400", func() { - Expect(resp.Code).To(Equal(http.StatusBadRequest)) - - errJson := &models.ErrorResponse{} - err = json.Unmarshal(resp.Body.Bytes(), errJson) - - Expect(err).ToNot(HaveOccurred()) - Expect(errJson).To(Equal(&models.ErrorResponse{ - Code: "Bad-Request", - Message: "InstanceIndex must be greater than or equal to 0", - })) - }) - }) - - Context("when there are multiple start pararmeters in query string", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?start=123&start=231", nil) - Expect(err).ToNot(HaveOccurred()) - - }) - - It("returns 400", func() { - Expect(resp.Code).To(Equal(http.StatusBadRequest)) - - errJson := &models.ErrorResponse{} - err = json.Unmarshal(resp.Body.Bytes(), errJson) - - Expect(err).ToNot(HaveOccurred()) - Expect(errJson).To(Equal(&models.ErrorResponse{ - Code: "Bad-Request", - Message: "Incorrect start parameter in query string", - })) - }) - }) - - Context("when start time is not a number", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?start=abc", nil) - Expect(err).ToNot(HaveOccurred()) - - }) - - It("returns 400", func() { - Expect(resp.Code).To(Equal(http.StatusBadRequest)) - - errJson := &models.ErrorResponse{} - err = json.Unmarshal(resp.Body.Bytes(), errJson) - - Expect(err).ToNot(HaveOccurred()) - Expect(errJson).To(Equal(&models.ErrorResponse{ - Code: "Bad-Request", - Message: "Error parsing start time", - })) - }) - }) - - Context("when there are multiple end parameters in query string", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?end=123&end=231", nil) - Expect(err).ToNot(HaveOccurred()) - - }) - - It("returns 400", func() { - Expect(resp.Code).To(Equal(http.StatusBadRequest)) - - errJson := &models.ErrorResponse{} - err = json.Unmarshal(resp.Body.Bytes(), errJson) - - Expect(err).ToNot(HaveOccurred()) - Expect(errJson).To(Equal(&models.ErrorResponse{ - Code: "Bad-Request", - Message: "Incorrect end parameter in query string", - })) - }) - }) - - Context("when end time is not a number", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?end=abc", nil) - Expect(err).ToNot(HaveOccurred()) - - }) - - It("returns 400", func() { - Expect(resp.Code).To(Equal(http.StatusBadRequest)) - - errJson := &models.ErrorResponse{} - err = json.Unmarshal(resp.Body.Bytes(), errJson) - - Expect(err).ToNot(HaveOccurred()) - Expect(errJson).To(Equal(&models.ErrorResponse{ - Code: "Bad-Request", - Message: "Error parsing end time", - })) - }) - }) - - Context("when there are multiple order parameters in query string", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?order=asc&order=asc", nil) - Expect(err).ToNot(HaveOccurred()) - - }) - - It("returns 400", func() { - Expect(resp.Code).To(Equal(http.StatusBadRequest)) - - errJson := &models.ErrorResponse{} - err = json.Unmarshal(resp.Body.Bytes(), errJson) - - Expect(err).ToNot(HaveOccurred()) - Expect(errJson).To(Equal(&models.ErrorResponse{ - Code: "Bad-Request", - Message: "Incorrect order parameter in query string", - })) - }) - }) - - Context("when order value is invalid", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?order=not-order-type", nil) - Expect(err).ToNot(HaveOccurred()) - - }) - - It("returns 400", func() { - Expect(resp.Code).To(Equal(http.StatusBadRequest)) - - errJson := &models.ErrorResponse{} - err = json.Unmarshal(resp.Body.Bytes(), errJson) - - Expect(err).ToNot(HaveOccurred()) - Expect(errJson).To(Equal(&models.ErrorResponse{ - Code: "Bad-Request", - Message: fmt.Sprintf("Incorrect order parameter in query string, the value can only be %s or %s", db.ASCSTR, db.DESCSTR), - })) - }) - }) - - }) - - Context("when request query string is valid", func() { - Context("when start,end and order are all in query string", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?instanceindex=0&start=123&end=567&order=desc", nil) - Expect(err).ToNot(HaveOccurred()) - }) - - It("queries metrics with the given start, end and order ", func() { - Expect(instanceIndex).To(Equal(0)) - Expect(appID).To(Equal("an-app-id")) - Expect(metricName).To(Equal("a-metric-type")) - Expect(startTime).To(Equal(int64(123))) - Expect(endTime).To(Equal(int64(567))) - Expect(queryOrder).To(Equal(db.DESC)) - }) - - }) - - Context("when there is no instanceindex in query string", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?start=123&end=567&order=desc", nil) - Expect(err).ToNot(HaveOccurred()) - }) - - It("queries metrics with instanceindex -1", func() { - Expect(instanceIndex).To(Equal(-1)) - }) - - }) - - Context("when there is no start time in query string", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?instanceindex=0&end=123&order=desc", nil) - Expect(err).ToNot(HaveOccurred()) - }) - - It("queries metrics with start time 0", func() { - Expect(startTime).To(Equal(int64(0))) - }) - - }) - - Context("when there is no end time in query string", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?instanceindex=0&start=123&order=desc", nil) - Expect(err).ToNot(HaveOccurred()) - }) - - It("queries metrics with end time -1 ", func() { - Expect(endTime).To(Equal(int64(-1))) - }) - - }) - - Context("when there is no order in query string", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?instanceindex=0&start=123&end=567", nil) - Expect(err).ToNot(HaveOccurred()) - }) - - It("queries metrics with end time -1 ", func() { - Expect(queryOrder).To(Equal(db.ASC)) - }) - - }) - - Context("when query succeeds", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?instanceindex=0&start=123&end=567&order=desc", nil) - Expect(err).ToNot(HaveOccurred()) - - metric1 = models.AppInstanceMetric{ - AppId: "an-app-id", - InstanceIndex: 0, - CollectedAt: 111, - Name: "a-metric-type", - Unit: "metric-unit", - Value: "12345678", - Timestamp: 345, - } - - metric2 = models.AppInstanceMetric{ - AppId: "an-app-id", - InstanceIndex: 0, - CollectedAt: 222, - Name: "a-metric-type", - Unit: "metric-unit", - Value: "87654321", - Timestamp: 456, - } - metrics = []*models.AppInstanceMetric{&metric1, &metric2} - }) - - It("returns 200 with metrics in message body", func() { - Expect(resp.Code).To(Equal(http.StatusOK)) - result := &[]models.AppInstanceMetric{} - err = json.Unmarshal(resp.Body.Bytes(), result) - - Expect(err).ToNot(HaveOccurred()) - Expect(*result).To(Equal([]models.AppInstanceMetric{metric1, metric2})) - }) - - }) - - Context("when requesting app metrics in other shard", func() { - BeforeEach(func() { - nodeIndex = 1 - nodeAddrs = []string{"localhost:8080", "localhost:9090"} - }) - Context("when it is not a redirect", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?instanceindex=0&start=123&end=567&order=desc", nil) - Expect(err).ToNot(HaveOccurred()) - }) - - It("Redirects to the right shard", func() { - Expect(resp.Code).To(Equal(http.StatusFound)) - Expect(resp.Header()["Location"]).To(HaveLen(1)) - redirecURL, err := url.Parse(resp.Header()["Location"][0]) - Expect(err).NotTo(HaveOccurred()) - Expect(redirecURL.Scheme).To(Equal("https")) - Expect(redirecURL.Host).To(Equal("localhost:8080")) - Expect(redirecURL.Path).To(Equal("/v1/apps/an-app-id/metric_histories/a-metric-type")) - Expect(redirecURL.Query()).To(BeEquivalentTo(map[string][]string{ - "instanceindex": {"0"}, - "start": {"123"}, - "end": {"567"}, - "order": {"desc"}, - "referer": {"localhost:9090"}, - })) - }) - }) - - Context("when it is a redirect from other node", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?instanceindex=0&start=123&end=567&order=desc&referer=another-node", nil) - Expect(err).ToNot(HaveOccurred()) - }) - - It("does not redirect again", func() { - Expect(resp.Code).To(Equal(http.StatusOK)) - }) - }) - }) - - Context("when query fails", func() { - BeforeEach(func() { - req, err = http.NewRequest(http.MethodGet, testUrlMetricHistories+"?instanceindex=0&start=123&end=567&order=desc", nil) - Expect(err).ToNot(HaveOccurred()) - queryErr = errors.New("query error") - }) - - It("returns 500", func() { - Expect(resp.Code).To(Equal(http.StatusInternalServerError)) - - errJson := &models.ErrorResponse{} - err = json.Unmarshal(resp.Body.Bytes(), errJson) - - Expect(err).ToNot(HaveOccurred()) - Expect(errJson).To(Equal(&models.ErrorResponse{ - Code: "Internal-Server-Error", - Message: "Error getting instance metric histories", - })) - }) - }) - - }) - }) -}) diff --git a/src/autoscaler/metricsserver/collector/server.go b/src/autoscaler/metricsserver/collector/server.go deleted file mode 100644 index c8d5c4361c..0000000000 --- a/src/autoscaler/metricsserver/collector/server.go +++ /dev/null @@ -1,38 +0,0 @@ -package collector - -import ( - "net/http" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/healthendpoint" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/routes" - "go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux" - - "code.cloudfoundry.org/lager/v3" - "github.com/gorilla/mux" - "github.com/tedsuo/ifrit" -) - -type VarsFunc func(w http.ResponseWriter, r *http.Request, vars map[string]string) - -func (vh VarsFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - vh(w, r, vars) -} - -func NewServer(logger lager.Logger, serverConfig *ServerConfig, query MetricQueryFunc, httpStatusCollector healthendpoint.HTTPStatusCollector) (ifrit.Runner, error) { - mh := NewMetricHandler(logger, serverConfig.NodeIndex, serverConfig.NodeAddrs, query) - httpStatusCollectMiddleware := healthendpoint.NewHTTPStatusCollectMiddleware(httpStatusCollector) - - r := routes.MetricsCollectorRoutes() - r.Use(otelmux.Middleware("metricsserver")) - r.Use(httpStatusCollectMiddleware.Collect) - r.Get(routes.GetMetricHistoriesRouteName).Handler(VarsFunc(mh.GetMetricHistories)) - - httpServerConfig := helpers.ServerConfig{ - Port: serverConfig.Port, - TLS: serverConfig.TLS, - } - - return helpers.NewHTTPServer(logger, httpServerConfig, r) -} diff --git a/src/autoscaler/metricsserver/collector/server_test.go b/src/autoscaler/metricsserver/collector/server_test.go deleted file mode 100644 index dca13bf0d2..0000000000 --- a/src/autoscaler/metricsserver/collector/server_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package collector_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "net/http" -) - -const TestPathMetricHistories = "/v1/apps/an-app-id/metric_histories/a-metric-type" - -var _ = Describe("Server", func() { - var ( - rsp *http.Response - err error - ) - - Context("when retrieving metrics history", func() { - BeforeEach(func() { - serverUrl.Path = TestPathMetricHistories - }) - - JustBeforeEach(func() { - rsp, err = http.Get(serverUrl.String()) - }) - - It("should return 200", func() { - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) - rsp.Body.Close() - }) - }) - - Context("when requesting the wrong path", func() { - BeforeEach(func() { - serverUrl.Path = "/not-exist-path" - }) - - JustBeforeEach(func() { - rsp, err = http.Get(serverUrl.String()) - }) - - It("should return 404", func() { - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusNotFound)) - rsp.Body.Close() - }) - }) - - Context("when using wrong method to retrieve metric histories", func() { - BeforeEach(func() { - serverUrl.Path = TestPathMetricHistories - }) - - JustBeforeEach(func() { - rsp, err = http.Post(serverUrl.String(), "garbage", nil) - }) - - It("should return 405", func() { - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusMethodNotAllowed)) - rsp.Body.Close() - }) - }) - -}) diff --git a/src/autoscaler/metricsserver/collector/ws_handler.go b/src/autoscaler/metricsserver/collector/ws_handler.go deleted file mode 100644 index 767d8191f7..0000000000 --- a/src/autoscaler/metricsserver/collector/ws_handler.go +++ /dev/null @@ -1,142 +0,0 @@ -package collector - -import ( - "net/http" - "sync" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "code.cloudfoundry.org/lager/v3" - "github.com/golang/protobuf/proto" //nolint - "github.com/gorilla/websocket" -) - -type wsMessageHandler struct { - logger lager.Logger - envelopeChannels []chan *loggregator_v2.Envelope - keepAlive time.Duration - lock *sync.Mutex -} - -func NewWSMessageHandler(logger lager.Logger, envelopeChannels []chan *loggregator_v2.Envelope, keepAlive time.Duration) *wsMessageHandler { - return &wsMessageHandler{ - logger: logger, - envelopeChannels: envelopeChannels, - keepAlive: keepAlive, - lock: &sync.Mutex{}, - } -} - -func (h *wsMessageHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - upgrader := websocket.Upgrader{ - CheckOrigin: func(*http.Request) bool { return true }, - } - - ws, err := upgrader.Upgrade(rw, r, nil) - if err != nil { - h.logger.Error("serve-websocket-upgrade", err) - return - } - defer ws.Close() - - closeCode, closeMessage := h.runWebsocketUntilClosed(ws) - err = ws.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(closeCode, closeMessage), time.Time{}) - if err != nil { - h.logger.Error("serve-websocket-close", err) - return - } -} - -func (h *wsMessageHandler) runWebsocketUntilClosed(ws *websocket.Conn) (closeCode int, closeMessage string) { - keepAliveExpired := make(chan struct{}) - clientWentAway := make(chan struct{}) - - go func() { - for { - _, bytes, err := ws.ReadMessage() - if err != nil { - h.logger.Error("run-websocket-read-message", err) - close(clientWentAway) - return - } - var envelop loggregator_v2.Envelope - err = proto.Unmarshal(bytes, &envelop) - if err != nil { - h.logger.Error("run-websocket-unmarshal", err) - } - h.envelopeChannels[helpers.FNVHash(envelop.GetSourceId())%uint32(len(h.envelopeChannels))] <- &envelop - } - }() - - go func() { - NewKeepAlive(h.lock, ws, h.keepAlive).Run() - close(keepAliveExpired) - }() - - closeCode = websocket.CloseNormalClosure - closeMessage = "" - for { - select { - case <-clientWentAway: - return closeCode, closeMessage - case <-keepAliveExpired: - closeCode = websocket.ClosePolicyViolation - closeMessage = "Client did not respond to ping before keep-alive timeout expired." - return closeCode, closeMessage - } - } -} - -type KeepAlive struct { - lock *sync.Mutex - conn *websocket.Conn - pongChan chan struct{} - keepAliveInterval time.Duration -} - -func NewKeepAlive(lock *sync.Mutex, conn *websocket.Conn, keepAliveInterval time.Duration) *KeepAlive { - return &KeepAlive{ - lock: lock, - conn: conn, - pongChan: make(chan struct{}, 1), - keepAliveInterval: keepAliveInterval, - } -} - -func (k *KeepAlive) Run() { - k.lock.Lock() - k.conn.SetPongHandler(k.pongHandler) - k.lock.Unlock() - - defer func() { - k.lock.Lock() - k.conn.SetPongHandler(nil) - k.lock.Unlock() - }() - - timeout := time.NewTimer(k.keepAliveInterval) - for { - err := k.conn.WriteControl(websocket.PingMessage, nil, time.Time{}) - if err != nil { - return - } - timeout.Reset(k.keepAliveInterval) - select { - case <-k.pongChan: - time.Sleep(k.keepAliveInterval / 2) - continue - case <-timeout.C: - return - } - } -} - -func (k *KeepAlive) pongHandler(string) error { - select { - case k.pongChan <- struct{}{}: - default: - } - return nil -} diff --git a/src/autoscaler/metricsserver/collector/ws_server.go b/src/autoscaler/metricsserver/collector/ws_server.go deleted file mode 100644 index b788575750..0000000000 --- a/src/autoscaler/metricsserver/collector/ws_server.go +++ /dev/null @@ -1,24 +0,0 @@ -package collector - -import ( - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/healthendpoint" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "code.cloudfoundry.org/lager/v3" - "github.com/tedsuo/ifrit" -) - -func NewWSServer(logger lager.Logger, tls models.TLSCerts, port int, keepAlive time.Duration, envelopeChannels []chan *loggregator_v2.Envelope, httpStatusCollector healthendpoint.HTTPStatusCollector) (ifrit.Runner, error) { - wsHandler := NewWSMessageHandler(logger.Session("ws_handler"), envelopeChannels, keepAlive) - - httpServerConfig := helpers.ServerConfig{ - Port: port, - TLS: tls, - } - - return helpers.NewHTTPServer(logger, httpServerConfig, wsHandler) -} diff --git a/src/autoscaler/metricsserver/config/config.go b/src/autoscaler/metricsserver/config/config.go deleted file mode 100644 index 85a5b671f3..0000000000 --- a/src/autoscaler/metricsserver/config/config.go +++ /dev/null @@ -1,157 +0,0 @@ -package config - -import ( - "fmt" - "io" - "strings" - "time" - - yaml "gopkg.in/yaml.v3" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" -) - -const ( - DefaultLoggingLevel = "info" - DefaultHttpClientTimeout = 5 * time.Second - DefaultWSKeepAliveTime = 1 * time.Minute - DefaultWSPort = 8443 - DefaultRefreshInterval = 60 * time.Second - DefaultCollectInterval = 30 * time.Second - DefaultSaveInterval = 5 * time.Second - DefaultMetricCacheSizePerApp = 1000 - DefaultIsMetricsPersistencySupported = true - DefaultEnvelopeProcessorCount = 5 - DefaultEnvelopeChannelSize = 1000 - DefaultMetricChannelSize = 1000 - DefaultHTTPServerPort = 8080 - DefaultHealthPort = 8081 -) - -type DBConfig struct { - PolicyDB db.DatabaseConfig `yaml:"policy_db"` - InstanceMetricsDB db.DatabaseConfig `yaml:"instance_metrics_db"` -} - -type CollectorConfig struct { - WSPort int `yaml:"port"` - WSKeepAliveTime time.Duration `yaml:"keep_alive_time"` - TLS models.TLSCerts `yaml:"tls"` - RefreshInterval time.Duration `yaml:"refresh_interval"` - CollectInterval time.Duration `yaml:"collect_interval"` - SaveInterval time.Duration `yaml:"save_interval"` - MetricCacheSizePerApp int `yaml:"metric_cache_size_per_app"` - PersistMetrics bool `yaml:"persist_metrics"` - EnvelopeProcessorCount int `yaml:"envelope_processor_count"` - EnvelopeChannelSize int `yaml:"envelope_channel_size"` - MetricChannelSize int `yaml:"metric_channel_size"` -} - -type Config struct { - Logging helpers.LoggingConfig `yaml:"logging"` - HttpClientTimeout time.Duration `yaml:"http_client_timeout"` - NodeAddrs []string `yaml:"node_addrs"` - NodeIndex int `yaml:"node_index"` - DB DBConfig `yaml:"db"` - Collector CollectorConfig `yaml:"collector"` - Server helpers.ServerConfig `yaml:"server"` - Health helpers.HealthConfig `yaml:"health"` -} - -func LoadConfig(reader io.Reader) (*Config, error) { - conf := &Config{ - Logging: helpers.LoggingConfig{ - Level: DefaultLoggingLevel, - }, - HttpClientTimeout: DefaultHttpClientTimeout, - Health: helpers.HealthConfig{ - ServerConfig: helpers.ServerConfig{ - Port: DefaultHealthPort, - }, - }, - Collector: CollectorConfig{ - WSPort: DefaultWSPort, - WSKeepAliveTime: DefaultWSKeepAliveTime, - RefreshInterval: DefaultRefreshInterval, - CollectInterval: DefaultCollectInterval, - SaveInterval: DefaultSaveInterval, - MetricCacheSizePerApp: DefaultMetricCacheSizePerApp, - PersistMetrics: DefaultIsMetricsPersistencySupported, - EnvelopeProcessorCount: DefaultEnvelopeProcessorCount, - EnvelopeChannelSize: DefaultEnvelopeChannelSize, - MetricChannelSize: DefaultMetricChannelSize, - }, - Server: helpers.ServerConfig{ - Port: DefaultHTTPServerPort, - }, - } - - dec := yaml.NewDecoder(reader) - dec.KnownFields(true) - err := dec.Decode(conf) - - if err != nil { - return nil, err - } - - conf.Logging.Level = strings.ToLower(conf.Logging.Level) - return conf, nil -} - -func (c *Config) Validate() error { - if c.HttpClientTimeout <= time.Duration(0) { - return fmt.Errorf("Configuration error: http_client_timeout is less-equal than 0") - } - - if (c.NodeIndex >= len(c.NodeAddrs)) || (c.NodeIndex < 0) { - return fmt.Errorf("Configuration error: node_index out of range") - } - - if c.DB.PolicyDB.URL == "" { - return fmt.Errorf("Configuration error: db.policy_db.url is empty") - } - - if c.DB.InstanceMetricsDB.URL == "" { - return fmt.Errorf("Configuration error: db.instance_metrics_db.url is empty") - } - - if c.Collector.WSKeepAliveTime == time.Duration(0) { - return fmt.Errorf("Configuration error: keep_alive_time is less-equal than 0") - } - - if c.Collector.CollectInterval == time.Duration(0) { - return fmt.Errorf("Configuration error: collector.collect_interval is 0") - } - - if c.Collector.RefreshInterval == time.Duration(0) { - return fmt.Errorf("Configuration error: collector.refresh_interval is 0") - } - - if c.Collector.SaveInterval == time.Duration(0) { - return fmt.Errorf("Configuration error: collector.save_interval is 0") - } - - if c.Collector.MetricCacheSizePerApp <= 0 { - return fmt.Errorf("Configuration error: invalid collector.metric_cache_size_per_app") - } - - if c.Collector.EnvelopeProcessorCount <= 0 { - return fmt.Errorf("Configuration error: envelope_processor_count is less-equal than 0") - } - - if c.Collector.EnvelopeChannelSize <= 0 { - return fmt.Errorf("Configuration error: envelope_channel_size is less-equal than 0") - } - - if c.Collector.MetricChannelSize <= 0 { - return fmt.Errorf("Configuration error: metric_channel_size is less-equal than 0") - } - - if err := c.Health.Validate(); err != nil { - return err - } - - return nil -} diff --git a/src/autoscaler/metricsserver/config/config_suite_test.go b/src/autoscaler/metricsserver/config/config_suite_test.go deleted file mode 100644 index 4d4d240f67..0000000000 --- a/src/autoscaler/metricsserver/config/config_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package config_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "testing" -) - -func TestConfig(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Config Suite") -} diff --git a/src/autoscaler/metricsserver/config/config_test.go b/src/autoscaler/metricsserver/config/config_test.go deleted file mode 100644 index 7dca81895e..0000000000 --- a/src/autoscaler/metricsserver/config/config_test.go +++ /dev/null @@ -1,352 +0,0 @@ -package config_test - -import ( - "bytes" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsserver/config" -) - -var _ = Describe("Config", func() { - - var ( - conf *Config - err error - configBytes []byte - ) - - Describe("LoadConfig", func() { - - JustBeforeEach(func() { - conf, err = LoadConfig(bytes.NewReader(configBytes)) - }) - - Context("with invalid yaml", func() { - BeforeEach(func() { - configBytes = []byte(` -node_addrs: - - 10.0.2.3:8080 - - 10.0.2.5:8080 -node_index: 0 - collector: - refresh_interval: 30s -`) - }) - - It("returns an error", func() { - Expect(err).To(MatchError(MatchRegexp("yaml: .*"))) - }) - }) - - Context("with valid yaml", func() { - BeforeEach(func() { - configBytes = []byte(` -logging: - level: DebuG -http_client_timeout: 10s -node_addrs: - - address1 - - address2 -node_index: 0 -db: - policy_db: - url: postgres://pqgotest:password@localhost/pqgotest - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s - instance_metrics_db: - url: postgres://pqgotest:password@localhost/pqgotest - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -collector: - port: 8989 - tls: - key_file: /var/vcap/jobs/autoscaler/config/certs/server.key - cert_file: /var/vcap/jobs/autoscaler/config/certs/server.crt - ca_file: /var/vcap/jobs/autoscaler/config/certs/ca.crt - keep_alive_time: 180s - refresh_interval: 20s - collect_interval: 10s - save_interval: 5s - metric_cache_size_per_app: 100 - persist_metrics: true - envelope_processor_count: 10 - envelope_channel_size: 500 - metric_channel_size: 500 -server: - port: 8888 - tls: - key_file: /var/vcap/jobs/autoscaler/config/certs/server.key - cert_file: /var/vcap/jobs/autoscaler/config/certs/server.crt - ca_file: /var/vcap/jobs/autoscaler/config/certs/ca.crt -health: - port: 9999 -`) - }) - - It("returns the config", func() { - Expect(err).NotTo(HaveOccurred()) - Expect(conf.Logging.Level).To(Equal("debug")) - Expect(conf.HttpClientTimeout).To(Equal(10 * time.Second)) - Expect(conf.NodeAddrs).To(Equal([]string{"address1", "address2"})) - Expect(conf.NodeIndex).To(Equal(0)) - - Expect(conf.DB.PolicyDB).To(Equal( - db.DatabaseConfig{ - URL: "postgres://pqgotest:password@localhost/pqgotest", - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - })) - Expect(conf.DB.InstanceMetricsDB).To(Equal( - db.DatabaseConfig{ - URL: "postgres://pqgotest:password@localhost/pqgotest", - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - })) - - Expect(conf.Collector.WSPort).To(Equal(8989)) - Expect(conf.Collector.TLS.KeyFile).To(Equal("/var/vcap/jobs/autoscaler/config/certs/server.key")) - Expect(conf.Collector.TLS.CertFile).To(Equal("/var/vcap/jobs/autoscaler/config/certs/server.crt")) - Expect(conf.Collector.TLS.CACertFile).To(Equal("/var/vcap/jobs/autoscaler/config/certs/ca.crt")) - Expect(conf.Collector.WSKeepAliveTime).To(Equal(180 * time.Second)) - Expect(conf.Collector.RefreshInterval).To(Equal(20 * time.Second)) - Expect(conf.Collector.CollectInterval).To(Equal(10 * time.Second)) - Expect(conf.Collector.SaveInterval).To(Equal(5 * time.Second)) - Expect(conf.Collector.MetricCacheSizePerApp).To(Equal(100)) - Expect(conf.Collector.PersistMetrics).To(BeTrue()) - Expect(conf.Collector.EnvelopeProcessorCount).To(Equal(10)) - Expect(conf.Collector.EnvelopeChannelSize).To(Equal(500)) - Expect(conf.Collector.MetricChannelSize).To(Equal(500)) - - Expect(conf.Server.Port).To(Equal(8888)) - Expect(conf.Server.TLS.KeyFile).To(Equal("/var/vcap/jobs/autoscaler/config/certs/server.key")) - Expect(conf.Server.TLS.CertFile).To(Equal("/var/vcap/jobs/autoscaler/config/certs/server.crt")) - Expect(conf.Server.TLS.CACertFile).To(Equal("/var/vcap/jobs/autoscaler/config/certs/ca.crt")) - - Expect(conf.Health.Port).To(Equal(9999)) - - }) - }) - - Context("with partial config", func() { - BeforeEach(func() { - configBytes = []byte(` -db: - policy_db: - url: postgres://pqgotest:password@localhost/pqgotest - instance_metrics_db: - url: postgres://pqgotest:password@localhost/pqgotest -`) - }) - - It("returns default values", func() { - Expect(err).NotTo(HaveOccurred()) - - Expect(conf.Logging.Level).To(Equal(DefaultLoggingLevel)) - Expect(conf.HttpClientTimeout).To(Equal(DefaultHttpClientTimeout)) - - Expect(conf.DB.PolicyDB).To(Equal( - db.DatabaseConfig{ - URL: "postgres://pqgotest:password@localhost/pqgotest", - MaxOpenConnections: 0, - MaxIdleConnections: 0, - ConnectionMaxLifetime: 0 * time.Second, - })) - Expect(conf.DB.InstanceMetricsDB).To(Equal( - db.DatabaseConfig{ - URL: "postgres://pqgotest:password@localhost/pqgotest", - MaxOpenConnections: 0, - MaxIdleConnections: 0, - ConnectionMaxLifetime: 0 * time.Second, - })) - Expect(conf.Collector.WSPort).To(Equal(DefaultWSPort)) - Expect(conf.Collector.RefreshInterval).To(Equal(DefaultRefreshInterval)) - Expect(conf.Collector.CollectInterval).To(Equal(DefaultCollectInterval)) - Expect(conf.Collector.WSKeepAliveTime).To(Equal(DefaultWSKeepAliveTime)) - Expect(conf.Collector.RefreshInterval).To(Equal(DefaultRefreshInterval)) - Expect(conf.Collector.MetricCacheSizePerApp).To(Equal(DefaultMetricCacheSizePerApp)) - Expect(conf.Collector.PersistMetrics).To(Equal(DefaultIsMetricsPersistencySupported)) - Expect(conf.Collector.EnvelopeProcessorCount).To(Equal(DefaultEnvelopeProcessorCount)) - Expect(conf.Collector.EnvelopeChannelSize).To(Equal(DefaultEnvelopeChannelSize)) - Expect(conf.Collector.MetricChannelSize).To(Equal(DefaultMetricChannelSize)) - - Expect(conf.Server.Port).To(Equal(DefaultHTTPServerPort)) - - Expect(conf.Health.Port).To(Equal(DefaultHealthPort)) - - }) - }) - }) - - Describe("Validate", func() { - BeforeEach(func() { - conf = &Config{} - conf.NodeAddrs = []string{"address1", "address2"} - conf.NodeIndex = 0 - conf.HttpClientTimeout = 10 * time.Second - conf.DB.PolicyDB = db.DatabaseConfig{ - URL: "postgres://pqgotest:password@localhost/pqgotest", - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - } - conf.DB.InstanceMetricsDB = db.DatabaseConfig{ - URL: "postgres://pqgotest:password@localhost/pqgotest", - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - } - conf.Collector.WSKeepAliveTime = 180 * time.Second - conf.Collector.RefreshInterval = 60 * time.Second - conf.Collector.CollectInterval = 30 * time.Second - conf.Collector.SaveInterval = 5 * time.Second - conf.Collector.MetricCacheSizePerApp = 100 - conf.Collector.EnvelopeProcessorCount = 5 - conf.Collector.EnvelopeChannelSize = 300 - conf.Collector.MetricChannelSize = 300 - conf.Health.Port = 8081 - }) - - JustBeforeEach(func() { - err = conf.Validate() - }) - - Context("when all the configs are valid", func() { - It("should not error", func() { - Expect(err).NotTo(HaveOccurred()) - }) - }) - - Context("when HttpClientTimeout is <= 0", func() { - BeforeEach(func() { - conf.HttpClientTimeout = 0 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: http_client_timeout is less-equal than 0")) - }) - }) - - Context("when node index is out of range", func() { - Context("when node index is negative", func() { - BeforeEach(func() { - conf.NodeIndex = -1 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: node_index out of range")) - }) - }) - - Context("when node index is >= number of nodes", func() { - BeforeEach(func() { - conf.NodeIndex = 2 - conf.NodeAddrs = []string{"address1", "address2"} - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: node_index out of range")) - }) - }) - }) - Context("when policy db url is not set", func() { - BeforeEach(func() { - conf.DB.PolicyDB.URL = "" - }) - - It("should error", func() { - Expect(err).To(MatchError("Configuration error: db.policy_db.url is empty")) - }) - }) - - Context("when metrics db url is not set", func() { - BeforeEach(func() { - conf.DB.InstanceMetricsDB.URL = "" - }) - - It("should error", func() { - Expect(err).To(MatchError("Configuration error: db.instance_metrics_db.url is empty")) - }) - }) - - Context("when collect interval is 0", func() { - BeforeEach(func() { - conf.Collector.CollectInterval = time.Duration(0) - }) - - It("should error", func() { - Expect(err).To(MatchError("Configuration error: collector.collect_interval is 0")) - }) - }) - - Context("when refresh interval is 0", func() { - BeforeEach(func() { - conf.Collector.RefreshInterval = time.Duration(0) - }) - - It("should error", func() { - Expect(err).To(MatchError("Configuration error: collector.refresh_interval is 0")) - }) - }) - - Context("when save interval is 0", func() { - BeforeEach(func() { - conf.Collector.SaveInterval = time.Duration(0) - }) - - It("should error", func() { - Expect(err).To(MatchError("Configuration error: collector.save_interval is 0")) - }) - }) - - Context("when metrics cache size per app is invalid", func() { - BeforeEach(func() { - conf.Collector.MetricCacheSizePerApp = 0 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: invalid collector.metric_cache_size_per_app")) - }) - }) - - Context("when KeepAliveTime is <= 0", func() { - BeforeEach(func() { - conf.Collector.WSKeepAliveTime = 0 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: keep_alive_time is less-equal than 0")) - }) - }) - - Context("when EnvelopeProcessorCount is <= 0", func() { - BeforeEach(func() { - conf.Collector.EnvelopeProcessorCount = 0 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: envelope_processor_count is less-equal than 0")) - }) - }) - - Context("when EnvelopeChannelSize is <= 0", func() { - BeforeEach(func() { - conf.Collector.EnvelopeChannelSize = 0 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: envelope_channel_size is less-equal than 0")) - }) - }) - - Context("when MetricChannelSize is <= 0", func() { - BeforeEach(func() { - conf.Collector.MetricChannelSize = 0 - }) - It("should error", func() { - Expect(err).To(MatchError("Configuration error: metric_channel_size is less-equal than 0")) - }) - }) - - }) -}) diff --git a/src/autoscaler/metricsserver/db/metricscollector.db.changelog.yml b/src/autoscaler/metricsserver/db/metricscollector.db.changelog.yml index 1a9292ec1b..71cbd9cfcc 100644 --- a/src/autoscaler/metricsserver/db/metricscollector.db.changelog.yml +++ b/src/autoscaler/metricsserver/db/metricscollector.db.changelog.yml @@ -159,3 +159,13 @@ databaseChangeLog: type: bigint indexName: idx_instance_metrics tableName: appinstancemetrics + - changeSet: + id: 5 + author: jgeiger + logicalFilePath: /var/vcap/packages/metricsserver/metricscollector.db.changelog.yml + preConditions: + - onFail: MARK_RAN + tableExists: + tableName: appinstancemetrics + dropTable: + tableName: appinstancemetrics diff --git a/src/autoscaler/metricsserver/exampleconfig/example.yml b/src/autoscaler/metricsserver/exampleconfig/example.yml deleted file mode 100644 index c425a11931..0000000000 --- a/src/autoscaler/metricsserver/exampleconfig/example.yml +++ /dev/null @@ -1,31 +0,0 @@ -logging: - level: debug -http_client_timeout: 30s -node_addrs: - - 10.0.2.3 - - 10.0.2.5 -node_index: 0 -db: - policy_db: - url: postgres://postgres:postgres@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s - instance_metrics_db: - url: postgres://postgres:postgres@localhost/autoscaler?sslmode=disable - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s -collector: - port: 8443 - keep_alive_time: 30s - refresh_interval: 30s - collect_interval: 10s - save_interval: 5s - persist_metrics: true - keep_alive_time: 120s - envelope_processor_count: 10 -server: - port: 8080 -health: - port: 8081 \ No newline at end of file diff --git a/src/autoscaler/operator/cmd/operator/main.go b/src/autoscaler/operator/cmd/operator/main.go index 3348a0ef32..16424db26b 100644 --- a/src/autoscaler/operator/cmd/operator/main.go +++ b/src/autoscaler/operator/cmd/operator/main.go @@ -57,13 +57,6 @@ func main() { logger := helpers.InitLoggerFromConfig(&conf.Logging, "operator") prClock := clock.NewClock() - instanceMetricsDB, err := sqldb.NewInstanceMetricsSQLDB(conf.InstanceMetricsDB.DB, logger.Session("instancemetrics-db")) - if err != nil { - logger.Error("failed to connect instancemetrics db", err, lager.Data{"dbConfig": conf.InstanceMetricsDB.DB}) - os.Exit(1) - } - defer instanceMetricsDB.Close() - appMetricsDB, err := sqldb.NewAppMetricSQLDB(conf.AppMetricsDB.DB, logger.Session("appmetrics-db")) if err != nil { logger.Error("failed to connect appmetrics db", err, lager.Data{"dbConfig": conf.AppMetricsDB.DB}) @@ -91,7 +84,7 @@ func main() { promRegistry := prometheus.NewRegistry() healthendpoint.RegisterCollectors(promRegistry, []prometheus.Collector{ healthendpoint.NewDatabaseStatusCollector("autoscaler", "operator", "policyDB", policyDb), - healthendpoint.NewDatabaseStatusCollector("autoscaler", "operator", "instanceMetricsDB", instanceMetricsDB), + healthendpoint.NewDatabaseStatusCollector("autoscaler", "operator", "appMetricsDB", appMetricsDB), healthendpoint.NewDatabaseStatusCollector("autoscaler", "operator", "scalingEngineDB", scalingEngineDB), }, true, logger.Session("operator-prometheus")) @@ -107,11 +100,7 @@ func main() { os.Exit(1) } - loggerSessionName := "instancemetrics-dbpruner" - instanceMetricDBPruner := operator.NewInstanceMetricsDbPruner(instanceMetricsDB, conf.InstanceMetricsDB.CutoffDuration, prClock, logger.Session(loggerSessionName)) - instanceMetricsDBOperatorRunner := operator.NewOperatorRunner(instanceMetricDBPruner, conf.InstanceMetricsDB.RefreshInterval, prClock, logger.Session(loggerSessionName)) - - loggerSessionName = "appmetrics-dbpruner" + loggerSessionName := "appmetrics-dbpruner" appMetricsDBPruner := operator.NewAppMetricsDbPruner(appMetricsDB, conf.AppMetricsDB.CutoffDuration, prClock, logger.Session(loggerSessionName)) appMetricsDBOperatorRunner := operator.NewOperatorRunner(appMetricsDBPruner, conf.AppMetricsDB.RefreshInterval, prClock, logger.Session(loggerSessionName)) @@ -131,7 +120,6 @@ func main() { applicationSyncRunner := operator.NewOperatorRunner(applicationSync, conf.AppSyncer.SyncInterval, prClock, logger.Session(loggerSessionName)) members := grouper.Members{ - {"instancemetrics-dbpruner", instanceMetricsDBOperatorRunner}, {"appmetrics-dbpruner", appMetricsDBOperatorRunner}, {"scalingEngine-dbpruner", scalingEngineDBOperatorRunner}, {"scalingEngine-sync", scalingEngineSyncRunner}, diff --git a/src/autoscaler/operator/cmd/operator/operator_suite_test.go b/src/autoscaler/operator/cmd/operator/operator_suite_test.go index 20d8308011..3faa160def 100644 --- a/src/autoscaler/operator/cmd/operator/operator_suite_test.go +++ b/src/autoscaler/operator/cmd/operator/operator_suite_test.go @@ -78,15 +78,6 @@ func initConfig() { cfg.Logging.Level = "debug" dbUrl := testhelpers.GetDbUrl() - cfg.InstanceMetricsDB.DB = db.DatabaseConfig{ - URL: dbUrl, - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 10 * time.Second, - } - cfg.InstanceMetricsDB.RefreshInterval = 12 * time.Hour - cfg.InstanceMetricsDB.CutoffDuration = 20 * 24 * time.Hour - cfg.AppMetricsDB.DB = db.DatabaseConfig{ URL: dbUrl, MaxOpenConnections: 10, diff --git a/src/autoscaler/operator/cmd/operator/operator_test.go b/src/autoscaler/operator/cmd/operator/operator_test.go index 4a13cda514..455a60838c 100644 --- a/src/autoscaler/operator/cmd/operator/operator_test.go +++ b/src/autoscaler/operator/cmd/operator/operator_test.go @@ -73,7 +73,7 @@ var _ = Describe("Operator", Serial, func() { Context("with missing/invalid configuration", func() { BeforeEach(func() { - cfg.InstanceMetricsDB.CutoffDuration = -1 + cfg.AppMetricsDB.CutoffDuration = -1 cfg := writeConfig(&cfg) runner.configPath = cfg.Name() @@ -236,16 +236,6 @@ var _ = Describe("Operator", Serial, func() { runner.Start() }) - It("should start instancemetrics dbpruner", func() { - Eventually(runner.Session.Buffer, 2*time.Second).Should(Say("operator.instancemetrics-dbpruner.started")) - Consistently(runner.Session).ShouldNot(Exit()) - }) - - It("should start appmetrics dbpruner", func() { - Eventually(runner.Session.Buffer, 2*time.Second).Should(Say("operator.appmetrics-dbpruner.started")) - Consistently(runner.Session).ShouldNot(Exit()) - }) - It("should start scalingengine dbpruner", func() { Eventually(runner.Session.Buffer, 2*time.Second).Should(Say("operator.scalingengine-dbpruner.started")) Consistently(runner.Session).ShouldNot(Exit()) @@ -263,25 +253,6 @@ var _ = Describe("Operator", Serial, func() { }) - Context("when connection to instancemetrics db fails", func() { - BeforeEach(func() { - cfg.InstanceMetricsDB.DB.URL = "postgres://not-exist-user:not-exist-password@localhost/autoscaler?sslmode=disable" - cfg := writeConfig(&cfg) - runner.configPath = cfg.Name() - runner.Start() - }) - - AfterEach(func() { - os.Remove(runner.configPath) - }) - - It("should error", func() { - Eventually(runner.Session).Should(Exit(1)) - Expect(runner.Session.Buffer()).To(Say("failed to connect instancemetrics db")) - }) - - }) - Context("when connection to appmetrics db fails", func() { BeforeEach(func() { cfg.AppMetricsDB.DB.URL = "postgres://not-exist-user:not-exist-password@localhost/autoscaler?sslmode=disable" @@ -363,8 +334,6 @@ var _ = Describe("Operator", Serial, func() { raw, _ := io.ReadAll(rsp.Body) healthData := string(raw) Expect(healthData).To(ContainSubstring("autoscaler_operator_policyDB")) - Expect(healthData).To(ContainSubstring("autoscaler_operator_instanceMetricsDB")) - Expect(healthData).To(ContainSubstring("autoscaler_operator_appMetricsDB")) Expect(healthData).To(ContainSubstring("autoscaler_operator_scalingEngineDB")) Expect(healthData).To(ContainSubstring("go_goroutines")) Expect(healthData).To(ContainSubstring("go_memstats_alloc_bytes")) diff --git a/src/autoscaler/operator/config/config.go b/src/autoscaler/operator/config/config.go index b1b25d025e..85c1591865 100644 --- a/src/autoscaler/operator/config/config.go +++ b/src/autoscaler/operator/config/config.go @@ -68,7 +68,6 @@ type Config struct { CF cf.Config `yaml:"cf"` Health helpers.HealthConfig `yaml:"health"` Logging helpers.LoggingConfig `yaml:"logging"` - InstanceMetricsDB DbPrunerConfig `yaml:"instance_metrics_db"` AppMetricsDB DbPrunerConfig `yaml:"app_metrics_db"` ScalingEngineDB DbPrunerConfig `yaml:"scaling_engine_db"` ScalingEngine ScalingEngineConfig `yaml:"scaling_engine"` @@ -84,10 +83,6 @@ var defaultConfig = Config{ }, Health: defaultHealthConfig, Logging: helpers.LoggingConfig{Level: DefaultLoggingLevel}, - InstanceMetricsDB: DbPrunerConfig{ - RefreshInterval: DefaultRefreshInterval, - CutoffDuration: DefaultCutoffDuration, - }, AppMetricsDB: DbPrunerConfig{ RefreshInterval: DefaultRefreshInterval, CutoffDuration: DefaultCutoffDuration, @@ -126,18 +121,6 @@ func LoadConfig(reader io.Reader) (*Config, error) { } func (c *Config) Validate() error { - if c.InstanceMetricsDB.DB.URL == "" { - return fmt.Errorf("Configuration error: instance_metrics_db.db.url is empty") - } - - if c.InstanceMetricsDB.RefreshInterval <= 0 { - return fmt.Errorf("Configuration error: instance_metrics_db.refresh_interval is less than or equal to 0") - } - - if c.InstanceMetricsDB.CutoffDuration <= 0 { - return fmt.Errorf("Configuration error: instance_metrics_db.cutoff_duration is less than or equal to 0") - } - if c.AppMetricsDB.DB.URL == "" { return fmt.Errorf("Configuration error: app_metrics_db.db.url is empty") } diff --git a/src/autoscaler/operator/config/config_test.go b/src/autoscaler/operator/config/config_test.go index 7ab76e9c43..936e3d19c0 100644 --- a/src/autoscaler/operator/config/config_test.go +++ b/src/autoscaler/operator/config/config_test.go @@ -53,15 +53,6 @@ var _ = Describe("Config", func() { Expect(conf.Health.Port).To(Equal(9999)) Expect(conf.Logging.Level).To(Equal("debug")) - Expect(conf.InstanceMetricsDB.DB).To(Equal(db.DatabaseConfig{ - URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", - MaxOpenConnections: 10, - MaxIdleConnections: 5, - ConnectionMaxLifetime: 60 * time.Second, - })) - Expect(conf.InstanceMetricsDB.RefreshInterval).To(Equal(12 * time.Hour)) - Expect(conf.InstanceMetricsDB.CutoffDuration).To(Equal(20 * time.Hour)) - Expect(conf.AppMetricsDB.DB).To(Equal(db.DatabaseConfig{ URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", MaxOpenConnections: 10, @@ -105,14 +96,6 @@ var _ = Describe("Config", func() { Expect(conf.Logging.Level).To(Equal(config.DefaultLoggingLevel)) Expect(conf.Health.Port).To(Equal(8081)) - Expect(conf.InstanceMetricsDB.DB).To(Equal(db.DatabaseConfig{ - URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", - MaxOpenConnections: 0, - MaxIdleConnections: 0, - ConnectionMaxLifetime: 0 * time.Second, - })) - Expect(conf.InstanceMetricsDB.RefreshInterval).To(Equal(config.DefaultRefreshInterval)) - Expect(conf.InstanceMetricsDB.CutoffDuration).To(Equal(config.DefaultCutoffDuration)) Expect(conf.AppMetricsDB.DB).To(Equal(db.DatabaseConfig{ URL: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable", MaxOpenConnections: 0, @@ -148,31 +131,6 @@ var _ = Describe("Config", func() { }) }) - Context("when cutoff_duration of instance_metrics_db is not a time.Duration", func() { - BeforeEach(func() { - configBytes = []byte(` -instance_metrics_db: - cutoff_duration: 7k -`) - }) - - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - }) - }) - - Context("when refresh_interval of instance_metrics_db is not a time duration", func() { - BeforeEach(func() { - configBytes = []byte(` -instance_metrics_db: - refresh_interval: 12k -`) - }) - - It("should error", func() { - Expect(err).To(BeAssignableToTypeOf(&yaml.TypeError{})) - }) - }) Context("when scaling engine sync interval is not a time duration", func() { BeforeEach(func() { @@ -219,10 +177,6 @@ scheduler: BeforeEach(func() { conf = &config.Config{} - conf.InstanceMetricsDB.DB.URL = "postgres://pqgotest:password@exampl.com/pqgotest" - conf.InstanceMetricsDB.RefreshInterval = 12 * time.Hour - conf.InstanceMetricsDB.CutoffDuration = 30 * time.Hour - conf.AppMetricsDB.DB.URL = "postgres://pqgotest:password@exampl.com/pqgotest" conf.AppMetricsDB.RefreshInterval = 10 * time.Hour conf.AppMetricsDB.CutoffDuration = 15 * time.Hour @@ -255,17 +209,6 @@ scheduler: }) }) - Context("when InstanceMetrics db url is not set", func() { - - BeforeEach(func() { - conf.InstanceMetricsDB.DB.URL = "" - }) - - It("should error", func() { - Expect(err).To(MatchError("Configuration error: instance_metrics_db.db.url is empty")) - }) - }) - Context("when AppMetrics db url is not set", func() { BeforeEach(func() { @@ -310,17 +253,6 @@ scheduler: }) }) - Context("when InstanceMetrics db refresh interval in hours is set to a negative value", func() { - - BeforeEach(func() { - conf.InstanceMetricsDB.RefreshInterval = -1 - }) - - It("should error", func() { - Expect(err).To(MatchError("Configuration error: instance_metrics_db.refresh_interval is less than or equal to 0")) - }) - }) - Context("when AppMetrics db refresh interval in hours is set to a negative value", func() { BeforeEach(func() { @@ -343,17 +275,6 @@ scheduler: }) }) - Context("when InstanceMetrics db cutoff duration is set to a negative value", func() { - - BeforeEach(func() { - conf.InstanceMetricsDB.CutoffDuration = -1 - }) - - It("should error", func() { - Expect(err).To(MatchError("Configuration error: instance_metrics_db.cutoff_duration is less than or equal to 0")) - }) - }) - Context("when AppMetrics db cutoff duration is set to a negative value", func() { BeforeEach(func() { diff --git a/src/autoscaler/operator/config/testdata/invalid.yml b/src/autoscaler/operator/config/testdata/invalid.yml index 05329069c2..0e0b671e1d 100644 --- a/src/autoscaler/operator/config/testdata/invalid.yml +++ b/src/autoscaler/operator/config/testdata/invalid.yml @@ -1,14 +1,6 @@ logging: level: "debug" -instance_metrics_db: - db: - url: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable" - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s - refresh_interval: 12h - cutoff_duration: 10h app_metrics_db: db: url: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable" diff --git a/src/autoscaler/operator/config/testdata/partial.yml b/src/autoscaler/operator/config/testdata/partial.yml index b1eb654eec..897ea84767 100644 --- a/src/autoscaler/operator/config/testdata/partial.yml +++ b/src/autoscaler/operator/config/testdata/partial.yml @@ -1,7 +1,3 @@ - -instance_metrics_db: - db: - url: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable" app_metrics_db: db: url: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable" diff --git a/src/autoscaler/operator/config/testdata/valid.yml b/src/autoscaler/operator/config/testdata/valid.yml index 50bc787f1a..08d141c610 100644 --- a/src/autoscaler/operator/config/testdata/valid.yml +++ b/src/autoscaler/operator/config/testdata/valid.yml @@ -8,14 +8,6 @@ health: port: 9999 logging: level: "debug" -instance_metrics_db: - db: - url: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable" - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s - refresh_interval: 12h - cutoff_duration: 20h app_metrics_db: db: url: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable" diff --git a/src/autoscaler/operator/exampleconfig/example.yml b/src/autoscaler/operator/exampleconfig/example.yml index f74833979d..8ac46ae325 100644 --- a/src/autoscaler/operator/exampleconfig/example.yml +++ b/src/autoscaler/operator/exampleconfig/example.yml @@ -2,14 +2,6 @@ logging: level: "debug" health: port: 8081 -instance_metrics_db: - db: - url: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable" - max_open_connections: 10 - max_idle_connections: 5 - connection_max_lifetime: 60s - refresh_interval: 12h - cutoff_duration: 24h app_metrics_db: db: url: "postgres://postgres:postgres@localhost/autoscaler?sslmode=disable" diff --git a/src/autoscaler/operator/instancemetricsdb.go b/src/autoscaler/operator/instancemetricsdb.go deleted file mode 100644 index b058b21858..0000000000 --- a/src/autoscaler/operator/instancemetricsdb.go +++ /dev/null @@ -1,40 +0,0 @@ -package operator - -import ( - "context" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/clock" - "code.cloudfoundry.org/lager/v3" -) - -type InstanceMetricsDbPruner struct { - instanceMetricsDb db.InstanceMetricsDB - cutoffDuration time.Duration - clock clock.Clock - logger lager.Logger -} - -func NewInstanceMetricsDbPruner(instanceMetricsDb db.InstanceMetricsDB, cutoffDuration time.Duration, clock clock.Clock, logger lager.Logger) *InstanceMetricsDbPruner { - return &InstanceMetricsDbPruner{ - instanceMetricsDb: instanceMetricsDb, - cutoffDuration: cutoffDuration, - clock: clock, - logger: logger.Session("instance_metrics_db_pruner"), - } -} - -func (idp InstanceMetricsDbPruner) Operate(ctx context.Context) { - timestamp := idp.clock.Now().Add(-idp.cutoffDuration).UnixNano() - - logger := idp.logger.Session("pruning-instance-metrics", lager.Data{"cutoff-time": timestamp}) - logger.Info("starting") - defer logger.Info("completed") - - err := idp.instanceMetricsDb.PruneInstanceMetrics(ctx, timestamp) - if err != nil { - idp.logger.Error("failed-prune-metrics", err) - return - } -} diff --git a/src/autoscaler/operator/instancemetricsdb_test.go b/src/autoscaler/operator/instancemetricsdb_test.go deleted file mode 100644 index 9521f278ee..0000000000 --- a/src/autoscaler/operator/instancemetricsdb_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package operator_test - -import ( - "context" - "errors" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/operator" - - "code.cloudfoundry.org/clock/fakeclock" - "code.cloudfoundry.org/lager/v3/lagertest" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/onsi/gomega/gbytes" -) - -var _ = Describe("InstanceMetricsDB Prune", func() { - var ( - instanceMetricsDb *fakes.FakeInstanceMetricsDB - fclock *fakeclock.FakeClock - cutoffDuration time.Duration - buffer *gbytes.Buffer - instanceMetricsDbPruner *operator.InstanceMetricsDbPruner - ) - - BeforeEach(func() { - - cutoffDuration = 20 * time.Hour - logger := lagertest.NewTestLogger("prune-test") - buffer = logger.Buffer() - - instanceMetricsDb = &fakes.FakeInstanceMetricsDB{} - fclock = fakeclock.NewFakeClock(time.Now()) - - instanceMetricsDbPruner = operator.NewInstanceMetricsDbPruner(instanceMetricsDb, cutoffDuration, fclock, logger) - - }) - - Describe("Prune", func() { - JustBeforeEach(func() { - instanceMetricsDbPruner.Operate(context.Background()) - }) - - Context("when pruning metrics records from instancemetrics db", func() { - It("prunes as per given cutoff days", func() { - Eventually(instanceMetricsDb.PruneInstanceMetricsCallCount).Should(Equal(1)) - _, cutoffTime := instanceMetricsDb.PruneInstanceMetricsArgsForCall(0) - Expect(cutoffTime).To(BeNumerically("==", fclock.Now().Add(-cutoffDuration).UnixNano())) - }) - }) - - Context("when pruning records from instancemetrics db fails", func() { - BeforeEach(func() { - instanceMetricsDb.PruneInstanceMetricsReturns(errors.New("test operator. error")) - }) - - It("should error", func() { - Eventually(instanceMetricsDb.PruneInstanceMetricsCallCount).Should(Equal(1)) - Eventually(buffer).Should(gbytes.Say("test operator. error")) - }) - }) - }) -}) diff --git a/src/autoscaler/testhelpers/fakeLoggregator.go b/src/autoscaler/testhelpers/fakeLoggregator.go deleted file mode 100644 index 1191791757..0000000000 --- a/src/autoscaler/testhelpers/fakeLoggregator.go +++ /dev/null @@ -1,226 +0,0 @@ -package testhelpers - -import ( - "crypto/tls" - "crypto/x509" - "errors" - "fmt" - "log" - "net" - "os" - "sync" - "time" - - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/status" -) - -type FakeEventProducer struct { - server *grpc.Server - addr string - mu sync.Mutex - connectionAttempts int - actualReq *loggregator_v2.EgressBatchRequest - envelops []*loggregator_v2.Envelope - certFile string - keyFile string - caCertFile string - emitInterval time.Duration - loggregator_v2.UnimplementedEgressServer -} - -func NewFakeEventProducer(certFile string, keyFile string, caCertFile string, emitInterval time.Duration) (*FakeEventProducer, error) { - if emitInterval == 0 { - emitInterval = 500 * time.Millisecond - } - f := &FakeEventProducer{ - certFile: certFile, - keyFile: keyFile, - caCertFile: caCertFile, - emitInterval: emitInterval, - } - - return f, nil -} - -func (f *FakeEventProducer) Receiver( - *loggregator_v2.EgressRequest, - loggregator_v2.Egress_ReceiverServer, -) error { - return status.Errorf(codes.Unimplemented, "use BatchedReceiver instead") -} - -func (f *FakeEventProducer) BatchedReceiver( - req *loggregator_v2.EgressBatchRequest, - srv loggregator_v2.Egress_BatchedReceiverServer, -) error { - f.mu.Lock() - f.connectionAttempts++ - f.actualReq = req - f.mu.Unlock() - var i int - for range time.NewTicker(f.emitInterval).C { - fpEnvs := []*loggregator_v2.Envelope{} - for _, e := range f.envelops { - fpEnvs = append(fpEnvs, &loggregator_v2.Envelope{ - SourceId: e.SourceId, - Message: e.Message, - Tags: e.Tags, - DeprecatedTags: e.DeprecatedTags, - Timestamp: time.Now().UnixNano(), - }) - } - err := srv.Send(&loggregator_v2.EnvelopeBatch{ - Batch: fpEnvs, - }) - if err != nil { - return err - } - i++ - } - return nil -} - -func (f *FakeEventProducer) Start() { - addr := f.addr - if addr == "" { - addr = "127.0.0.1:0" - } - var lis net.Listener - for i := 0; ; i++ { - var err error - lis, err = net.Listen("tcp", addr) - if err != nil { - // This can happen if the port is already in use... - if i < 50 { - log.Printf("failed to bind for fake producer. Trying again (%d/50)...: %s", i+1, err) - time.Sleep(100 * time.Millisecond) - continue - } - panic(err) - } - break - } - f.addr = lis.Addr().String() - c, err := NewServerMutualTLSConfig(f.certFile, f.keyFile, f.caCertFile) - if err != nil { - panic(err) - } - opt := grpc.Creds(credentials.NewTLS(c)) - f.server = grpc.NewServer(opt) - loggregator_v2.RegisterEgressServer(f.server, f) - - go f.listen(lis) -} - -func (f *FakeEventProducer) listen(lis net.Listener) { - _ = f.server.Serve(lis) -} - -func (f *FakeEventProducer) Stop() bool { - if f.server == nil { - return false - } - - f.server.Stop() - f.server = nil - return true -} - -func (f *FakeEventProducer) GetActualReq() *loggregator_v2.EgressBatchRequest { - f.mu.Lock() - defer f.mu.Unlock() - return f.actualReq -} - -func (f *FakeEventProducer) GetConnectionAttempts() int { - f.mu.Lock() - defer f.mu.Unlock() - return f.connectionAttempts -} -func (f *FakeEventProducer) GetAddr() string { - return f.addr -} - -func (f *FakeEventProducer) SetEnvelops(envelops []*loggregator_v2.Envelope) { - f.mu.Lock() - defer f.mu.Unlock() - f.envelops = envelops -} - -func NewServerMutualTLSConfig(certFile, keyFile, caCertFile string) (*tls.Config, error) { - tlsCert, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - return nil, fmt.Errorf("failed to load keypair: %w", err) - } - - certBytes, err := os.ReadFile(caCertFile) - if err != nil { - return nil, fmt.Errorf("failed to read ca cert file: %w", err) - } - - caCertPool := x509.NewCertPool() - if ok := caCertPool.AppendCertsFromPEM(certBytes); !ok { - return nil, errors.New("unable to load ca cert file") - } - - tlsConfig := &tls.Config{ - InsecureSkipVerify: false, - MinVersion: tls.VersionTLS12, - Certificates: []tls.Certificate{tlsCert}, - ClientCAs: caCertPool, - ClientAuth: tls.RequireAndVerifyClientCert, - } - - return tlsConfig, nil -} - -func NewClientMutualTLSConfig( - certFile string, - keyFile string, - caCertFile string, - serverName string, -) (*tls.Config, error) { - tlsCert, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - return nil, fmt.Errorf("failed to load keypair: %w", err) - } - - certBytes, err := os.ReadFile(caCertFile) - if err != nil { - return nil, fmt.Errorf("failed to read ca cert file: %w", err) - } - - caCertPool := x509.NewCertPool() - if ok := caCertPool.AppendCertsFromPEM(certBytes); !ok { - return nil, errors.New("unable to load ca cert file") - } - - certificate, err := x509.ParseCertificate(tlsCert.Certificate[0]) - if err != nil { - return nil, err - } - - verifyOptions := x509.VerifyOptions{ - Roots: caCertPool, - KeyUsages: []x509.ExtKeyUsage{ - x509.ExtKeyUsageAny, - }, - } - if _, err := certificate.Verify(verifyOptions); err != nil { - return nil, fmt.Errorf("failed to verify certificate: %w", err) - } - - tlsConfig := &tls.Config{ - InsecureSkipVerify: false, - MinVersion: tls.VersionTLS12, - Certificates: []tls.Certificate{tlsCert}, - ServerName: serverName, - RootCAs: caCertPool, - } - - return tlsConfig, err -} From 46399ea7febeea3fa8ce93ab31ea1c16aa1981f8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 21:41:03 +0000 Subject: [PATCH 034/186] fix(deps): update code.cloudfoundry.org/loggregator-agent-release/src digest to fde7ff8 --- src/autoscaler/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 5f4fa13412..f60db2e50a 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,7 +8,7 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240606143910-b2e6ea608429 + code.cloudfoundry.org/loggregator-agent-release/src fde7ff8f79b7 code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65 dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 From 034f7cb9914d896519049140ab56fee4080d67e7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 22:47:54 +0100 Subject: [PATCH 035/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/autoscaler/go.mod | 4 ++-- src/autoscaler/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index f60db2e50a..2d550ec8b5 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,8 +8,8 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src fde7ff8f79b7 - code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65 + code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240607070054-fde7ff8f79b7 + code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/go-chi/chi/v5 v5.0.12 diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 9384c98980..992db1ebcc 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -606,10 +606,10 @@ code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a h1:X code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a/go.mod h1:/Be8VtLiCeMUoYdUzFtmW8GGkk89HAy3zD79KUXzbhs= code.cloudfoundry.org/lager/v3 v3.0.3 h1:/UTmadZfIaKuT/whEinSxK1mzRfNu1uPfvjFfGqiwzM= code.cloudfoundry.org/lager/v3 v3.0.3/go.mod h1:Zn5q1SrIuuHjEUE7xerMKt3ztunrJQCZETAo7rV0CH8= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240606143910-b2e6ea608429 h1:wZjrTg9Xf3urmfVip8TKz8NLFR4VEcu87aGwBOWDVOE= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240606143910-b2e6ea608429/go.mod h1:5YRZP5jmXZ+VEuVT0qhXKuchToXgE1r1L9K1d2KQE8c= -code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65 h1:0wXfAs/32wbpMKDGiV3XEHev1l9dLgIPjda+WRwJhCc= -code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65/go.mod h1:R4odSd3iXVYlIlesDHaQi9IZx2gXQryfvJmJSjPS/ug= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240607070054-fde7ff8f79b7 h1:2NXdiGqiv6XaAKpEAJBlzOkWjI2FWw/dXlSiyClCCVI= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240607070054-fde7ff8f79b7/go.mod h1:djkW8FecJqemEchtA9hUzp0QsB3V7MG3xSXADV3JL3g= +code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea h1:PjyR5Os5IVETrZkgATzSNl4R++/h+r6uQchSJ3dZsgc= +code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea/go.mod h1:RwJBA7HrvmyPNY8pRzAwlJ56WFmqJiPjxywZpxCZdWQ= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= From 4c783dcd10f2647df5384c095cb2699a4e3b9f99 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 8 Jun 2024 00:20:03 +0000 Subject: [PATCH 036/186] chore(deps): update dependency bosh-bootloader to v9.0.22 --- ci/dockerfiles/autoscaler-tools/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/dockerfiles/autoscaler-tools/Dockerfile b/ci/dockerfiles/autoscaler-tools/Dockerfile index da4327377a..67ef38628a 100644 --- a/ci/dockerfiles/autoscaler-tools/Dockerfile +++ b/ci/dockerfiles/autoscaler-tools/Dockerfile @@ -70,7 +70,7 @@ RUN wget -q https://github.com/cloudfoundry/bosh-cli/releases/download/v${BOSH_V # Install bbl # renovate: datasource=github-releases depName=bosh-bootloader lookupName=cloudfoundry/bosh-bootloader -ARG BBL_VERSION=v9.0.21 +ARG BBL_VERSION=v9.0.22 RUN wget -q https://github.com/cloudfoundry/bosh-bootloader/releases/download/${BBL_VERSION}/bbl-${BBL_VERSION}_linux_amd64 && \ mv bbl-* /usr/local/bin/bbl &&\ chmod +x /usr/local/bin/bbl &&\ From b830cad839e9449f8e923f8e5ada7984fc0f3228 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 8 Jun 2024 04:14:15 +0000 Subject: [PATCH 037/186] chore(deps): update dependency golang to v1.22.4 --- ci/dockerfiles/autoscaler-tools/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/dockerfiles/autoscaler-tools/Dockerfile b/ci/dockerfiles/autoscaler-tools/Dockerfile index 67ef38628a..52c8aa53ca 100644 --- a/ci/dockerfiles/autoscaler-tools/Dockerfile +++ b/ci/dockerfiles/autoscaler-tools/Dockerfile @@ -46,7 +46,7 @@ RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg m # install golang # renovate: datasource=golang-version depName=golang -ARG GO_VERSION=1.22.3 +ARG GO_VERSION=1.22.4 ENV GOPATH $HOME/go ENV PATH $HOME/go/bin:/usr/local/go/bin:$PATH RUN wget -q https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz -P /tmp &&\ From 2934420449c681506d9f74948611099604ec5f71 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 8 Jun 2024 07:03:45 +0000 Subject: [PATCH 038/186] chore(deps): update dependency python to v3.12.4 --- ci/terragrunt/.tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/terragrunt/.tool-versions b/ci/terragrunt/.tool-versions index a8c7e256b6..bd00aeb9d8 100644 --- a/ci/terragrunt/.tool-versions +++ b/ci/terragrunt/.tool-versions @@ -1,6 +1,6 @@ gcloud 473.0.0 helm 3.14.4 -python 3.12.3 +python 3.12.4 terraform 1.8.4 terraform-lsp terragrunt 0.58.14 From 249743183c7afd97ef231e2f23cef388bdd7d912 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 8 Jun 2024 10:18:30 +0000 Subject: [PATCH 039/186] chore(deps): update dependency ruby to v3.3.2 --- .tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index 00e3964005..0a5a8db414 100644 --- a/.tool-versions +++ b/.tool-versions @@ -12,7 +12,7 @@ golangci-lint 1.57.2 java temurin-21.0.3+9.0.LTS make 4.4 maven 3.8.6 -ruby 3.3.1 +ruby 3.3.2 shellcheck 0.10.0 yq 4.43.1 gum 0.13.0 From e1bb16a27cb562b2a5d29e096a469e52d185d1eb Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 8 Jun 2024 10:20:36 +0000 Subject: [PATCH 040/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20scripts/asdf2devbox.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- devbox.json | 6 +++--- devbox.lock | 53 ++++++++++++++++++++++++++--------------------------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/devbox.json b/devbox.json index 1767886ad7..603acd39d5 100644 --- a/devbox.json +++ b/devbox.json @@ -29,15 +29,15 @@ "python": "latest", "cloudfoundry-cli": "8.7.10", "shellcheck": "0.10.0", - "ruby": "3.3.1", "ginkgo": "2.19.0", "bundix": "latest", "oha": "latest", "gum": "0.13.0", "act": "0.2.63", - "google-cloud-sdk": "latest", "go": "1.21.6", - "temurin-bin-17": "latest" + "google-cloud-sdk": "latest", + "temurin-bin-17": "latest", + "ruby": "latest" }, "shell": { "init_hook": [ diff --git a/devbox.lock b/devbox.lock index 490b9eab58..3e35758057 100644 --- a/devbox.lock +++ b/devbox.lock @@ -554,50 +554,50 @@ } }, "google-cloud-sdk@latest": { - "last_modified": "2024-05-22T06:18:38Z", - "resolved": "github:NixOS/nixpkgs/3f316d2a50699a78afe5e77ca486ad553169061e#google-cloud-sdk", + "last_modified": "2024-06-06T04:00:50Z", + "resolved": "github:NixOS/nixpkgs/818dbe2f96df233d2041739d6079bb616d3e5597#google-cloud-sdk", "source": "devbox-search", - "version": "475.0.0", + "version": "478.0.0", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/s3hak6b0mmb2hx86h3nw0ycw8yb00d8y-google-cloud-sdk-475.0.0", + "path": "/nix/store/6vw818ni5lym496xy4d18pn2j272c5iv-google-cloud-sdk-478.0.0", "default": true } ], - "store_path": "/nix/store/s3hak6b0mmb2hx86h3nw0ycw8yb00d8y-google-cloud-sdk-475.0.0" + "store_path": "/nix/store/6vw818ni5lym496xy4d18pn2j272c5iv-google-cloud-sdk-478.0.0" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/5p94d16aqg8a04bin4pcz19zmsmadp99-google-cloud-sdk-475.0.0", + "path": "/nix/store/y133y7aaq6knvxgq6w5y0ml9y16d7811-google-cloud-sdk-478.0.0", "default": true } ], - "store_path": "/nix/store/5p94d16aqg8a04bin4pcz19zmsmadp99-google-cloud-sdk-475.0.0" + "store_path": "/nix/store/y133y7aaq6knvxgq6w5y0ml9y16d7811-google-cloud-sdk-478.0.0" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/a55rqgn3b6ycvkl8i9radrfkp7p1yisn-google-cloud-sdk-475.0.0", + "path": "/nix/store/cazzvzlwf34nnmnh2ilb2bc8b5g6jg5q-google-cloud-sdk-478.0.0", "default": true } ], - "store_path": "/nix/store/a55rqgn3b6ycvkl8i9radrfkp7p1yisn-google-cloud-sdk-475.0.0" + "store_path": "/nix/store/cazzvzlwf34nnmnh2ilb2bc8b5g6jg5q-google-cloud-sdk-478.0.0" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/yfkm1jl2frrvgzyxqc53f3szvnk0xgmj-google-cloud-sdk-475.0.0", + "path": "/nix/store/l04ghn5n5crhz7bzmkqi4ycs9770vw7j-google-cloud-sdk-478.0.0", "default": true } ], - "store_path": "/nix/store/yfkm1jl2frrvgzyxqc53f3szvnk0xgmj-google-cloud-sdk-475.0.0" + "store_path": "/nix/store/l04ghn5n5crhz7bzmkqi4ycs9770vw7j-google-cloud-sdk-478.0.0" } } }, @@ -1007,10 +1007,9 @@ } } }, - "ruby@3.3.1": { - "last_modified": "2024-05-13T18:06:16Z", - "plugin_version": "0.0.2", - "resolved": "github:NixOS/nixpkgs/1daef0114a6074be56586f2cf81898ed142e1e44#ruby_3_3", + "ruby@latest": { + "last_modified": "2024-05-22T06:18:38Z", + "resolved": "github:NixOS/nixpkgs/3f316d2a50699a78afe5e77ca486ad553169061e#ruby_3_3", "source": "devbox-search", "version": "3.3.1", "systems": { @@ -1018,57 +1017,57 @@ "outputs": [ { "name": "out", - "path": "/nix/store/g84fk6i3alpkd8m7gq31fcwykihmfdm9-ruby-3.3.1", + "path": "/nix/store/pkp3jxxrv5k2q30d7dlrhi3iavf7yx30-ruby-3.3.1", "default": true }, { "name": "devdoc", - "path": "/nix/store/h8qihxdi9jh6x6zn7wx6880z5xh856s6-ruby-3.3.1-devdoc" + "path": "/nix/store/kd1jzkciy029zvr9z93flqvj5jv3p4fg-ruby-3.3.1-devdoc" } ], - "store_path": "/nix/store/g84fk6i3alpkd8m7gq31fcwykihmfdm9-ruby-3.3.1" + "store_path": "/nix/store/pkp3jxxrv5k2q30d7dlrhi3iavf7yx30-ruby-3.3.1" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/c7g40p7m9qblj7g79bmbgs2fdwzpxvbf-ruby-3.3.1", + "path": "/nix/store/nz9ad6vkijjqb4a472d9wdsnr2v8zpzk-ruby-3.3.1", "default": true }, { "name": "devdoc", - "path": "/nix/store/fsg4x9g1sc8vnfp9nnd681gbgw00vi95-ruby-3.3.1-devdoc" + "path": "/nix/store/fdg456ic2642jzhrc2xx77930xw2zml0-ruby-3.3.1-devdoc" } ], - "store_path": "/nix/store/c7g40p7m9qblj7g79bmbgs2fdwzpxvbf-ruby-3.3.1" + "store_path": "/nix/store/nz9ad6vkijjqb4a472d9wdsnr2v8zpzk-ruby-3.3.1" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/k3np8iw8rgvnvzahw90k692kvvk7arzc-ruby-3.3.1", + "path": "/nix/store/52f08h8gxjd0i0kf321ka6iyyfv17cyw-ruby-3.3.1", "default": true }, { "name": "devdoc", - "path": "/nix/store/v9m76yxmckdsvy4yk77fbsah38yvf678-ruby-3.3.1-devdoc" + "path": "/nix/store/bj3s7wngp91256scm3x6x7jivncgynrx-ruby-3.3.1-devdoc" } ], - "store_path": "/nix/store/k3np8iw8rgvnvzahw90k692kvvk7arzc-ruby-3.3.1" + "store_path": "/nix/store/52f08h8gxjd0i0kf321ka6iyyfv17cyw-ruby-3.3.1" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/l7i7nszmnx5dxn0bvdf838r2nzviydwn-ruby-3.3.1", + "path": "/nix/store/nwbbnrwrzhk552p11lcxkxjfxi7z69sx-ruby-3.3.1", "default": true }, { "name": "devdoc", - "path": "/nix/store/57c2xsirc0nz182gcl62ab14igpxsndj-ruby-3.3.1-devdoc" + "path": "/nix/store/iqrnl5h2rfczsrkiqpskg9m86r5l51gl-ruby-3.3.1-devdoc" } ], - "store_path": "/nix/store/l7i7nszmnx5dxn0bvdf838r2nzviydwn-ruby-3.3.1" + "store_path": "/nix/store/nwbbnrwrzhk552p11lcxkxjfxi7z69sx-ruby-3.3.1" } } }, From 984064e148974036127f81dc5c1df431c53d7de2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 8 Jun 2024 13:18:43 +0000 Subject: [PATCH 041/186] chore(deps): update dependency terraform to v1.8.5 --- ci/terragrunt/.tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/terragrunt/.tool-versions b/ci/terragrunt/.tool-versions index bd00aeb9d8..a7673f263b 100644 --- a/ci/terragrunt/.tool-versions +++ b/ci/terragrunt/.tool-versions @@ -1,6 +1,6 @@ gcloud 473.0.0 helm 3.14.4 python 3.12.4 -terraform 1.8.4 +terraform 1.8.5 terraform-lsp terragrunt 0.58.14 From d08cf252f3ff479e0163de90ea63cc6269b2a84b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 8 Jun 2024 13:18:46 +0000 Subject: [PATCH 042/186] chore(deps): update github/codeql-action action to v3.25.8 --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index dbfab61af5..bd5c98a1f7 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -54,6 +54,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6 + uses: github/codeql-action/upload-sarif@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # v3.25.8 with: sarif_file: results.sarif From 96f4f849b9504606414f681ba7ce7813729ca2d1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 8 Jun 2024 15:18:55 +0000 Subject: [PATCH 043/186] chore(deps): update dependency com.puppycrawl.tools:checkstyle to v10.17.0 --- src/scheduler/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scheduler/pom.xml b/src/scheduler/pom.xml index 8197c62ba3..16b46b22c1 100644 --- a/src/scheduler/pom.xml +++ b/src/scheduler/pom.xml @@ -184,7 +184,7 @@ com.puppycrawl.tools checkstyle - 10.15.0 + 10.17.0 From cc41b3b2546803fb57c7563713be28820bc140dd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 06:56:42 +0000 Subject: [PATCH 044/186] chore(deps): update dependency bosh-cli to v7.6.1 --- ci/dockerfiles/autoscaler-tools/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/dockerfiles/autoscaler-tools/Dockerfile b/ci/dockerfiles/autoscaler-tools/Dockerfile index 52c8aa53ca..6eac2e2f77 100644 --- a/ci/dockerfiles/autoscaler-tools/Dockerfile +++ b/ci/dockerfiles/autoscaler-tools/Dockerfile @@ -63,7 +63,7 @@ RUN apt-get update && \ # Install bosh_cli # renovate: datasource=github-releases depName=bosh-cli lookupName=cloudfoundry/bosh-cli -ARG BOSH_VERSION=7.5.7 +ARG BOSH_VERSION=7.6.1 RUN wget -q https://github.com/cloudfoundry/bosh-cli/releases/download/v${BOSH_VERSION}/bosh-cli-${BOSH_VERSION}-linux-amd64 && \ mv bosh-cli-* /usr/local/bin/bosh && \ chmod +x /usr/local/bin/bosh From b8b47c6b3828689bee138a26a0c293b0587df88f Mon Sep 17 00:00:00 2001 From: app-autoscaler-ci-bot Date: Mon, 10 Jun 2024 09:19:06 +0100 Subject: [PATCH 045/186] Updated release version to 13.2.0 in golangapiserver --- jobs/golangapiserver/spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/golangapiserver/spec b/jobs/golangapiserver/spec index d783ff4492..4cce6c9f1e 100644 --- a/jobs/golangapiserver/spec +++ b/jobs/golangapiserver/spec @@ -92,7 +92,7 @@ properties: default: "Autoscaler" autoscaler.apiserver.info.build: description: "Autoscaler application build number" - default: "13.1.1" + default: "13.2.0" autoscaler.apiserver.info.support_url: description: "Autoscaler application support page URL" default: "https://github.com/cloudfoundry/app-autoscaler-release" From d9fcff67d578710da6cd6956f39dcff3ff9b0626 Mon Sep 17 00:00:00 2001 From: app-autoscaler-ci-bot Date: Mon, 10 Jun 2024 09:21:53 +0100 Subject: [PATCH 046/186] created release v13.2.0 --- .final_builds/jobs/golangapiserver/index.yml | 4 + .final_builds/jobs/metricsforwarder/index.yml | 4 + .../packages/changeloglockcleaner/index.yml | 4 + .final_builds/packages/db/index.yml | 4 + .../packages/eventgenerator/index.yml | 4 + .../packages/golangapiserver/index.yml | 4 + .../packages/metricsforwarder/index.yml | 4 + .../packages/metricsgateway/index.yml | 4 + .../packages/metricsserver/index.yml | 4 + .final_builds/packages/operator/index.yml | 4 + .../packages/scalingengine/index.yml | 4 + .final_builds/packages/scheduler/index.yml | 4 + .../app-autoscaler/app-autoscaler-13.2.0.yml | 163 ++++++++++++++++++ releases/app-autoscaler/index.yml | 2 + 14 files changed, 213 insertions(+) create mode 100644 releases/app-autoscaler/app-autoscaler-13.2.0.yml diff --git a/.final_builds/jobs/golangapiserver/index.yml b/.final_builds/jobs/golangapiserver/index.yml index cab3a7f9d6..0164f3a677 100644 --- a/.final_builds/jobs/golangapiserver/index.yml +++ b/.final_builds/jobs/golangapiserver/index.yml @@ -175,6 +175,10 @@ builds: version: 7a59bfb3465fe34f6e504114858897242d8944bf588bcc681a3bc38e0aebe1b1 blobstore_id: 4298b79b-3ee1-4603-50fe-c753e56b0e13 sha1: sha256:494718ec9cbb8a5a6ade0616c0ec25c264b1c4cef5bb94447b4249aa441c78a7 + 7b50daca9c9526ea96c9631baf2a794411a5ff112fffce5a90cd0638322c1197: + version: 7b50daca9c9526ea96c9631baf2a794411a5ff112fffce5a90cd0638322c1197 + blobstore_id: b41ce626-1203-4431-493d-cd6115224af2 + sha1: sha256:7c641f775487decfa25266869e6f60066f4011a1339e2c73b31978ad491471b8 7d50dbe2fdfbbeae0d1a01f84cb6ebfea1c5e409c8468890226d932e54c7017b: version: 7d50dbe2fdfbbeae0d1a01f84cb6ebfea1c5e409c8468890226d932e54c7017b blobstore_id: 97d6f574-93b3-4774-4fb6-595ee05d0fd9 diff --git a/.final_builds/jobs/metricsforwarder/index.yml b/.final_builds/jobs/metricsforwarder/index.yml index 7cfb09ac07..94291564d3 100644 --- a/.final_builds/jobs/metricsforwarder/index.yml +++ b/.final_builds/jobs/metricsforwarder/index.yml @@ -11,6 +11,10 @@ builds: version: 8af6f7ef09c0f11be0bd4972f0ca2b9074a8eb8b5045ad7aca43cf4b177152d6 blobstore_id: c1469768-5a5e-4daa-7196-507659a1f256 sha1: sha256:6d857a2ddd988cbee9c1870b1d2f2b3f331916576f38371f445d04124e9ac0e8 + 8d0b8d5a1f17f01079172163d6a78147cde52d4b0c81ea2771b3af1f5bd0e649: + version: 8d0b8d5a1f17f01079172163d6a78147cde52d4b0c81ea2771b3af1f5bd0e649 + blobstore_id: 42f834f2-052a-4657-4d34-5ea15c8542d1 + sha1: sha256:07bcd92da7f7f66257110d6e1f91d9c624dbb7937b9bd11c9b26784317c8c641 8ec2a304ce88d6b18d65cb68ebfae935f7e07a32d3c99bc396bada47ac278e3d: version: 8ec2a304ce88d6b18d65cb68ebfae935f7e07a32d3c99bc396bada47ac278e3d blobstore_id: afa32046-18f4-4783-4b63-6fd305cfabf8 diff --git a/.final_builds/packages/changeloglockcleaner/index.yml b/.final_builds/packages/changeloglockcleaner/index.yml index 4cf0df8c90..2fdc47d06d 100644 --- a/.final_builds/packages/changeloglockcleaner/index.yml +++ b/.final_builds/packages/changeloglockcleaner/index.yml @@ -147,6 +147,10 @@ builds: version: bbd6b1bc309c9aee0cb7276885b7465bbf6009bc4c31bfcbf3eb882e6ec96783 blobstore_id: 52632ede-a58a-4e41-6b18-f8dd06e9e260 sha1: sha256:ed67f19a318df287ec938ea3d5a18c43f838cad33fe1c23bdfc724c153333bb5 + ce9be5cce207009a889a305a1dc96774bd60fe319dbe7c97ce4b1f3b5248f654: + version: ce9be5cce207009a889a305a1dc96774bd60fe319dbe7c97ce4b1f3b5248f654 + blobstore_id: fb9e7c5e-e5fb-49da-738e-c23463e0093e + sha1: sha256:34b6f5db7aad87bae7bf13308b2fdd46cf31cccb2ae69079fe992317d96095b6 cfbc3e5733735c5c8220f649bca20a79e937edc99178078efb11b6dcd5c1e670: version: cfbc3e5733735c5c8220f649bca20a79e937edc99178078efb11b6dcd5c1e670 blobstore_id: 14d4b82f-81db-420a-7b87-3e8696f85ce4 diff --git a/.final_builds/packages/db/index.yml b/.final_builds/packages/db/index.yml index f8e8c667c6..9ee0602fdd 100644 --- a/.final_builds/packages/db/index.yml +++ b/.final_builds/packages/db/index.yml @@ -43,6 +43,10 @@ builds: version: 26aec38d8aac24e6b83d3ab782975eff0b68fc883f60452dd198668bed77f081 blobstore_id: 6c709146-39fc-4edf-69f9-8ad64d892490 sha1: sha256:36469b9620b2bf990e6f12db1eb22c46a6357bf72d582e12166116db841af37c + 30314772b27e47ea89fdddafc21b9f2ba058c24c5631c280e19c182153d6d162: + version: 30314772b27e47ea89fdddafc21b9f2ba058c24c5631c280e19c182153d6d162 + blobstore_id: 5e327315-ff0e-41f0-688b-ea6bebf3b128 + sha1: sha256:f64b2f4bedfbde1d895e0372f1d13ab04e0416dcc412baa2e29c5bfc680408c1 3e3baa755320452467b16b73f1c34d239b4b974a55d99482b0d8cfa173e4e932: version: 3e3baa755320452467b16b73f1c34d239b4b974a55d99482b0d8cfa173e4e932 blobstore_id: 43169a01-a728-402e-6df8-be89d7e236bd diff --git a/.final_builds/packages/eventgenerator/index.yml b/.final_builds/packages/eventgenerator/index.yml index 1996b69fcc..6e47f46411 100644 --- a/.final_builds/packages/eventgenerator/index.yml +++ b/.final_builds/packages/eventgenerator/index.yml @@ -167,6 +167,10 @@ builds: version: c6800c8b79da99afab00effa688fbf4625fb0be0f906bfaa1a7fc728118ca81b blobstore_id: d82d9cf8-519b-42b0-6405-78963f8138b2 sha1: sha256:476d587aa54d7de9f70718bf2d445c1335c2607b4a109282b101673fa2d64ec5 + c833815d3567b331a94b76a5e10c3bcbc3434c1def3d4d5b188e1b00fc15fec7: + version: c833815d3567b331a94b76a5e10c3bcbc3434c1def3d4d5b188e1b00fc15fec7 + blobstore_id: 2236cf40-8689-4acf-4956-acca1828f6d2 + sha1: sha256:99242ca53585808d1a1043b052018e9d982e7112662dd735ff32fc80e0ecb0e6 c8f50b5331c5edce9b41d2acf747452711eec2a02fbd3bcd11159ad58f3b34e0: version: c8f50b5331c5edce9b41d2acf747452711eec2a02fbd3bcd11159ad58f3b34e0 blobstore_id: fa62f106-34cf-44b9-494d-ea3d19b58709 diff --git a/.final_builds/packages/golangapiserver/index.yml b/.final_builds/packages/golangapiserver/index.yml index 23c8768cab..3da2f01dac 100644 --- a/.final_builds/packages/golangapiserver/index.yml +++ b/.final_builds/packages/golangapiserver/index.yml @@ -1,4 +1,8 @@ builds: + 056422919536f140a2db2ca13d43edc7e455c76553d4fd45e7fe3ad3a438bfce: + version: 056422919536f140a2db2ca13d43edc7e455c76553d4fd45e7fe3ad3a438bfce + blobstore_id: 0bd76d23-53d8-4de7-6c2a-7de9e5784aa7 + sha1: sha256:820a6b92406dee9e9b8456a7015f2369cd2ad00231e705e190d4095763a223ab 0c5e23ae94504bc03a18d4583be01b7e998c06b3bd993a1bc44e8e8b5b6091e7: version: 0c5e23ae94504bc03a18d4583be01b7e998c06b3bd993a1bc44e8e8b5b6091e7 blobstore_id: 8739f362-8515-4a6b-7cc3-11a326790a4c diff --git a/.final_builds/packages/metricsforwarder/index.yml b/.final_builds/packages/metricsforwarder/index.yml index bb166a763f..cb639ccdef 100644 --- a/.final_builds/packages/metricsforwarder/index.yml +++ b/.final_builds/packages/metricsforwarder/index.yml @@ -139,6 +139,10 @@ builds: version: 90f1e81596099b56867ee5fbaecc7017340d8e96b600295fdf0e5435ae8f8415 blobstore_id: d6400d9e-ac27-4889-5e16-9d29f380d100 sha1: sha256:de9bc7020a79395fef2642343cda9b2527e78af76c30e3d4ee9b0b5c6e18f1ee + 9958d98181a2916a7b25df8a31242d0eec47da2a55a47c251e362e4e6d4afc01: + version: 9958d98181a2916a7b25df8a31242d0eec47da2a55a47c251e362e4e6d4afc01 + blobstore_id: a5cbf516-d3c8-4ce6-596a-a0fbaee1fc9a + sha1: sha256:04c46596e5880db917c91e509046778c2876821cfbec0b190f91b1fae18877a9 a09649f8917221e3310e6b043d0d2ded5c1767855470105b5dde93bbce052827: version: a09649f8917221e3310e6b043d0d2ded5c1767855470105b5dde93bbce052827 blobstore_id: c0530c49-6523-4a28-54a4-4d88b3e44db7 diff --git a/.final_builds/packages/metricsgateway/index.yml b/.final_builds/packages/metricsgateway/index.yml index d98243845b..8965425be4 100644 --- a/.final_builds/packages/metricsgateway/index.yml +++ b/.final_builds/packages/metricsgateway/index.yml @@ -55,6 +55,10 @@ builds: version: 33abbce712b80dcbcecac92bff6db70b2a103eaa2adc01f2e032dd7cddf0c32a blobstore_id: fa876268-39aa-45ff-4494-5d86930fc152 sha1: sha256:f66fc83e4b2810d536438ebc956833e43ace69034a85a9a631118b9a6446090c + 347565f1d9a82e1b4450abee06497681bdd66767776e01a4f0ad95afc75a87ca: + version: 347565f1d9a82e1b4450abee06497681bdd66767776e01a4f0ad95afc75a87ca + blobstore_id: 08339505-7b7a-4134-78d4-33e15700236c + sha1: sha256:182d3664acb9e7bc38e437e8df4c8c6d52705028dbc02054fce46703a74c4051 3960d2e8102c72a53d1ef3dadb2c478c61e6a843fd3d51d79fbcb8f60dea18ea: version: 3960d2e8102c72a53d1ef3dadb2c478c61e6a843fd3d51d79fbcb8f60dea18ea blobstore_id: 8a26cf47-ef74-4fa3-6920-d3dc9a55a43c diff --git a/.final_builds/packages/metricsserver/index.yml b/.final_builds/packages/metricsserver/index.yml index f926345dc5..52188d2552 100644 --- a/.final_builds/packages/metricsserver/index.yml +++ b/.final_builds/packages/metricsserver/index.yml @@ -107,6 +107,10 @@ builds: version: 5c4c9715ad065a9a0d1019c865dc97213d0fc6b95c0da09a0809f6c8db562ed9 blobstore_id: d64cdbcd-bb69-4e51-62ce-7a5596cc4d3e sha1: sha256:a6cb25a60e0141b959a7798940c27f48ce21f029b43496c919d4bb6d48049b36 + 75a07fb457011ba841e4e95aa1b381ffdec649323ae315ea7eb8ad03f8571542: + version: 75a07fb457011ba841e4e95aa1b381ffdec649323ae315ea7eb8ad03f8571542 + blobstore_id: e9a0bed1-6979-40a4-6176-74999e39343e + sha1: sha256:844247ef773d339af7f8b1941ceb2280356b38dcdccd7de0476ab63ea0f9be85 7a33e815844bf49dbafe9c438df4035aba2b0f14ac52cb02358f09dfbdf63e99: version: 7a33e815844bf49dbafe9c438df4035aba2b0f14ac52cb02358f09dfbdf63e99 blobstore_id: dbbc94d4-461c-4c85-4a58-8bd1ade25696 diff --git a/.final_builds/packages/operator/index.yml b/.final_builds/packages/operator/index.yml index f5ead18976..5399661818 100644 --- a/.final_builds/packages/operator/index.yml +++ b/.final_builds/packages/operator/index.yml @@ -211,6 +211,10 @@ builds: version: e2edcb48f5cc1c196cc842f21bf34ae40058777297698a335b8eb0fef27a65df blobstore_id: 0fcfe8fb-fdd2-4a37-477d-41bfee0371c4 sha1: sha256:995a3de5ee16e112e394091cd982ff77bf61a2198c3f23b65ad7ea295c465b15 + e6c7ea00e3353318766f56c0ae34a1909286a6aba9ac6560ec9246dc4bee716b: + version: e6c7ea00e3353318766f56c0ae34a1909286a6aba9ac6560ec9246dc4bee716b + blobstore_id: 3e0058ad-410e-4f4f-6d73-2181a10434fc + sha1: sha256:e1273be12b2c5f95a5befe5b148ebe4cf9d80a4443c59da9733c83e3780705e5 ebb7e628ad51d51a4b24c362cd2616ca7bb9cdd029cfdb0d5cf2d44e00313cac: version: ebb7e628ad51d51a4b24c362cd2616ca7bb9cdd029cfdb0d5cf2d44e00313cac blobstore_id: 26c51761-a592-4369-5502-251c20740bdb diff --git a/.final_builds/packages/scalingengine/index.yml b/.final_builds/packages/scalingengine/index.yml index b1c3986a57..f24e27d9ff 100644 --- a/.final_builds/packages/scalingengine/index.yml +++ b/.final_builds/packages/scalingengine/index.yml @@ -247,6 +247,10 @@ builds: version: f372cb9da65e0a7915464bc738622996dd4ed31c949e616698388ec0012f1307 blobstore_id: 88017568-6ca8-4ffe-6e90-6400dd3b3ec8 sha1: sha256:4fcb79b199aee14f163c2654833a63a1225ecedc13b477ea7e7d4933737b2436 + fd565f3f53503af495ae10bc2f79cd332fdbfc7dc4bc7d4398ac64503cbcf481: + version: fd565f3f53503af495ae10bc2f79cd332fdbfc7dc4bc7d4398ac64503cbcf481 + blobstore_id: c3cec662-ed22-4474-4a06-b68ea84088ef + sha1: sha256:f52622d8a699843c3dfd62b6718a6e34f97dc00aa69f277dea99b7d44fa60917 fdcbc49f7dc37f35c388c7b0e09bc027365bb374336f4098848052c235f6fed0: version: fdcbc49f7dc37f35c388c7b0e09bc027365bb374336f4098848052c235f6fed0 blobstore_id: 3a6e91ba-b905-40dd-5066-bd849a9f5bd8 diff --git a/.final_builds/packages/scheduler/index.yml b/.final_builds/packages/scheduler/index.yml index 3e87770877..563675b1f6 100644 --- a/.final_builds/packages/scheduler/index.yml +++ b/.final_builds/packages/scheduler/index.yml @@ -119,6 +119,10 @@ builds: version: 676dddf54ba166daf8a0426b4ee82dd0845fe75b51d50ca7beef053272cf898b blobstore_id: 0e9619eb-ed99-4081-7e76-cce4524a2db6 sha1: sha256:7516818790ada62125bf48f35441ffd09523aaa23eb80b235b7d5031f00dabf0 + 68c993e8cd12782c3ecc81a1dc40705962aeee86b1d0bfd2fef9b612f0ce5778: + version: 68c993e8cd12782c3ecc81a1dc40705962aeee86b1d0bfd2fef9b612f0ce5778 + blobstore_id: dec959b0-748a-4a8f-422d-36588f80e479 + sha1: sha256:6b75c282c14d3e967a27c259b4fe29691b597a761c4ca49d1195618f6daaa828 68d4db110b15bf25b4d1ddfa4829a7c21055f5c4afe172f1403bfda3d40d00ea: version: 68d4db110b15bf25b4d1ddfa4829a7c21055f5c4afe172f1403bfda3d40d00ea blobstore_id: dc57e212-0dc4-4df9-5529-e64e031e6048 diff --git a/releases/app-autoscaler/app-autoscaler-13.2.0.yml b/releases/app-autoscaler/app-autoscaler-13.2.0.yml new file mode 100644 index 0000000000..28f682a868 --- /dev/null +++ b/releases/app-autoscaler/app-autoscaler-13.2.0.yml @@ -0,0 +1,163 @@ +name: app-autoscaler +version: 13.2.0 +commit_hash: 2f0b382dd +uncommitted_changes: false +jobs: +- name: eventgenerator + version: 292a7e23aaaeabdf009f1aa735e7ab98517faecdaadf387a8309832c8404deda + fingerprint: 292a7e23aaaeabdf009f1aa735e7ab98517faecdaadf387a8309832c8404deda + sha1: sha256:a55c01ef25e26d61a43fc7cd035ac13156520db3da4e510f527ac2be67767953 + packages: + - openjdk-17 + - db + - eventgenerator + - common + - changeloglockcleaner +- name: golangapiserver + version: 7b50daca9c9526ea96c9631baf2a794411a5ff112fffce5a90cd0638322c1197 + fingerprint: 7b50daca9c9526ea96c9631baf2a794411a5ff112fffce5a90cd0638322c1197 + sha1: sha256:7c641f775487decfa25266869e6f60066f4011a1339e2c73b31978ad491471b8 + packages: + - golang-1-linux + - golangapiserver + - db + - openjdk-17 + - common + - changeloglockcleaner +- name: metricsforwarder + version: 8d0b8d5a1f17f01079172163d6a78147cde52d4b0c81ea2771b3af1f5bd0e649 + fingerprint: 8d0b8d5a1f17f01079172163d6a78147cde52d4b0c81ea2771b3af1f5bd0e649 + sha1: sha256:07bcd92da7f7f66257110d6e1f91d9c624dbb7937b9bd11c9b26784317c8c641 + packages: + - db + - metricsforwarder + - common +- name: metricsgateway + version: 6b2fd6c39c12b3ce1b774a4f3eb634939347df02d4921b1a374bca027dae112c + fingerprint: 6b2fd6c39c12b3ce1b774a4f3eb634939347df02d4921b1a374bca027dae112c + sha1: sha256:83f07a5b73bd18a48bce412b258e0361a87bc5a1db6975a04166850a11dfd588 + packages: + - openjdk-17 + - db + - metricsgateway + - common +- name: metricsserver + version: fb43940973167724049f2524b3e35f93eba92f576fb7c4665db1b309b6944b25 + fingerprint: fb43940973167724049f2524b3e35f93eba92f576fb7c4665db1b309b6944b25 + sha1: sha256:de4dfcbfe7f28689377ee83f74886d76f7014dbd677f1bca02267fcc58d59572 + packages: + - openjdk-17 + - db + - metricsserver + - common + - changeloglockcleaner +- name: operator + version: 11ffdb19acd96f1099e42bdce522013ee7440519f746be68d99d7b1c37581375 + fingerprint: 11ffdb19acd96f1099e42bdce522013ee7440519f746be68d99d7b1c37581375 + sha1: sha256:3502913539aebf8f2e0b3a117ca5dc10c69bb87ead1d4939d538e1c869361bc1 + packages: + - operator + - common + - db + - openjdk-17 + - changeloglockcleaner +- name: scalingengine + version: fbd7ee3f304cf1ac2ec8fdc10e67f003d50d949ab25a58df838f0497ac503f5f + fingerprint: fbd7ee3f304cf1ac2ec8fdc10e67f003d50d949ab25a58df838f0497ac503f5f + sha1: sha256:d55638d6a6f8ebc26ea9fd29a5aa3dd83db59be9aa3f0fad2a4d3dda3721941f + packages: + - openjdk-17 + - db + - scalingengine + - common + - changeloglockcleaner +- name: scheduler + version: 81741f07889700443355701c5afdaa04c492acf69d5f89b94dc32b4b997bc288 + fingerprint: 81741f07889700443355701c5afdaa04c492acf69d5f89b94dc32b4b997bc288 + sha1: sha256:57067b1b7c6884b17db77bc691cce8de2a71db5ca5076d619d158a9be608e514 + packages: + - scheduler + - db + - openjdk-17 + - common + - changeloglockcleaner +packages: +- name: changeloglockcleaner + version: ce9be5cce207009a889a305a1dc96774bd60fe319dbe7c97ce4b1f3b5248f654 + fingerprint: ce9be5cce207009a889a305a1dc96774bd60fe319dbe7c97ce4b1f3b5248f654 + sha1: sha256:34b6f5db7aad87bae7bf13308b2fdd46cf31cccb2ae69079fe992317d96095b6 + dependencies: + - golang-1-linux +- name: common + version: 9d7726aa454227e3087cd4a04425acae72d883a5bb19d314930ab58e83f04d0e + fingerprint: 9d7726aa454227e3087cd4a04425acae72d883a5bb19d314930ab58e83f04d0e + sha1: sha256:979ff40c6e44031bec55344aa4a522cac3d265ab23deaea0f7bc7541aa2b4124 + dependencies: [] +- name: db + version: 30314772b27e47ea89fdddafc21b9f2ba058c24c5631c280e19c182153d6d162 + fingerprint: 30314772b27e47ea89fdddafc21b9f2ba058c24c5631c280e19c182153d6d162 + sha1: sha256:f64b2f4bedfbde1d895e0372f1d13ab04e0416dcc412baa2e29c5bfc680408c1 + dependencies: + - openjdk-17 +- name: eventgenerator + version: c833815d3567b331a94b76a5e10c3bcbc3434c1def3d4d5b188e1b00fc15fec7 + fingerprint: c833815d3567b331a94b76a5e10c3bcbc3434c1def3d4d5b188e1b00fc15fec7 + sha1: sha256:99242ca53585808d1a1043b052018e9d982e7112662dd735ff32fc80e0ecb0e6 + dependencies: + - golang-1-linux +- name: golang-1-linux + version: c2342901fca75f4c7ec3f32e6a757e923089c6c50d8eb3effd2c25eac1009e31 + fingerprint: c2342901fca75f4c7ec3f32e6a757e923089c6c50d8eb3effd2c25eac1009e31 + sha1: sha256:d354b084ed879adc3218f89b9109b474a68c3eafb5c0ea204c4b77963e4736b0 + dependencies: [] +- name: golangapiserver + version: 056422919536f140a2db2ca13d43edc7e455c76553d4fd45e7fe3ad3a438bfce + fingerprint: 056422919536f140a2db2ca13d43edc7e455c76553d4fd45e7fe3ad3a438bfce + sha1: sha256:820a6b92406dee9e9b8456a7015f2369cd2ad00231e705e190d4095763a223ab + dependencies: + - golang-1-linux +- name: metricsforwarder + version: 9958d98181a2916a7b25df8a31242d0eec47da2a55a47c251e362e4e6d4afc01 + fingerprint: 9958d98181a2916a7b25df8a31242d0eec47da2a55a47c251e362e4e6d4afc01 + sha1: sha256:04c46596e5880db917c91e509046778c2876821cfbec0b190f91b1fae18877a9 + dependencies: + - golang-1-linux +- name: metricsgateway + version: 347565f1d9a82e1b4450abee06497681bdd66767776e01a4f0ad95afc75a87ca + fingerprint: 347565f1d9a82e1b4450abee06497681bdd66767776e01a4f0ad95afc75a87ca + sha1: sha256:182d3664acb9e7bc38e437e8df4c8c6d52705028dbc02054fce46703a74c4051 + dependencies: + - golang-1-linux +- name: metricsserver + version: 75a07fb457011ba841e4e95aa1b381ffdec649323ae315ea7eb8ad03f8571542 + fingerprint: 75a07fb457011ba841e4e95aa1b381ffdec649323ae315ea7eb8ad03f8571542 + sha1: sha256:844247ef773d339af7f8b1941ceb2280356b38dcdccd7de0476ab63ea0f9be85 + dependencies: + - golang-1-linux +- name: openjdk-17 + version: be13f95495365f8879a75d9237320cd587774f6cbc087020bd6508fb8a82184a + fingerprint: be13f95495365f8879a75d9237320cd587774f6cbc087020bd6508fb8a82184a + sha1: sha256:99e2ef396973d7bbbfddffad6968b419627c20bb426117f8c3115cad5b206227 + dependencies: [] +- name: operator + version: e6c7ea00e3353318766f56c0ae34a1909286a6aba9ac6560ec9246dc4bee716b + fingerprint: e6c7ea00e3353318766f56c0ae34a1909286a6aba9ac6560ec9246dc4bee716b + sha1: sha256:e1273be12b2c5f95a5befe5b148ebe4cf9d80a4443c59da9733c83e3780705e5 + dependencies: + - golang-1-linux +- name: scalingengine + version: fd565f3f53503af495ae10bc2f79cd332fdbfc7dc4bc7d4398ac64503cbcf481 + fingerprint: fd565f3f53503af495ae10bc2f79cd332fdbfc7dc4bc7d4398ac64503cbcf481 + sha1: sha256:f52622d8a699843c3dfd62b6718a6e34f97dc00aa69f277dea99b7d44fa60917 + dependencies: + - golang-1-linux +- name: scheduler + version: 68c993e8cd12782c3ecc81a1dc40705962aeee86b1d0bfd2fef9b612f0ce5778 + fingerprint: 68c993e8cd12782c3ecc81a1dc40705962aeee86b1d0bfd2fef9b612f0ce5778 + sha1: sha256:6b75c282c14d3e967a27c259b4fe29691b597a761c4ca49d1195618f6daaa828 + dependencies: + - openjdk-17 +license: + version: 15eb386a0db48f190d29bd28058f5d0d6605f5acde7c6cfc6f014d57747e41f9 + fingerprint: 15eb386a0db48f190d29bd28058f5d0d6605f5acde7c6cfc6f014d57747e41f9 + sha1: sha256:6c7a834097b96efc340787fbe5876c1e366de0235def5e2b93fa016c8655535c diff --git a/releases/app-autoscaler/index.yml b/releases/app-autoscaler/index.yml index 00944573d7..a6967c59ba 100644 --- a/releases/app-autoscaler/index.yml +++ b/releases/app-autoscaler/index.yml @@ -23,6 +23,8 @@ builds: version: 6.0.0 28082e04-3605-4919-7a2c-3a2939c7e47d: version: 12.0.0 + 2ffe5558-cd6f-4be3-43ab-8fe4fc4f9005: + version: 13.2.0 4087e599-6e4b-4a10-6595-eec8e187fb5f: version: 7.1.0 482559bc-07e0-47a6-71b8-e38c35497490: From df75d1a4243bf3325cfad6b1b0b9b97890fa33e9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 09:04:59 +0000 Subject: [PATCH 047/186] fix(deps): update module github.com/gorilla/websocket to v1.5.2 --- src/autoscaler/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 2d550ec8b5..8d3a801a27 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -19,7 +19,7 @@ require ( github.com/go-sql-driver/mysql v1.8.1 github.com/golang/protobuf v1.5.4 github.com/gorilla/mux v1.8.1 - github.com/gorilla/websocket v1.5.1 + github.com/gorilla/websocket v1.5.2 github.com/hashicorp/go-retryablehttp v0.7.7 github.com/jackc/pgx/v5 v5.6.0 github.com/jmoiron/sqlx v1.4.0 From 583b76572862818aa0d71f2a272f878b40aac7a8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 10:11:55 +0100 Subject: [PATCH 048/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/autoscaler/go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 992db1ebcc..2ee0538170 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -849,8 +849,8 @@ github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gorilla/websocket v1.5.2 h1:qoW6V1GT3aZxybsbC6oLnailWnB+qTMVwMreOso9XUw= +github.com/gorilla/websocket v1.5.2/go.mod h1:0n9H61RBAcf5/38py2MCYbxzPIY9rOkpvvMT24Rqs30= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= From fa329b37dd118ed81d52c108377bf2d63f9115d1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:53:41 +0000 Subject: [PATCH 049/186] chore(deps): update jetify-com/devbox-install-action action to v0.11.0 --- .github/workflows/asdf2devbox.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/asdf2devbox.yaml b/.github/workflows/asdf2devbox.yaml index a131d4a25a..15162d7f88 100644 --- a/.github/workflows/asdf2devbox.yaml +++ b/.github/workflows/asdf2devbox.yaml @@ -44,7 +44,7 @@ jobs: echo "commit_subject=${commit_subject}" >> "$GITHUB_OUTPUT" - name: Install devbox - uses: jetify-com/devbox-install-action@v0.9.0 + uses: jetify-com/devbox-install-action@a03caf5813591bc882139eba6ae947930a83a427 # v0.11.0 with: enable-cache: 'true' From 07cb14715c66f0ae940eb8c7177506ad009f9592 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 22:48:17 +0000 Subject: [PATCH 050/186] chore(deps): update dependency terragrunt to v0.58.15 --- ci/terragrunt/.tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/terragrunt/.tool-versions b/ci/terragrunt/.tool-versions index a7673f263b..c50c3d9ae8 100644 --- a/ci/terragrunt/.tool-versions +++ b/ci/terragrunt/.tool-versions @@ -3,4 +3,4 @@ helm 3.14.4 python 3.12.4 terraform 1.8.5 terraform-lsp -terragrunt 0.58.14 +terragrunt 0.58.15 From ffcd3e233ea2766999ac0541d0902c1693a0f27d Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 7 Jun 2024 20:44:29 +0200 Subject: [PATCH 051/186] remove metricsserver and metricsgateway from drawings, delete outdated docu --- README.md | 2 -- docs/components-db-usage.drawio | 1 - docs/components-db-usage.svg | 4 +-- docs/developer_guide.md | 16 --------- docs/images/autoscaler.svg | 4 +-- docs/images/components-db-usage.svg | 4 --- docs/images/eventgenerator.svg | 4 +-- docs/images/metrics_gateway.svg | 4 --- docs/images/metrics_server.svg | 4 --- docs/log-cache.md | 53 ----------------------------- 10 files changed, 6 insertions(+), 90 deletions(-) delete mode 100644 docs/components-db-usage.drawio delete mode 100644 docs/images/components-db-usage.svg delete mode 100644 docs/images/metrics_gateway.svg delete mode 100644 docs/images/metrics_server.svg delete mode 100644 docs/log-cache.md diff --git a/README.md b/README.md index 70c52987bb..26c63ef229 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,6 @@ are listed in the table below, but the passwords are available in credhub. |------------------|--------------------------------------------------------------|------------------|----------------------------------------------| | eventgenerator | https://autoscaler-eventgenerator.((system_domain))/health | eventgenerator | /autoscaler_eventgenerator_health_password | | metricsforwarder | https://autoscaler-metricsforwarder.((system_domain))/health | metricsforwarder | /autoscaler_metricsforwarder_health_password | -| metricsgateway | https://autoscaler-metricsgateway.((system_domain))/health | metricsgateway | /autoscaler_metricsgateway_health_password | -| metricsserver | https://autoscaler-metricsserver.((system_domain))/health | metricsserver | /autoscaler_metricsserver_health_password | | scalingengine | https://autoscaler-scalingengine.((system_domain))/health | scalingengine | /autoscaler_scalingengine_health_password | | operator | https://autoscaler-operator.((system_domain))/health | operator | /autoscaler_operator_health_password | | scheduler | https://autoscaler-scheduler.((system_domain))/health | scheduler | /autoscaler_scheduler_health_password | diff --git a/docs/components-db-usage.drawio b/docs/components-db-usage.drawio deleted file mode 100644 index 27ff44f89b..0000000000 --- a/docs/components-db-usage.drawio +++ /dev/null @@ -1 +0,0 @@ -5Vxfk5o6FP80PtYBQgI8qutup91OO+PcubdPnSxklRaJN8a63k9/EwkqSVbdLQju2pkuHOBAzvnl/A30wGj+dMfwYvaFJiTreU7y1AM3Pc9zXQeJP5KyKSihGxaEKUsTddKeMEn/I4roKOoqTciyciKnNOPpokqMaZ6TmFdomDG6rp72SLPqXRd4SgzCJMaZSf07TfhMjQI6e/pHkk5n5Z19Rx2Z4/JkRVjOcELXByQw7oERo5QXW/OnEcmk8Eq5FNfdPnN092CM5PycCz77P9fhMBn4n5P7+/nHT6OvH50PblCw+Y2zlRqxelq+KUXA6CpPiOTi9MBwPUs5mSxwLI+uhdIFbcbnmdhzxeYjzfmIZpRtrwXO9ifo6jaEcfL07ADcnVgEngidE8424hR1AYz6sLhGYcnzlWjXe814gaLNDrVSqgsrNEx3zPcCExtKZi+RH7ge+fmBVxVfYBGfaxOf25j4DOnd0Qzn08G3VJAnhIkBH5Gne1qeNcgt0HHn2nDnQFNwqCm5eYbcFjRL482P5MEQl7A7C7kZb7JUyI2B00J7KCR8/7Aj4PjXdCv3rysu2BBFXxb22oUSuWmWHSD39jasDbmGBrwIGhpAFuSGTSnAnPYPQrhpPu2QBoYwANFNPRpwfUMFyJwEF1WBb6hgwikjyYLRmCQrRjqkChQMhtFtPaoQ4Ddmg6kKmxuMmlIFNFQx/i1Gc0dywrBQSusmXHd9u4CgNQuO3pcFh44We4QtG/AypDlQAF4sfkjmabxsVQvPRICadoLxAI1ri6y7ph0zwDH0QfJkIBM8qZIML5dpXBV+Nex+TqrkKeX/yHP6PgJq/7vk0HdAuX/zpJhsdzYHO98IS8WICVO04iFJYuSUmibEQOiKxeSUTTU1dqARaNFISWMkwzz9XX0Mm5rUHb7RVDzgHhBAA4Svabp4fHXVYdKpM0JBlVGgMeKYTQk3GG1Rsxv2HwAp7ACQRJL9UiAJuLCNYgdRSZD8Pjh9x4MlZc9xu7c53GsQnM+Yk8uAEwA9AALOK+HpBrDve87+p6EVoMuiNTqN1q5UE0plKlHB0BJR2XyG6zWWFJsu/Uvhzh8pW2OWdKCWEFbtqm8rwVy2kmCWYN50IBrptgM5Lcc6nllLeCeJrOtpmmg5jfXMksJVBAs1OvaDzkQbnl1L0xF4tV+vOnKoz96GHblnVkQ668h9LdaHvmkQgWfz5M05JbM68nXRlVKS7sIt4rqwC7f18FAmbjtcMFKRFvp3JduNWzh9eMTzNBOjGPSkdhGeS2GpEz4RPmQ4zZfi2BeaU/14zxPP6szFkWUh7OGeudiaqr/bhxBn5NanMC8RN+Q4j0mlDlNwEaIpGFWZC/J2kCW1G27ydvsT9Aw/kGy4Y1CeklPJpdFZDbVZbYnPLxvm2DLx60TpMsaZbPWQfCrQcL0YvYkQahOjEEV9rxqNQ0tXyYd9mzmFTQHVloRfJ1D1ivYVonR0M/CG4zZRijRL2nqaAsw6h6GtLqYpHa9plkFnkRqcDO7bypECX4ejZgjPzZECrekDocao4RwJmJWnI8F9x5aehbYGMvJsjsotywr1GwLPkKAqeC67sXIq8jWxBa333YFZbXvT9U5XK7KFbbd2wRmlkS56rxo9TTFv2yuyVWtj0aubZ2Fkt4mXciBmlehaA/X3UvdIyCNeZfzIpK3B5HmeZvIsTbbLmjyz9KH89BRzssab1h01dJx+oLnq1luTwEzE37SrRiHQyyFh64tsrz/X3OWV3w+PPZNk1unl1aw/mU/6rYYDgYm6SF+icfZqrwgaCL7wgi//jBdypHonapcyPqNTmuNsvKdqkN2fc0/pQqH6J+F8o95XwytOX5KL1oKvsg5wEl+FHTXxdTZw/syCmElqJ6K0bW0U53gq8+SzYqoXFSNeV65cckZ/kTNW/2qBFpWLm7R71BEWgODZ+XxgnVxbd7ixsMC3hf0drRtpvWLXLVf4nXxjETQ2H81u8aToZxXtrCOyvFDRyHjdzvHaDkV9M4J/06GoqQMXuX2zcgStFU+EmlJDR1tzy3hGklVGWDVv72CCPkbyX02lRcO6AXOe2iJXPSSs7z00W7bSAXy8fo3BtS8xGImf5mL3NaDjMVINCD2wTzuMQmSzY4HNjgVN+WB4Ruur41l1Fzu4RWjTWl3daBK6bhBVuZxdWo+Ovpci4sjL9mrLJRDXEHP7yNSDByyT3v6ti3Jk9U96szKxixuOCPNSL0hrCHM8S6Zy0Zgbmqvx33TMDbVOrZjjvgW1Fy3/QluztgMB1fsMuJGjr6foBEbO+IRUF4OZGgOPwrS3VsE33od13TB8XeAB9Q9lGK9rNhxpIDOFG6w4lemQdJPOiM4XIj/IuTQfOE/E/zeY4we8lObor6X8nJuOPjHjeBVvIrea5hKMgpEExFDOy1TcY6AOzNMkKToBRExu/LBlJVGzkAPfigIOe1B+XUcW/0sDYJRwy3rvgVGwlICrbXHVVJD6riVND4wirmeaDN9mM/TXr+r7+oktnuyAX8lo/KvzLmUYjZ0BPJk/N7kc0uwLIBumImtW/XJQid39dxILS7P/2iQY/w8= \ No newline at end of file diff --git a/docs/components-db-usage.svg b/docs/components-db-usage.svg index 4c4a68542f..5654e6e6ec 100644 --- a/docs/components-db-usage.svg +++ b/docs/components-db-usage.svg @@ -1,4 +1,4 @@ - + -
GolangAPi Server
GolangAPi Server
policy_db
policy_db
binding_db
binding_db
Storedprocedure_db
Storedproce...
EventGenerator
EventGenerator
policy_db
policy_db
app_metrics_db
app_metric...
Metricsforwarder
Metricsforwarder
policy_db
policy_db
Storedprocedure_db
Storedproce...
Operator
Operator
instance_metrics_db
instance_m...
scaling_engine_db
scaling...
app_metrics_db
app_metrics...
Metricsserver
Metricsserver
policy_db
policy_db
instance_metrics_db
instance_m...
Metricsgateway
Metricsgateway
policy_db
policy_db
app_manager
app_manager
Scalingengine
Scalingengine
policy_db
policy_db
scheduler_db
schedule...
scalingengine_db
scalingen...
scheduler
scheduler
policy_db
policy_db
scheduler_db
scheduler_...
Autoscaler Components and Database Usage
Autoscaler Components and Database Usage
lock_db
lock_dbText is not SVG - cannot display \ No newline at end of file +
GolangAPi Server
GolangAPi Server
policy_db
policy_db
binding_db
binding_db
Storedprocedure_db
Storedproce...
EventGenerator
EventGenerator
policy_db
policy_db
app_metrics_db
app_metric...
Operator
Operator
instance_metrics_db
instance_m...
scaling_engine_db
scaling...
app_metrics_db
app_metrics...
Scalingengine
Scalingengine
policy_db
policy_db
scheduler_db
schedule...
scalingengine_db
scalingen...
scheduler
scheduler
policy_db
policy_db
scheduler_db
scheduler_...
Autoscaler Components and Database Usage
Autoscaler Components and Database Usage
lock_db
lock_dbText is not SVG - cannot display \ No newline at end of file diff --git a/docs/developer_guide.md b/docs/developer_guide.md index 1d41a6c3ff..96d4f8da00 100644 --- a/docs/developer_guide.md +++ b/docs/developer_guide.md @@ -22,19 +22,3 @@ - Evaluates app policies rules and generates scaling events based on metrics cache. - Manages coolDown threshold for scaling events. -### MetricsServer (To be Deprecated) - -![Alt text](./images/metrics_server.svg) - -**Responsabilities:** - -- For Timer metrics it caches and compiles httpStartStop events to collect a average response time and throughput metric for a configured interval, by default 60 Seconds. -- Keeps track of current metrics sharded by node. -- if persistence is enabled, it stores metrics in DB. -- Provides HTTPServer GET endpoint to retrieve metrics_history by appid/metrictype. -- Transforms GAUGE envelopes into autoscaler compatible metrics (memoryutil, ) - -### MetricsGateway (To be Deprecated) - -![Alt text](./images/metrics_gateway.svg) - diff --git a/docs/images/autoscaler.svg b/docs/images/autoscaler.svg index ecb5943d80..53c8910909 100644 --- a/docs/images/autoscaler.svg +++ b/docs/images/autoscaler.svg @@ -1,4 +1,4 @@ - + -
Log Cache
Log Cache
retrive envelops 
via Grpc
retrive envelops...
eventGenerator
eventGenerator
syslog
syslog
MetricsForwarder
MetricsForwarder
envelops via WSS
envelops via WSS
MetricsServer
MetricsServer
envelope
via nozzle subs
envelope...
MetricsGateway
MetricsGateway
AppMetrics
 via HTTP
AppMetrics...
Loggregator
Loggregator
When log-cache enabled no request are performed to metricServer
When log-cache enabled no reque...
CF Deployment
CF Deployment
App Autoscaler
App Autosc...
scalingEngine
scalingEngine
ApiServer
ApiServerText is not SVG - cannot display \ No newline at end of file + \ No newline at end of file diff --git a/docs/images/components-db-usage.svg b/docs/images/components-db-usage.svg deleted file mode 100644 index 3d85eb5f43..0000000000 --- a/docs/images/components-db-usage.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
GolangAPi Server
GolangAPi Server
policy_db
policy_db
binding_db
binding_db
Storedprocedure_db
Storedproce...
EventGenerator
EventGenerator
policy_db
policy_db
app_metrics_db
app_metric...
Metricsforwarder
Metricsforwarder
policy_db
policy_db
Storedprocedure_db
Storedproce...
Operator
Operator
instance_metrics_db
instance_m...
scaling_engine_db
scaling...
app_metrics_db
app_metrics...
Metricsserver
Metricsserver
policy_db
policy_db
instance_metrics_db
instance_m...
Metricsgateway
Metricsgateway
policy_db
policy_db
app_manager
app_manager
Scalingengine
Scalingengine
policy_db
policy_db
scheduler_db
schedule...
scalingengine_db
scalingen...
scheduler
scheduler
policy_db
policy_db
scheduler_db
scheduler_...
Autoscaler Components and Database Usage
Autoscaler Components and Database Usage
lock_db
lock_dbText is not SVG - cannot display \ No newline at end of file diff --git a/docs/images/eventgenerator.svg b/docs/images/eventgenerator.svg index 8aa6ed8d75..43a6548fa7 100644 --- a/docs/images/eventgenerator.svg +++ b/docs/images/eventgenerator.svg @@ -1,4 +1,4 @@ - + -
SaveMetricToCache(...)
SaveMetricToCache(...)
Aggregator
Aggregator

(PolicyDB, appMetricsDB)
 (PolicyDB, appMetricsDB)
AppManager
AppManager
MetricPoller
MetricPoller
1..config.MetricPollerCount
1..config.MetricPollerCount
MetricPoller
MetricPoller
MetricPoller
MetricPoller
The aggregator does not aggregate but only say what and how much will be aggregated. The real aggregation is performed by the MetricPoller and send to the Aggregator.
The aggregator does not aggrega...
Start(): Listen to the instructions in form of *models.AppMonitor.

For each of these metric-collection-instructions, poll the MetricServer and do some form of aggregation. Send the result to the Aggregator via the channel “appMetricChan”.
Start(): Listen to the instruct...
Server
Server
eventgen_handler
eventgen_handler
Server
Server
Start(): Caches and refresh app policies managed by this node every conf.Aggregator.PolicyPollerInterval duration internal. It shards policies by node.
Start(): Caches and refresh app...
AppManager.QueryAppMetrics(...)
AppManager.QueryAppMetrics(...)
AppMetrics[]
AppMetrics[]
conf.aggregatorInterval
conf.aggre...
Autoscaler - Eventgenerator
Autoscaler - Eventgenerator
Generator
Generator
1..config.EvaluatorCount
1..config.EvaluatorCount
TriggersChan:
 *[]Triggers
TriggersChan:...
evaluator
evaluator
evaluator
evaluator
evaluator
evaluatoruuu
AppEvaluationManager
AppEvaluationManager
ScalingEngine
ScalingEngine
POST https
POST https
doEvaluate():
- Create any missing circuite
 breaker for the appPolicies
retrived by AppManager.
- writes on triggers channel
an updated list per app of all
existing rules on that app
policy. This happens 
 every 60s by default.
%3CmxGraphModel%3E%3Croot%3E%3CmxCell%20id%3D%220%22%2F%3E%3CmxCell%20id%3D%221%22%20parent%3D%220%22%2F%3E%3CmxCell%20id%3D%222%22%20value%3D%22%22%20style%3D%22group%22%20vertex%3D%221%22%20connectable%3D%220%22%20parent%3D%221%22%3E%3CmxGeometry%20x%3D%2210%22%20y%3D%2275%22%20width%3D%22230%22%20height%3D%2289.99999999999999%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%223%22%20value%3D%22%22%20style%3D%22verticalLabelPosition%3Dbottom%3Bsketch%3D0%3Baspect%3Dfixed%3Bhtml%3D1%3BverticalAlign%3Dtop%3BstrokeColor%3Dnone%3Balign%3Dcenter%3BoutlineConnect%3D0%3Bshape%3Dmxgraph.citrix.process%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20y%3D%2221.67%22%20width%3D%2226.84%22%20height%3D%2232%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%224%22%20value%3D%22Start()%3A%20Caches%20and%20refresh%20app%20%26lt%3Bbr%26gt%3Bpolicies%20managed%20by%20this%20%26lt%3Bbr%26gt%3BeventGenerator%20node%20every%20x%20time%20%26lt%3Bbr%26gt%3Binternal.%20It%20shards%20policies%20by%20%26lt%3Bbr%26gt%3BeventGenerator%26amp%3Bnbsp%3Bnode.%26lt%3Bbr%26gt%3B%26amp%3Bnbsp%3B%22%20style%3D%22text%3Bhtml%3D1%3Balign%3Dleft%3BverticalAlign%3Dmiddle%3Bresizable%3D0%3Bpoints%3D%5B%5D%3Bautosize%3D1%3BstrokeColor%3Dnone%3BfillColor%3Dnone%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20x%3D%2229.999999999999943%22%20width%3D%22200%22%20height%3D%2290%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3C%2Froot%3E%3C%2FmxGraphModel%3ECra
doEvaluate():...
doEvaluate():
- sets default break duration if 
duration is missing for specific 
triggers
%3CmxGraphModel%3E%3Croot%3E%3CmxCell%20id%3D%220%22%2F%3E%3CmxCell%20id%3D%221%22%20parent%3D%220%22%2F%3E%3CmxCell%20id%3D%222%22%20value%3D%22%22%20style%3D%22group%22%20vertex%3D%221%22%20connectable%3D%220%22%20parent%3D%221%22%3E%3CmxGeometry%20x%3D%2210%22%20y%3D%2275%22%20width%3D%22230%22%20height%3D%2289.99999999999999%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%223%22%20value%3D%22%22%20style%3D%22verticalLabelPosition%3Dbottom%3Bsketch%3D0%3Baspect%3Dfixed%3Bhtml%3D1%3BverticalAlign%3Dtop%3BstrokeColor%3Dnone%3Balign%3Dcenter%3BoutlineConnect%3D0%3Bshape%3Dmxgraph.citrix.process%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20y%3D%2221.67%22%20width%3D%2226.84%22%20height%3D%2232%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%224%22%20value%3D%22Start()%3A%20Caches%20and%20refresh%20app%20%26lt%3Bbr%26gt%3Bpolicies%20managed%20by%20this%20%26lt%3Bbr%26gt%3BeventGenerator%20node%20every%20x%20time%20%26lt%3Bbr%26gt%3Binternal.%20It%20shards%20policies%20by%20%26lt%3Bbr%26gt%3BeventGenerator%26amp%3Bnbsp%3Bnode.%26lt%3Bbr%26gt%3B%26amp%3Bnbsp%3B%22%20style%3D%22text%3Bhtml%3D1%3Balign%3Dleft%3BverticalAlign%3Dmiddle%3Bresizable%3D0%3Bpoints%3D%5B%5D%3Bautosize%3D1%3BstrokeColor%3Dnone%3BfillColor%3Dnone%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20x%3D%2229.999999999999943%22%20width%3D%22200%22%20height%3D%2290%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3C%2Froot%3E%3C%2FmxGraphModel%3ECra
doEvaluate():...
GET /v1/apps/{appid}/metric_histories/{metrictype}
GET /v1/apps/{appid}/metric_histories/{metrictype}
MetricsServer
MetricsServer
Aggregator
Aggregator
Start():
Pulls all policies via AppManager and turns them into AppMonitor rules that get streamed to the a MetricPoller.
Caches appMetrics and periodically saves them in Bulk via AppManager 

Start():...
appMonitorChan:
*models.AppMonitor
appMonitorChan:...
AppManager.QueryAppMetrics(...)
AppManager.QueryAppMetrics(...)
GET "/v1/apps/{appid}/aggregated_metric_histories/{metrictype}"
GET "/v1/apps/{appid}/aggregated_metric_histories/{metrictype}"
Instance metrics 
Instance metrics 
App Metrics
App Metrics
apiserver
apiserver
MetricServerClient
MetricServerClient
log-cache
log-cache
LogCacheClient
LogCacheClient
via HTTPS or gRPC
via HTTPS or gRPCText is not SVG - cannot display \ No newline at end of file +
SaveMetricToCache(...)
SaveMetricToCache(...)
Aggregator
Aggregator

(PolicyDB, appMetricsDB)
 (PolicyDB, appMetricsDB)
AppManager
AppManager
MetricPoller
MetricPoller
1..config.MetricPollerCount
1..config.MetricPollerCount
MetricPoller
MetricPoller
MetricPoller
MetricPoller
The aggregator does not aggregate but only say what and how much will be aggregated. The real aggregation is performed by the MetricPoller and send to the Aggregator.
The aggregator does not aggrega...
Start(): Listen to the instructions in form of *models.AppMonitor.

For each of these metric-collection-instructions, poll the log-cache and do some form of aggregation. Send the result to the Aggregator via the channel “appMetricChan”.
Start(): Listen to the instruct...
Server
Server
eventgen_handler
eventgen_handler
Server
Server
Start(): Caches and refresh app policies managed by this node every conf.Aggregator.PolicyPollerInterval duration internal. It shards policies by node.
Start(): Caches and refresh app...
AppManager.QueryAppMetrics(...)
AppManager.QueryAppMetrics(...)
AppMetrics[]
AppMetrics[]
conf.aggregatorInterval
conf.aggre...
Autoscaler - Eventgenerator
Autoscaler - Eventgenerator
Generator
Generator
1..config.EvaluatorCount
1..config.EvaluatorCount
TriggersChan:
 *[]Triggers
TriggersChan:...
evaluator
evaluator
evaluator
evaluator
evaluator
evaluatoruuu
AppEvaluationManager
AppEvaluationManager
ScalingEngine
ScalingEngine
POST https
POST https
doEvaluate():
- Create any missing circuite
 breaker for the appPolicies
retrived by AppManager.
- writes on triggers channel
an updated list per app of all
existing rules on that app
policy. This happens 
 every 60s by default.
%3CmxGraphModel%3E%3Croot%3E%3CmxCell%20id%3D%220%22%2F%3E%3CmxCell%20id%3D%221%22%20parent%3D%220%22%2F%3E%3CmxCell%20id%3D%222%22%20value%3D%22%22%20style%3D%22group%22%20vertex%3D%221%22%20connectable%3D%220%22%20parent%3D%221%22%3E%3CmxGeometry%20x%3D%2210%22%20y%3D%2275%22%20width%3D%22230%22%20height%3D%2289.99999999999999%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%223%22%20value%3D%22%22%20style%3D%22verticalLabelPosition%3Dbottom%3Bsketch%3D0%3Baspect%3Dfixed%3Bhtml%3D1%3BverticalAlign%3Dtop%3BstrokeColor%3Dnone%3Balign%3Dcenter%3BoutlineConnect%3D0%3Bshape%3Dmxgraph.citrix.process%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20y%3D%2221.67%22%20width%3D%2226.84%22%20height%3D%2232%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%224%22%20value%3D%22Start()%3A%20Caches%20and%20refresh%20app%20%26lt%3Bbr%26gt%3Bpolicies%20managed%20by%20this%20%26lt%3Bbr%26gt%3BeventGenerator%20node%20every%20x%20time%20%26lt%3Bbr%26gt%3Binternal.%20It%20shards%20policies%20by%20%26lt%3Bbr%26gt%3BeventGenerator%26amp%3Bnbsp%3Bnode.%26lt%3Bbr%26gt%3B%26amp%3Bnbsp%3B%22%20style%3D%22text%3Bhtml%3D1%3Balign%3Dleft%3BverticalAlign%3Dmiddle%3Bresizable%3D0%3Bpoints%3D%5B%5D%3Bautosize%3D1%3BstrokeColor%3Dnone%3BfillColor%3Dnone%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20x%3D%2229.999999999999943%22%20width%3D%22200%22%20height%3D%2290%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3C%2Froot%3E%3C%2FmxGraphModel%3ECra
doEvaluate():...
doEvaluate():
- sets default break duration if 
duration is missing for specific 
triggers
%3CmxGraphModel%3E%3Croot%3E%3CmxCell%20id%3D%220%22%2F%3E%3CmxCell%20id%3D%221%22%20parent%3D%220%22%2F%3E%3CmxCell%20id%3D%222%22%20value%3D%22%22%20style%3D%22group%22%20vertex%3D%221%22%20connectable%3D%220%22%20parent%3D%221%22%3E%3CmxGeometry%20x%3D%2210%22%20y%3D%2275%22%20width%3D%22230%22%20height%3D%2289.99999999999999%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%223%22%20value%3D%22%22%20style%3D%22verticalLabelPosition%3Dbottom%3Bsketch%3D0%3Baspect%3Dfixed%3Bhtml%3D1%3BverticalAlign%3Dtop%3BstrokeColor%3Dnone%3Balign%3Dcenter%3BoutlineConnect%3D0%3Bshape%3Dmxgraph.citrix.process%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20y%3D%2221.67%22%20width%3D%2226.84%22%20height%3D%2232%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%224%22%20value%3D%22Start()%3A%20Caches%20and%20refresh%20app%20%26lt%3Bbr%26gt%3Bpolicies%20managed%20by%20this%20%26lt%3Bbr%26gt%3BeventGenerator%20node%20every%20x%20time%20%26lt%3Bbr%26gt%3Binternal.%20It%20shards%20policies%20by%20%26lt%3Bbr%26gt%3BeventGenerator%26amp%3Bnbsp%3Bnode.%26lt%3Bbr%26gt%3B%26amp%3Bnbsp%3B%22%20style%3D%22text%3Bhtml%3D1%3Balign%3Dleft%3BverticalAlign%3Dmiddle%3Bresizable%3D0%3Bpoints%3D%5B%5D%3Bautosize%3D1%3BstrokeColor%3Dnone%3BfillColor%3Dnone%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20x%3D%2229.999999999999943%22%20width%3D%22200%22%20height%3D%2290%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3C%2Froot%3E%3C%2FmxGraphModel%3ECra
doEvaluate():...
GET /v1/apps/{appid}/metric_histories/{metrictype}
GET /v1/apps/{appid}/metric_histories/{metrictype}
Aggregator
Aggregator
Start():
Pulls all policies via AppManager and turns them into AppMonitor rules that get streamed to the a MetricPoller.
Caches appMetrics and periodically saves them in Bulk via AppManager 

Start():...
appMonitorChan:
*models.AppMonitor
appMonitorChan:...
AppManager.QueryAppMetrics(...)
AppManager.QueryAppMetrics(...)
GET "/v1/apps/{appid}/aggregated_metric_histories/{metrictype}"
GET "/v1/apps/{appid}/aggregated_metric_histories/{metrictype}"
Instance metrics 
Instance metrics 
App Metrics
App Metrics
apiserver
apiserver
log-cache
log-cache
LogCacheClient
LogCacheClient
via HTTPS or gRPC
via HTTPS or gRPCText is not SVG - cannot display \ No newline at end of file diff --git a/docs/images/metrics_gateway.svg b/docs/images/metrics_gateway.svg deleted file mode 100644 index 7b7f569960..0000000000 --- a/docs/images/metrics_gateway.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
MetricServer via WSServer
MetricServer via WSS...
Gauge(memory, memory_usg, CPU)
Gauge(memory, memory_usg, CPU)
Timer (httpStartStop)
Timer (httpStartStop)
Autoscaler - Metrics Gateway
Autoscaler - Metrics Gateway
Dispatcher
Dispatcher
Dispatch: When an envelope is
 received it send it to the right emmiter
Dispatch: When an envelope is...
StartEmitEnvelope:
Listents on envelopeChan
and emits the message
StartEmitEnvelope:...
SetupConn: Connects
 with WSServer in metrics server
SetupConn: Connects...
EnvelopProcessor
EnvelopProcessor
EnvelopProcessor
EnvelopProcessor
EnvelopProcessor
EnvelopProcessor
emiter
emiter
- Accept(envelop)
- Emit(envelop)
- Accept(envelop)...
1..MetricServerCount
1..MetricServerCount
Nozzle
Nozzle
Nozzle
Nozzle
Nozzle
Nozzle
Nozzle
Nozzle
1..NozzleCount
1..NozzleCount
Start: Retrieves and
caches appIDs every
 x interval
Start: Retrieves and...
AppManager
AppManager
Gauge(memory, memory_usg, CPU)
Gauge(memory, memory_usg, CPU)
Timer (httpStartStop)
Timer (httpStartStop)
Loggregator via gRPC
Loggregator via gRPC
by shardID (SubscriptionID)
by shardID (SubscriptionID)
GetAppIDs()
GetAppIDs()
EnvelopChan
EnvelopChanText is not SVG - cannot display \ No newline at end of file diff --git a/docs/images/metrics_server.svg b/docs/images/metrics_server.svg deleted file mode 100644 index 48829bd984..0000000000 --- a/docs/images/metrics_server.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
EnvelopProcessor
EnvelopProcessor
EnvelopProcessor
EnvelopProcessor
EnvelopProcessor
EnvelopProcessor
EnvelopProcessor
(envelopChan[I], AppMetricsChan)
EnvelopProcessor...
EnvelopeChans[4]
EnvelopeChans[4]
WSServer
WSServer
MetricsChan
MetricsChan
Collector
Collector
Gauge(memory, memory_usg, CPU)
Gauge(memory, memory_usg, CPU)
Timer (httpStartStop)
Timer (httpStartStop)
Every minute: Process
 Throughput, Response time
Every minute: Process...
OnEnvelop: Process
 App Metrics
OnEnvelop: Process...
Autoscaler - Metrics Server
Autoscaler - Metrics Server
Loggregator via metricsGateway
Loggregator via metr...
AppRefresh: cleanup/sync
 policies and listening app ids
AppRefresh: cleanup/sync...
SaveMetrics: RealTime cache storage,
batch persistence of metrics
SaveMetrics: RealTime cache storage,...
EventGenerator
EventGenerator
#QueryMetrics (From
 cache is possible || Persistence)
#QueryMetrics (From...
Server
Server
GET /v1/apps/{appid}/metric_histories/{metrictype}"
GET /v1/apps/{appid}/metric_histories/{metrictype}"
AppInstanceMetrics
AppInstanceMetrics
envelopeProcessor
envelopeProcessor
Envelope[]
Envelope[]Text is not SVG - cannot display \ No newline at end of file diff --git a/docs/log-cache.md b/docs/log-cache.md deleted file mode 100644 index 0d9cbe3bbb..0000000000 --- a/docs/log-cache.md +++ /dev/null @@ -1,53 +0,0 @@ -# Log Cache and Autoscaler - -Log Cache allows you to view logs and metrics over a specified period of time. It could work as a replacement for -MetricsGateway and MetricsServer components. See [developers docs](developer_guide.md). - -Autoscaler uses 2 main group of metrics, TIMER and GAUGE. it is theoretically possible for autoscaler to consume -metrics via log-cache instead of firehose. - -__TIMER__ - -- httpsStartStop.start -- httpsStartStop.stop - -__GAUGE__ - -- cpu -- diks -- disk_quota -- memory -- memory_quota -- "custom_metric" - -## Autoscaler metrics - -Autoscaler turns Loggregator instance metrics into Autoscaler app metrics, these calculations currently happen in -MetricsServer's envelop_processor. - -### Memory and CPU - -**memoryutil** : int(math.Ceil(memory/memory_quota*100)) % - -**memoryused** : int(math.Ceil(memory/(1024*1024))) MB - -**cpu** : int64(math.Ceil(cpu)) % - -### Throughput and ResponseTime - -**throughput** : Count(httsStartStop metrics for the last x seconds) - -**responsetime** : sum(stop-start)/numReq - -If no timer metrics were received during the collection period for a given app, one evenlop_processor -per app will generate 2 metric and stream them to the pipe: - -- 1 throughput metrics with instance_index: 0, value: 0 -- 1 responsetime metrics with instance_index: 0, value: 0 - -## Retrieve data from log-cache - -``` -curl --header "Authorization: $(cf oauth-token)" \ - "https://log-cache.((SYSTEM_DOMAIN))/api/v1/read/((APP_ID))?envelope_types=[GAUGE|TIMER]&start_time=$(($(date --utc '+%s%N') - SECONDS * 1000000000))" -``` From d1e8966e4cc644438ccc7feedd917fcf58a56ea4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 08:42:52 +0000 Subject: [PATCH 052/186] chore(deps): update dependency rhysd/actionlint to v1.7.1 --- .tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index 0a5a8db414..67a361b72d 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,5 +1,5 @@ act 0.2.63 -actionlint 1.6.27 +actionlint 1.7.1 bosh 7.3.1 credhub 2.9.29 cf 8.7.10 From b186415d279a446f8426a3583e031190f1ddb0a2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 08:49:05 +0000 Subject: [PATCH 053/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20scripts/asdf2devbox.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- devbox.json | 2 +- devbox.lock | 44 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/devbox.json b/devbox.json index 603acd39d5..232a677b98 100644 --- a/devbox.json +++ b/devbox.json @@ -15,7 +15,6 @@ "rubocop": "latest", "rubyPackages.solargraph": "latest", "swagger-cli": "latest", - "actionlint": "1.6.27", "direnv": "2.34.0", "gnumake": "4.4", "maven": "3.8.6", @@ -35,6 +34,7 @@ "gum": "0.13.0", "act": "0.2.63", "go": "1.21.6", + "actionlint": "1.7.1", "google-cloud-sdk": "latest", "temurin-bin-17": "latest", "ruby": "latest" diff --git a/devbox.lock b/devbox.lock index 3e35758057..e2840f1999 100644 --- a/devbox.lock +++ b/devbox.lock @@ -49,23 +49,51 @@ } } }, - "actionlint@1.6.27": { - "last_modified": "2024-03-17T01:03:25Z", - "resolved": "github:NixOS/nixpkgs/299d4668ba61600311553920d9fd9c102145b2cb#actionlint", + "actionlint@1.7.1": { + "last_modified": "2024-05-29T18:25:13Z", + "resolved": "github:NixOS/nixpkgs/f7a63cf975cc66559d5f488ffe6367c987a79826#actionlint", "source": "devbox-search", - "version": "1.6.27", + "version": "1.7.1", "systems": { "aarch64-darwin": { - "store_path": "/nix/store/nnhjp60w14pdy1hjgxn9dfsg4nbfs93v-actionlint-1.6.27" + "outputs": [ + { + "name": "out", + "path": "/nix/store/04x5bylwlkyh8bj0qz5w9qdvhdwcgfkp-actionlint-1.7.1", + "default": true + } + ], + "store_path": "/nix/store/04x5bylwlkyh8bj0qz5w9qdvhdwcgfkp-actionlint-1.7.1" }, "aarch64-linux": { - "store_path": "/nix/store/p8rc44x83sld3wk1m4y27za9n1mjyk48-actionlint-1.6.27" + "outputs": [ + { + "name": "out", + "path": "/nix/store/prc9nslsi3j574ff3n5fsqhp0lbl3k5a-actionlint-1.7.1", + "default": true + } + ], + "store_path": "/nix/store/prc9nslsi3j574ff3n5fsqhp0lbl3k5a-actionlint-1.7.1" }, "x86_64-darwin": { - "store_path": "/nix/store/7v1lh7c3pwlax5fy7pw233czn4hs89hl-actionlint-1.6.27" + "outputs": [ + { + "name": "out", + "path": "/nix/store/vh08g8rz4h1545yzg2vf4mikbi3w77wm-actionlint-1.7.1", + "default": true + } + ], + "store_path": "/nix/store/vh08g8rz4h1545yzg2vf4mikbi3w77wm-actionlint-1.7.1" }, "x86_64-linux": { - "store_path": "/nix/store/56xxvs3yijdmhhxbfvps38nz95xdxgyl-actionlint-1.6.27" + "outputs": [ + { + "name": "out", + "path": "/nix/store/8y9djdrxa2drp19vk0kb4n9c10i6d9y2-actionlint-1.7.1", + "default": true + } + ], + "store_path": "/nix/store/8y9djdrxa2drp19vk0kb4n9c10i6d9y2-actionlint-1.7.1" } } }, From 0789f6443cdc899ccf62122fd270e37fcf4714f9 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Tue, 11 Jun 2024 10:00:51 +0200 Subject: [PATCH 054/186] remove websockethandler that has no usage --- src/autoscaler/go.mod | 1 - src/autoscaler/go.sum | 2 - .../testhelpers/websocket_handler.go | 151 ------------------ 3 files changed, 154 deletions(-) delete mode 100644 src/autoscaler/testhelpers/websocket_handler.go diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index bb301da425..89890df9cb 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -18,7 +18,6 @@ require ( github.com/go-logr/logr v1.4.2 github.com/go-sql-driver/mysql v1.8.1 github.com/gorilla/mux v1.8.1 - github.com/gorilla/websocket v1.5.1 github.com/hashicorp/go-retryablehttp v0.7.7 github.com/jackc/pgx/v5 v5.6.0 github.com/jmoiron/sqlx v1.4.0 diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 5ea19cd648..27b8defd70 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -847,8 +847,6 @@ github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= diff --git a/src/autoscaler/testhelpers/websocket_handler.go b/src/autoscaler/testhelpers/websocket_handler.go deleted file mode 100644 index e5086a9895..0000000000 --- a/src/autoscaler/testhelpers/websocket_handler.go +++ /dev/null @@ -1,151 +0,0 @@ -package testhelpers - -import ( - "log" - "net/http" - "sync" - "time" - - "github.com/gorilla/websocket" -) - -type WebsocketHandler struct { - messages chan []byte - pingPongChan chan int - keepAlive time.Duration - lock *sync.Mutex - conLock *sync.Mutex - wsConn *websocket.Conn -} - -func NewWebsocketHandler(m chan []byte, pingPongChan chan int, keepAlive time.Duration) *WebsocketHandler { - return &WebsocketHandler{ - messages: m, - pingPongChan: pingPongChan, - keepAlive: keepAlive, - lock: &sync.Mutex{}, - conLock: &sync.Mutex{}, - } -} -func (h *WebsocketHandler) CloseWSConnection() { - h.conLock.Lock() - defer h.conLock.Unlock() - if h.wsConn != nil { - h.wsConn.Close() - } -} -func (h *WebsocketHandler) ServeWebsocket(rw http.ResponseWriter, r *http.Request) { - upgrader := websocket.Upgrader{ - CheckOrigin: func(*http.Request) bool { return true }, - } - - ws, err := upgrader.Upgrade(rw, r, nil) - ws.SetPingHandler(func(message string) error { - h.pingPongChan <- 1 - return nil - }) - if err != nil { - log.Printf("websocket handler: Not a websocket handshake: %s", err) - return - } - h.conLock.Lock() - h.wsConn = ws - h.conLock.Unlock() - defer ws.Close() - - closeCode, closeMessage := h.runWebsocketUntilClosed(ws) - err = ws.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(closeCode, closeMessage), time.Time{}) - if err != nil { - log.Printf("websocket handler: failed to close: %s", err) - return - } -} - -func (h *WebsocketHandler) runWebsocketUntilClosed(ws *websocket.Conn) (closeCode int, closeMessage string) { - keepAliveExpired := make(chan struct{}) - clientWentAway := make(chan struct{}) - - go func() { - for { - h.lock.Lock() - _, m, err := ws.ReadMessage() - h.messages <- m - h.lock.Unlock() - - if err != nil { - close(clientWentAway) - return - } - } - }() - - go func() { - NewKeepAlive(h.lock, ws, h.keepAlive).Run() - close(keepAliveExpired) - }() - - closeCode = websocket.CloseNormalClosure - closeMessage = "" - for { - select { - case <-clientWentAway: - return closeCode, closeMessage - case <-keepAliveExpired: - closeCode = websocket.ClosePolicyViolation - closeMessage = "Client did not respond to ping before keep-alive timeout expired." - return closeCode, closeMessage - } - } -} - -type KeepAlive struct { - lock *sync.Mutex - conn *websocket.Conn - pongChan chan struct{} - keepAliveInterval time.Duration -} - -func NewKeepAlive(lock *sync.Mutex, conn *websocket.Conn, keepAliveInterval time.Duration) *KeepAlive { - return &KeepAlive{ - lock: lock, - conn: conn, - pongChan: make(chan struct{}, 1), - keepAliveInterval: keepAliveInterval, - } -} - -func (k *KeepAlive) Run() { - k.lock.Lock() - k.conn.SetPongHandler(k.pongHandler) - k.lock.Unlock() - - defer func() { - k.lock.Lock() - k.conn.SetPongHandler(nil) - k.lock.Unlock() - }() - - timeout := time.NewTimer(k.keepAliveInterval) - for { - err := k.conn.WriteControl(websocket.PingMessage, nil, time.Time{}) - if err != nil { - return - } - timeout.Reset(k.keepAliveInterval) - select { - case <-k.pongChan: - time.Sleep(k.keepAliveInterval / 2) - continue - case <-timeout.C: - return - } - } -} - -func (k *KeepAlive) pongHandler(string) error { - select { - case k.pongChan <- struct{}{}: - default: - } - return nil -} From b5d19afbcd8b32f9b8179ac83dc6ad5c407bdf12 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Tue, 11 Jun 2024 10:03:07 +0200 Subject: [PATCH 055/186] remove use_log_cache feature flag, make metricclientfactory always return a logcacheclient and remove meticsserverclient code, adjust failing tests to test against a mock logcache server instead of a fake metricserver --- jobs/eventgenerator/spec | 4 - .../templates/eventgenerator.yml.erb | 7 +- scripts/generate_test_certs.sh | 8 - .../eventgenerator/eventgenerator_spec.rb | 91 +++---- .../aggregator/metric_poller_test.go | 232 +++++++++++------- .../client/metric_client_factory.go | 16 +- .../client/metric_client_factory_test.go | 80 +++--- .../client/metric_server_client.go | 75 ------ .../client/metric_server_client_test.go | 26 -- .../eventgenerator_suite_test.go | 183 +++++++++----- .../cmd/eventgenerator/eventgenerator_test.go | 172 +------------ .../eventgenerator/config/config.go | 1 - .../eventgenerator/config/config_test.go | 102 ++++---- src/autoscaler/generate-fakes.go | 1 - src/autoscaler/integration/components_test.go | 1 - .../integration/integration_suite_test.go | 7 +- src/autoscaler/routes/routes.go | 8 - src/autoscaler/routes/routes_test.go | 10 - src/autoscaler/testhelpers/clients.go | 4 - .../mock => testhelpers}/log_cache_mock.go | 84 ++++--- templates/app-autoscaler.yml | 1 - 21 files changed, 441 insertions(+), 672 deletions(-) delete mode 100644 src/autoscaler/eventgenerator/client/metric_server_client.go delete mode 100644 src/autoscaler/eventgenerator/client/metric_server_client_test.go rename src/autoscaler/{integration/mock => testhelpers}/log_cache_mock.go (55%) diff --git a/jobs/eventgenerator/spec b/jobs/eventgenerator/spec index 1a8363b1eb..dc816f076e 100644 --- a/jobs/eventgenerator/spec +++ b/jobs/eventgenerator/spec @@ -173,10 +173,6 @@ properties: autoscaler.eventgenerator.scaling_engine.client_key: description: "PEM-encoded client key" - autoscaler.eventgenerator.metricscollector.use_log_cache: - description: "Uses log cache instead of metric server" - default: false - autoscaler.eventgenerator.metricscollector.host: description: "Host where the metrics collector is running" default: "metricscollector.service.cf.internal" diff --git a/jobs/eventgenerator/templates/eventgenerator.yml.erb b/jobs/eventgenerator/templates/eventgenerator.yml.erb index 570d5809b1..2b302a7115 100644 --- a/jobs/eventgenerator/templates/eventgenerator.yml.erb +++ b/jobs/eventgenerator/templates/eventgenerator.yml.erb @@ -41,11 +41,7 @@ end app_metrics_db_url = build_db_url('appmetrics_db', job_name) - unless p("autoscaler.eventgenerator.metricscollector.use_log_cache") - metric_collector_url = "https://" + p("autoscaler.eventgenerator.metricscollector.host") - else - metric_collector_url = p("autoscaler.eventgenerator.metricscollector.host") - end + metric_collector_url = p("autoscaler.eventgenerator.metricscollector.host") metrics_collector_port = p("autoscaler.eventgenerator.metricscollector.port").to_s unless metrics_collector_port.empty? @@ -118,7 +114,6 @@ scalingEngine: metricCollector: metric_collector_url: <%= metric_collector_url %> - use_log_cache: <%= p("autoscaler.eventgenerator.metricscollector.use_log_cache") %> tls: key_file: /var/vcap/jobs/eventgenerator/config/certs/metricscollector/client.key cert_file: /var/vcap/jobs/eventgenerator/config/certs/metricscollector/client.crt diff --git a/scripts/generate_test_certs.sh b/scripts/generate_test_certs.sh index 5a119d30a7..ba4fc8b1ed 100755 --- a/scripts/generate_test_certs.sh +++ b/scripts/generate_test_certs.sh @@ -82,14 +82,6 @@ ${CERTSTRAP} --depot-path "${depot_path}" sign reverselogproxy --CA autoscaler-c ${CERTSTRAP} --depot-path "${depot_path}" request-cert --passphrase '' --domain reverselogproxy_client ${CERTSTRAP} --depot-path "${depot_path}" sign reverselogproxy_client --CA autoscaler-ca --years "20" -# metricserver test client certificate -${CERTSTRAP} --depot-path "${depot_path}" request-cert --passphrase '' --domain metricserver --ip 127.0.0.1 -${CERTSTRAP} --depot-path "${depot_path}" sign metricserver --CA autoscaler-ca --years "20" - -# metricserver test client certificate -${CERTSTRAP} --depot-path "${depot_path}" request-cert --passphrase '' --domain metricserver_client -${CERTSTRAP} --depot-path "${depot_path}" sign metricserver_client --CA autoscaler-ca --years "20" - # metricsforwarder certificate for loggregator_agent ${CERTSTRAP} --depot-path "${depot_path}" request-cert --passphrase '' --domain metron ${CERTSTRAP} --depot-path "${depot_path}" sign metron --CA loggregator-ca --years "20" diff --git a/spec/jobs/eventgenerator/eventgenerator_spec.rb b/spec/jobs/eventgenerator/eventgenerator_spec.rb index ac1966b804..9627477cfd 100644 --- a/spec/jobs/eventgenerator/eventgenerator_spec.rb +++ b/spec/jobs/eventgenerator/eventgenerator_spec.rb @@ -52,71 +52,52 @@ ) end - describe "use_log_cache feature" do - it "keeps log cache off by default" do - expect(rendered_template["metricCollector"]) - .to include({"use_log_cache" => false}) + describe "when using log-cache via grpc" do + before do + properties["autoscaler"]["eventgenerator"] = { + "metricscollector" => { + "host" => "logcache", + "port" => "8080" + } + } end - it "should add https protocol to metric_collector_url" do + it "should not add https protocol to metric_collector_url" do expect(rendered_template["metricCollector"]["metric_collector_url"]) - .to include("http") + .not_to include("http") end + end - describe "when log cache on" do - before do - properties["autoscaler"]["eventgenerator"] = { - "metricscollector" => { - "host" => "logcache", - "port" => "8080", - "use_log_cache" => true + describe "when using log-cache via https/uaa" do + before do + properties["autoscaler"]["eventgenerator"] = { + "metricscollector" => { + "host" => "logcache.cf.test.com", + "port" => "", + "uaa" => { + "client_id" => "logs_admin_client_id", + "client_secret" => "logs_admin_client_secret", + "url" => "uaa.cf.test.com" } } - end - - it "check eventgenerator use log cache" do - expect(rendered_template["metricCollector"]) - .to include({"use_log_cache" => true}) - end - - it "should not add https protocol to metric_collector_url" do - expect(rendered_template["metricCollector"]["metric_collector_url"]) - .not_to include("http") - end - - describe "when using log-cache via https/uaa" do - before do - properties["autoscaler"]["eventgenerator"] = { - "metricscollector" => { - "host" => "logcache.cf.test.com", - "port" => "", - "use_log_cache" => true, - "uaa" => { - "client_id" => "logs_admin_client_id", - "client_secret" => "logs_admin_client_secret", - "url" => "uaa.cf.test.com" - } - } - } - end + } + end - it "should not add metric_collector_port to metric_collector_url" do - expect(rendered_template["metricCollector"]["metric_collector_url"]) - .not_to include(":") - end + it "should not add metric_collector_port to metric_collector_url" do + expect(rendered_template["metricCollector"]["metric_collector_url"]) + .not_to include(":") + end - it "should add uaa credentials" do - expect(rendered_template["metricCollector"]["uaa"]).to include({ - "client_id" => "logs_admin_client_id", - "client_secret" => "logs_admin_client_secret", - "url" => "uaa.cf.test.com" - }) - end + it "should add uaa credentials" do + expect(rendered_template["metricCollector"]["uaa"]).to include({ + "client_id" => "logs_admin_client_id", + "client_secret" => "logs_admin_client_secret", + "url" => "uaa.cf.test.com" + }) + end - it "should set uaa skip ssl validation to false by default" do - expect(rendered_template["metricCollector"]["uaa"]["skip_ssl_validation"]).to be_falsey - end - end + it "should set uaa skip ssl validation to false by default" do + expect(rendered_template["metricCollector"]["uaa"]["skip_ssl_validation"]).to be_falsey end end diff --git a/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go b/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go index f91626afe2..b1a0f90fa1 100644 --- a/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go +++ b/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go @@ -1,20 +1,22 @@ package aggregator_test import ( + "errors" + "path/filepath" + + "time" + . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/aggregator" . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/client" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/routes" - "code.cloudfoundry.org/lager/v3" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" + rpc "code.cloudfoundry.org/go-log-cache/v2/rpc/logcache_v1" + "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" "code.cloudfoundry.org/lager/v3/lagertest" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gbytes" - "github.com/onsi/gomega/ghttp" - - "net/http" - "time" ) var _ = Describe("MetricPoller", func() { @@ -28,48 +30,8 @@ var _ = Describe("MetricPoller", func() { appMetricChan chan *models.AppMetric metricPoller *MetricPoller metricClient MetricClient - metricServer *ghttp.Server - metrics = []*models.AppInstanceMetric{ - { - AppId: testAppId, - InstanceIndex: 0, - CollectedAt: 111111, - Name: testMetricType, - Unit: testMetricUnit, - Value: "100", - Timestamp: 111100, - }, - { - AppId: testAppId, - InstanceIndex: 1, - CollectedAt: 111111, - Name: testMetricType, - Unit: testMetricUnit, - Value: "200", - Timestamp: 110000, - }, - - { - AppId: testAppId, - InstanceIndex: 0, - CollectedAt: 222222, - Name: testMetricType, - Unit: testMetricUnit, - Value: "300", - Timestamp: 222200, - }, - { - AppId: testAppId, - InstanceIndex: 1, - CollectedAt: 222222, - Name: testMetricType, - Unit: testMetricUnit, - Value: "401", - Timestamp: 220000, - }, - } - urlPath string - appMonitor *models.AppMonitor + mockLogCache *testhelpers.MockLogCache + appMonitor *models.AppMonitor ) BeforeEach(func() { logger = lagertest.NewTestLogger("MetricPoller-test") @@ -77,11 +39,7 @@ var _ = Describe("MetricPoller", func() { appMonitorsChan = make(chan *models.AppMonitor, 1) appMetricChan = make(chan *models.AppMetric, 1) - metricServer = nil - path, err := routes.MetricsCollectorRoutes().Get(routes.GetMetricHistoriesRouteName).URLPath("appid", testAppId, "metrictype", testMetricType) - Expect(err).NotTo(HaveOccurred()) - urlPath = path.Path appMonitor = &models.AppMonitor{ AppId: testAppId, MetricType: testMetricType, @@ -93,11 +51,12 @@ var _ = Describe("MetricPoller", func() { Context("When metric-collector is not running", func() { BeforeEach(func() { - metricServer = ghttp.NewUnstartedServer() + metricClient = NewMetricClientFactory().GetMetricClient(logger, &config.Config{ + MetricCollector: config.MetricCollectorConfig{ + MetricCollectorURL: "this.endpoint.does.not.exist:1234", + }, + }) - httpClient, err := helpers.CreateHTTPClient(nil, helpers.DefaultClientConfig(), lager.NewLogger("metrics_server")) - Expect(err).ToNot(HaveOccurred()) - metricClient = NewMetricServerClient(logger, metricServer.URL(), httpClient) metricPoller = NewMetricPoller(logger, metricClient, appMonitorsChan, appMetricChan) metricPoller.Start() @@ -106,12 +65,11 @@ var _ = Describe("MetricPoller", func() { AfterEach(func() { metricPoller.Stop() - metricServer.Close() }) It("logs an error", func() { //TODO this should be a prometheus counter not a log statement check - Eventually(logger.Buffer).Should(Say("Failed to retrieve metric")) + Eventually(logger.Buffer).Should(Say("retrieveMetric Failed")) }) It("does not save any metrics", func() { @@ -123,14 +81,122 @@ var _ = Describe("MetricPoller", func() { var appMetric *models.AppMetric BeforeEach(func() { - - metricServer = ghttp.NewServer() - metricServer.RouteToHandler("GET", urlPath, ghttp.RespondWithJSONEncoded(http.StatusOK, - &metrics)) - - httpClient, err := helpers.CreateHTTPClient(nil, helpers.DefaultClientConfig(), lager.NewLogger("metric_server")) + testCertDir := testhelpers.TestCertFolder() + tlsConfig, err := testhelpers.NewTLSConfig( + filepath.Join(testCertDir, "autoscaler-ca.crt"), + filepath.Join(testCertDir, "log-cache.crt"), + filepath.Join(testCertDir, "log-cache.key"), + "log-cache", + ) Expect(err).ToNot(HaveOccurred()) - metricClient = NewMetricServerClient(logger, metricServer.URL(), httpClient) + + mockLogCache = testhelpers.NewMockLogCache(tlsConfig) + mockLogCache.ReadReturns(testAppId, &rpc.ReadResponse{ + Envelopes: &loggregator_v2.EnvelopeBatch{ + Batch: []*loggregator_v2.Envelope{ + { + SourceId: testAppId, + InstanceId: "0", + Timestamp: 111100, + DeprecatedTags: map[string]*loggregator_v2.Value{ + "origin": { + Data: &loggregator_v2.Value_Text{ + Text: "autoscaler_metrics_forwarder", + }, + }, + }, + Message: &loggregator_v2.Envelope_Gauge{ + Gauge: &loggregator_v2.Gauge{ + Metrics: map[string]*loggregator_v2.GaugeValue{ + testMetricType: { + Unit: testMetricUnit, + Value: 100, + }, + }, + }, + }, + }, + { + SourceId: testAppId, + InstanceId: "1", + Timestamp: 110000, + DeprecatedTags: map[string]*loggregator_v2.Value{ + "origin": { + Data: &loggregator_v2.Value_Text{ + Text: "autoscaler_metrics_forwarder", + }, + }, + }, + Message: &loggregator_v2.Envelope_Gauge{ + Gauge: &loggregator_v2.Gauge{ + Metrics: map[string]*loggregator_v2.GaugeValue{ + testMetricType: { + Unit: testMetricUnit, + Value: 200, + }, + }, + }, + }, + }, + { + SourceId: testAppId, + InstanceId: "0", + Timestamp: 222200, + DeprecatedTags: map[string]*loggregator_v2.Value{ + "origin": { + Data: &loggregator_v2.Value_Text{ + Text: "autoscaler_metrics_forwarder", + }, + }, + }, + Message: &loggregator_v2.Envelope_Gauge{ + Gauge: &loggregator_v2.Gauge{ + Metrics: map[string]*loggregator_v2.GaugeValue{ + testMetricType: { + Unit: testMetricUnit, + Value: 300, + }, + }, + }, + }, + }, + { + SourceId: testAppId, + InstanceId: "1", + Timestamp: 220000, + DeprecatedTags: map[string]*loggregator_v2.Value{ + "origin": { + Data: &loggregator_v2.Value_Text{ + Text: "autoscaler_metrics_forwarder", + }, + }, + }, + Message: &loggregator_v2.Envelope_Gauge{ + Gauge: &loggregator_v2.Gauge{ + Metrics: map[string]*loggregator_v2.GaugeValue{ + testMetricType: { + Unit: testMetricUnit, + Value: 401, + }, + }, + }, + }, + }, + }, + }, + }, nil) + mockLogCacheEndpoint := mockLogCache.Start(3000 + GinkgoParallelProcess()) + + metricClient = NewMetricClientFactory().GetMetricClient(logger, &config.Config{ + MetricCollector: config.MetricCollectorConfig{ + MetricCollectorURL: mockLogCacheEndpoint, + TLSClientCerts: models.TLSCerts{ + KeyFile: filepath.Join(testCertDir, "log-cache.key"), + CertFile: filepath.Join(testCertDir, "log-cache.crt"), + CACertFile: filepath.Join(testCertDir, "autoscaler-ca.crt"), + }, + }, + }) }) JustBeforeEach(func() { @@ -142,7 +208,7 @@ var _ = Describe("MetricPoller", func() { AfterEach(func() { metricPoller.Stop() - metricServer.Close() + mockLogCache.Stop() }) Context("when metrics are successfully retrieved", func() { @@ -159,14 +225,13 @@ var _ = Describe("MetricPoller", func() { }) }) - Context("when the metrics are not valid JSON", func() { + Context("when an error occurs during metric retrieval", func() { BeforeEach(func() { - metricServer.RouteToHandler("GET", urlPath, ghttp.RespondWith(http.StatusOK, - "{[}")) + mockLogCache.ReadReturns(testAppId, &rpc.ReadResponse{}, errors.New("error")) }) It("logs an error", func() { - Eventually(logger.Buffer).Should(Say("Failed to parse response")) + Eventually(logger.Buffer).Should(Say("retrieveMetric Failed")) }) It("should not send any metrics to appmetric channel", func() { @@ -176,8 +241,11 @@ var _ = Describe("MetricPoller", func() { Context("when empty metrics are retrieved", func() { BeforeEach(func() { - metricServer.RouteToHandler("GET", urlPath, ghttp.RespondWithJSONEncoded(http.StatusOK, - &[]*models.AppInstanceMetric{})) + mockLogCache.ReadReturns(testAppId, &rpc.ReadResponse{ + Envelopes: &loggregator_v2.EnvelopeBatch{ + Batch: []*loggregator_v2.Envelope{}, + }, + }, nil) }) It("send the average metrics with no value to appmetric channel", func() { @@ -192,28 +260,10 @@ var _ = Describe("MetricPoller", func() { Timestamp: timestamp})) }) }) - - Context("when an error ocurrs retrieving metrics", func() { - BeforeEach(func() { - metricServer.RouteToHandler("GET", urlPath, ghttp.RespondWithJSONEncoded(http.StatusBadRequest, - models.ErrorResponse{ - Code: "Internal-Server-Error", - Message: "Error"})) - }) - - It("should not send any metrics to appmetric channel", func() { - Consistently(appMetricChan).ShouldNot(Receive()) - }) - }) - }) Context("Stop", func() { BeforeEach(func() { - metricServer = ghttp.NewServer() - metricServer.RouteToHandler("GET", urlPath, ghttp.RespondWithJSONEncoded(http.StatusOK, - &metrics)) - metricPoller = NewMetricPoller(logger, metricClient, appMonitorsChan, appMetricChan) metricPoller.Start() metricPoller.Stop() diff --git a/src/autoscaler/eventgenerator/client/metric_client_factory.go b/src/autoscaler/eventgenerator/client/metric_client_factory.go index cfeaafd218..3a3307d307 100644 --- a/src/autoscaler/eventgenerator/client/metric_client_factory.go +++ b/src/autoscaler/eventgenerator/client/metric_client_factory.go @@ -5,7 +5,6 @@ import ( "code.cloudfoundry.org/app-autoscaler/src/autoscaler/envelopeprocessor" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/config" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" "code.cloudfoundry.org/lager/v3" ) @@ -26,11 +25,7 @@ func NewMetricClientFactory() *MetricClientFactory { } func (mc *MetricClientFactory) GetMetricClient(logger lager.Logger, conf *config.Config) MetricClient { - if conf.MetricCollector.UseLogCache { - return mc.createLogCacheMetricClient(logger, conf) - } else { - return mc.createMetricServerMetricClient(logger, conf) - } + return mc.createLogCacheMetricClient(logger, conf) } func (mc *MetricClientFactory) createLogCacheMetricClient(logger lager.Logger, conf *config.Config) MetricClient { @@ -58,15 +53,6 @@ func (mc *MetricClientFactory) createLogCacheMetricClient(logger lager.Logger, c return c } -func (mc *MetricClientFactory) createMetricServerMetricClient(logger lager.Logger, conf *config.Config) MetricClient { - httpClient, err := helpers.CreateHTTPClient(&conf.MetricCollector.TLSClientCerts, helpers.DefaultClientConfig(), logger.Session("metric_client")) - - if err != nil { - logger.Error("failed to create http client for MetricCollector", err, lager.Data{"metriccollectorTLS": httpClient}) - } - return NewMetricServerClient(logger, conf.MetricCollector.MetricCollectorURL, httpClient) -} - func hasUAACreds(conf *config.Config) bool { return conf.MetricCollector.UAACreds.URL != "" && conf.MetricCollector.UAACreds.ClientSecret != "" && conf.MetricCollector.UAACreds.ClientID != "" diff --git a/src/autoscaler/eventgenerator/client/metric_client_factory_test.go b/src/autoscaler/eventgenerator/client/metric_client_factory_test.go index 25d9569b7f..bf0dc1295d 100644 --- a/src/autoscaler/eventgenerator/client/metric_client_factory_test.go +++ b/src/autoscaler/eventgenerator/client/metric_client_factory_test.go @@ -31,7 +31,6 @@ var _ = Describe("MetricClientFactory", func() { expectedMetricCollectorURL string tlsCerts models.TLSCerts uaaCreds models.UAACreds - useLogCache bool caCertFilePath string certFilePath string keyFilePath string @@ -55,19 +54,18 @@ var _ = Describe("MetricClientFactory", func() { AggregatorExecuteInterval: 51 * time.Second, }, MetricCollector: config.MetricCollectorConfig{ - UseLogCache: useLogCache, MetricCollectorURL: expectedMetricCollectorURL, TLSClientCerts: tlsCerts, UAACreds: uaaCreds, }, } - logger = lagertest.NewTestLogger("MetricServer") + logger = lagertest.NewTestLogger("LogCache") metricClient = metricClientFactory.GetMetricClient(logger, &conf) }) Describe("GetMetricClient", func() { BeforeEach(func() { - expectedMetricCollectorURL = "some-metric-server-url" + expectedMetricCollectorURL = "some-log-cache-url:8080" tlsCerts = models.TLSCerts{ KeyFile: keyFilePath, CertFile: certFilePath, @@ -75,65 +73,43 @@ var _ = Describe("MetricClientFactory", func() { } }) - Describe("when logCacheEnabled is false", func() { - BeforeEach(func() { - useLogCache = false - }) - - It("should create a MetricServerClient by default", func() { - Expect(metricClient).To(BeAssignableToTypeOf(&MetricServerClient{})) - Expect(metricClient).NotTo(BeAssignableToTypeOf(&LogCacheClient{})) - - actualURL := metricClient.(*MetricServerClient).GetUrl() - Expect(actualURL).To(Equal("some-metric-server-url")) - }) + It("Should create a LogCacheClient", func() { + Expect(metricClient).To(BeAssignableToTypeOf(&LogCacheClient{})) + actualURL := metricClient.(*LogCacheClient).GetUrl() + Expect(actualURL).To(Equal(expectedMetricCollectorURL)) }) - Describe("when logCacheEnabled is true", func() { + Describe("when uaa client and secret are not provided", func() { BeforeEach(func() { - expectedMetricCollectorURL = "some-log-cache-url:8080" - useLogCache = true + uaaCreds = models.UAACreds{} }) - It("Should create a LogCacheClient and not a metricserver client", func() { - Expect(metricClient).To(BeAssignableToTypeOf(&LogCacheClient{})) - Expect(metricClient).NotTo(BeAssignableToTypeOf(&MetricServerClient{})) - actualURL := metricClient.(*LogCacheClient).GetUrl() - Expect(actualURL).To(Equal(expectedMetricCollectorURL)) + It("Should set TLSConfig from config opts", func() { + expectedTlSCreds := &models.TLSCerts{KeyFile: keyFilePath, CertFile: certFilePath, CACertFile: caCertFilePath} + expectedTLSConfig, err := expectedTlSCreds.CreateClientConfig() + Expect(err).NotTo(HaveOccurred()) + actualTLSConfig := metricClient.(*LogCacheClient).GetTlsConfig() + Expect(actualTLSConfig.Certificates).To(Equal(expectedTLSConfig.Certificates)) }) + }) - Describe("when uaa client and secret are not provided", func() { - BeforeEach(func() { - uaaCreds = models.UAACreds{} - }) - - It("Should set TLSConfig from config opts", func() { - expectedTlSCreds := &models.TLSCerts{KeyFile: keyFilePath, CertFile: certFilePath, CACertFile: caCertFilePath} - expectedTLSConfig, err := expectedTlSCreds.CreateClientConfig() - Expect(err).NotTo(HaveOccurred()) - actualTLSConfig := metricClient.(*LogCacheClient).GetTlsConfig() - Expect(actualTLSConfig.Certificates).To(Equal(expectedTLSConfig.Certificates)) - }) + Describe("when uaa client and secret are provided", func() { + BeforeEach(func() { + uaaCreds = models.UAACreds{ + URL: "log-cache.some-url.com", + ClientID: "some-id", + ClientSecret: "some-secret", + } }) - Describe("when uaa client and secret are provided", func() { - BeforeEach(func() { - uaaCreds = models.UAACreds{ - URL: "log-cache.some-url.com", - ClientID: "some-id", - ClientSecret: "some-secret", - } - }) - - It("should set uaa creds from config", func() { - actualUAACreds := metricClient.(*LogCacheClient).GetUAACreds() - Expect(actualUAACreds).To(Equal(uaaCreds)) - }) + It("should set uaa creds from config", func() { + actualUAACreds := metricClient.(*LogCacheClient).GetUAACreds() + Expect(actualUAACreds).To(Equal(uaaCreds)) }) + }) - It("Should set AggregatorExecuteInterval as collectionInterval on LogCacheClient", func() { - Expect(metricClient.(*LogCacheClient).CollectionInterval()).To(Equal(conf.Aggregator.AggregatorExecuteInterval)) - }) + It("Should set AggregatorExecuteInterval as collectionInterval on LogCacheClient", func() { + Expect(metricClient.(*LogCacheClient).CollectionInterval()).To(Equal(conf.Aggregator.AggregatorExecuteInterval)) }) }) }) diff --git a/src/autoscaler/eventgenerator/client/metric_server_client.go b/src/autoscaler/eventgenerator/client/metric_server_client.go deleted file mode 100644 index 90669ffe97..0000000000 --- a/src/autoscaler/eventgenerator/client/metric_server_client.go +++ /dev/null @@ -1,75 +0,0 @@ -package client - -import ( - "encoding/json" - "errors" - "fmt" - "net/http" - "strconv" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/routes" - "code.cloudfoundry.org/lager/v3" -) - -type MetricServerClient struct { - httpClient *http.Client - logger lager.Logger - url string -} - -type NewMetricsServerClientFunc func(logger lager.Logger, metricCollectorUrl string, httpClient *http.Client) MetricClient - -type MetricServerClientCreator interface { - NewMetricServerClient(logger lager.Logger, metricCollectorUrl string, httpClient *http.Client) MetricClient -} - -func NewMetricServerClient(logger lager.Logger, url string, httpClient *http.Client) *MetricServerClient { - return &MetricServerClient{ - logger: logger.Session("MetricServerClient"), - url: url, - httpClient: httpClient, - } -} -func (c *MetricServerClient) GetMetrics(appId string, metricType string, startTime time.Time, endTime time.Time) ([]models.AppInstanceMetric, error) { - c.logger.Debug("GetMetrics") - var url string - path, err := routes.MetricsCollectorRoutes().Get(routes.GetMetricHistoriesRouteName).URLPath("appid", appId, "metrictype", metricType) - if err != nil { - c.logger.Error("Failed to retrieve metric from metrics collector. Request failed", err, lager.Data{"appId": appId, "metricType": metricType}) - return nil, fmt.Errorf("Failed to retrieve metric from metrics collector. Request failed: %w", err) - } - parameters := path.Query() - parameters.Add("start", strconv.FormatInt(startTime.UnixNano(), 10)) - parameters.Add("end", strconv.FormatInt(endTime.UnixNano(), 10)) - parameters.Add("order", db.ASCSTR) - - url = c.url + path.RequestURI() + "?" + parameters.Encode() - resp, err := c.httpClient.Get(url) - if err != nil { - c.logger.Error("Failed to retrieve metric from metrics collector. Request failed", err, lager.Data{"appId": appId, "metricType": metricType, "url": url}) - return nil, fmt.Errorf("failed to retrieve metric from metrics collector: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - c.logger.Error("Failed to retrieve metric from metrics collector", nil, - lager.Data{"appId": appId, "metricType": metricType, "statusCode": resp.StatusCode}) - return nil, errors.New("Failed to retrieve metric from metrics collector") - } - - var metrics []models.AppInstanceMetric - err = json.NewDecoder(resp.Body).Decode(&metrics) - if err != nil { - c.logger.Error("Failed to parse response", err, lager.Data{"appId": appId, "metricType": metricType}) - return nil, errors.New("Failed to parse response") - } - - return metrics, nil -} - -func (c *MetricServerClient) GetUrl() string { - return c.url -} diff --git a/src/autoscaler/eventgenerator/client/metric_server_client_test.go b/src/autoscaler/eventgenerator/client/metric_server_client_test.go deleted file mode 100644 index 7aa4b80b1d..0000000000 --- a/src/autoscaler/eventgenerator/client/metric_server_client_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package client_test - -import ( - "net/http" - - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/client" - "code.cloudfoundry.org/lager/v3" - "code.cloudfoundry.org/lager/v3/lagertest" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("MetricServerClient", func() { - var ( - msc *MetricServerClient - logger lager.Logger - ) - BeforeEach(func() { - logger = lagertest.NewTestLogger("MetricPoller-test") - msc = NewMetricServerClient(logger, "https://some-metric-server-url/", &http.Client{}) - }) - - It("Can be assign to a Metric Client type", func() { - Expect(msc).To(BeAssignableToTypeOf(&MetricServerClient{})) - }) -}) diff --git a/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_suite_test.go b/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_suite_test.go index 9f30510557..4daa6613b3 100644 --- a/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_suite_test.go +++ b/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_suite_test.go @@ -2,6 +2,8 @@ package main_test import ( "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" + rpc "code.cloudfoundry.org/go-log-cache/v2/rpc/logcache_v1" + "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -33,55 +35,17 @@ var ( metricType = "a-metric-type" metricUnit = "a-metric-unit" - regPath = regexp.MustCompile(`^/v1/apps/.*/scale$`) - configFile *os.File - conf config.Config - egPort int - healthport int - httpClient *http.Client - healthHttpClient *http.Client - metricCollector *ghttp.Server - scalingEngine *ghttp.Server - breachDurationSecs = 10 - metrics = []*models.AppInstanceMetric{ - { - AppId: testAppId, - InstanceIndex: 0, - CollectedAt: 111111, - Name: metricType, - Unit: metricUnit, - Value: "500", - Timestamp: 111100, - }, - { - AppId: testAppId, - InstanceIndex: 1, - CollectedAt: 111111, - Name: metricType, - Unit: metricUnit, - Value: "600", - Timestamp: 110000, - }, - - { - AppId: testAppId, - InstanceIndex: 0, - CollectedAt: 222222, - Name: metricType, - Unit: metricUnit, - Value: "700", - Timestamp: 222200, - }, - { - AppId: testAppId, - InstanceIndex: 1, - CollectedAt: 222222, - Name: metricType, - Unit: metricUnit, - Value: "800", - Timestamp: 220000, - }, - } + regPath = regexp.MustCompile(`^/v1/apps/.*/scale$`) + configFile *os.File + conf config.Config + egPort int + healthport int + httpClient *http.Client + healthHttpClient *http.Client + mockLogCache *testhelpers.MockLogCache + mockLogCacheEndpoint string + mockScalingEngine *ghttp.Server + breachDurationSecs = 10 scalingResult = &models.AppScalingResult{ AppId: testAppId, @@ -158,17 +122,116 @@ func initDB() { } func initHttpEndPoints() { - metricCollector = ghttp.NewUnstartedServer() - metricCollector.HTTPTestServer.TLS = testhelpers.ServerTlsConfig("metricscollector") - metricCollector.HTTPTestServer.StartTLS() + testCertDir := testhelpers.TestCertFolder() + tlsConfig, err := testhelpers.NewTLSConfig( + filepath.Join(testCertDir, "autoscaler-ca.crt"), + filepath.Join(testCertDir, "log-cache.crt"), + filepath.Join(testCertDir, "log-cache.key"), + "log-cache", + ) + Expect(err).ToNot(HaveOccurred()) + mockLogCache = testhelpers.NewMockLogCache(tlsConfig) + mockLogCache.ReadReturns(testAppId, &rpc.ReadResponse{ + Envelopes: &loggregator_v2.EnvelopeBatch{ + Batch: []*loggregator_v2.Envelope{ + { + SourceId: testAppId, + InstanceId: "0", + Timestamp: 111100, + DeprecatedTags: map[string]*loggregator_v2.Value{ + "origin": { + Data: &loggregator_v2.Value_Text{ + Text: "autoscaler_metrics_forwarder", + }, + }, + }, + Message: &loggregator_v2.Envelope_Gauge{ + Gauge: &loggregator_v2.Gauge{ + Metrics: map[string]*loggregator_v2.GaugeValue{ + metricType: { + Unit: metricUnit, + Value: 500, + }, + }, + }, + }, + }, + { + SourceId: testAppId, + InstanceId: "1", + Timestamp: 110000, + DeprecatedTags: map[string]*loggregator_v2.Value{ + "origin": { + Data: &loggregator_v2.Value_Text{ + Text: "autoscaler_metrics_forwarder", + }, + }, + }, + Message: &loggregator_v2.Envelope_Gauge{ + Gauge: &loggregator_v2.Gauge{ + Metrics: map[string]*loggregator_v2.GaugeValue{ + metricType: { + Unit: metricUnit, + Value: 600, + }, + }, + }, + }, + }, + { + SourceId: testAppId, + InstanceId: "0", + Timestamp: 222200, + DeprecatedTags: map[string]*loggregator_v2.Value{ + "origin": { + Data: &loggregator_v2.Value_Text{ + Text: "autoscaler_metrics_forwarder", + }, + }, + }, + Message: &loggregator_v2.Envelope_Gauge{ + Gauge: &loggregator_v2.Gauge{ + Metrics: map[string]*loggregator_v2.GaugeValue{ + metricType: { + Unit: metricUnit, + Value: 700, + }, + }, + }, + }, + }, + { + SourceId: testAppId, + InstanceId: "1", + Timestamp: 220000, + DeprecatedTags: map[string]*loggregator_v2.Value{ + "origin": { + Data: &loggregator_v2.Value_Text{ + Text: "autoscaler_metrics_forwarder", + }, + }, + }, + Message: &loggregator_v2.Envelope_Gauge{ + Gauge: &loggregator_v2.Gauge{ + Metrics: map[string]*loggregator_v2.GaugeValue{ + metricType: { + Unit: metricUnit, + Value: 800, + }, + }, + }, + }, + }, + }, + }, + }, nil) + mockLogCacheEndpoint = mockLogCache.Start(10000 + GinkgoParallelProcess()) - scalingEngine = ghttp.NewUnstartedServer() - scalingEngine.HTTPTestServer.TLS = testhelpers.ServerTlsConfig("scalingengine") - scalingEngine.HTTPTestServer.StartTLS() + mockScalingEngine = ghttp.NewUnstartedServer() + mockScalingEngine.HTTPTestServer.TLS = testhelpers.ServerTlsConfig("scalingengine") + mockScalingEngine.HTTPTestServer.StartTLS() - metricCollector.RouteToHandler("GET", "/v1/apps/"+testAppId+"/metric_histories/"+metricType, ghttp.RespondWithJSONEncoded(http.StatusOK, - &metrics)) - scalingEngine.RouteToHandler("POST", regPath, ghttp.RespondWithJSONEncoded(http.StatusOK, &scalingResult)) + mockScalingEngine.RouteToHandler("POST", regPath, ghttp.RespondWithJSONEncoded(http.StatusOK, &scalingResult)) } func initConfig() { @@ -222,7 +285,7 @@ func initConfig() { }, }, ScalingEngine: config.ScalingEngineConfig{ - ScalingEngineURL: scalingEngine.URL(), + ScalingEngineURL: mockScalingEngine.URL(), TLSClientCerts: models.TLSCerts{ KeyFile: filepath.Join(testCertDir, "eventgenerator.key"), CertFile: filepath.Join(testCertDir, "eventgenerator.crt"), @@ -230,7 +293,7 @@ func initConfig() { }, }, MetricCollector: config.MetricCollectorConfig{ - MetricCollectorURL: metricCollector.URL(), + MetricCollectorURL: mockLogCacheEndpoint, TLSClientCerts: models.TLSCerts{ KeyFile: filepath.Join(testCertDir, "eventgenerator.key"), CertFile: filepath.Join(testCertDir, "eventgenerator.crt"), diff --git a/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go b/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go index 21e07bcde5..207eeaf869 100644 --- a/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go +++ b/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go @@ -1,25 +1,12 @@ package main_test import ( - "context" - "crypto/rand" - "crypto/tls" "fmt" "io" - "net" "net/http" "os" - "path/filepath" - "strings" - "sync" "time" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - - rpc "code.cloudfoundry.org/go-log-cache/v2/rpc/logcache_v1" - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "google.golang.org/grpc" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" @@ -32,8 +19,7 @@ import ( var _ = Describe("Eventgenerator", func() { var ( - runner *EventGeneratorRunner - testCertDir = "../../../../../test-certs" + runner *EventGeneratorRunner ) BeforeEach(func() { @@ -51,10 +37,17 @@ var _ = Describe("Eventgenerator", func() { It("Starts successfully, retrives metrics and generates events", func() { Consistently(runner.Session).ShouldNot(Exit()) - Eventually(func() bool { return len(metricCollector.ReceivedRequests()) >= 1 }, 5*time.Second).Should(BeTrue()) - Eventually(func() bool { return len(scalingEngine.ReceivedRequests()) >= 1 }, time.Duration(2*breachDurationSecs)*time.Second).Should(BeTrue()) + Eventually(func() bool { return mockLogCache.ReadRequestsCount() >= 1 }, 5*time.Second).Should(BeTrue()) + Eventually(func() bool { return len(mockScalingEngine.ReceivedRequests()) >= 1 }, time.Duration(2*breachDurationSecs)*time.Second).Should(BeTrue()) }) + It("Should create a LogCacheClient", func() { + Eventually(runner.Session.Buffer(), 2).Should(Say("eventgenerator.LogCacheClient.GetMetrics")) + }) + + It("Should initialized an envelopeProcessor", func() { + Eventually(runner.Session.Buffer(), 2).Should(Say("eventgenerator.EnvelopeProcessor.GetGaugeMetrics")) + }) }) Context("with a missing config file", func() { @@ -247,149 +240,4 @@ var _ = Describe("Eventgenerator", func() { }) }) }) - - Describe("", func() { - Context("when logCache is enabled", func() { - var fakeLogCacheServer *stubGrpcLogCache - BeforeEach(func() { - caCert := filepath.Join(testCertDir, "autoscaler-ca.crt") - certFile := filepath.Join(testCertDir, "metricserver.crt") - keyFile := filepath.Join(testCertDir, "metricserver.key") - fakeLogCacheServer = newStubGrpcLogCache(caCert, certFile, keyFile) - logCacheConfig := conf - logCacheConfig.MetricCollector.UseLogCache = true - logCacheConfig.MetricCollector.TLSClientCerts.CACertFile = caCert - logCacheConfig.MetricCollector.TLSClientCerts.CertFile = certFile - logCacheConfig.MetricCollector.TLSClientCerts.KeyFile = keyFile - logCacheConfig.MetricCollector.MetricCollectorURL = strings.ReplaceAll(fakeLogCacheServer.addr(), "http://", "") - runner.configPath = writeConfig(&logCacheConfig).Name() - runner.Start() - }) - - It("Should create a LogCacheClient", func() { - Eventually(runner.Session.Buffer()).ShouldNot(Say("eventgenerator.MetricServerClient.GetMetrics")) - Eventually(runner.Session.Buffer(), 2).Should(Say("eventgenerator.LogCacheClient.GetMetrics")) - }) - - It("Should initialized an envelopeProcessor", func() { - Eventually(runner.Session.Buffer(), 2).Should(Say("eventgenerator.EnvelopeProcessor.GetGaugeMetrics")) - }) - AfterEach(func() { - fakeLogCacheServer.stop() - }) - }) - }) }) - -type stubGrpcLogCache struct { - mu sync.Mutex - reqs []*rpc.ReadRequest - promInstantReqs []*rpc.PromQL_InstantQueryRequest - promRangeReqs []*rpc.PromQL_RangeQueryRequest - lis net.Listener - srv *grpc.Server - block bool - rpc.UnimplementedEgressServer - rpc.UnimplementedPromQLQuerierServer -} - -func newStubGrpcLogCache(caCert, certFile, keyFile string) *stubGrpcLogCache { - s := &stubGrpcLogCache{} - tlsCerts := &models.TLSCerts{CACertFile: caCert, CertFile: certFile, KeyFile: keyFile} - config, err := tlsCerts.CreateClientConfig() - Expect(err).NotTo(HaveOccurred()) - - config.Rand = rand.Reader - lis, err := tls.Listen("tcp", "127.0.0.1:0", config) - Expect(err).ToNot(HaveOccurred()) - - s.lis = lis - s.srv = grpc.NewServer() - - rpc.RegisterEgressServer(s.srv, s) - rpc.RegisterPromQLQuerierServer(s.srv, s) - go func() { - err := s.srv.Serve(lis) - Expect(err).NotTo(HaveOccurred()) - }() - - return s -} -func (s *stubGrpcLogCache) stop() { - s.srv.Stop() -} - -func (s *stubGrpcLogCache) addr() string { - return s.lis.Addr().String() -} - -func (s *stubGrpcLogCache) Read(c context.Context, r *rpc.ReadRequest) (*rpc.ReadResponse, error) { - if s.block { - var block chan struct{} - <-block - } - - s.mu.Lock() - defer s.mu.Unlock() - s.reqs = append(s.reqs, r) - - return &rpc.ReadResponse{ - Envelopes: &loggregator_v2.EnvelopeBatch{ - Batch: []*loggregator_v2.Envelope{ - {Timestamp: 99, SourceId: "some-id"}, - {Timestamp: 100, SourceId: "some-id"}, - }, - }, - }, nil -} - -func (s *stubGrpcLogCache) InstantQuery(c context.Context, r *rpc.PromQL_InstantQueryRequest) (*rpc.PromQL_InstantQueryResult, error) { - if s.block { - var block chan struct{} - <-block - } - - s.mu.Lock() - defer s.mu.Unlock() - s.promInstantReqs = append(s.promInstantReqs, r) - - return &rpc.PromQL_InstantQueryResult{ - Result: &rpc.PromQL_InstantQueryResult_Scalar{ - Scalar: &rpc.PromQL_Scalar{ - Time: "99.000", - Value: 101, - }, - }, - }, nil -} - -func (s *stubGrpcLogCache) RangeQuery(c context.Context, r *rpc.PromQL_RangeQueryRequest) (*rpc.PromQL_RangeQueryResult, error) { - if s.block { - var block chan struct{} - <-block - } - - s.mu.Lock() - defer s.mu.Unlock() - s.promRangeReqs = append(s.promRangeReqs, r) - - return &rpc.PromQL_RangeQueryResult{ - Result: &rpc.PromQL_RangeQueryResult_Matrix{ - Matrix: &rpc.PromQL_Matrix{ - Series: []*rpc.PromQL_Series{ - { - Metric: map[string]string{ - "__name__": "test", - }, - Points: []*rpc.PromQL_Point{ - { - Time: "99.000", - Value: 101, - }, - }, - }, - }, - }, - }, - }, nil -} diff --git a/src/autoscaler/eventgenerator/config/config.go b/src/autoscaler/eventgenerator/config/config.go index 2374263ae6..a35b6691b5 100644 --- a/src/autoscaler/eventgenerator/config/config.go +++ b/src/autoscaler/eventgenerator/config/config.go @@ -65,7 +65,6 @@ type ScalingEngineConfig struct { } type MetricCollectorConfig struct { - UseLogCache bool `yaml:"use_log_cache"` MetricCollectorURL string `yaml:"metric_collector_url"` TLSClientCerts models.TLSCerts `yaml:"tls"` UAACreds models.UAACreds `yaml:"uaa"` diff --git a/src/autoscaler/eventgenerator/config/config_test.go b/src/autoscaler/eventgenerator/config/config_test.go index 9396f53254..05f3108054 100644 --- a/src/autoscaler/eventgenerator/config/config_test.go +++ b/src/autoscaler/eventgenerator/config/config_test.go @@ -54,7 +54,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 30s policy_poller_interval: 30s save_interval: 30s @@ -73,8 +73,7 @@ scalingEngine: cert_file: /var/vcap/jobs/autoscaler/config/certs/se.crt ca_file: /var/vcap/jobs/autoscaler/config/certs/autoscaler-ca.crt metricCollector: - use_log_cache: false - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 tls: key_file: /var/vcap/jobs/autoscaler/config/certs/mc.key cert_file: /var/vcap/jobs/autoscaler/config/certs/mc.crt @@ -146,7 +145,7 @@ circuitBreaker: }, }, MetricCollector: MetricCollectorConfig{ - MetricCollectorURL: "http://localhost:8083", + MetricCollectorURL: "log-cache:1234", TLSClientCerts: models.TLSCerts{ KeyFile: "/var/vcap/jobs/autoscaler/config/certs/mc.key", CertFile: "/var/vcap/jobs/autoscaler/config/certs/mc.crt", @@ -179,7 +178,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 30s policy_poller_interval: 30s save_interval: 30s @@ -193,7 +192,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -214,7 +213,7 @@ db: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -268,10 +267,11 @@ defaultBreachDurationSecs: 600 TriggerArrayChannelSize: DefaultTriggerArrayChannelSize, }, ScalingEngine: ScalingEngineConfig{ - ScalingEngineURL: "http://localhost:8082"}, + ScalingEngineURL: "http://localhost:8082", + }, MetricCollector: MetricCollectorConfig{ - UseLogCache: false, - MetricCollectorURL: "http://localhost:8083"}, + MetricCollectorURL: "log-cache:1234", + }, DefaultBreachDurationSecs: 600, DefaultStatWindowSecs: 300, CircuitBreaker: CircuitBreakerConfig{ @@ -299,7 +299,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 60s policy_poller_interval: 30s metric_poller_count: 10 @@ -311,7 +311,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -339,7 +339,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 60s policy_poller_interval: 30s metric_poller_count: 10 @@ -351,7 +351,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -379,7 +379,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 60s policy_poller_interval: 30s metric_poller_count: 10 @@ -391,7 +391,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -419,7 +419,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 60s policy_poller_interval: 30s metric_poller_count: 10 @@ -431,7 +431,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -459,7 +459,7 @@ db: max_open_connections: NOT-INTEGER-VALUE max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 60s policy_poller_interval: 30s metric_poller_count: 10 @@ -471,7 +471,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -499,7 +499,7 @@ db: max_open_connections: 10 max_idle_connections: NOT-INTEGER-VALUE connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 60s policy_poller_interval: 30s metric_poller_count: 10 @@ -511,7 +511,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -539,7 +539,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 6k -aggregator: +aggregator: aggregator_execute_interval: 60s policy_poller_interval: 30s metric_poller_count: 10 @@ -551,7 +551,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -579,7 +579,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 5k policy_poller_interval: 30s metric_poller_count: 10 @@ -591,7 +591,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -619,7 +619,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 30s policy_poller_interval: 7u metric_poller_count: 10 @@ -631,7 +631,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -659,7 +659,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 30s policy_poller_interval: 30s save_interval: 7u @@ -672,7 +672,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -700,7 +700,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 30s policy_poller_interval: 30s metric_poller_count: NOT-INTEGER-VALUE @@ -712,7 +712,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -740,7 +740,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 30s policy_poller_interval: 30s metric_poller_count: 10 @@ -752,7 +752,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -772,7 +772,7 @@ logging: db: policy_db_url: postgres://postgres:password@localhost/autoscaler?sslmode=disable app_metrics_db_url: postgres://postgres:password@localhost/autoscaler?sslmode=disable -aggregator: +aggregator: aggregator_execute_interval: 30s policy_poller_interval: 30s metric_poller_count: 10 @@ -785,7 +785,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -805,7 +805,7 @@ logging: db: policy_db_url: postgres://postgres:password@localhost/autoscaler?sslmode=disable app_metrics_db_url: postgres://postgres:password@localhost/autoscaler?sslmode=disable -aggregator: +aggregator: aggregator_execute_interval: 30s policy_poller_interval: 30s metric_poller_count: 10 @@ -819,7 +819,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -846,7 +846,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 30s policy_poller_interval: 30s metric_poller_count: 10 @@ -858,7 +858,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -886,7 +886,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 30s policy_poller_interval: 30s metric_poller_count: 10 @@ -898,7 +898,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -925,7 +925,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 30s policy_poller_interval: 30s metric_poller_count: 10 @@ -937,7 +937,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 600 `) @@ -965,7 +965,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 30s policy_poller_interval: 30s metric_poller_count: 10 @@ -977,7 +977,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: NOT-INTEGER-VALUE defaultBreachDurationSecs: 600 `) @@ -1004,7 +1004,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 30s policy_poller_interval: 30s metric_poller_count: 10 @@ -1016,7 +1016,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: NOT-INTEGER-VALUE `) @@ -1044,7 +1044,7 @@ db: max_open_connections: 10 max_idle_connections: 5 connection_max_lifetime: 60s -aggregator: +aggregator: aggregator_execute_interval: 30s policy_poller_interval: 30s metric_poller_count: 10 @@ -1056,7 +1056,7 @@ evaluator: scalingEngine: scaling_engine_url: http://localhost:8082 metricCollector: - metric_collector_url: http://localhost:8083 + metric_collector_url: log-cache:1234 defaultStatWindowSecs: 300 defaultBreachDurationSecs: 300 health: @@ -1110,7 +1110,7 @@ health: ScalingEngine: ScalingEngineConfig{ ScalingEngineURL: "http://localhost:8082"}, MetricCollector: MetricCollectorConfig{ - MetricCollectorURL: "http://localhost:8083", + MetricCollectorURL: "log-cache:1234", }, DefaultBreachDurationSecs: 600, DefaultStatWindowSecs: 300, diff --git a/src/autoscaler/generate-fakes.go b/src/autoscaler/generate-fakes.go index d6f7f4f4fa..4342d78076 100644 --- a/src/autoscaler/generate-fakes.go +++ b/src/autoscaler/generate-fakes.go @@ -17,7 +17,6 @@ package fakes //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_log_cache_client.go ./eventgenerator/client LogCacheClientReader //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_envelope_processor.go ./envelopeprocessor EnvelopeProcessor //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_log_cache_creator.go ./eventgenerator/client LogCacheClientCreator -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_metric_server_creator_creator.go ./eventgenerator/client MetricServerClientCreator //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_go_log_cache_client.go ./eventgenerator/client GoLogCacheClient //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_grpc.go ./eventgenerator/client GRPCOptions //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_envelope_processor_creator.go ./envelopeprocessor EnvelopeProcessorCreator diff --git a/src/autoscaler/integration/components_test.go b/src/autoscaler/integration/components_test.go index 5ff75f2a1f..9e78efcdd3 100644 --- a/src/autoscaler/integration/components_test.go +++ b/src/autoscaler/integration/components_test.go @@ -368,7 +368,6 @@ func (components *Components) PrepareEventGeneratorConfig(dbUri string, port int }, }, MetricCollector: egConfig.MetricCollectorConfig{ - UseLogCache: true, MetricCollectorURL: metricsCollectorURL, TLSClientCerts: models.TLSCerts{ KeyFile: filepath.Join(testCertDir, "eventgenerator.key"), diff --git a/src/autoscaler/integration/integration_suite_test.go b/src/autoscaler/integration/integration_suite_test.go index 212a422b10..004c94cecf 100644 --- a/src/autoscaler/integration/integration_suite_test.go +++ b/src/autoscaler/integration/integration_suite_test.go @@ -17,7 +17,6 @@ import ( "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cf" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cf/mocks" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/integration/mock" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" @@ -55,7 +54,7 @@ var ( fakeCCNOAAUAA *mocks.Server testUserScope = []string{"cloud_controller.read", "cloud_controller.write", "password.write", "openid", "network.admin", "network.write", "uaa.user"} processMap = map[string]ifrit.Process{} - mockLogCache = &mock.LogCache{} + mockLogCache = &MockLogCache{} mockLogCachePort = 20000 + GinkgoParallelProcess() defaultHttpClientTimeout = 10 * time.Second @@ -194,7 +193,7 @@ func startOperator() { } func startMockLogCache() { - tlsConfig, err := mock.NewTLSConfig( + tlsConfig, err := NewTLSConfig( filepath.Join(testCertDir, "autoscaler-ca.crt"), filepath.Join(testCertDir, "log-cache.crt"), filepath.Join(testCertDir, "log-cache.key"), @@ -202,7 +201,7 @@ func startMockLogCache() { ) Expect(err).ToNot(HaveOccurred()) - mockLogCache = mock.NewLogCache(tlsConfig) + mockLogCache = NewMockLogCache(tlsConfig) mockLogCache.Start(mockLogCachePort) } diff --git a/src/autoscaler/routes/routes.go b/src/autoscaler/routes/routes.go index 3cf31a7875..d845239d11 100644 --- a/src/autoscaler/routes/routes.go +++ b/src/autoscaler/routes/routes.go @@ -64,7 +64,6 @@ type AutoScalerRoute struct { metricsCollectorRoutes *mux.Router eventGeneratorRoutes *mux.Router scalingEngineRoutes *mux.Router - metricServerRoutes *mux.Router metricsForwarderRoutes *mux.Router apiOpenRoutes *mux.Router apiRoutes *mux.Router @@ -80,7 +79,6 @@ func newRouters() *AutoScalerRoute { metricsCollectorRoutes: mux.NewRouter(), eventGeneratorRoutes: mux.NewRouter(), scalingEngineRoutes: mux.NewRouter(), - metricServerRoutes: mux.NewRouter(), metricsForwarderRoutes: mux.NewRouter(), apiOpenRoutes: mux.NewRouter(), apiRoutes: mux.NewRouter(), @@ -101,8 +99,6 @@ func newRouters() *AutoScalerRoute { instance.metricsForwarderRoutes.Path(CustomMetricsPath).Methods(http.MethodPost).Name(PostCustomMetricsRouteName) - instance.metricServerRoutes.Path(EnvelopePath).Name(EnvelopeReportRouteName) - instance.schedulerRoutes.Path(SchedulePath).Methods(http.MethodPut).Name(UpdateScheduleRouteName) instance.schedulerRoutes.Path(SchedulePath).Methods(http.MethodDelete).Name(DeleteScheduleRouteName) instance.apiOpenRoutes.Path(PublicApiInfoPath).Methods(http.MethodGet).Name(PublicApiInfoRouteName) @@ -132,10 +128,6 @@ func ScalingEngineRoutes() *mux.Router { return autoScalerRouteInstance.scalingEngineRoutes } -func MetricServerRoutes() *mux.Router { - return autoScalerRouteInstance.metricServerRoutes -} - func MetricsForwarderRoutes() *mux.Router { return autoScalerRouteInstance.metricsForwarderRoutes } diff --git a/src/autoscaler/routes/routes_test.go b/src/autoscaler/routes/routes_test.go index 2d40b1486d..4d5c3bbd65 100644 --- a/src/autoscaler/routes/routes_test.go +++ b/src/autoscaler/routes/routes_test.go @@ -353,16 +353,6 @@ var _ = Describe("Routes", func() { }) }) - Describe("MetricServerRoutes", func() { - Context("EnvelopeReportRouteName", func() { - It("should return the correct path", func() { - path, err := routes.MetricServerRoutes().Get(routes.EnvelopeReportRouteName).URLPath() - Expect(err).NotTo(HaveOccurred()) - Expect(path.Path).To(Equal("/v1/envelopes")) - }) - }) - }) - Context("PostCustomMetricsRoute", func() { Context("when provide correct route variable", func() { It("should return the correct path", func() { diff --git a/src/autoscaler/testhelpers/clients.go b/src/autoscaler/testhelpers/clients.go index cd8e1a3373..eecf376d1b 100644 --- a/src/autoscaler/testhelpers/clients.go +++ b/src/autoscaler/testhelpers/clients.go @@ -20,10 +20,6 @@ func NewPublicApiClient() *http.Client { return CreateClientFor("api_public") } -func NewMetricServerClient() *http.Client { - return CreateClientFor("metricserver") -} - func NewEventGeneratorClient() *http.Client { return CreateClientFor("eventgenerator") } diff --git a/src/autoscaler/integration/mock/log_cache_mock.go b/src/autoscaler/testhelpers/log_cache_mock.go similarity index 55% rename from src/autoscaler/integration/mock/log_cache_mock.go rename to src/autoscaler/testhelpers/log_cache_mock.go index e6c9bdd990..613594ebab 100644 --- a/src/autoscaler/integration/mock/log_cache_mock.go +++ b/src/autoscaler/testhelpers/log_cache_mock.go @@ -1,4 +1,4 @@ -package mock +package testhelpers import ( "crypto/tls" @@ -18,13 +18,14 @@ import ( "google.golang.org/grpc/credentials" ) -type LogCache struct { +type MockLogCache struct { mutex sync.Mutex lis net.Listener - readResponses map[string]*rpc.ReadResponse - readErrors map[string]error + readRequestsCount int + readResponses map[string]*rpc.ReadResponse + readErrors map[string]error instantQueryResults map[string]*rpc.PromQL_InstantQueryResult instantQueryErrors map[string]error @@ -36,8 +37,8 @@ type LogCache struct { rpc.UnimplementedPromQLQuerierServer } -func NewLogCache(tlsConfig *tls.Config) *LogCache { - return &LogCache{ +func NewMockLogCache(tlsConfig *tls.Config) *MockLogCache { + return &MockLogCache{ tlsConfig: tlsConfig, readResponses: map[string]*rpc.ReadResponse{}, @@ -48,80 +49,89 @@ func NewLogCache(tlsConfig *tls.Config) *LogCache { } } -func (s *LogCache) Start(port int) string { +func (m *MockLogCache) Start(port int) string { var err error - s.lis, err = net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) + m.lis, err = net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) if err != nil { log.Fatalf("failed to listen: %v", err) } var srv *grpc.Server - if s.tlsConfig == nil { + if m.tlsConfig == nil { srv = grpc.NewServer() } else { - srv = grpc.NewServer(grpc.Creds(credentials.NewTLS(s.tlsConfig))) + srv = grpc.NewServer(grpc.Creds(credentials.NewTLS(m.tlsConfig))) } - rpc.RegisterIngressServer(srv, s) - rpc.RegisterEgressServer(srv, s) - rpc.RegisterPromQLQuerierServer(srv, s) + rpc.RegisterIngressServer(srv, m) + rpc.RegisterEgressServer(srv, m) + rpc.RegisterPromQLQuerierServer(srv, m) //nolint:errcheck - go srv.Serve(s.lis) + go srv.Serve(m.lis) - return s.lis.Addr().String() + return m.lis.Addr().String() } -func (s *LogCache) Stop() { - s.lis.Close() +func (m *MockLogCache) Stop() { + m.lis.Close() } -func (s *LogCache) Read(ctx context.Context, r *rpc.ReadRequest) (*rpc.ReadResponse, error) { - s.mutex.Lock() - defer s.mutex.Unlock() +func (m *MockLogCache) Read(ctx context.Context, r *rpc.ReadRequest) (*rpc.ReadResponse, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + m.readRequestsCount++ response := &rpc.ReadResponse{Envelopes: &loggregator_v2.EnvelopeBatch{Batch: []*loggregator_v2.Envelope{}}} - if storedResponse, ok := s.readResponses[r.SourceId]; ok { + if storedResponse, ok := m.readResponses[r.SourceId]; ok { response = storedResponse } err := errors.New("default-error") - if storedError, ok := s.readErrors[r.SourceId]; ok { + if storedError, ok := m.readErrors[r.SourceId]; ok { err = storedError } return response, err } -func (s *LogCache) ReadReturns(sourceId string, response *rpc.ReadResponse, err error) { - s.mutex.Lock() - defer s.mutex.Unlock() +func (m *MockLogCache) ReadRequestsCount() int { + m.mutex.Lock() + defer m.mutex.Unlock() + + return m.readRequestsCount +} + +func (m *MockLogCache) ReadReturns(sourceId string, response *rpc.ReadResponse, err error) { + m.mutex.Lock() + defer m.mutex.Unlock() - s.readResponses[sourceId] = response - s.readErrors[sourceId] = err + m.readResponses[sourceId] = response + m.readErrors[sourceId] = err } -func (s *LogCache) InstantQuery(ctx context.Context, r *rpc.PromQL_InstantQueryRequest) (*rpc.PromQL_InstantQueryResult, error) { - s.mutex.Lock() - defer s.mutex.Unlock() +func (m *MockLogCache) InstantQuery(ctx context.Context, r *rpc.PromQL_InstantQueryRequest) (*rpc.PromQL_InstantQueryResult, error) { + m.mutex.Lock() + defer m.mutex.Unlock() // every query carries the source_id, example "http{source_id='foo'}" // find stored mock-result or mock-error by checking if any of the stored source ids is contained in the query query := r.Query - for sourceId := range s.instantQueryResults { + for sourceId := range m.instantQueryResults { if strings.Contains(query, sourceId) { - return s.instantQueryResults[sourceId], s.instantQueryErrors[sourceId] + return m.instantQueryResults[sourceId], m.instantQueryErrors[sourceId] } } return nil, fmt.Errorf("default-error(query: %s)", query) } -func (s *LogCache) InstantQueryReturns(sourceId string, result *rpc.PromQL_InstantQueryResult, err error) { - s.mutex.Lock() - defer s.mutex.Unlock() +func (m *MockLogCache) InstantQueryReturns(sourceId string, result *rpc.PromQL_InstantQueryResult, err error) { + m.mutex.Lock() + defer m.mutex.Unlock() - s.instantQueryResults[sourceId] = result - s.instantQueryErrors[sourceId] = err + m.instantQueryResults[sourceId] = result + m.instantQueryErrors[sourceId] = err } func NewTLSConfig(caPath, certPath, keyPath, cn string) (*tls.Config, error) { diff --git a/templates/app-autoscaler.yml b/templates/app-autoscaler.yml index e7b1148987..0814be9d7e 100644 --- a/templates/app-autoscaler.yml +++ b/templates/app-autoscaler.yml @@ -446,7 +446,6 @@ instance_groups: client_key: ((!scalingengine_client_cert.private_key)) host: *scalingengine_domain metricscollector: - use_log_cache: true ca_cert: ((/bosh-autoscaler/cf/log_cache.ca)) client_cert: ((/bosh-autoscaler/cf/log_cache.certificate)) client_key: ((/bosh-autoscaler/cf/log_cache.private_key)) From a04faaf36d8ada24bae3eef5c2b478362876168d Mon Sep 17 00:00:00 2001 From: bonzofenix <317403+bonzofenix@users.noreply.github.com> Date: Tue, 11 Jun 2024 13:19:51 +0200 Subject: [PATCH 056/186] Add netcat to dockerfile to use bosh scp bosh scp needs the `nc` binary in order to run. This is provided in the netcat package --- ci/dockerfiles/autoscaler-tools/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/dockerfiles/autoscaler-tools/Dockerfile b/ci/dockerfiles/autoscaler-tools/Dockerfile index 6eac2e2f77..ed4ec976a8 100644 --- a/ci/dockerfiles/autoscaler-tools/Dockerfile +++ b/ci/dockerfiles/autoscaler-tools/Dockerfile @@ -36,6 +36,7 @@ RUN apt-get update && \ cf8-cli \ gnupg \ gnupg2 \ + netcat \ gh \ make \ mysql-client && \ From 9e081f1be31434fa68e9c868dd88bfb16d755f73 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 11 Jun 2024 13:45:29 +0200 Subject: [PATCH 057/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/autoscaler/go.mod | 53 ++++++++++--------- src/autoscaler/go.sum | 120 +++++++++++++++++++++++------------------- 2 files changed, 93 insertions(+), 80 deletions(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 8d3a801a27..3f8b2aa0f5 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -1,15 +1,15 @@ module code.cloudfoundry.org/app-autoscaler/src/autoscaler -go 1.21.6 +go 1.21.5 require ( - code.cloudfoundry.org/cfhttp/v2 v2.1.0 + code.cloudfoundry.org/cfhttp/v2 v2.0.1 code.cloudfoundry.org/clock v1.1.0 code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240607070054-fde7ff8f79b7 - code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea + code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240522070055-f1179c4954a5 + code.cloudfoundry.org/tlsconfig v0.0.0-20240522170710-79df114af82a dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/go-chi/chi/v5 v5.0.12 @@ -19,21 +19,21 @@ require ( github.com/go-sql-driver/mysql v1.8.1 github.com/golang/protobuf v1.5.4 github.com/gorilla/mux v1.8.1 - github.com/gorilla/websocket v1.5.2 - github.com/hashicorp/go-retryablehttp v0.7.7 - github.com/jackc/pgx/v5 v5.6.0 + github.com/gorilla/websocket v1.5.1 + github.com/hashicorp/go-retryablehttp v0.7.5 + github.com/jackc/pgx/v5 v5.5.5 github.com/jmoiron/sqlx v1.4.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d - github.com/ogen-go/ogen v1.1.1 - github.com/onsi/ginkgo/v2 v2.19.0 + github.com/ogen-go/ogen v1.1.0 + github.com/onsi/ginkgo/v2 v2.18.0 github.com/onsi/gomega v1.33.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pivotal-cf/brokerapi/v11 v11.0.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.1 github.com/rubyist/circuitbreaker v2.2.1+incompatible - github.com/steinfletcher/apitest v1.5.16 + github.com/steinfletcher/apitest v1.5.15 github.com/stretchr/testify v1.9.0 github.com/tedsuo/ifrit v0.0.0-20230516164442-7862c310ad26 github.com/uptrace/opentelemetry-go-extra/otelsql v0.2.4 @@ -44,17 +44,17 @@ require ( go.opentelemetry.io/otel/metric v1.26.0 go.opentelemetry.io/otel/sdk v1.26.0 go.opentelemetry.io/otel/trace v1.26.0 - golang.org/x/crypto v0.24.0 - golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 - golang.org/x/net v0.26.0 + golang.org/x/crypto v0.23.0 + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 + golang.org/x/net v0.25.0 golang.org/x/time v0.5.0 google.golang.org/grpc v1.64.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - code.cloudfoundry.org/go-diodes v0.0.0-20240604201846-c756bfed2ed3 // indirect - code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a // indirect + code.cloudfoundry.org/go-diodes v0.0.0-20240515174142-71582f284718 // indirect + code.cloudfoundry.org/go-metric-registry v0.0.0-20240507185439-f0b17e194563 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -64,7 +64,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect - github.com/fatih/color v1.17.0 // indirect + github.com/fatih/color v1.16.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/ghodss/yaml v1.0.0 // indirect @@ -72,10 +72,11 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect + github.com/google/pprof v0.0.0-20240521024322-9665fa269a30 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-hclog v1.2.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect @@ -87,22 +88,22 @@ require ( github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.54.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect + github.com/prometheus/common v0.53.0 // indirect + github.com/prometheus/procfs v0.15.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.54.0 // indirect + github.com/valyala/fasthttp v1.53.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/mod v0.18.0 // indirect + golang.org/x/mod v0.17.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect - golang.org/x/tools v0.22.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/tools v0.21.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 2ee0538170..66b3e30177 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -592,24 +592,24 @@ cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoIS cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= -code.cloudfoundry.org/cfhttp/v2 v2.1.0 h1:HbQ5H2R+HEKG/rcB6Gk3okeC3h2fAC4PPnLQoMHvzZM= -code.cloudfoundry.org/cfhttp/v2 v2.1.0/go.mod h1:k9R36Y/9dUc9OsX4dfDuEjHZ7Q00ttklKQj6HD6h6+U= +code.cloudfoundry.org/cfhttp/v2 v2.0.1 h1:xCbkvDVUDuFCquDo66QnvM9hG6vxM2cvTMjj/A8BcNA= +code.cloudfoundry.org/cfhttp/v2 v2.0.1/go.mod h1:yD0n22D4KP4xamgTxbz82KsWzK5kcfUrmJNrrmd4YBg= code.cloudfoundry.org/clock v1.1.0 h1:XLzC6W3Ah/Y7ht1rmZ6+QfPdt1iGWEAAtIZXgiaj57c= code.cloudfoundry.org/clock v1.1.0/go.mod h1:yA3fxddT9RINQL2XHS7PS+OXxKCGhfrZmlNUCIM6AKo= -code.cloudfoundry.org/go-diodes v0.0.0-20240604201846-c756bfed2ed3 h1:4WCYwJmqSfV7ChDohsJB8Z0aDVklIE+n8OTBJxpif0c= -code.cloudfoundry.org/go-diodes v0.0.0-20240604201846-c756bfed2ed3/go.mod h1:8O5g1DEzJU9ktEmykKPhY4mZOM/dBENWVHKVInuuch8= +code.cloudfoundry.org/go-diodes v0.0.0-20240515174142-71582f284718 h1:6wazSuHaJjatGy8pvchSy6L+4M67WPfjgK9yh7cJLMs= +code.cloudfoundry.org/go-diodes v0.0.0-20240515174142-71582f284718/go.mod h1:eVHabU/rcpC5ocoIAXvnaySkdH6+PgORRVlY5l3SDys= code.cloudfoundry.org/go-log-cache/v2 v2.0.7 h1:yR/JjQ/RscO1n4xVAT9HDYcpx5ET/3Cq2/RhpJml6ZU= code.cloudfoundry.org/go-log-cache/v2 v2.0.7/go.mod h1:6KQe2FeeaqRheD5vCvpyTa80YoJojB/r21E54mT97Mc= code.cloudfoundry.org/go-loggregator/v9 v9.2.1 h1:S6Lgg5UJbhh2bt2TGQxs6R00CF8PrUA3GFPYDxy56Fk= code.cloudfoundry.org/go-loggregator/v9 v9.2.1/go.mod h1:FTFFruqGeOhVCDFvyLgl8EV8YW63NNwRzLhxJcporu8= -code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a h1:XpebbxgIBBy7SrwIGW+gREZuAtnJ9PHWC4Y+k7yje2I= -code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a/go.mod h1:/Be8VtLiCeMUoYdUzFtmW8GGkk89HAy3zD79KUXzbhs= +code.cloudfoundry.org/go-metric-registry v0.0.0-20240507185439-f0b17e194563 h1:mkCsrZ0h4ZF9jcVTEtLiSRJ0Mu2AIQANEWVDcXrOSVY= +code.cloudfoundry.org/go-metric-registry v0.0.0-20240507185439-f0b17e194563/go.mod h1:zYhHvL+6LLy/lGiIe/T5Ma6ae0Tv5Qxm6TNWOvsQVrQ= code.cloudfoundry.org/lager/v3 v3.0.3 h1:/UTmadZfIaKuT/whEinSxK1mzRfNu1uPfvjFfGqiwzM= code.cloudfoundry.org/lager/v3 v3.0.3/go.mod h1:Zn5q1SrIuuHjEUE7xerMKt3ztunrJQCZETAo7rV0CH8= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240607070054-fde7ff8f79b7 h1:2NXdiGqiv6XaAKpEAJBlzOkWjI2FWw/dXlSiyClCCVI= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240607070054-fde7ff8f79b7/go.mod h1:djkW8FecJqemEchtA9hUzp0QsB3V7MG3xSXADV3JL3g= -code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea h1:PjyR5Os5IVETrZkgATzSNl4R++/h+r6uQchSJ3dZsgc= -code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea/go.mod h1:RwJBA7HrvmyPNY8pRzAwlJ56WFmqJiPjxywZpxCZdWQ= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240522070055-f1179c4954a5 h1:5ae8G6wQss4vqKkBQQ8fNQoXEhdV5vEkKStGlPQ68Rc= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240522070055-f1179c4954a5/go.mod h1:2cFY9sx7jyRwEglCfSLMV57cWE0opfIBQhgXHaHIx+0= +code.cloudfoundry.org/tlsconfig v0.0.0-20240522170710-79df114af82a h1:yWneME1ZPxsWkZT1R9e0n6CymukD5pU/ceEYo4ZVMak= +code.cloudfoundry.org/tlsconfig v0.0.0-20240522170710-79df114af82a/go.mod h1:TI9y0bK+FJ9OKSLaJmBo5Z2tr+/VULZ2jiQwSg1+Z5c= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -696,8 +696,9 @@ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6Ni github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= -github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= -github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -821,8 +822,8 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20240528025155-186aa0362fba h1:ql1qNgCyOB7iAEk8JTNM+zJrgIbnyCKX/wdlyPufP5g= -github.com/google/pprof v0.0.0-20240528025155-186aa0362fba/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20240521024322-9665fa269a30 h1:r6YdmbD41tGHeCWDyHF691LWtL7D1iSTyJaKejTWwVU= +github.com/google/pprof v0.0.0-20240521024322-9665fa269a30/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -849,8 +850,8 @@ github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/websocket v1.5.2 h1:qoW6V1GT3aZxybsbC6oLnailWnB+qTMVwMreOso9XUw= -github.com/gorilla/websocket v1.5.2/go.mod h1:0n9H61RBAcf5/38py2MCYbxzPIY9rOkpvvMT24Rqs30= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= @@ -858,10 +859,11 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= -github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= -github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= +github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= +github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -872,8 +874,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= +github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= @@ -904,8 +906,11 @@ github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -922,8 +927,8 @@ github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/ogen-go/ogen v1.1.1 h1:ccHc/vgS+WbOLLgM4JYW7EDiBv7DPfbvKCnbo5fctSY= -github.com/ogen-go/ogen v1.1.1/go.mod h1:ZmPa2LwJrwXKPo3s3p/XFMg5znV60IX5950UftdNmb4= +github.com/ogen-go/ogen v1.1.0 h1:hM4osoOPGWvx8pcdqE8jb5VvzHA+wi3u+DwCvZoW4dg= +github.com/ogen-go/ogen v1.1.0/go.mod h1:d5Ph7PWpJz453ZdGbVe+DBVPS150idePr5dZ+S0JG00= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= @@ -941,8 +946,8 @@ github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxm github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM= -github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= -github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/ginkgo/v2 v2.18.0 h1:W9Y7IWXxPUpAit9ieMOLI7PJZGaW22DTKgiVAuhDTLc= +github.com/onsi/ginkgo/v2 v2.18.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= @@ -988,10 +993,10 @@ github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8= -github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= +github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= +github.com/prometheus/procfs v0.15.0 h1:A82kmvXJq2jTu5YUhSGNlYoxh85zLnKgPz4bMZgI5Ek= +github.com/prometheus/procfs v0.15.0/go.mod h1:Y0RJ/Y5g5wJpkTisOtqwDSo4HwhGmLB4VQSw2sQJLHk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1013,8 +1018,8 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/square/certstrap v1.3.0 h1:N9P0ZRA+DjT8pq5fGDj0z3FjafRKnBDypP0QHpMlaAk= github.com/square/certstrap v1.3.0/go.mod h1:wGZo9eE1B7WX2GKBn0htJ+B3OuRl2UsdCFySNooy9hU= -github.com/steinfletcher/apitest v1.5.16 h1:J/ZoBmhgdzH4qfxPSw9kaXRBgzy3OsCoKh1gcc1h2zM= -github.com/steinfletcher/apitest v1.5.16/go.mod h1:mF+KnYaIkuHM0C4JgGzkIIOJAEjo+EA5tTjJ+bHXnQc= +github.com/steinfletcher/apitest v1.5.15 h1:AAdTN0yMbf0VMH/PMt9uB2I7jljepO6i+5uhm1PjH3c= +github.com/steinfletcher/apitest v1.5.15/go.mod h1:mF+KnYaIkuHM0C4JgGzkIIOJAEjo+EA5tTjJ+bHXnQc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -1038,8 +1043,8 @@ github.com/uptrace/opentelemetry-go-extra/otelsqlx v0.2.4 h1:Pt/+CUTRusJb471SBXw github.com/uptrace/opentelemetry-go-extra/otelsqlx v0.2.4/go.mod h1:kQgNoghy4K/wguxbOd/u0OJw/Y0maNPc7PF4JpEGeUc= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.54.0 h1:cCL+ZZR3z3HPLMVfEYVUMtJqVaui0+gu7Lx63unHwS0= -github.com/valyala/fasthttp v1.54.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM= +github.com/valyala/fasthttp v1.53.0 h1:lW/+SUkOxCx2vlIu0iaImv4JLrVRnbbkpCoaawvA4zc= +github.com/valyala/fasthttp v1.53.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -1077,8 +1082,13 @@ go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZu go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -go.step.sm/crypto v0.47.0 h1:LWxiKWiN0Y/A5+dq+fTIAvFYAL8oe3PQmCurjtn6ZBU= -go.step.sm/crypto v0.47.0/go.mod h1:0NMEfYrFfV5jqs8aJY5wRqIShBV8y/fyDLTseyv5xhY= +<<<<<<< HEAD +go.step.sm/crypto v0.45.0 h1:Z0WYAaaOYrJmKP9sJkPW+6wy3pgN3Ija8ek/D4serjc= +go.step.sm/crypto v0.45.0/go.mod h1:6IYlT0L2jfj81nVyCPpvA5cORy0EVHPhieSgQyuwHIY= +======= +go.step.sm/crypto v0.45.1 h1:Xb8XldsbqT6pDYsg46BVPP1euASNbeNAhzrlvUP3QWo= +go.step.sm/crypto v0.45.1/go.mod h1:XtJBuMuZb11YeJpG8uP3fyBl2MerXWJ/pWQX/Au+Kt8= +>>>>>>> 9f4a84771 (🤖🦾🛠️ go mod tidy & make package-specs) go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -1095,8 +1105,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1112,8 +1122,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM= -golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1159,8 +1169,8 @@ golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1224,8 +1234,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1276,6 +1286,7 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1286,6 +1297,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1359,8 +1371,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1387,8 +1399,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1463,8 +1475,8 @@ golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= -golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1677,10 +1689,10 @@ google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= From 27cf2062b3be887d41be6c84403b382e070137c3 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 22 May 2024 14:39:31 +0200 Subject: [PATCH 058/186] Adds helpers to fetch metrics forwarder config Pulls metricsforwarder config from vm and swap configuration to reference to cf container instead of bosh vm. --- src/autoscaler/metricsforwarder/Makefile | 37 +++++++++++++++++++ .../metricsforwarder/security-group.json | 14 +++++++ 2 files changed, 51 insertions(+) create mode 100644 src/autoscaler/metricsforwarder/Makefile create mode 100644 src/autoscaler/metricsforwarder/security-group.json diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile new file mode 100644 index 0000000000..b52d96ad66 --- /dev/null +++ b/src/autoscaler/metricsforwarder/Makefile @@ -0,0 +1,37 @@ +PR_NUMBER := $(shell gh pr view --json number --jq '.number') +DEPLOYMENT_NAME ?= autoscaler-$(PR_NUMBER) +VM_NAME := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq ".Tables | .[0] | .Rows | .[] | .instance" -r | grep metricsforwarder) + +fetch-metricsforwarder-config: + # how to define variables in deployment name + mkdir -p assets/certs/policy_db assets/certs/storedprocedure_db assets/certs/syslog_client + + $(eval POSTGRES_IP := $(shell bosh -d ${DEPLOYMENT_NAME} vms --json | jq -r '.Tables | .[] | .Rows | .[] | select(.instance|test("postgres")) | .ips' )) + echo "Postgres IP: $(POSTGRES_IP)" + + + @echo "Pulling metricforwarder config from $(VM_NAME)..." + bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/metricsforwarder.yml assets/metricsforwarder.yml + + @echo "Pulling policy db certs from $(VM_NAME)..." + bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/ca.crt assets/certs/policy_db/. + bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/crt assets/certs/policy_db/. + bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/key assets/certs/policy_db/. + + @echo "Pulling storeprocedure db certs from $(VM_NAME)..." + bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/ca.crt assets/certs/storedprocedure_db/. + bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/crt assets/certs/storedprocedure_db/. + bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/key assets/certs/storedprocedure_db/. + + @echo "Pulling syslog-client certs from $(VM_NAME)..." + bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/ca.crt assets/certs/syslog_client/. + bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.crt assets/certs/syslog_client/. + bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.key assets/certs/syslog_client/. + + @echo "Build metricsforwarder config yaml" + cp assets/metricsforwarder.yml metricsforwarder.yml + + sed -i '' 's|\/var\/vcap\/jobs\/metricsforwarder\/config|\/home\/vcap\/app/assets|g' metricsforwarder.yml + sed -i '' 's|$(DEPLOYMENT_NAME).autoscalerpostgres.service.cf.internal|$(POSTGRES_IP)|g' metricsforwarder.yml + + diff --git a/src/autoscaler/metricsforwarder/security-group.json b/src/autoscaler/metricsforwarder/security-group.json new file mode 100644 index 0000000000..00725113d8 --- /dev/null +++ b/src/autoscaler/metricsforwarder/security-group.json @@ -0,0 +1,14 @@ + [ + { + "protocol": "tcp", + "destination": "10.0.1.0/24", + "ports": "5432", + "description": "Allow postgress traffic from" + }, + { + "protocol": "tcp", + "destination": "10.0.1.0/24", + "ports": "6067", + "description": "Allow syslog traffic from" + } + ] From 8bb3387a4f4f98ac1cc2a953d0c929c83475b75e Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 22 May 2024 17:01:19 +0200 Subject: [PATCH 059/186] Adds build cf task --- src/autoscaler/Makefile | 4 ++++ src/autoscaler/metricsforwarder/Makefile | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/autoscaler/Makefile b/src/autoscaler/Makefile index abc9e08758..1d5eebc9be 100644 --- a/src/autoscaler/Makefile +++ b/src/autoscaler/Makefile @@ -83,6 +83,10 @@ go-mod-vendor: ${go-vendoring-folder} ${go-vendored-files} ${go-vendoring-folder} ${go-vendored-files} &: ${app-fakes-dir} ${app-fakes-files} go mod vendor +build-cf-%: + @echo "# building for cf $*" + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o $*/$* $*/cmd/$*/main.go + # CGO_ENABLED := 1 is required to enforce dynamic linking which is a requirement of dynatrace. build-%: ${openapi-generated-clients-and-servers-dir} ${openapi-generated-clients-and-servers-files} @echo "# building $*" diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index b52d96ad66..e5eaa2aaf9 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -34,4 +34,5 @@ fetch-metricsforwarder-config: sed -i '' 's|\/var\/vcap\/jobs\/metricsforwarder\/config|\/home\/vcap\/app/assets|g' metricsforwarder.yml sed -i '' 's|$(DEPLOYMENT_NAME).autoscalerpostgres.service.cf.internal|$(POSTGRES_IP)|g' metricsforwarder.yml - +build-cf: + @cd ../; make build-cf-metricsforwarder From a7895cd05ae1a2413797205edc6148f976d4a152 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 22 May 2024 17:01:49 +0200 Subject: [PATCH 060/186] Adds security group task --- src/autoscaler/metricsforwarder/Makefile | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index e5eaa2aaf9..8e446b7bf8 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -2,7 +2,18 @@ PR_NUMBER := $(shell gh pr view --json number --jq '.number') DEPLOYMENT_NAME ?= autoscaler-$(PR_NUMBER) VM_NAME := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq ".Tables | .[0] | .Rows | .[] | .instance" -r | grep metricsforwarder) -fetch-metricsforwarder-config: + + + + +set-security-group: + $(eval ORG := $(shell cf target |grep "org\:" |cut -d':' -f2 | xargs)) + $(eval SPACE := $(shell cf target |grep "space\:" |cut -d':' -f2 | xargs)) + + cf create-security-group metricsforwarder security-group.json + cf bind-security-group metricsforwarder $(ORG) + +fetch-config: # how to define variables in deployment name mkdir -p assets/certs/policy_db assets/certs/storedprocedure_db assets/certs/syslog_client From 17d5a7bad4a36465af65c2b7c8438bafa6019124 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 22 May 2024 17:07:44 +0200 Subject: [PATCH 061/186] Uses logcache ip for cf app connection --- src/autoscaler/metricsforwarder/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index 8e446b7bf8..3c03aa3470 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -20,6 +20,9 @@ fetch-config: $(eval POSTGRES_IP := $(shell bosh -d ${DEPLOYMENT_NAME} vms --json | jq -r '.Tables | .[] | .Rows | .[] | select(.instance|test("postgres")) | .ips' )) echo "Postgres IP: $(POSTGRES_IP)" + $(eval LOG_CACHE_IP := $(shell bosh -d cf vms --json | jq -r '.Tables | .[] | .Rows | .[] | select(.instance|test("log-cache")) | .ips' )) + echo "LOG_CACHE IP: $(LOG_CACHE_IP)" + @echo "Pulling metricforwarder config from $(VM_NAME)..." bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/metricsforwarder.yml assets/metricsforwarder.yml @@ -44,6 +47,7 @@ fetch-config: sed -i '' 's|\/var\/vcap\/jobs\/metricsforwarder\/config|\/home\/vcap\/app/assets|g' metricsforwarder.yml sed -i '' 's|$(DEPLOYMENT_NAME).autoscalerpostgres.service.cf.internal|$(POSTGRES_IP)|g' metricsforwarder.yml + sed -i '' 's|log-cache.service.cf.internal|$(LOG_CACHE_IP)|g' metricsforwarder.yml build-cf: @cd ../; make build-cf-metricsforwarder From 0d02d532c9f13d797e37c1ab82e117ac348d63e8 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 22 May 2024 17:08:13 +0200 Subject: [PATCH 062/186] Disable ssl for postgres --- operations/remove-postgres-tls.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 operations/remove-postgres-tls.yml diff --git a/operations/remove-postgres-tls.yml b/operations/remove-postgres-tls.yml new file mode 100644 index 0000000000..fb94ad9909 --- /dev/null +++ b/operations/remove-postgres-tls.yml @@ -0,0 +1,6 @@ +- type: remove + path: /instance_groups/name=postgres/jobs/name=postgres/properties/databases/tls + +- type: replace + path: /instance_groups/name=postgres/jobs/name=postgres/properties/databases/sslmode + value: disable From b3396f260c3de2622ab58cef760ce11a1aa56980 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 22 May 2024 17:10:23 +0200 Subject: [PATCH 063/186] Removes postgres ssl when deploying autoscaler --- ci/autoscaler/scripts/deploy-autoscaler.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index d8e0351339..09e21c4f4b 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -21,6 +21,7 @@ ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\ ${autoscaler_dir}/operations/remove-metricsgateway.yml\ ${autoscaler_dir}/operations/enable-log-cache-via-uaa.yml\ ${autoscaler_dir}/operations/enable-metricsforwarder-via-syslog-agent.yml\ + ${autoscaler_dir}/operations/remove-postgres-tls.yml\ ${autoscaler_dir}/operations/enable-scheduler-logging.yml"} From 57d229c82765aeb6cd8d253c77c7a1d834bf199e Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 22 May 2024 17:30:44 +0200 Subject: [PATCH 064/186] disable scalingeingine ssl --- operations/remove-postgres-tls.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/operations/remove-postgres-tls.yml b/operations/remove-postgres-tls.yml index fb94ad9909..7df0350610 100644 --- a/operations/remove-postgres-tls.yml +++ b/operations/remove-postgres-tls.yml @@ -4,3 +4,15 @@ - type: replace path: /instance_groups/name=postgres/jobs/name=postgres/properties/databases/sslmode value: disable + +- type: replace + path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/policy_db/sslmode + value: disable + +- type: replace + path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/scalingengine_db/sslmode + value: disable + +- type: replace + path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/scheduler_db/sslmode + value: disable From c552d71c0ce337c0c91f626233acf712c8cf6f0e Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 23 May 2024 12:18:11 +0200 Subject: [PATCH 065/186] Fix disable tls for postgres db --- operations/remove-postgres-tls.yml | 57 ++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/operations/remove-postgres-tls.yml b/operations/remove-postgres-tls.yml index 7df0350610..56a402ae71 100644 --- a/operations/remove-postgres-tls.yml +++ b/operations/remove-postgres-tls.yml @@ -5,6 +5,14 @@ path: /instance_groups/name=postgres/jobs/name=postgres/properties/databases/sslmode value: disable +- type: replace + path: /instance_groups/name=metricsforwarder/jobs/name=metricsforwarder/properties/autoscaler/policy_db/sslmode + value: disable + +- type: replace + path: /instance_groups/name=metricsforwarder/jobs/name=metricsforwarder/properties/autoscaler/storedprocedure_db/sslmode + value: disable + - type: replace path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/policy_db/sslmode value: disable @@ -16,3 +24,52 @@ - type: replace path: /instance_groups/name=scalingengine/jobs/name=scalingengine/properties/autoscaler/scheduler_db/sslmode value: disable + +- type: replace + path: /instance_groups/name=apiserver/jobs/name=golangapiserver/properties/autoscaler/policy_db/sslmode + value: disable + +- type: replace + path: /instance_groups/name=apiserver/jobs/name=golangapiserver/properties/autoscaler/binding_db/sslmode + value: disable + +- type: replace + path: /instance_groups/name=scheduler/jobs/name=scheduler/properties/autoscaler/policy_db/sslmode + value: disable + +- type: replace + path: /instance_groups/name=scheduler/jobs/name=scheduler/properties/autoscaler/scheduler_db/sslmode + value: disable + +- type: replace + path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/appmetrics_db/sslmode + value: disable + +- type: replace + path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/instancemetrics_db/sslmode + value: disable + +- type: replace + path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/lock_db/sslmode + value: disable + +- type: replace + path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/policy_db/sslmode + value: disable + +- type: replace + path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/scalingengine_db/sslmode + value: disable + +- type: replace + path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/policy_db/sslmode + value: disable + +- type: replace + path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/appmetrics_db/sslmode + value: disable + +- type: replace + path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/lock_db/sslmode + value: disable + From 021633c9f9ddc8b94c5cc42162c917f0f8ed7f90 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 30 May 2024 13:55:33 +0200 Subject: [PATCH 066/186] Updates metricsforwarder Makefile Adds task to set security groups Adds cf login task --- src/autoscaler/metricsforwarder/Makefile | 26 +++++++++++++----------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index 3c03aa3470..f101c5cac7 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -2,18 +2,7 @@ PR_NUMBER := $(shell gh pr view --json number --jq '.number') DEPLOYMENT_NAME ?= autoscaler-$(PR_NUMBER) VM_NAME := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq ".Tables | .[0] | .Rows | .[] | .instance" -r | grep metricsforwarder) - - - - -set-security-group: - $(eval ORG := $(shell cf target |grep "org\:" |cut -d':' -f2 | xargs)) - $(eval SPACE := $(shell cf target |grep "space\:" |cut -d':' -f2 | xargs)) - - cf create-security-group metricsforwarder security-group.json - cf bind-security-group metricsforwarder $(ORG) - -fetch-config: +fetch-config: cf-login # how to define variables in deployment name mkdir -p assets/certs/policy_db assets/certs/storedprocedure_db assets/certs/syslog_client @@ -49,5 +38,18 @@ fetch-config: sed -i '' 's|$(DEPLOYMENT_NAME).autoscalerpostgres.service.cf.internal|$(POSTGRES_IP)|g' metricsforwarder.yml sed -i '' 's|log-cache.service.cf.internal|$(LOG_CACHE_IP)|g' metricsforwarder.yml +set-security-group: + $(eval ORG := $(shell cf target |grep "org\:" |cut -d':' -f2 | xargs)) + $(eval SPACE := $(shell cf target |grep "space\:" |cut -d':' -f2 | xargs)) + + cf create-security-group metricsforwarder security-group.json + cf bind-security-group metricsforwarder $(ORG) + build-cf: @cd ../; make build-cf-metricsforwarder + + + +cf-login: + @cd ../../../; make cf-login + From bacda51b54e430ec377fee6ebeb4f9b86f47fc07 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Fri, 24 May 2024 16:59:14 +0200 Subject: [PATCH 067/186] Revert "Remove gitignore" This reverts commit 0cf0175f6a8448cc037fd820089291d93dba2d30. --- src/autoscaler/metricsforwarder/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/autoscaler/metricsforwarder/.gitignore diff --git a/src/autoscaler/metricsforwarder/.gitignore b/src/autoscaler/metricsforwarder/.gitignore new file mode 100644 index 0000000000..1007fd4802 --- /dev/null +++ b/src/autoscaler/metricsforwarder/.gitignore @@ -0,0 +1,4 @@ +certs +metricsforwarder +working.yml +metricsforwarder.yml From fedd1e879f421547b5eac8720b92a71963884b9e Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 11:56:33 +0200 Subject: [PATCH 068/186] Revert devbox changes from rebase --- devbox.json | 79 ++++++++++++++++++++++++++--------------------------- devbox.lock | 72 ++++++++++++++++++++++++------------------------ 2 files changed, 75 insertions(+), 76 deletions(-) diff --git a/devbox.json b/devbox.json index 232a677b98..793988cef6 100644 --- a/devbox.json +++ b/devbox.json @@ -1,44 +1,43 @@ { - "packages": { - "path:local-flake#bosh-bootloader": "", - "path:local-flake#app-autoscaler-cli-plugin": "", - "path:local-flake#log-cache-cli-plugin": "", - "path:local-flake#cloud-mta-build-tool": "", - "path:local-flake#uaac": "", - "credhub-cli": "2.9.29", - "delve": "latest", - "gh": "latest", - "go-tools": "latest", - "gopls": "latest", - "nodejs": "latest", - "nodePackages.yaml-language-server": "latest", - "rubocop": "latest", - "rubyPackages.solargraph": "latest", - "swagger-cli": "latest", - "direnv": "2.34.0", - "gnumake": "4.4", - "maven": "3.8.6", - "pre-commit": "latest", - "bosh-cli": "7.3.1", - "golangci-lint": "1.57.2", - "yq-go": "4.43.1", - "which": "latest", - "jq": "latest", - "fly": "7.10.0", - "python": "latest", - "cloudfoundry-cli": "8.7.10", - "shellcheck": "0.10.0", - "ginkgo": "2.19.0", - "bundix": "latest", - "oha": "latest", - "gum": "0.13.0", - "act": "0.2.63", - "go": "1.21.6", - "actionlint": "1.7.1", - "google-cloud-sdk": "latest", - "temurin-bin-17": "latest", - "ruby": "latest" - }, + "packages": [ + "path:local-flake#bosh-bootloader", + "path:local-flake#app-autoscaler-cli-plugin", + "path:local-flake#log-cache-cli-plugin", + "path:local-flake#uaac", + "delve@latest", + "gh@latest", + "go-tools@latest", + "gopls@latest", + "nodejs@latest", + "nodePackages.yaml-language-server@latest", + "rubocop@latest", + "rubyPackages.solargraph@latest", + "swagger-cli@latest", + "actionlint@1.6.27", + "direnv@2.34.0", + "gnumake@4.4", + "maven@3.8.6", + "pre-commit@latest", + "bosh-cli@7.3.1", + "golangci-lint@1.57.2", + "yq-go@4.43.1", + "which@latest", + "jq@latest", + "fly@7.10.0", + "python@latest", + "cloudfoundry-cli@8.7.10", + "shellcheck@0.10.0", + "act@0.2.62", + "go@1.21.5", + "ruby@3.3.1", + "bundix@latest", + "oha@latest", + "credhub-cli@2.9.29", + "gum@0.13.0", + "google-cloud-sdk@latest", + "ginkgo@latest", + "temurin-bin-17@latest" + ], "shell": { "init_hook": [ "cf install-plugin -f $(which app-autoscaler-cli-plugin)", diff --git a/devbox.lock b/devbox.lock index e2840f1999..d4d0f48ef3 100644 --- a/devbox.lock +++ b/devbox.lock @@ -1,51 +1,51 @@ { "lockfile_version": "1", "packages": { - "act@0.2.63": { - "last_modified": "2024-06-03T07:19:07Z", - "resolved": "github:NixOS/nixpkgs/4a4ecb0ab415c9fccfb005567a215e6a9564cdf5#act", + "act@0.2.62": { + "last_modified": "2024-05-13T18:06:16Z", + "resolved": "github:NixOS/nixpkgs/1daef0114a6074be56586f2cf81898ed142e1e44#act", "source": "devbox-search", - "version": "0.2.63", + "version": "0.2.62", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/fhn4kf6qa88l2pl0n202b4mplyc5pcyf-act-0.2.63", + "path": "/nix/store/75bq6y1b2yzhd0ikpl1dh2qy8n1jl6f6-act-0.2.62", "default": true } ], - "store_path": "/nix/store/fhn4kf6qa88l2pl0n202b4mplyc5pcyf-act-0.2.63" + "store_path": "/nix/store/75bq6y1b2yzhd0ikpl1dh2qy8n1jl6f6-act-0.2.62" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/5pmgbyh36x428ivx8dg2rq3rvaicjb66-act-0.2.63", + "path": "/nix/store/gvv6azi73f613vjpaqf359bba4xqdwlq-act-0.2.62", "default": true } ], - "store_path": "/nix/store/5pmgbyh36x428ivx8dg2rq3rvaicjb66-act-0.2.63" + "store_path": "/nix/store/gvv6azi73f613vjpaqf359bba4xqdwlq-act-0.2.62" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/iwjzs1r05xrdk4xsw6a2c33436s08x18-act-0.2.63", + "path": "/nix/store/wy1cd4gmf0rm2j4mylswmrmnb02ymj6n-act-0.2.62", "default": true } ], - "store_path": "/nix/store/iwjzs1r05xrdk4xsw6a2c33436s08x18-act-0.2.63" + "store_path": "/nix/store/wy1cd4gmf0rm2j4mylswmrmnb02ymj6n-act-0.2.62" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/4d5qhzbrwm9k9yxnwlkwjn8740mh2hlz-act-0.2.63", + "path": "/nix/store/8jrwy52sp5ffzcf1mw7za92jd3vxlmw9-act-0.2.62", "default": true } ], - "store_path": "/nix/store/4d5qhzbrwm9k9yxnwlkwjn8740mh2hlz-act-0.2.63" + "store_path": "/nix/store/8jrwy52sp5ffzcf1mw7za92jd3vxlmw9-act-0.2.62" } } }, @@ -397,51 +397,51 @@ } } }, - "ginkgo@2.19.0": { - "last_modified": "2024-05-29T10:04:41Z", - "resolved": "github:NixOS/nixpkgs/ac82a513e55582291805d6f09d35b6d8b60637a1#ginkgo", + "ginkgo@latest": { + "last_modified": "2024-05-23T08:10:22Z", + "resolved": "github:NixOS/nixpkgs/3305b2b25e4ae4baee872346eae133cf6f611783#ginkgo", "source": "devbox-search", - "version": "2.19.0", + "version": "2.18.0", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/i6ic1hfkrnccaahf87zirrvhaqafkbpj-ginkgo-2.19.0", + "path": "/nix/store/qll95lrxb39cbxlyd1d5rzl8b5p5rpkq-ginkgo-2.18.0", "default": true } ], - "store_path": "/nix/store/i6ic1hfkrnccaahf87zirrvhaqafkbpj-ginkgo-2.19.0" + "store_path": "/nix/store/qll95lrxb39cbxlyd1d5rzl8b5p5rpkq-ginkgo-2.18.0" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/rvds7fvg6drqrbf8wl86yjq78abi40i0-ginkgo-2.19.0", + "path": "/nix/store/sfczfr41wj9vy7iasyjnx983hwl8yvi0-ginkgo-2.18.0", "default": true } ], - "store_path": "/nix/store/rvds7fvg6drqrbf8wl86yjq78abi40i0-ginkgo-2.19.0" + "store_path": "/nix/store/sfczfr41wj9vy7iasyjnx983hwl8yvi0-ginkgo-2.18.0" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/mxywmkg46h6wj5p5811zpdlkrng74nps-ginkgo-2.19.0", + "path": "/nix/store/w6vwpkg3sz63f14dkr609xhqpjj3j5p0-ginkgo-2.18.0", "default": true } ], - "store_path": "/nix/store/mxywmkg46h6wj5p5811zpdlkrng74nps-ginkgo-2.19.0" + "store_path": "/nix/store/w6vwpkg3sz63f14dkr609xhqpjj3j5p0-ginkgo-2.18.0" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/6gjxwi4ygxlr98mn4qy629yicdrfp1vw-ginkgo-2.19.0", + "path": "/nix/store/gbrg7rnqhsjvqi5z2p8pqq6synplyssq-ginkgo-2.18.0", "default": true } ], - "store_path": "/nix/store/6gjxwi4ygxlr98mn4qy629yicdrfp1vw-ginkgo-2.19.0" + "store_path": "/nix/store/gbrg7rnqhsjvqi5z2p8pqq6synplyssq-ginkgo-2.18.0" } } }, @@ -485,51 +485,51 @@ } } }, - "go@1.21.6": { - "last_modified": "2024-02-10T18:15:24Z", - "resolved": "github:NixOS/nixpkgs/10b813040df67c4039086db0f6eaf65c536886c6#go", + "go@1.21.5": { + "last_modified": "2024-01-14T03:55:27Z", + "resolved": "github:NixOS/nixpkgs/dd5621df6dcb90122b50da5ec31c411a0de3e538#go", "source": "devbox-search", - "version": "1.21.6", + "version": "1.21.5", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/44g95npjwgjyrxm1sx54g22w5sfmkz1i-go-1.21.6", + "path": "/nix/store/z6p2j8shdwi74kbm86jwdh03vxq91l0q-go-1.21.5", "default": true } ], - "store_path": "/nix/store/44g95npjwgjyrxm1sx54g22w5sfmkz1i-go-1.21.6" + "store_path": "/nix/store/z6p2j8shdwi74kbm86jwdh03vxq91l0q-go-1.21.5" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/bq5scmlrr463qwx4yfbid9p3w7009r57-go-1.21.6", + "path": "/nix/store/1p00q366rk2j7w9lnflrl7xaks7wfq5d-go-1.21.5", "default": true } ], - "store_path": "/nix/store/bq5scmlrr463qwx4yfbid9p3w7009r57-go-1.21.6" + "store_path": "/nix/store/1p00q366rk2j7w9lnflrl7xaks7wfq5d-go-1.21.5" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/f0mzci3szxjf0cws1a1vlls6d4zbd8s4-go-1.21.6", + "path": "/nix/store/brv7d6mlrclkzywf1vaf35wqhq4c0c82-go-1.21.5", "default": true } ], - "store_path": "/nix/store/f0mzci3szxjf0cws1a1vlls6d4zbd8s4-go-1.21.6" + "store_path": "/nix/store/brv7d6mlrclkzywf1vaf35wqhq4c0c82-go-1.21.5" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/zg65r8ys8y5865lcwmmybrq5gn30n1az-go-1.21.6", + "path": "/nix/store/rccpgf2bnh58hhh4908p78i1rln9gzvp-go-1.21.5", "default": true } ], - "store_path": "/nix/store/zg65r8ys8y5865lcwmmybrq5gn30n1az-go-1.21.6" + "store_path": "/nix/store/rccpgf2bnh58hhh4908p78i1rln9gzvp-go-1.21.5" } } }, From 6f113027e53dd02ed9873574583ae642012267c2 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 30 May 2024 14:03:27 +0200 Subject: [PATCH 069/186] Revert go mod changes from rebase --- src/autoscaler/go.mod | 22 +++++++++---------- src/autoscaler/go.sum | 49 +++++++++++++++++++------------------------ 2 files changed, 33 insertions(+), 38 deletions(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 3f8b2aa0f5..e172a183b2 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,7 +8,7 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240522070055-f1179c4954a5 + code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240529070211-563f04117420 code.cloudfoundry.org/tlsconfig v0.0.0-20240522170710-79df114af82a dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 @@ -21,12 +21,12 @@ require ( github.com/gorilla/mux v1.8.1 github.com/gorilla/websocket v1.5.1 github.com/hashicorp/go-retryablehttp v0.7.5 - github.com/jackc/pgx/v5 v5.5.5 + github.com/jackc/pgx/v5 v5.6.0 github.com/jmoiron/sqlx v1.4.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d - github.com/ogen-go/ogen v1.1.0 - github.com/onsi/ginkgo/v2 v2.18.0 + github.com/ogen-go/ogen v1.1.1 + github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pivotal-cf/brokerapi/v11 v11.0.0 @@ -45,7 +45,7 @@ require ( go.opentelemetry.io/otel/sdk v1.26.0 go.opentelemetry.io/otel/trace v1.26.0 golang.org/x/crypto v0.23.0 - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 + golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 golang.org/x/net v0.25.0 golang.org/x/time v0.5.0 google.golang.org/grpc v1.64.0 @@ -54,7 +54,7 @@ require ( require ( code.cloudfoundry.org/go-diodes v0.0.0-20240515174142-71582f284718 // indirect - code.cloudfoundry.org/go-metric-registry v0.0.0-20240507185439-f0b17e194563 // indirect + code.cloudfoundry.org/go-metric-registry v0.0.0-20240523160243-6c152ef80e25 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -64,7 +64,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect - github.com/fatih/color v1.16.0 // indirect + github.com/fatih/color v1.17.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/ghodss/yaml v1.0.0 // indirect @@ -72,7 +72,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/pprof v0.0.0-20240521024322-9665fa269a30 // indirect + github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -92,7 +92,7 @@ require ( github.com/prometheus/procfs v0.15.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.53.0 // indirect + github.com/valyala/fasthttp v1.54.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect go.uber.org/multierr v1.11.0 // indirect @@ -102,8 +102,8 @@ require ( golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect golang.org/x/tools v0.21.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 66b3e30177..e5b4d54e25 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -602,12 +602,12 @@ code.cloudfoundry.org/go-log-cache/v2 v2.0.7 h1:yR/JjQ/RscO1n4xVAT9HDYcpx5ET/3Cq code.cloudfoundry.org/go-log-cache/v2 v2.0.7/go.mod h1:6KQe2FeeaqRheD5vCvpyTa80YoJojB/r21E54mT97Mc= code.cloudfoundry.org/go-loggregator/v9 v9.2.1 h1:S6Lgg5UJbhh2bt2TGQxs6R00CF8PrUA3GFPYDxy56Fk= code.cloudfoundry.org/go-loggregator/v9 v9.2.1/go.mod h1:FTFFruqGeOhVCDFvyLgl8EV8YW63NNwRzLhxJcporu8= -code.cloudfoundry.org/go-metric-registry v0.0.0-20240507185439-f0b17e194563 h1:mkCsrZ0h4ZF9jcVTEtLiSRJ0Mu2AIQANEWVDcXrOSVY= -code.cloudfoundry.org/go-metric-registry v0.0.0-20240507185439-f0b17e194563/go.mod h1:zYhHvL+6LLy/lGiIe/T5Ma6ae0Tv5Qxm6TNWOvsQVrQ= +code.cloudfoundry.org/go-metric-registry v0.0.0-20240523160243-6c152ef80e25 h1:p1RGKEtKpZJ4CtjLoQudjEpvMLi+l2MZCyheWthmaNg= +code.cloudfoundry.org/go-metric-registry v0.0.0-20240523160243-6c152ef80e25/go.mod h1:C+P03mf5ZSJVpcigPZ3tH5Vb66HIfCTKxatA4llc298= code.cloudfoundry.org/lager/v3 v3.0.3 h1:/UTmadZfIaKuT/whEinSxK1mzRfNu1uPfvjFfGqiwzM= code.cloudfoundry.org/lager/v3 v3.0.3/go.mod h1:Zn5q1SrIuuHjEUE7xerMKt3ztunrJQCZETAo7rV0CH8= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240522070055-f1179c4954a5 h1:5ae8G6wQss4vqKkBQQ8fNQoXEhdV5vEkKStGlPQ68Rc= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240522070055-f1179c4954a5/go.mod h1:2cFY9sx7jyRwEglCfSLMV57cWE0opfIBQhgXHaHIx+0= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240529070211-563f04117420 h1:yAhwG+MCi7WJNLhcRUVdgAtQD7yim2QV95laV7cSg6E= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240529070211-563f04117420/go.mod h1:5YWSzpaF0s4rYqzYJuWnTudX0jCcpgFeBmwcbsh2VBY= code.cloudfoundry.org/tlsconfig v0.0.0-20240522170710-79df114af82a h1:yWneME1ZPxsWkZT1R9e0n6CymukD5pU/ceEYo4ZVMak= code.cloudfoundry.org/tlsconfig v0.0.0-20240522170710-79df114af82a/go.mod h1:TI9y0bK+FJ9OKSLaJmBo5Z2tr+/VULZ2jiQwSg1+Z5c= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= @@ -697,8 +697,8 @@ github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6Ni github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -822,8 +822,8 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20240521024322-9665fa269a30 h1:r6YdmbD41tGHeCWDyHF691LWtL7D1iSTyJaKejTWwVU= -github.com/google/pprof v0.0.0-20240521024322-9665fa269a30/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/pprof v0.0.0-20240528025155-186aa0362fba h1:ql1qNgCyOB7iAEk8JTNM+zJrgIbnyCKX/wdlyPufP5g= +github.com/google/pprof v0.0.0-20240528025155-186aa0362fba/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -874,8 +874,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= -github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= @@ -927,8 +927,8 @@ github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/ogen-go/ogen v1.1.0 h1:hM4osoOPGWvx8pcdqE8jb5VvzHA+wi3u+DwCvZoW4dg= -github.com/ogen-go/ogen v1.1.0/go.mod h1:d5Ph7PWpJz453ZdGbVe+DBVPS150idePr5dZ+S0JG00= +github.com/ogen-go/ogen v1.1.1 h1:ccHc/vgS+WbOLLgM4JYW7EDiBv7DPfbvKCnbo5fctSY= +github.com/ogen-go/ogen v1.1.1/go.mod h1:ZmPa2LwJrwXKPo3s3p/XFMg5znV60IX5950UftdNmb4= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= @@ -946,8 +946,8 @@ github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxm github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM= -github.com/onsi/ginkgo/v2 v2.18.0 h1:W9Y7IWXxPUpAit9ieMOLI7PJZGaW22DTKgiVAuhDTLc= -github.com/onsi/ginkgo/v2 v2.18.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= @@ -1043,8 +1043,8 @@ github.com/uptrace/opentelemetry-go-extra/otelsqlx v0.2.4 h1:Pt/+CUTRusJb471SBXw github.com/uptrace/opentelemetry-go-extra/otelsqlx v0.2.4/go.mod h1:kQgNoghy4K/wguxbOd/u0OJw/Y0maNPc7PF4JpEGeUc= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.53.0 h1:lW/+SUkOxCx2vlIu0iaImv4JLrVRnbbkpCoaawvA4zc= -github.com/valyala/fasthttp v1.53.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM= +github.com/valyala/fasthttp v1.54.0 h1:cCL+ZZR3z3HPLMVfEYVUMtJqVaui0+gu7Lx63unHwS0= +github.com/valyala/fasthttp v1.54.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -1082,13 +1082,8 @@ go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZu go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -<<<<<<< HEAD -go.step.sm/crypto v0.45.0 h1:Z0WYAaaOYrJmKP9sJkPW+6wy3pgN3Ija8ek/D4serjc= -go.step.sm/crypto v0.45.0/go.mod h1:6IYlT0L2jfj81nVyCPpvA5cORy0EVHPhieSgQyuwHIY= -======= go.step.sm/crypto v0.45.1 h1:Xb8XldsbqT6pDYsg46BVPP1euASNbeNAhzrlvUP3QWo= go.step.sm/crypto v0.45.1/go.mod h1:XtJBuMuZb11YeJpG8uP3fyBl2MerXWJ/pWQX/Au+Kt8= ->>>>>>> 9f4a84771 (🤖🦾🛠️ go mod tidy & make package-specs) go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -1122,8 +1117,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 h1:vpzMC/iZhYFAjJzHU0Cfuq+w1vLLsF2vLkDrPjzKYck= +golang.org/x/exp v0.0.0-20240529005216-23cca8864a10/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1689,10 +1684,10 @@ google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= -google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= From d59654f319f374aee3d1abf10aeca5c93883d304 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 30 May 2024 15:16:40 +0200 Subject: [PATCH 070/186] Reads metricforwarder port from environment variable --- .../metricsforwarder/config/config.go | 15 ++++++++++ .../metricsforwarder/config/config_test.go | 28 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/src/autoscaler/metricsforwarder/config/config.go b/src/autoscaler/metricsforwarder/config/config.go index ded7468bb8..374c50511e 100644 --- a/src/autoscaler/metricsforwarder/config/config.go +++ b/src/autoscaler/metricsforwarder/config/config.go @@ -3,6 +3,8 @@ package config import ( "fmt" "io" + "os" + "strconv" "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" @@ -12,6 +14,9 @@ import ( "gopkg.in/yaml.v3" ) +// define an error when environment variable is not a number +var ErrInvalidPort = fmt.Errorf("Invalid port number in PORT environment variable") + const ( DefaultMetronAddress = "127.0.0.1:3458" DefaultCacheTTL = 15 * time.Minute @@ -86,6 +91,16 @@ func LoadConfig(reader io.Reader) (*Config, error) { dec.KnownFields(true) err := dec.Decode(conf) + if os.Getenv("PORT") != "" { + port := os.Getenv("PORT") + portNumber, err := strconv.Atoi(port) + if err != nil { + return nil, ErrInvalidPort + } + conf.Server.Port = portNumber + + } + if err != nil { return nil, err } diff --git a/src/autoscaler/metricsforwarder/config/config_test.go b/src/autoscaler/metricsforwarder/config/config_test.go index e62442df83..fe02004a8b 100644 --- a/src/autoscaler/metricsforwarder/config/config_test.go +++ b/src/autoscaler/metricsforwarder/config/config_test.go @@ -2,6 +2,7 @@ package config_test import ( "bytes" + "os" "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" @@ -116,6 +117,33 @@ health: Expect(conf.CacheCleanupInterval).To(Equal(DefaultCacheCleanupInterval)) Expect(conf.Health.Port).To(Equal(8081)) }) + + When("PORT env variable is set", func() { + + AfterEach(func() { + os.Setenv("PORT", "") + }) + + When("PORT env is a number", func() { + BeforeEach(func() { + os.Setenv("PORT", "3333") + }) + It("prioritize env variable over config file", func() { + Expect(conf.Server.Port).NotTo(Equal(6110)) + Expect(conf.Server.Port).To(Equal(3333)) + }) + }) + + When("PORT env is not number", func() { + BeforeEach(func() { + os.Setenv("PORT", "NAN") + }) + + It("return invalid port error", func() { + Expect(err).To(MatchError(ErrInvalidPort)) + }) + }) + }) }) When("it gives a non integer port", func() { From ce9b4ef8766f03293e087a981e216c18f464e0f3 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Fri, 31 May 2024 15:03:34 +0200 Subject: [PATCH 071/186] Adds logging to test app --- .../assets/app/go_app/internal/app/app.go | 3 ++- .../app/go_app/internal/app/custom_metrics.go | 14 ++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/acceptance/assets/app/go_app/internal/app/app.go b/src/acceptance/assets/app/go_app/internal/app/app.go index 371e08533a..42286a67f4 100644 --- a/src/acceptance/assets/app/go_app/internal/app/app.go +++ b/src/acceptance/assets/app/go_app/internal/app/app.go @@ -16,7 +16,8 @@ import ( "go.uber.org/zap/zapcore" ) -func Router(logger *zap.Logger, timewaster TimeWaster, memoryTest MemoryGobbler, cpuTest CPUWaster, diskOccupier DiskOccupier, customMetricTest CustomMetricClient) *gin.Engine { +func Router(logger *zap.Logger, timewaster TimeWaster, memoryTest MemoryGobbler, + cpuTest CPUWaster, diskOccupier DiskOccupier, customMetricTest CustomMetricClient) *gin.Engine { r := gin.New() otel.SetTracerProvider(sdktrace.NewTracerProvider()) diff --git a/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go b/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go index de48c817e0..c4a05f492c 100644 --- a/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go +++ b/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go @@ -22,35 +22,41 @@ type CustomMetricClient interface { PostCustomMetric(ctx context.Context, appConfig *cfenv.App, metricsValue float64, metricName string, useMtls bool) error } -type CustomMetricAPIClient struct{} +type CustomMetricAPIClient struct { + logger logr.Logger +} var _ CustomMetricClient = &CustomMetricAPIClient{} func CustomMetricsTests(logger logr.Logger, r *gin.RouterGroup, customMetricTest CustomMetricClient) *gin.RouterGroup { - r.GET("/mtls/:name/:value", handleCustomMetricsEndpoint(customMetricTest, true)) - r.GET("/:name/:value", handleCustomMetricsEndpoint(customMetricTest, false)) + r.GET("/mtls/:name/:value", handleCustomMetricsEndpoint(logger, customMetricTest, true)) + r.GET("/:name/:value", handleCustomMetricsEndpoint(logger, customMetricTest, false)) return r } -func handleCustomMetricsEndpoint(customMetricTest CustomMetricClient, useMtls bool) func(c *gin.Context) { +func handleCustomMetricsEndpoint(logger logr.Logger, customMetricTest CustomMetricClient, useMtls bool) func(c *gin.Context) { return func(c *gin.Context) { var ( metricName string metricValue uint64 ) var err error + if metricName = c.Param("name"); metricName == "" { + logger.Error(err, "empty metric name") Error(c, http.StatusBadRequest, "empty metric name") return } if metricValue, err = strconv.ParseUint(c.Param("value"), 10, 64); err != nil { + logger.Error(err, "invalid metric value") Error(c, http.StatusBadRequest, "invalid metric value: %s", err.Error()) return } err = customMetricTest.PostCustomMetric(c, nil, float64(metricValue), metricName, useMtls) if err != nil { + logger.Error(err, "failed to submit custom metric") Error(c, http.StatusInternalServerError, "failed to submit custom metric: %s", err.Error()) return } From 7f6d134d59277e78ab8041a671ae9470e480b9b5 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 11:57:57 +0200 Subject: [PATCH 072/186] Remove unecessary changes on devbox --- devbox.json | 4 ++-- devbox.lock | 46 +++++++++++++++++++++++----------------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/devbox.json b/devbox.json index 793988cef6..963a033274 100644 --- a/devbox.json +++ b/devbox.json @@ -27,15 +27,15 @@ "python@latest", "cloudfoundry-cli@8.7.10", "shellcheck@0.10.0", - "act@0.2.62", "go@1.21.5", "ruby@3.3.1", "bundix@latest", "oha@latest", "credhub-cli@2.9.29", "gum@0.13.0", + "act@latest", "google-cloud-sdk@latest", - "ginkgo@latest", + "ginkgo@2.19.0", "temurin-bin-17@latest" ], "shell": { diff --git a/devbox.lock b/devbox.lock index d4d0f48ef3..6f2399a947 100644 --- a/devbox.lock +++ b/devbox.lock @@ -1,9 +1,9 @@ { "lockfile_version": "1", "packages": { - "act@0.2.62": { - "last_modified": "2024-05-13T18:06:16Z", - "resolved": "github:NixOS/nixpkgs/1daef0114a6074be56586f2cf81898ed142e1e44#act", + "act@latest": { + "last_modified": "2024-05-29T10:04:41Z", + "resolved": "github:NixOS/nixpkgs/ac82a513e55582291805d6f09d35b6d8b60637a1#act", "source": "devbox-search", "version": "0.2.62", "systems": { @@ -11,41 +11,41 @@ "outputs": [ { "name": "out", - "path": "/nix/store/75bq6y1b2yzhd0ikpl1dh2qy8n1jl6f6-act-0.2.62", + "path": "/nix/store/anliky2rlgzcicrhn1yyv9q4nkz7p8wp-act-0.2.62", "default": true } ], - "store_path": "/nix/store/75bq6y1b2yzhd0ikpl1dh2qy8n1jl6f6-act-0.2.62" + "store_path": "/nix/store/anliky2rlgzcicrhn1yyv9q4nkz7p8wp-act-0.2.62" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/gvv6azi73f613vjpaqf359bba4xqdwlq-act-0.2.62", + "path": "/nix/store/jz4lqgv3amm5vg2kd9qxcw43vzrbcnxv-act-0.2.62", "default": true } ], - "store_path": "/nix/store/gvv6azi73f613vjpaqf359bba4xqdwlq-act-0.2.62" + "store_path": "/nix/store/jz4lqgv3amm5vg2kd9qxcw43vzrbcnxv-act-0.2.62" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/wy1cd4gmf0rm2j4mylswmrmnb02ymj6n-act-0.2.62", + "path": "/nix/store/g4zs1xipwgxx1prvi7bpvymibhiq4cm5-act-0.2.62", "default": true } ], - "store_path": "/nix/store/wy1cd4gmf0rm2j4mylswmrmnb02ymj6n-act-0.2.62" + "store_path": "/nix/store/g4zs1xipwgxx1prvi7bpvymibhiq4cm5-act-0.2.62" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/8jrwy52sp5ffzcf1mw7za92jd3vxlmw9-act-0.2.62", + "path": "/nix/store/xl5x5p5axgrqr9h6p2a3idpmcgmj9407-act-0.2.62", "default": true } ], - "store_path": "/nix/store/8jrwy52sp5ffzcf1mw7za92jd3vxlmw9-act-0.2.62" + "store_path": "/nix/store/xl5x5p5axgrqr9h6p2a3idpmcgmj9407-act-0.2.62" } } }, @@ -397,51 +397,51 @@ } } }, - "ginkgo@latest": { - "last_modified": "2024-05-23T08:10:22Z", - "resolved": "github:NixOS/nixpkgs/3305b2b25e4ae4baee872346eae133cf6f611783#ginkgo", + "ginkgo@2.19.0": { + "last_modified": "2024-05-29T10:04:41Z", + "resolved": "github:NixOS/nixpkgs/ac82a513e55582291805d6f09d35b6d8b60637a1#ginkgo", "source": "devbox-search", - "version": "2.18.0", + "version": "2.19.0", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/qll95lrxb39cbxlyd1d5rzl8b5p5rpkq-ginkgo-2.18.0", + "path": "/nix/store/i6ic1hfkrnccaahf87zirrvhaqafkbpj-ginkgo-2.19.0", "default": true } ], - "store_path": "/nix/store/qll95lrxb39cbxlyd1d5rzl8b5p5rpkq-ginkgo-2.18.0" + "store_path": "/nix/store/i6ic1hfkrnccaahf87zirrvhaqafkbpj-ginkgo-2.19.0" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/sfczfr41wj9vy7iasyjnx983hwl8yvi0-ginkgo-2.18.0", + "path": "/nix/store/rvds7fvg6drqrbf8wl86yjq78abi40i0-ginkgo-2.19.0", "default": true } ], - "store_path": "/nix/store/sfczfr41wj9vy7iasyjnx983hwl8yvi0-ginkgo-2.18.0" + "store_path": "/nix/store/rvds7fvg6drqrbf8wl86yjq78abi40i0-ginkgo-2.19.0" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/w6vwpkg3sz63f14dkr609xhqpjj3j5p0-ginkgo-2.18.0", + "path": "/nix/store/mxywmkg46h6wj5p5811zpdlkrng74nps-ginkgo-2.19.0", "default": true } ], - "store_path": "/nix/store/w6vwpkg3sz63f14dkr609xhqpjj3j5p0-ginkgo-2.18.0" + "store_path": "/nix/store/mxywmkg46h6wj5p5811zpdlkrng74nps-ginkgo-2.19.0" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/gbrg7rnqhsjvqi5z2p8pqq6synplyssq-ginkgo-2.18.0", + "path": "/nix/store/6gjxwi4ygxlr98mn4qy629yicdrfp1vw-ginkgo-2.19.0", "default": true } ], - "store_path": "/nix/store/gbrg7rnqhsjvqi5z2p8pqq6synplyssq-ginkgo-2.18.0" + "store_path": "/nix/store/6gjxwi4ygxlr98mn4qy629yicdrfp1vw-ginkgo-2.19.0" } } }, From 90c94229f864498ae3342e9c606fc22233b1c023 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 11:59:23 +0200 Subject: [PATCH 073/186] Remove unecessary changes on go mods --- src/autoscaler/go.mod | 8 ++++---- src/autoscaler/go.sum | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index e172a183b2..f876accb84 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -3,13 +3,13 @@ module code.cloudfoundry.org/app-autoscaler/src/autoscaler go 1.21.5 require ( - code.cloudfoundry.org/cfhttp/v2 v2.0.1 + code.cloudfoundry.org/cfhttp/v2 v2.1.0 code.cloudfoundry.org/clock v1.1.0 code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240529070211-563f04117420 - code.cloudfoundry.org/tlsconfig v0.0.0-20240522170710-79df114af82a + code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240531070121-45846c83d791 + code.cloudfoundry.org/tlsconfig v0.0.0-20240530171334-2593348de0c6 dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/go-chi/chi/v5 v5.0.12 @@ -45,7 +45,7 @@ require ( go.opentelemetry.io/otel/sdk v1.26.0 go.opentelemetry.io/otel/trace v1.26.0 golang.org/x/crypto v0.23.0 - golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 + golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc golang.org/x/net v0.25.0 golang.org/x/time v0.5.0 google.golang.org/grpc v1.64.0 diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index e5b4d54e25..4bca9cf1cd 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -592,8 +592,8 @@ cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoIS cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= -code.cloudfoundry.org/cfhttp/v2 v2.0.1 h1:xCbkvDVUDuFCquDo66QnvM9hG6vxM2cvTMjj/A8BcNA= -code.cloudfoundry.org/cfhttp/v2 v2.0.1/go.mod h1:yD0n22D4KP4xamgTxbz82KsWzK5kcfUrmJNrrmd4YBg= +code.cloudfoundry.org/cfhttp/v2 v2.1.0 h1:HbQ5H2R+HEKG/rcB6Gk3okeC3h2fAC4PPnLQoMHvzZM= +code.cloudfoundry.org/cfhttp/v2 v2.1.0/go.mod h1:k9R36Y/9dUc9OsX4dfDuEjHZ7Q00ttklKQj6HD6h6+U= code.cloudfoundry.org/clock v1.1.0 h1:XLzC6W3Ah/Y7ht1rmZ6+QfPdt1iGWEAAtIZXgiaj57c= code.cloudfoundry.org/clock v1.1.0/go.mod h1:yA3fxddT9RINQL2XHS7PS+OXxKCGhfrZmlNUCIM6AKo= code.cloudfoundry.org/go-diodes v0.0.0-20240515174142-71582f284718 h1:6wazSuHaJjatGy8pvchSy6L+4M67WPfjgK9yh7cJLMs= @@ -606,10 +606,10 @@ code.cloudfoundry.org/go-metric-registry v0.0.0-20240523160243-6c152ef80e25 h1:p code.cloudfoundry.org/go-metric-registry v0.0.0-20240523160243-6c152ef80e25/go.mod h1:C+P03mf5ZSJVpcigPZ3tH5Vb66HIfCTKxatA4llc298= code.cloudfoundry.org/lager/v3 v3.0.3 h1:/UTmadZfIaKuT/whEinSxK1mzRfNu1uPfvjFfGqiwzM= code.cloudfoundry.org/lager/v3 v3.0.3/go.mod h1:Zn5q1SrIuuHjEUE7xerMKt3ztunrJQCZETAo7rV0CH8= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240529070211-563f04117420 h1:yAhwG+MCi7WJNLhcRUVdgAtQD7yim2QV95laV7cSg6E= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240529070211-563f04117420/go.mod h1:5YWSzpaF0s4rYqzYJuWnTudX0jCcpgFeBmwcbsh2VBY= -code.cloudfoundry.org/tlsconfig v0.0.0-20240522170710-79df114af82a h1:yWneME1ZPxsWkZT1R9e0n6CymukD5pU/ceEYo4ZVMak= -code.cloudfoundry.org/tlsconfig v0.0.0-20240522170710-79df114af82a/go.mod h1:TI9y0bK+FJ9OKSLaJmBo5Z2tr+/VULZ2jiQwSg1+Z5c= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240531070121-45846c83d791 h1:yHi0SOEzBu92AIYWD6LiK5Mh5jtDAqKqAJyqQubYMyg= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240531070121-45846c83d791/go.mod h1:Ynx9d3+fKEc1XA6MVWuO99TWdmM1KUEYJHJjO2enHWs= +code.cloudfoundry.org/tlsconfig v0.0.0-20240530171334-2593348de0c6 h1:K5EpewdQg/ln6+GOV4Ef4783ZxJ1cho459mt9EsO58c= +code.cloudfoundry.org/tlsconfig v0.0.0-20240530171334-2593348de0c6/go.mod h1:Tc6/rvNklulglqxjDqqwapYFYpd4K2c92h98o9Lu5ss= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -1082,8 +1082,8 @@ go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZu go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -go.step.sm/crypto v0.45.1 h1:Xb8XldsbqT6pDYsg46BVPP1euASNbeNAhzrlvUP3QWo= -go.step.sm/crypto v0.45.1/go.mod h1:XtJBuMuZb11YeJpG8uP3fyBl2MerXWJ/pWQX/Au+Kt8= +go.step.sm/crypto v0.46.0 h1:cuVZMpDbmEsUX+atC24+VineQr4gO+zO46MxbIVai4Y= +go.step.sm/crypto v0.46.0/go.mod h1:hcr0oTS2vGRTGSZxoVYxE+RRcsd4xP3rpqt3wPwYGqc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -1117,8 +1117,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 h1:vpzMC/iZhYFAjJzHU0Cfuq+w1vLLsF2vLkDrPjzKYck= -golang.org/x/exp v0.0.0-20240529005216-23cca8864a10/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc h1:O9NuF4s+E/PvMIy+9IUZB9znFwUIXEWSstNjek6VpVg= +golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= From 7360c00e3a0dc0dd0b6e5302542dca1a746c0535 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 12:00:16 +0200 Subject: [PATCH 074/186] Remove unused logger --- src/acceptance/assets/app/go_app/internal/app/custom_metrics.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go b/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go index c4a05f492c..dae8838657 100644 --- a/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go +++ b/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go @@ -23,7 +23,6 @@ type CustomMetricClient interface { } type CustomMetricAPIClient struct { - logger logr.Logger } var _ CustomMetricClient = &CustomMetricAPIClient{} From 37daad31d0164d7cee466ce4ad2d777f81f47e9c Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 13:04:34 +0200 Subject: [PATCH 075/186] Adds task to delete-pipeline, updates readme --- ci/Makefile | 10 ++++++++-- ci/README.md | 2 +- ci/scripts/delete-pipeline.sh | 29 +++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) create mode 100755 ci/scripts/delete-pipeline.sh diff --git a/ci/Makefile b/ci/Makefile index 3a5c795c41..e913edd03b 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -1,11 +1,17 @@ .PHONY: set-autoscaler-pipeline - set-autoscaler-pipeline: @./autoscaler/set-pipeline.sh .PHONY: unpause-pipeline - unpause-pipeline: @./scripts/unpause-pipeline.sh + +.PHONY: delete-pipeline +delete-pipeline: + @./scripts/delete-pipeline.sh + +.PHONY: set-target +set-target: + @fly --target app-autoscaler-release login --team-name app-autoscaler --concourse-url https://concourse.app-runtime-interfaces.ci.cloudfoundry.org diff --git a/ci/README.md b/ci/README.md index eb8813cbb0..8cf5e2fd7f 100644 --- a/ci/README.md +++ b/ci/README.md @@ -28,7 +28,7 @@ This directory contains the terragrunt managed stacks of resouces in account app __Setup__ ``` -fly --target app-autoscaler-release login --team-name app-autoscaler --concourse-url https://concourse.app-runtime-interfaces.ci.cloudfoundry.org +make set-target make set-autoscaler-pipeline ``` diff --git a/ci/scripts/delete-pipeline.sh b/ci/scripts/delete-pipeline.sh new file mode 100755 index 0000000000..abf0c76e8c --- /dev/null +++ b/ci/scripts/delete-pipeline.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2086 +# + +target="app-autoscaler-release" + +function delete-pipeline(){ + payload=$(fly -t "$target" pipelines --json) + + pipelines=$(echo "$payload" | jq ".[] |.name" -r | sort) + # ignore shellcheck warning + pipeline=$(gum choose $pipelines ) + + if [ ! -z "$pipeline" ]; then + fly -t "$target" destroy-pipeline -p "$pipeline" + fi +} + +function check-login(){ + if ! fly -t "$target" status; then + echo + echo "fly -t $target login" + echo + exit 1 + fi +} + +check-login +delete-pipeline "${@:-}" From 0b83c4e71fbb6e954342ecabcee1e65b2464357d Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 13:09:34 +0200 Subject: [PATCH 076/186] Deploys metrics forwarder after deploying autoscaler cluster --- ci/autoscaler/pipeline.yml | 98 +++++++++++++++++++--- ci/autoscaler/scripts/deploy-apps.sh | 49 +++++++++++ ci/autoscaler/scripts/deploy-autoscaler.sh | 1 + ci/autoscaler/scripts/vars.source.sh | 5 ++ ci/autoscaler/tasks/deploy-apps.yml | 25 ++++++ operations/use-cf-services.yml | 7 ++ src/autoscaler/metricsforwarder/Makefile | 24 ++++-- 7 files changed, 190 insertions(+), 19 deletions(-) create mode 100644 ci/autoscaler/scripts/deploy-apps.sh create mode 100644 ci/autoscaler/tasks/deploy-apps.yml create mode 100644 operations/use-cf-services.yml diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index d93fc3973b..972a2b2ad1 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -42,6 +42,24 @@ anchors: operations/remove-metricsgateway.yml operations/enable-scheduler-logging.yml + app-autoscaler-ops-files-log-cache-syslog-cf: &app-autoscaler-ops-files-log-cache-syslog-cf + OPS_FILES: | + operations/add-releases.yml + operations/instance-identity-cert-from-cf.yml + operations/add-postgres-variables.yml + operations/enable-nats-tls.yml + operations/loggregator-certs-from-cf.yml + operations/add-extra-plan.yml + operations/set-release-version.yml + operations/configure-log-cache-and-forward-metrics-via-mtls.yml + operations/enable-metricsforwarder-via-syslog-agent.yml + operations/remove-metricsserver.yml + operations/remove-metricsgateway.yml + operations/enable-scheduler-logging.yml + operations/remove-postgres-tls.yml + operations/use-cf-services.yml + + app-autoscaler-ops-files-upgrade: &app-autoscaler-ops-files-upgrade OPS_FILES: | operations/add-releases.yml @@ -60,6 +78,7 @@ groups: - acceptance - acceptance-log-cache-metron - acceptance-log-cache-syslog + - acceptance-log-cache-syslog-cf - cleanup-autoscaler-deployments - fetch-latest-stemcell - draft @@ -197,6 +216,7 @@ jobs: acceptance_deployment_name: ((acceptance_deployment_name)) acceptance_deployment_name_logcache_metron: ((acceptance_deployment_name_logcache_metron)) acceptance_deployment_name_logcache_syslog: ((acceptance_deployment_name_logcache_syslog)) + acceptance_deployment_name_logcache_syslog_cf: ((acceptance_deployment_name_logcache_syslog_cf)) - name: unit-tests public: true @@ -286,7 +306,7 @@ jobs: on_success: task: cleanup file: ci/ci/autoscaler/tasks/cleanup-autoscaler.yml - params: &acceptance-log-cache-params + params: &acceptance-log-cache-metron-params DEPLOYMENT_NAME: ((acceptance_deployment_name_logcache_metron)) plan: - in_parallel: @@ -304,31 +324,31 @@ jobs: - task: deploy-autoscaler file: ci/ci/autoscaler/tasks/deploy-autoscaler.yml params: - <<: *acceptance-log-cache-params + <<: *acceptance-log-cache-metron-params <<: *app-autoscaler-ops-files-log-cache-metron timeout: 30m - task: register-broker file: ci/ci/autoscaler/tasks/register-broker.yml params: - <<: *acceptance-log-cache-params + <<: *acceptance-log-cache-metron-params timeout: 5m - in_parallel: - task: autoscaler-acceptance-api file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml params: - <<: *acceptance-log-cache-params + <<: *acceptance-log-cache-metron-params SUITES: api timeout: 15m - task: autoscaler-acceptance-app file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml params: - <<: *acceptance-log-cache-params + <<: *acceptance-log-cache-metron-params SUITES: app timeout: 45m - task: autoscaler-acceptance-broker file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml params: - <<: *acceptance-log-cache-params + <<: *acceptance-log-cache-metron-params SUITES: broker timeout: 15m @@ -339,7 +359,7 @@ jobs: on_success: task: cleanup file: ci/ci/autoscaler/tasks/cleanup-autoscaler.yml - params: &acceptance-log-cache-params + params: &acceptance-log-cache-syslog-params DEPLOYMENT_NAME: ((acceptance_deployment_name_logcache_syslog)) plan: - in_parallel: @@ -357,34 +377,88 @@ jobs: - task: deploy-autoscaler file: ci/ci/autoscaler/tasks/deploy-autoscaler.yml params: - <<: *acceptance-log-cache-params + <<: *acceptance-log-cache-syslog-params <<: *app-autoscaler-ops-files-log-cache-syslog timeout: 30m - task: register-broker file: ci/ci/autoscaler/tasks/register-broker.yml params: - <<: *acceptance-log-cache-params + <<: *acceptance-log-cache-syslog-params timeout: 5m - in_parallel: - task: autoscaler-acceptance-api file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml params: - <<: *acceptance-log-cache-params + <<: *acceptance-log-cache-syslog-params SUITES: api timeout: 15m - task: autoscaler-acceptance-app file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml params: - <<: *acceptance-log-cache-params + <<: *acceptance-log-cache-syslog-params SUITES: app timeout: 45m - task: autoscaler-acceptance-broker file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml params: - <<: *acceptance-log-cache-params + <<: *acceptance-log-cache-syslog-params SUITES: broker timeout: 15m +- name: acceptance-log-cache-syslog-cf + public: true + build_logs_to_retain: 100 + serial: true + on_success: + task: cleanup + file: ci/ci/autoscaler/tasks/cleanup-autoscaler.yml + params: &acceptance-log-cache-syslog-cf-params + DEPLOYMENT_NAME: ((acceptance_deployment_name_logcache_syslog_cf)) + plan: + - in_parallel: + - get: bbl-state + - get: app-autoscaler-release + passed: [unit-tests, integration-tests] + trigger: true + - get: ci + - task: make-prerelease + file: ci/ci/autoscaler/tasks/make/make.yaml + params: + # ⚠️ Here it is used that make officially guarantees to reach the goals in the provided order. + TARGETS: generate-fakes generate-openapi-generated-clients-and-servers go-mod-tidy go-mod-vendor db scheduler + timeout: 15m + - task: deploy-autoscaler + file: ci/ci/autoscaler/tasks/deploy-autoscaler.yml + params: + <<: *acceptance-log-cache-syslog-cf-params + <<: *app-autoscaler-ops-files-log-cache-syslog-cf + - task: deploy-apps + file: ci/ci/autoscaler/tasks/deploy-apps.yml + timeout: 30m + - task: register-broker + file: ci/ci/autoscaler/tasks/register-broker.yml + params: + <<: *acceptance-log-cache-syslog-cf-params + timeout: 5m + - in_parallel: + - task: autoscaler-acceptance-api + file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml + params: + <<: *acceptance-log-cache-syslog-cf-params + SUITES: api + timeout: 15m + - task: autoscaler-acceptance-app + file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml + params: + <<: *acceptance-log-cache-syslog-cf-params + SUITES: app + timeout: 45m + - task: autoscaler-acceptance-broker + file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml + params: + <<: *acceptance-log-cache-syslog-cf-params + SUITES: broker + timeout: 15m - name: performance public: true build_logs_to_retain: 100 diff --git a/ci/autoscaler/scripts/deploy-apps.sh b/ci/autoscaler/scripts/deploy-apps.sh new file mode 100644 index 0000000000..f6415c3a21 --- /dev/null +++ b/ci/autoscaler/scripts/deploy-apps.sh @@ -0,0 +1,49 @@ +#! /usr/bin/env bash +# shellcheck disable=SC2086,SC2034,SC2155 +set -euo pipefail +script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${script_dir}/vars.source.sh" + + +CURRENT_COMMIT_HASH=$(cd "${autoscaler_dir}"; git log -1 --pretty=format:"%H") +bosh_release_version=${RELEASE_VERSION:-${CURRENT_COMMIT_HASH}-${deployment_name}} + +pushd "${bbl_state_path}" > /dev/null + eval "$(bbl print-env)" +popd > /dev/null + +function setup_autoscaler_uaac(){ + local uaac_authorities="cloud_controller.read,cloud_controller.admin,uaa.resource,routing.routes.write,routing.routes.read,routing.router_groups.read" + local autoscaler_secret="autoscaler_client_secret" + local uaa_client_secret=$(credhub get -n /bosh-autoscaler/cf/uaa_admin_client_secret --quiet) + uaac target "https://uaa.${system_domain}" --skip-ssl-validation > /dev/null + uaac token client get admin -s "${uaa_client_secret}" > /dev/null + + if uaac client get autoscaler_client_id >/dev/null; then + step "updating autoscaler uaac client" + uaac client update "autoscaler_client_id" \ + --authorities "$uaac_authorities" > /dev/null + else + step "creating autoscaler uaac client" + uaac client add "autoscaler_client_id" \ + --authorized_grant_types "client_credentials" \ + --authorities "$uaac_authorities" \ + --secret "$autoscaler_secret" > /dev/null + fi +} + + +function deploy() { + pushd "${autoscaler_dir}/src/autoscaler/metricsforwarder" > /dev/null + log "Deploying metricsforwarder" + make cf-push + make stop-metricsforwarder-vm + popd > /dev/null +} + + +log "Deploying autoscaler apps for bosh deployment '${deployment_name}' " + +pushd "${autoscaler_dir}" > /dev/null + deploy +popd > /dev/null diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index 09e21c4f4b..3e11dc951a 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -99,6 +99,7 @@ function create_manifest(){ -v eventgenerator_uaa_client_id=firehose_exporter \ -v eventgenerator_uaa_client_secret="$(credhub get -n /bosh-autoscaler/cf/uaa_clients_firehose_exporter_secret --quiet)"\ -v eventgenerator_uaa_skip_ssl_validation=true \ + -v metricsforwarder_host="${metricsforwarder_host}" \ -v skip_ssl_validation=true \ > "${tmp_manifest_file}" diff --git a/ci/autoscaler/scripts/vars.source.sh b/ci/autoscaler/scripts/vars.source.sh index 7a9bd62e53..3c1be50d7a 100644 --- a/ci/autoscaler/scripts/vars.source.sh +++ b/ci/autoscaler/scripts/vars.source.sh @@ -60,6 +60,11 @@ debug "SYSTEM_DOMAIN: ${SYSTEM_DOMAIN}" # shellcheck disable=SC2034 system_domain="${SYSTEM_DOMAIN}" +export METRICSFORWARDER_HOST="${METRICSFORWARDER_HOST:-"metricsforwarder.$SYSTEM_DOMAIN"}" +debug "METRICSFORWARDER_HOST: ${METRICSFORWARDER_HOST}" +# shellcheck disable=SC2034 +metricsforwarder_host="${METRICSFORWARDER_HOST}" + BBL_STATE_PATH="${BBL_STATE_PATH:-$( realpath -e "${root_dir}/../app-autoscaler-env-bbl-state/bbl-state" 2> /dev/null || echo "${root_dir}/../bbl-state/bbl-state" )}" BBL_STATE_PATH="$(realpath -e "${BBL_STATE_PATH}" || echo "ERR_invalid_state_path" )" export BBL_STATE_PATH diff --git a/ci/autoscaler/tasks/deploy-apps.yml b/ci/autoscaler/tasks/deploy-apps.yml new file mode 100644 index 0000000000..d656afd8d1 --- /dev/null +++ b/ci/autoscaler/tasks/deploy-apps.yml @@ -0,0 +1,25 @@ +--- +platform: linux + +image_resource: + type: registry-image + source: + repository: ghcr.io/cloudfoundry/app-autoscaler-release-tools + tag: main + + +params: + # Optional: override the user to access bosh + BOSH_USERNAME: admin + + AUTOSCALER_DIR: app-autoscaler-release + +inputs: +- name: app-autoscaler-release +- name: ci +- name: bbl-state + +outputs: + +run: + path: ci/ci/autoscaler/scripts/deploy-apps.sh diff --git a/operations/use-cf-services.yml b/operations/use-cf-services.yml new file mode 100644 index 0000000000..2ea51e42f3 --- /dev/null +++ b/operations/use-cf-services.yml @@ -0,0 +1,7 @@ + + +- type: replace + path: /instance_groups/name=apiserver/jobs/name=golangapiserver/properties/autoscaler/apiserver/metrics_forwarder + value: + host: ((metricsforwarder_host)) + mtls_host: ((metricsforwarder_host)) diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index f101c5cac7..2813a5ae95 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -1,8 +1,15 @@ -PR_NUMBER := $(shell gh pr view --json number --jq '.number') +PR_NUMBER ?= $(shell gh pr view --json number --jq '.number') DEPLOYMENT_NAME ?= autoscaler-$(PR_NUMBER) -VM_NAME := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq ".Tables | .[0] | .Rows | .[] | .instance" -r | grep metricsforwarder) +VM_NAME := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq '.Tables | .[] | .Rows | .[] | select(.instance|test("metricsforwarder")) | .instance') -fetch-config: cf-login +.PHONY: cf-push +cf-push: fetch-config build-cf + cf push metricsforwarder -b binary_buildpack -p . --no-start + # cf set-env metricsforwarder METRICSFORWARDER_CONFIG "$(shell cat metricsforwarder.yml)" + cf start metricsforwarder + +.PHONY: fetch-config +fetch-config: # how to define variables in deployment name mkdir -p assets/certs/policy_db assets/certs/storedprocedure_db assets/certs/syslog_client @@ -36,8 +43,9 @@ fetch-config: cf-login sed -i '' 's|\/var\/vcap\/jobs\/metricsforwarder\/config|\/home\/vcap\/app/assets|g' metricsforwarder.yml sed -i '' 's|$(DEPLOYMENT_NAME).autoscalerpostgres.service.cf.internal|$(POSTGRES_IP)|g' metricsforwarder.yml - sed -i '' 's|log-cache.service.cf.internal|$(LOG_CACHE_IP)|g' metricsforwarder.yml + +PHONY: set-security-group set-security-group: $(eval ORG := $(shell cf target |grep "org\:" |cut -d':' -f2 | xargs)) $(eval SPACE := $(shell cf target |grep "space\:" |cut -d':' -f2 | xargs)) @@ -45,11 +53,13 @@ set-security-group: cf create-security-group metricsforwarder security-group.json cf bind-security-group metricsforwarder $(ORG) +PHONY: build-cf build-cf: @cd ../; make build-cf-metricsforwarder +PHONY: stop-metricsforwarder-vm +stop-metricsforwarder-vm: + $(eval VM_NAME := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq '.Tables | .[] | .Rows | .[] | select(.instance|test("metricsforwarder")) | .instance')) + bosh -d $(DEPLOYMENT_NAME) stop $(VM_NAME) -cf-login: - @cd ../../../; make cf-login - From caca6670915c4469a2658d515c4a3227402cb9c9 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 13:13:20 +0200 Subject: [PATCH 077/186] Adds missing variable for concourse pipeline --- ci/autoscaler/set-pipeline.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/autoscaler/set-pipeline.sh b/ci/autoscaler/set-pipeline.sh index 3142b37252..7a9c7415a0 100755 --- a/ci/autoscaler/set-pipeline.sh +++ b/ci/autoscaler/set-pipeline.sh @@ -33,10 +33,12 @@ function set_pipeline(){ add_var acceptance_deployment_name "acceptance" add_var acceptance_deployment_name_logcache_metron "acceptance-lc" add_var acceptance_deployment_name_logcache_syslog "acceptance-lc-sl" + add_var acceptance_deployment_name_logcache_syslog_cf "acceptance-lc-sl-cf" else add_var acceptance_deployment_name "${PR_NUMBER}-acceptance" add_var acceptance_deployment_name_logcache_metron "${PR_NUMBER}-acceptance-lc" add_var acceptance_deployment_name_logcache_syslog "${PR_NUMBER}-acceptance-lc-sl" + add_var acceptance_deployment_name_logcache_syslog_cf "${PR_NUMBER}-acceptance-lc-sl-cf" fi # shellcheck disable=SC2086 From 01b7951cc9f75b1873245a4a9c98e9f401fa350c Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 13:46:00 +0200 Subject: [PATCH 078/186] Logs to cf org and space before trying to set up apps --- ci/autoscaler/pipeline.yml | 3 ++- ci/autoscaler/scripts/deploy-apps.sh | 33 +++++++++++++++++++++++++--- ci/autoscaler/scripts/vars.source.sh | 7 ++++++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index 972a2b2ad1..d9714f1b1b 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -418,7 +418,7 @@ jobs: - in_parallel: - get: bbl-state - get: app-autoscaler-release - passed: [unit-tests, integration-tests] + # passed: [unit-tests, integration-tests] trigger: true - get: ci - task: make-prerelease @@ -459,6 +459,7 @@ jobs: <<: *acceptance-log-cache-syslog-cf-params SUITES: broker timeout: 15m + - name: performance public: true build_logs_to_retain: 100 diff --git a/ci/autoscaler/scripts/deploy-apps.sh b/ci/autoscaler/scripts/deploy-apps.sh index f6415c3a21..8a33f499a1 100644 --- a/ci/autoscaler/scripts/deploy-apps.sh +++ b/ci/autoscaler/scripts/deploy-apps.sh @@ -5,13 +5,32 @@ script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source "${script_dir}/vars.source.sh" -CURRENT_COMMIT_HASH=$(cd "${autoscaler_dir}"; git log -1 --pretty=format:"%H") -bosh_release_version=${RELEASE_VERSION:-${CURRENT_COMMIT_HASH}-${deployment_name}} - pushd "${bbl_state_path}" > /dev/null eval "$(bbl print-env)" popd > /dev/null +function find_or_create_org(){ + local org_name="$1" + if ! cf orgs | grep -q "${org_name}"; then + cf create-org "${org_name}" + fi + cf target -o "${org_name}" +} + +function find_or_create_space(){ + local space_name="$1" + if ! cf spaces | grep -q "${space_name}"; then + cf create-space "${space_name}" + fi + cf target -s "${space_name}" +} + +function cf_login(){ + cf_admin_password="$(credhub get -n /bosh-autoscaler/cf/cf_admin_password -q)" + cf auth admin "${cf_admin_password}" +} + + function setup_autoscaler_uaac(){ local uaac_authorities="cloud_controller.read,cloud_controller.admin,uaa.resource,routing.routes.write,routing.routes.read,routing.router_groups.read" local autoscaler_secret="autoscaler_client_secret" @@ -32,6 +51,12 @@ function setup_autoscaler_uaac(){ fi } +function cf_target(){ + local org_name="$1" + local space_name="$2" + find_or_create_org "${org_name}" + find_or_create_space "${space_name}" +} function deploy() { pushd "${autoscaler_dir}/src/autoscaler/metricsforwarder" > /dev/null @@ -45,5 +70,7 @@ function deploy() { log "Deploying autoscaler apps for bosh deployment '${deployment_name}' " pushd "${autoscaler_dir}" > /dev/null + cf_login + cf_target "${autoscaler_org}" "${autoscaler_space}" deploy popd > /dev/null diff --git a/ci/autoscaler/scripts/vars.source.sh b/ci/autoscaler/scripts/vars.source.sh index 3c1be50d7a..bf2fdc6493 100644 --- a/ci/autoscaler/scripts/vars.source.sh +++ b/ci/autoscaler/scripts/vars.source.sh @@ -55,6 +55,13 @@ log "set up vars: DEPLOYMENT_NAME=${DEPLOYMENT_NAME}" # shellcheck disable=SC2034 deployment_name="${DEPLOYMENT_NAME}" +export AUTOSCALER_ORG="${DEPLOYMENT_NAME:-"autoscaler-${PR_NUMBER}"}" +[ "${AUTOSCALER_ORG}" = "autoscaler-" ] && DEPLOYMENT_NAME="${user}" +debug "AUTOSCALER_ORG: ${DEPLOYMENT_NAME}" +log "set up vars: AUTOSCALER_ORG=${DEPLOYMENT_NAME}" +# shellcheck disable=SC2034 +autoscaler_org="${AUTOSCALER_ORG}" + export SYSTEM_DOMAIN="${SYSTEM_DOMAIN:-"autoscaler.app-runtime-interfaces.ci.cloudfoundry.org"}" debug "SYSTEM_DOMAIN: ${SYSTEM_DOMAIN}" # shellcheck disable=SC2034 From 5b4d1e58dba6f5e4c1a1301c7711ba71ea0aef99 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 13:59:03 +0200 Subject: [PATCH 079/186] Adds missing permission to deploy app task --- ci/autoscaler/tasks/deploy-apps.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/autoscaler/tasks/deploy-apps.yml diff --git a/ci/autoscaler/tasks/deploy-apps.yml b/ci/autoscaler/tasks/deploy-apps.yml old mode 100644 new mode 100755 From 265022cb742b56b0bdc05d6c65aa36826e514fe5 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 14:26:37 +0200 Subject: [PATCH 080/186] Adds missing AUTOSCALER_SPACE variable definition --- ci/Makefile | 10 ++++++++++ ci/autoscaler/scripts/vars.source.sh | 14 ++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/ci/Makefile b/ci/Makefile index e913edd03b..403b06cc05 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -15,3 +15,13 @@ delete-pipeline: .PHONY: set-target set-target: @fly --target app-autoscaler-release login --team-name app-autoscaler --concourse-url https://concourse.app-runtime-interfaces.ci.cloudfoundry.org + +.phony: lint +lint: + shellcheck -x -s bash autoscaler/**/**/*.sh + +.phony: fix-lint +fix-lint: + shellcheck -x -s bash -f diff autoscaler/**/**/*.sh + + diff --git a/ci/autoscaler/scripts/vars.source.sh b/ci/autoscaler/scripts/vars.source.sh index bf2fdc6493..bffdf35a24 100644 --- a/ci/autoscaler/scripts/vars.source.sh +++ b/ci/autoscaler/scripts/vars.source.sh @@ -55,13 +55,19 @@ log "set up vars: DEPLOYMENT_NAME=${DEPLOYMENT_NAME}" # shellcheck disable=SC2034 deployment_name="${DEPLOYMENT_NAME}" -export AUTOSCALER_ORG="${DEPLOYMENT_NAME:-"autoscaler-${PR_NUMBER}"}" -[ "${AUTOSCALER_ORG}" = "autoscaler-" ] && DEPLOYMENT_NAME="${user}" -debug "AUTOSCALER_ORG: ${DEPLOYMENT_NAME}" -log "set up vars: AUTOSCALER_ORG=${DEPLOYMENT_NAME}" +export AUTOSCALER_ORG="${AUTOSCALER_ORG:-"autoscaler-${PR_NUMBER}"}" +[ "${AUTOSCALER_ORG}" = "autoscaler-" ] && AUTOSCALER_ORG="${user}" +debug "AUTOSCALER_ORG: ${AUTOSCALER_ORG}" +log "set up vars: AUTOSCALER_ORG=${AUTOSCALER_ORG}" # shellcheck disable=SC2034 autoscaler_org="${AUTOSCALER_ORG}" +export AUTOSCALER_SPACE="${AUTOSCALER_SPACE:-"develop"}" +debug "AUTOSCALER_SPACE: ${AUTOSCALER_SPACE}" +log "set up vars: AUTOSCALER_SPACE=${AUTOSCALER_SPACE}" +# shellcheck disable=SC2034 +autoscaler_space="${AUTOSCALER_SPACE}" + export SYSTEM_DOMAIN="${SYSTEM_DOMAIN:-"autoscaler.app-runtime-interfaces.ci.cloudfoundry.org"}" debug "SYSTEM_DOMAIN: ${SYSTEM_DOMAIN}" # shellcheck disable=SC2034 From 9f82246c91aef22ea2106856bc19a19dc1761b81 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 14:32:31 +0200 Subject: [PATCH 081/186] Remove unecessary whiteline --- src/autoscaler/metricsforwarder/config/config.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/autoscaler/metricsforwarder/config/config.go b/src/autoscaler/metricsforwarder/config/config.go index 374c50511e..9809dd08d7 100644 --- a/src/autoscaler/metricsforwarder/config/config.go +++ b/src/autoscaler/metricsforwarder/config/config.go @@ -98,7 +98,6 @@ func LoadConfig(reader io.Reader) (*Config, error) { return nil, ErrInvalidPort } conf.Server.Port = portNumber - } if err != nil { From 55801418bf3bb160b985d9288564cb94d6ef012d Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 14:34:48 +0200 Subject: [PATCH 082/186] Fix docs for metricsforwarder config.go --- src/autoscaler/metricsforwarder/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/metricsforwarder/config/config.go b/src/autoscaler/metricsforwarder/config/config.go index 9809dd08d7..217d43d42e 100644 --- a/src/autoscaler/metricsforwarder/config/config.go +++ b/src/autoscaler/metricsforwarder/config/config.go @@ -14,7 +14,7 @@ import ( "gopkg.in/yaml.v3" ) -// define an error when environment variable is not a number +// ErrInvalidPort is returned when the PORT environment variable is not a valid port number var ErrInvalidPort = fmt.Errorf("Invalid port number in PORT environment variable") const ( From a38743f6b3545e965a6c772a5e2f6120e58524d4 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 14:36:12 +0200 Subject: [PATCH 083/186] Apply shellcheck suggestion on delete pipeline --- ci/scripts/delete-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/delete-pipeline.sh b/ci/scripts/delete-pipeline.sh index abf0c76e8c..4347e50d4e 100755 --- a/ci/scripts/delete-pipeline.sh +++ b/ci/scripts/delete-pipeline.sh @@ -11,7 +11,7 @@ function delete-pipeline(){ # ignore shellcheck warning pipeline=$(gum choose $pipelines ) - if [ ! -z "$pipeline" ]; then + if [ -n "$pipeline" ]; then fly -t "$target" destroy-pipeline -p "$pipeline" fi } From ade5a9a84deb53dab069b5edaad080bb5d0796ba Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 14:38:40 +0200 Subject: [PATCH 084/186] Remove unused newline --- .../assets/app/go_app/internal/app/custom_metrics.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go b/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go index dae8838657..82dfdc079c 100644 --- a/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go +++ b/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go @@ -22,8 +22,7 @@ type CustomMetricClient interface { PostCustomMetric(ctx context.Context, appConfig *cfenv.App, metricsValue float64, metricName string, useMtls bool) error } -type CustomMetricAPIClient struct { -} +type CustomMetricAPIClient struct{} var _ CustomMetricClient = &CustomMetricAPIClient{} From a24fbe441b7f8cabc0367938f2ea506e4a017444 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 14:48:48 +0200 Subject: [PATCH 085/186] Fix lint issues in update java packages shell scripts --- ci/Makefile | 4 ++-- ci/autoscaler/tasks/update-sdk/update_java_package.sh | 5 +++-- ci/autoscaler/tasks/update-sdk/vendor_package.sh | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ci/Makefile b/ci/Makefile index 403b06cc05..abf7810662 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -18,10 +18,10 @@ set-target: .phony: lint lint: - shellcheck -x -s bash autoscaler/**/**/*.sh + shellcheck --external-sources -x -s bash autoscaler/**/**/*.sh .phony: fix-lint fix-lint: - shellcheck -x -s bash -f diff autoscaler/**/**/*.sh + shellcheck --external-sources -x -s bash -f diff autoscaler/**/**/*.sh diff --git a/ci/autoscaler/tasks/update-sdk/update_java_package.sh b/ci/autoscaler/tasks/update-sdk/update_java_package.sh index 0340b9a518..f1b9d4ddf8 100755 --- a/ci/autoscaler/tasks/update-sdk/update_java_package.sh +++ b/ci/autoscaler/tasks/update-sdk/update_java_package.sh @@ -8,10 +8,11 @@ source "${script_dir}/vars.source.sh" source "${script_dir}/vendor_package.sh" java_dir=${JAVA_DIR:-"${autoscaler_dir}/../java-release"} -export java_dir="$(realpath -e "${java_dir}")" +java_dir=$(realpath -e "${java_dir}") +export java_dir # shellcheck disable=SC2154 java_version=$(grep "${java_dir}/packages/openjdk-17/spec" -e "- jdk-" | sed -E 's/- jdk-(.*)\.tar\.gz/\1/g') -echo -n "${java_version}" > ${autoscaler_dir}/version +echo -n "${java_version}" > "${autoscaler_dir}/version" vendor-package "${java_dir}" openjdk-17 "${java_version}" diff --git a/ci/autoscaler/tasks/update-sdk/vendor_package.sh b/ci/autoscaler/tasks/update-sdk/vendor_package.sh index 239d4f5135..0393c5519a 100755 --- a/ci/autoscaler/tasks/update-sdk/vendor_package.sh +++ b/ci/autoscaler/tasks/update-sdk/vendor_package.sh @@ -32,7 +32,7 @@ EOF export UPLOADER_KEY=${UPLOADER_KEY:-$(cat "${HOME}/.ssh/autoscaler_blobstore_uploader.key")} yq eval -i '.blobstore.options.json_key = strenv(UPLOADER_KEY)' "${config_file}" - pushd ${autoscaler_dir} > /dev/null + pushd "${autoscaler_dir}" > /dev/null step "vendoring package ${package}" bosh vendor-package "${package}" "${package_location}" From 09c87d6a75394e20013d21cd996bc3da45c6a475 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 16:03:01 +0200 Subject: [PATCH 086/186] Adds missing permissions to deploy apps script --- ci/autoscaler/scripts/deploy-apps.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/autoscaler/scripts/deploy-apps.sh diff --git a/ci/autoscaler/scripts/deploy-apps.sh b/ci/autoscaler/scripts/deploy-apps.sh old mode 100644 new mode 100755 From 7fda7903bc4cc32e5e472e3dc8f0268713babf40 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Mon, 3 Jun 2024 16:36:17 +0200 Subject: [PATCH 087/186] Performs cf login before trying to deploy apps --- ci/autoscaler/scripts/deploy-apps.sh | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/ci/autoscaler/scripts/deploy-apps.sh b/ci/autoscaler/scripts/deploy-apps.sh index 8a33f499a1..fef5fca6e1 100755 --- a/ci/autoscaler/scripts/deploy-apps.sh +++ b/ci/autoscaler/scripts/deploy-apps.sh @@ -1,10 +1,10 @@ #! /usr/bin/env bash # shellcheck disable=SC2086,SC2034,SC2155 set -euo pipefail +set -x script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source "${script_dir}/vars.source.sh" - pushd "${bbl_state_path}" > /dev/null eval "$(bbl print-env)" popd > /dev/null @@ -26,34 +26,15 @@ function find_or_create_space(){ } function cf_login(){ + cf api "https://api.${system_domain}" --skip-ssl-validation cf_admin_password="$(credhub get -n /bosh-autoscaler/cf/cf_admin_password -q)" cf auth admin "${cf_admin_password}" } - -function setup_autoscaler_uaac(){ - local uaac_authorities="cloud_controller.read,cloud_controller.admin,uaa.resource,routing.routes.write,routing.routes.read,routing.router_groups.read" - local autoscaler_secret="autoscaler_client_secret" - local uaa_client_secret=$(credhub get -n /bosh-autoscaler/cf/uaa_admin_client_secret --quiet) - uaac target "https://uaa.${system_domain}" --skip-ssl-validation > /dev/null - uaac token client get admin -s "${uaa_client_secret}" > /dev/null - - if uaac client get autoscaler_client_id >/dev/null; then - step "updating autoscaler uaac client" - uaac client update "autoscaler_client_id" \ - --authorities "$uaac_authorities" > /dev/null - else - step "creating autoscaler uaac client" - uaac client add "autoscaler_client_id" \ - --authorized_grant_types "client_credentials" \ - --authorities "$uaac_authorities" \ - --secret "$autoscaler_secret" > /dev/null - fi -} - function cf_target(){ local org_name="$1" local space_name="$2" + find_or_create_org "${org_name}" find_or_create_space "${space_name}" } From 519d08ab60419291e30051e9bdcdfa7b09b800d1 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 4 Jun 2024 11:49:22 +0200 Subject: [PATCH 088/186] Uses bonzofenix docker image to test netcat for bosh scp command --- ci/autoscaler/tasks/deploy-apps.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/autoscaler/tasks/deploy-apps.yml b/ci/autoscaler/tasks/deploy-apps.yml index d656afd8d1..858df217e9 100755 --- a/ci/autoscaler/tasks/deploy-apps.yml +++ b/ci/autoscaler/tasks/deploy-apps.yml @@ -4,8 +4,8 @@ platform: linux image_resource: type: registry-image source: - repository: ghcr.io/cloudfoundry/app-autoscaler-release-tools - tag: main + repository: docker.io/bonzofenix/app-autoscaler-release-tools + tag: latest params: From 48b5141933b529782d07d3d1c6e9e2e7befe9ea8 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 4 Jun 2024 12:36:47 +0200 Subject: [PATCH 089/186] Update sed params to make it portable Sed command use in metricsforwarder makefile works both in BSD and GNU. --- src/autoscaler/metricsforwarder/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index 2813a5ae95..fd9ed3e311 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -41,8 +41,8 @@ fetch-config: @echo "Build metricsforwarder config yaml" cp assets/metricsforwarder.yml metricsforwarder.yml - sed -i '' 's|\/var\/vcap\/jobs\/metricsforwarder\/config|\/home\/vcap\/app/assets|g' metricsforwarder.yml - sed -i '' 's|$(DEPLOYMENT_NAME).autoscalerpostgres.service.cf.internal|$(POSTGRES_IP)|g' metricsforwarder.yml + sed -i 's|\/var\/vcap\/jobs\/metricsforwarder\/config|\/home\/vcap\/app/assets|g' metricsforwarder.yml + sed -i 's|$(DEPLOYMENT_NAME).autoscalerpostgres.service.cf.internal|$(POSTGRES_IP)|g' metricsforwarder.yml PHONY: set-security-group From 62f6e24b6eee50621a5a65e7c89bbd24166831b8 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 4 Jun 2024 13:03:05 +0200 Subject: [PATCH 090/186] Sets deployment name when runnning deploy apps task in concourse --- ci/autoscaler/pipeline.yml | 2 ++ src/autoscaler/metricsforwarder/Makefile | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index d9714f1b1b..4bf86bf2dd 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -434,6 +434,8 @@ jobs: <<: *app-autoscaler-ops-files-log-cache-syslog-cf - task: deploy-apps file: ci/ci/autoscaler/tasks/deploy-apps.yml + params: + <<: *acceptance-log-cache-syslog-cf-params timeout: 30m - task: register-broker file: ci/ci/autoscaler/tasks/register-broker.yml diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index fd9ed3e311..00183ebe63 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -41,8 +41,8 @@ fetch-config: @echo "Build metricsforwarder config yaml" cp assets/metricsforwarder.yml metricsforwarder.yml - sed -i 's|\/var\/vcap\/jobs\/metricsforwarder\/config|\/home\/vcap\/app/assets|g' metricsforwarder.yml - sed -i 's|$(DEPLOYMENT_NAME).autoscalerpostgres.service.cf.internal|$(POSTGRES_IP)|g' metricsforwarder.yml + sed -i'' -e 's|\/var\/vcap\/jobs\/metricsforwarder\/config|\/home\/vcap\/app/assets|g' metricsforwarder.yml + sed -i'' -e 's|$(DEPLOYMENT_NAME).autoscalerpostgres.service.cf.internal|$(POSTGRES_IP)|g' metricsforwarder.yml PHONY: set-security-group From 91a430c60cec69b3940ce12c283c0d25ffee2361 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 4 Jun 2024 13:03:37 +0200 Subject: [PATCH 091/186] Remove debug from deploy apps --- ci/autoscaler/scripts/deploy-apps.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/autoscaler/scripts/deploy-apps.sh b/ci/autoscaler/scripts/deploy-apps.sh index fef5fca6e1..beb87141e6 100755 --- a/ci/autoscaler/scripts/deploy-apps.sh +++ b/ci/autoscaler/scripts/deploy-apps.sh @@ -1,7 +1,6 @@ #! /usr/bin/env bash # shellcheck disable=SC2086,SC2034,SC2155 set -euo pipefail -set -x script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source "${script_dir}/vars.source.sh" From a44c3a3805aa4ab3aee017589792c2521bac6492 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 4 Jun 2024 13:04:25 +0200 Subject: [PATCH 092/186] Sets a dynamic metricforwarder_host when running in concourse to support pr work --- ci/autoscaler/scripts/vars.source.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/autoscaler/scripts/vars.source.sh b/ci/autoscaler/scripts/vars.source.sh index bffdf35a24..f6e592146b 100644 --- a/ci/autoscaler/scripts/vars.source.sh +++ b/ci/autoscaler/scripts/vars.source.sh @@ -73,7 +73,8 @@ debug "SYSTEM_DOMAIN: ${SYSTEM_DOMAIN}" # shellcheck disable=SC2034 system_domain="${SYSTEM_DOMAIN}" -export METRICSFORWARDER_HOST="${METRICSFORWARDER_HOST:-"metricsforwarder.$SYSTEM_DOMAIN"}" +# Metricsfowrder follows the same pattern that deployment name but instead of autoscaler is metricsforwarder. +export METRICSFORWARDER_HOST="${DEPLOYMENT_NAME/autoscaler/metricsforwarder}.$SYSTEM_DOMAIN" debug "METRICSFORWARDER_HOST: ${METRICSFORWARDER_HOST}" # shellcheck disable=SC2034 metricsforwarder_host="${METRICSFORWARDER_HOST}" From b1534f203eb7b09c8a7164f34a928e85648b6fb5 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 4 Jun 2024 13:04:53 +0200 Subject: [PATCH 093/186] Adds nc tooling to the concourse tools dockerimage --- ci/dockerfiles/autoscaler-tools/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/dockerfiles/autoscaler-tools/Dockerfile b/ci/dockerfiles/autoscaler-tools/Dockerfile index 6eac2e2f77..ed4ec976a8 100644 --- a/ci/dockerfiles/autoscaler-tools/Dockerfile +++ b/ci/dockerfiles/autoscaler-tools/Dockerfile @@ -36,6 +36,7 @@ RUN apt-get update && \ cf8-cli \ gnupg \ gnupg2 \ + netcat \ gh \ make \ mysql-client && \ From 25f2db5239f65ffb4e419a102fe99893b4209ad7 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 4 Jun 2024 13:05:18 +0200 Subject: [PATCH 094/186] Allows dynamic routes when pushing metricforwarder app --- src/autoscaler/metricsforwarder/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index 00183ebe63..019c214fea 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -1,11 +1,11 @@ PR_NUMBER ?= $(shell gh pr view --json number --jq '.number') DEPLOYMENT_NAME ?= autoscaler-$(PR_NUMBER) VM_NAME := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq '.Tables | .[] | .Rows | .[] | select(.instance|test("metricsforwarder")) | .instance') +METRICSFORWARDER_HOST ?= "metricsforwarder.autoscaler.app-runtime-interfaces.ci.cloudfoundry.org" .PHONY: cf-push cf-push: fetch-config build-cf - cf push metricsforwarder -b binary_buildpack -p . --no-start - # cf set-env metricsforwarder METRICSFORWARDER_CONFIG "$(shell cat metricsforwarder.yml)" + cf push metricsforwarder -b binary_buildpack -p . --no-start --route ${METRICSFORWARDER_HOST} cf start metricsforwarder .PHONY: fetch-config From d7a240530e1ab46baf8e54c76ba6837fd6fa1693 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 4 Jun 2024 13:10:26 +0200 Subject: [PATCH 095/186] Adds missing logging on var metricforwarder host --- ci/autoscaler/scripts/vars.source.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/autoscaler/scripts/vars.source.sh b/ci/autoscaler/scripts/vars.source.sh index f6e592146b..c717366e1a 100644 --- a/ci/autoscaler/scripts/vars.source.sh +++ b/ci/autoscaler/scripts/vars.source.sh @@ -76,9 +76,11 @@ system_domain="${SYSTEM_DOMAIN}" # Metricsfowrder follows the same pattern that deployment name but instead of autoscaler is metricsforwarder. export METRICSFORWARDER_HOST="${DEPLOYMENT_NAME/autoscaler/metricsforwarder}.$SYSTEM_DOMAIN" debug "METRICSFORWARDER_HOST: ${METRICSFORWARDER_HOST}" +log "set up vars: METRICSFORWRDER_HOST=${METRICSFORWRDER_HOST}" # shellcheck disable=SC2034 metricsforwarder_host="${METRICSFORWARDER_HOST}" + BBL_STATE_PATH="${BBL_STATE_PATH:-$( realpath -e "${root_dir}/../app-autoscaler-env-bbl-state/bbl-state" 2> /dev/null || echo "${root_dir}/../bbl-state/bbl-state" )}" BBL_STATE_PATH="$(realpath -e "${BBL_STATE_PATH}" || echo "ERR_invalid_state_path" )" export BBL_STATE_PATH From 1b8e41e95a6a1e3b97589532a7b96f25157e4ea8 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 4 Jun 2024 14:31:40 +0200 Subject: [PATCH 096/186] Binds route before starting metricsforwarder app --- ci/autoscaler/scripts/vars.source.sh | 10 ++++++++-- src/autoscaler/metricsforwarder/Makefile | 10 +++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/ci/autoscaler/scripts/vars.source.sh b/ci/autoscaler/scripts/vars.source.sh index c717366e1a..47cec14e48 100644 --- a/ci/autoscaler/scripts/vars.source.sh +++ b/ci/autoscaler/scripts/vars.source.sh @@ -73,10 +73,16 @@ debug "SYSTEM_DOMAIN: ${SYSTEM_DOMAIN}" # shellcheck disable=SC2034 system_domain="${SYSTEM_DOMAIN}" +# Configure Metrics forwarder variables + +export METRICSFORWARDER_APPNAME="${METRICSFORWARDER_APPNAME:-"metricsforwarder-${DEPLOYMENT_NAME}"}" +debug "METRICSFORWARDER_APPNAME: ${METRICSFORWARDER_APPNAME}" +log "set up vars: METRICSFORWRDER_APPNAME=${METRICSFORWARDER_APPNAME}" + # Metricsfowrder follows the same pattern that deployment name but instead of autoscaler is metricsforwarder. -export METRICSFORWARDER_HOST="${DEPLOYMENT_NAME/autoscaler/metricsforwarder}.$SYSTEM_DOMAIN" +export METRICSFORWARDER_HOST="${METRICSFORWARDER_HOST:-"${METRICSFORWARDER_APPNAME}.${SYSTEM_DOMAIN}"}" debug "METRICSFORWARDER_HOST: ${METRICSFORWARDER_HOST}" -log "set up vars: METRICSFORWRDER_HOST=${METRICSFORWRDER_HOST}" +log "set up vars: METRICSFORWRDER_HOST=${METRICSFORWARDER_HOST}" # shellcheck disable=SC2034 metricsforwarder_host="${METRICSFORWARDER_HOST}" diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index 019c214fea..98cfcd6f61 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -1,12 +1,16 @@ PR_NUMBER ?= $(shell gh pr view --json number --jq '.number') DEPLOYMENT_NAME ?= autoscaler-$(PR_NUMBER) VM_NAME := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq '.Tables | .[] | .Rows | .[] | select(.instance|test("metricsforwarder")) | .instance') -METRICSFORWARDER_HOST ?= "metricsforwarder.autoscaler.app-runtime-interfaces.ci.cloudfoundry.org" +METRICSFORWARDER_APPNAME ?= "metricsforwarder" +METRICSFORWARDER_HOSTNAME ?= $(METRICSFORWARDER_APPNAME) +SYSTEM_DOMAIN ?= "autoscaler.app-runtime-interfaces.ci.cloudfoundry.org" .PHONY: cf-push cf-push: fetch-config build-cf - cf push metricsforwarder -b binary_buildpack -p . --no-start --route ${METRICSFORWARDER_HOST} - cf start metricsforwarder + cf push $(METRICSFORWARDER_APPNAME) -b binary_buildpack -p . --no-start --no-route -c "./metricsforwarder -c metricsforwarder.yml" + cf create-route "$(SYSTEM_DOMAIN)" --hostname "$(METRICSFORWARDER_HOSTNAME)" + cf start "$(METRICSFORWARDER_APPNAME)" + cf bind-route-service "$(SYSTEM_DOMAIN)" --hostname "$(METRICSFORWARDER_APPNAME)" "$(METRICSFORWARDER_HOSTNAME)" .PHONY: fetch-config fetch-config: From 3f014cdda275037fd8df04186d93b40dae63aa70 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 5 Jun 2024 10:57:02 +0200 Subject: [PATCH 097/186] Creates or update security group before pushing metricsforwarder app --- src/autoscaler/metricsforwarder/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index 98cfcd6f61..78065f2bbf 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -6,7 +6,7 @@ METRICSFORWARDER_HOSTNAME ?= $(METRICSFORWARDER_APPNAME) SYSTEM_DOMAIN ?= "autoscaler.app-runtime-interfaces.ci.cloudfoundry.org" .PHONY: cf-push -cf-push: fetch-config build-cf +cf-push: fetch-config build-cf set-security-group cf push $(METRICSFORWARDER_APPNAME) -b binary_buildpack -p . --no-start --no-route -c "./metricsforwarder -c metricsforwarder.yml" cf create-route "$(SYSTEM_DOMAIN)" --hostname "$(METRICSFORWARDER_HOSTNAME)" cf start "$(METRICSFORWARDER_APPNAME)" From ec52383676279f32d6dc404f9416a67b29bfe708 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 5 Jun 2024 11:08:20 +0200 Subject: [PATCH 098/186] Fix issue when mapping route --- src/autoscaler/metricsforwarder/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index 78065f2bbf..e78b3df0ea 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -9,8 +9,8 @@ SYSTEM_DOMAIN ?= "autoscaler.app-runtime-interfaces.ci.cloudfoundry.org" cf-push: fetch-config build-cf set-security-group cf push $(METRICSFORWARDER_APPNAME) -b binary_buildpack -p . --no-start --no-route -c "./metricsforwarder -c metricsforwarder.yml" cf create-route "$(SYSTEM_DOMAIN)" --hostname "$(METRICSFORWARDER_HOSTNAME)" + cf map-route "$(METRICSFORWARDER_APPNAME)" "$(SYSTEM_DOMAIN)" --hostname "$(METRICSFORWARDER_HOSTNAME)" cf start "$(METRICSFORWARDER_APPNAME)" - cf bind-route-service "$(SYSTEM_DOMAIN)" --hostname "$(METRICSFORWARDER_APPNAME)" "$(METRICSFORWARDER_HOSTNAME)" .PHONY: fetch-config fetch-config: From 0c008dc3b96a1c1003dceb6a38e177d882c5b09f Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 5 Jun 2024 11:15:13 +0200 Subject: [PATCH 099/186] Starts and stops metricsforwarder VM when fetching configs --- src/autoscaler/metricsforwarder/Makefile | 41 +++++++++++++----------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index e78b3df0ea..826ec484a4 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -1,19 +1,19 @@ PR_NUMBER ?= $(shell gh pr view --json number --jq '.number') DEPLOYMENT_NAME ?= autoscaler-$(PR_NUMBER) -VM_NAME := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq '.Tables | .[] | .Rows | .[] | select(.instance|test("metricsforwarder")) | .instance') +METIRCSFORWARDER_VM := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq '.Tables | .[] | .Rows | .[] | select(.instance|test("metricsforwarder")) | .instance') METRICSFORWARDER_APPNAME ?= "metricsforwarder" METRICSFORWARDER_HOSTNAME ?= $(METRICSFORWARDER_APPNAME) SYSTEM_DOMAIN ?= "autoscaler.app-runtime-interfaces.ci.cloudfoundry.org" .PHONY: cf-push -cf-push: fetch-config build-cf set-security-group +cf-push: fetch-config stop-metricsforwarder-vm build-cf set-security-group cf push $(METRICSFORWARDER_APPNAME) -b binary_buildpack -p . --no-start --no-route -c "./metricsforwarder -c metricsforwarder.yml" cf create-route "$(SYSTEM_DOMAIN)" --hostname "$(METRICSFORWARDER_HOSTNAME)" cf map-route "$(METRICSFORWARDER_APPNAME)" "$(SYSTEM_DOMAIN)" --hostname "$(METRICSFORWARDER_HOSTNAME)" cf start "$(METRICSFORWARDER_APPNAME)" .PHONY: fetch-config -fetch-config: +fetch-config: start-metricsforwarder-vm # how to define variables in deployment name mkdir -p assets/certs/policy_db assets/certs/storedprocedure_db assets/certs/syslog_client @@ -24,23 +24,23 @@ fetch-config: echo "LOG_CACHE IP: $(LOG_CACHE_IP)" - @echo "Pulling metricforwarder config from $(VM_NAME)..." - bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/metricsforwarder.yml assets/metricsforwarder.yml + @echo "Pulling metricforwarder config from $(METIRCSFORWARDER_VM)..." + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/metricsforwarder.yml assets/metricsforwarder.yml - @echo "Pulling policy db certs from $(VM_NAME)..." - bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/ca.crt assets/certs/policy_db/. - bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/crt assets/certs/policy_db/. - bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/key assets/certs/policy_db/. + @echo "Pulling policy db certs from $(METIRCSFORWARDER_VM)..." + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/ca.crt assets/certs/policy_db/. + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/crt assets/certs/policy_db/. + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/policy_db/key assets/certs/policy_db/. - @echo "Pulling storeprocedure db certs from $(VM_NAME)..." - bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/ca.crt assets/certs/storedprocedure_db/. - bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/crt assets/certs/storedprocedure_db/. - bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/key assets/certs/storedprocedure_db/. + @echo "Pulling storeprocedure db certs from $(METIRCSFORWARDER_VM)..." + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/ca.crt assets/certs/storedprocedure_db/. + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/crt assets/certs/storedprocedure_db/. + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/storedprocedure_db/key assets/certs/storedprocedure_db/. - @echo "Pulling syslog-client certs from $(VM_NAME)..." - bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/ca.crt assets/certs/syslog_client/. - bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.crt assets/certs/syslog_client/. - bosh -d $(DEPLOYMENT_NAME) scp $(VM_NAME):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.key assets/certs/syslog_client/. + @echo "Pulling syslog-client certs from $(METIRCSFORWARDER_VM)..." + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/ca.crt assets/certs/syslog_client/. + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.crt assets/certs/syslog_client/. + bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/certs/syslog_client/client.key assets/certs/syslog_client/. @echo "Build metricsforwarder config yaml" cp assets/metricsforwarder.yml metricsforwarder.yml @@ -61,9 +61,12 @@ PHONY: build-cf build-cf: @cd ../; make build-cf-metricsforwarder +PHONY: start-metricsforwarder-vm +start-metricsforwarder-vm: + bosh -d $(DEPLOYMENT_NAME) -n start $(METIRCSFORWARDER_VM) + PHONY: stop-metricsforwarder-vm stop-metricsforwarder-vm: - $(eval VM_NAME := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq '.Tables | .[] | .Rows | .[] | select(.instance|test("metricsforwarder")) | .instance')) - bosh -d $(DEPLOYMENT_NAME) stop $(VM_NAME) + bosh -d $(DEPLOYMENT_NAME) -n stop $(METIRCSFORWARDER_VM) From b2cfa0fb21f120fb996a16b01f0e6617fdaacb42 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 5 Jun 2024 11:31:02 +0200 Subject: [PATCH 100/186] Refactor --- ci/autoscaler/scripts/deploy-apps.sh | 1 - src/autoscaler/metricsforwarder/Makefile | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/autoscaler/scripts/deploy-apps.sh b/ci/autoscaler/scripts/deploy-apps.sh index beb87141e6..58c2b4d8fb 100755 --- a/ci/autoscaler/scripts/deploy-apps.sh +++ b/ci/autoscaler/scripts/deploy-apps.sh @@ -42,7 +42,6 @@ function deploy() { pushd "${autoscaler_dir}/src/autoscaler/metricsforwarder" > /dev/null log "Deploying metricsforwarder" make cf-push - make stop-metricsforwarder-vm popd > /dev/null } diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index 826ec484a4..1870176ba2 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -6,11 +6,12 @@ METRICSFORWARDER_HOSTNAME ?= $(METRICSFORWARDER_APPNAME) SYSTEM_DOMAIN ?= "autoscaler.app-runtime-interfaces.ci.cloudfoundry.org" .PHONY: cf-push -cf-push: fetch-config stop-metricsforwarder-vm build-cf set-security-group +cf-push: fetch-config build-cf set-security-group cf push $(METRICSFORWARDER_APPNAME) -b binary_buildpack -p . --no-start --no-route -c "./metricsforwarder -c metricsforwarder.yml" cf create-route "$(SYSTEM_DOMAIN)" --hostname "$(METRICSFORWARDER_HOSTNAME)" cf map-route "$(METRICSFORWARDER_APPNAME)" "$(SYSTEM_DOMAIN)" --hostname "$(METRICSFORWARDER_HOSTNAME)" cf start "$(METRICSFORWARDER_APPNAME)" + @$(MAKE) stop-metricsforwarder-vm .PHONY: fetch-config fetch-config: start-metricsforwarder-vm From 9ad984136b5f2eec5780f7274a5b607131437eb6 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 5 Jun 2024 11:43:27 +0200 Subject: [PATCH 101/186] Change metricsforwarder appname format --- ci/autoscaler/scripts/deploy-apps.sh | 2 +- ci/autoscaler/scripts/vars.source.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/autoscaler/scripts/deploy-apps.sh b/ci/autoscaler/scripts/deploy-apps.sh index 58c2b4d8fb..782a482317 100755 --- a/ci/autoscaler/scripts/deploy-apps.sh +++ b/ci/autoscaler/scripts/deploy-apps.sh @@ -40,7 +40,7 @@ function cf_target(){ function deploy() { pushd "${autoscaler_dir}/src/autoscaler/metricsforwarder" > /dev/null - log "Deploying metricsforwarder" + log "Deploying autoscaler apps" make cf-push popd > /dev/null } diff --git a/ci/autoscaler/scripts/vars.source.sh b/ci/autoscaler/scripts/vars.source.sh index 47cec14e48..684bc4de65 100644 --- a/ci/autoscaler/scripts/vars.source.sh +++ b/ci/autoscaler/scripts/vars.source.sh @@ -75,7 +75,7 @@ system_domain="${SYSTEM_DOMAIN}" # Configure Metrics forwarder variables -export METRICSFORWARDER_APPNAME="${METRICSFORWARDER_APPNAME:-"metricsforwarder-${DEPLOYMENT_NAME}"}" +export METRICSFORWARDER_APPNAME="${METRICSFORWARDER_APPNAME:-"${DEPLOYMENT_NAME}-metricsforwarder"}" debug "METRICSFORWARDER_APPNAME: ${METRICSFORWARDER_APPNAME}" log "set up vars: METRICSFORWRDER_APPNAME=${METRICSFORWARDER_APPNAME}" From d678ced33d60793c5b9fc8c96fa125f85b2b7db5 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 5 Jun 2024 11:50:42 +0200 Subject: [PATCH 102/186] Refactor vars --- ci/autoscaler/scripts/vars.source.sh | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/ci/autoscaler/scripts/vars.source.sh b/ci/autoscaler/scripts/vars.source.sh index 684bc4de65..e09d018475 100644 --- a/ci/autoscaler/scripts/vars.source.sh +++ b/ci/autoscaler/scripts/vars.source.sh @@ -3,7 +3,9 @@ # Moved to ci/ *DO NOT MODIFY MANUALLY* # NOTE: to turn on debug use DEBUG=true -# shellcheck disable=SC2155 +# shellcheck disable=SC2155,SC2034 +# + if [ -z "${BASH_SOURCE[0]}" ]; then echo "### Source this from inside a script only! " echo "### =======================================" @@ -52,73 +54,60 @@ export DEPLOYMENT_NAME="${DEPLOYMENT_NAME:-"autoscaler-${PR_NUMBER}"}" [ "${DEPLOYMENT_NAME}" = "autoscaler-" ] && DEPLOYMENT_NAME="${user}" debug "DEPLOYMENT_NAME: ${DEPLOYMENT_NAME}" log "set up vars: DEPLOYMENT_NAME=${DEPLOYMENT_NAME}" -# shellcheck disable=SC2034 deployment_name="${DEPLOYMENT_NAME}" export AUTOSCALER_ORG="${AUTOSCALER_ORG:-"autoscaler-${PR_NUMBER}"}" [ "${AUTOSCALER_ORG}" = "autoscaler-" ] && AUTOSCALER_ORG="${user}" debug "AUTOSCALER_ORG: ${AUTOSCALER_ORG}" log "set up vars: AUTOSCALER_ORG=${AUTOSCALER_ORG}" -# shellcheck disable=SC2034 autoscaler_org="${AUTOSCALER_ORG}" export AUTOSCALER_SPACE="${AUTOSCALER_SPACE:-"develop"}" debug "AUTOSCALER_SPACE: ${AUTOSCALER_SPACE}" log "set up vars: AUTOSCALER_SPACE=${AUTOSCALER_SPACE}" -# shellcheck disable=SC2034 autoscaler_space="${AUTOSCALER_SPACE}" export SYSTEM_DOMAIN="${SYSTEM_DOMAIN:-"autoscaler.app-runtime-interfaces.ci.cloudfoundry.org"}" debug "SYSTEM_DOMAIN: ${SYSTEM_DOMAIN}" -# shellcheck disable=SC2034 system_domain="${SYSTEM_DOMAIN}" -# Configure Metrics forwarder variables - +# Configure cloudfoundry app variables export METRICSFORWARDER_APPNAME="${METRICSFORWARDER_APPNAME:-"${DEPLOYMENT_NAME}-metricsforwarder"}" debug "METRICSFORWARDER_APPNAME: ${METRICSFORWARDER_APPNAME}" log "set up vars: METRICSFORWRDER_APPNAME=${METRICSFORWARDER_APPNAME}" +metricsforwarder_appname="${METRICSFORWARDER_APPNAME}" -# Metricsfowrder follows the same pattern that deployment name but instead of autoscaler is metricsforwarder. export METRICSFORWARDER_HOST="${METRICSFORWARDER_HOST:-"${METRICSFORWARDER_APPNAME}.${SYSTEM_DOMAIN}"}" debug "METRICSFORWARDER_HOST: ${METRICSFORWARDER_HOST}" log "set up vars: METRICSFORWRDER_HOST=${METRICSFORWARDER_HOST}" -# shellcheck disable=SC2034 metricsforwarder_host="${METRICSFORWARDER_HOST}" - BBL_STATE_PATH="${BBL_STATE_PATH:-$( realpath -e "${root_dir}/../app-autoscaler-env-bbl-state/bbl-state" 2> /dev/null || echo "${root_dir}/../bbl-state/bbl-state" )}" BBL_STATE_PATH="$(realpath -e "${BBL_STATE_PATH}" || echo "ERR_invalid_state_path" )" export BBL_STATE_PATH debug "BBL_STATE_PATH: ${BBL_STATE_PATH}" -# shellcheck disable=SC2034 bbl_state_path="${BBL_STATE_PATH}" AUTOSCALER_DIR="${AUTOSCALER_DIR:-${root_dir}}" export AUTOSCALER_DIR="$(realpath -e "${AUTOSCALER_DIR}" )" debug "AUTOSCALER_DIR: ${AUTOSCALER_DIR}" -# shellcheck disable=SC2034 autoscaler_dir="${AUTOSCALER_DIR}" CI_DIR="${CI_DIR:-$(realpath -e "${root_dir}/ci")}" export CI_DIR="$(realpath -e "${CI_DIR}")" debug "CI_DIR: ${CI_DIR}" -# shellcheck disable=SC2034 ci_dir="${CI_DIR}" export SERVICE_NAME="${DEPLOYMENT_NAME}" debug "SERVICE_NAME: ${SERVICE_NAME}" -# shellcheck disable=SC2034 service_name="%{SERVICE_NAME" export SERVICE_BROKER_NAME="${DEPLOYMENT_NAME}servicebroker" debug "SERVICE_BROKER_NAME: ${SERVICE_BROKER_NAME}" -# shellcheck disable=SC2034 service_broker_name="${SERVICE_BROKER_NAME}" export NAME_PREFIX="${NAME_PREFIX:-"${DEPLOYMENT_NAME}-TESTS"}" debug "NAME_PREFIX: ${NAME_PREFIX}" -# shellcheck disable=SC2034 name_prefix="${NAME_PREFIX}" export GINKGO_OPTS=${GINKGO_OPTS:-"--fail-fast"} @@ -137,5 +126,4 @@ debug "PERFORMANCE_TEARDOWN: ${PERFORMANCE_TEARDOWN}" export CPU_UPPER_THRESHOLD=${CPU_UPPER_THRESHOLD:-100} debug "CPU_UPPER_THRESHOLD: ${CPU_UPPER_THRESHOLD}" -# shellcheck disable=SC2034 cpu_upper_threshold=${CPU_UPPER_THRESHOLD} From 86b497708d9143cc0535717e2336f5f068fa84c4 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 5 Jun 2024 13:04:52 +0200 Subject: [PATCH 103/186] Edit gitignore for metricsforwarder --- src/autoscaler/metricsforwarder/.gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/autoscaler/metricsforwarder/.gitignore b/src/autoscaler/metricsforwarder/.gitignore index 1007fd4802..7995a170f2 100644 --- a/src/autoscaler/metricsforwarder/.gitignore +++ b/src/autoscaler/metricsforwarder/.gitignore @@ -1,4 +1,3 @@ -certs +assets metricsforwarder -working.yml metricsforwarder.yml From 5fdab29e4687f9e7696d5a968949edaaaade5e94 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 5 Jun 2024 13:11:53 +0200 Subject: [PATCH 104/186] Revert go mod changes --- src/autoscaler/go.mod | 19 +++++++++-------- src/autoscaler/go.sum | 47 ++++++++++++++++++------------------------- 2 files changed, 29 insertions(+), 37 deletions(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index f876accb84..56f13c5f21 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,8 +8,8 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240531070121-45846c83d791 - code.cloudfoundry.org/tlsconfig v0.0.0-20240530171334-2593348de0c6 + code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240604192159-0c056d8148bf + code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65 dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/go-chi/chi/v5 v5.0.12 @@ -20,7 +20,7 @@ require ( github.com/golang/protobuf v1.5.4 github.com/gorilla/mux v1.8.1 github.com/gorilla/websocket v1.5.1 - github.com/hashicorp/go-retryablehttp v0.7.5 + github.com/hashicorp/go-retryablehttp v0.7.7 github.com/jackc/pgx/v5 v5.6.0 github.com/jmoiron/sqlx v1.4.0 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 @@ -44,7 +44,7 @@ require ( go.opentelemetry.io/otel/metric v1.26.0 go.opentelemetry.io/otel/sdk v1.26.0 go.opentelemetry.io/otel/trace v1.26.0 - golang.org/x/crypto v0.23.0 + golang.org/x/crypto v0.24.0 golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc golang.org/x/net v0.25.0 golang.org/x/time v0.5.0 @@ -76,7 +76,6 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-hclog v1.2.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect @@ -88,8 +87,8 @@ require ( github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.53.0 // indirect - github.com/prometheus/procfs v0.15.0 // indirect + github.com/prometheus/common v0.54.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.54.0 // indirect @@ -99,9 +98,9 @@ require ( go.uber.org/zap v1.27.0 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - golang.org/x/tools v0.21.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect google.golang.org/protobuf v1.34.1 // indirect diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 4bca9cf1cd..707272e824 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -606,10 +606,10 @@ code.cloudfoundry.org/go-metric-registry v0.0.0-20240523160243-6c152ef80e25 h1:p code.cloudfoundry.org/go-metric-registry v0.0.0-20240523160243-6c152ef80e25/go.mod h1:C+P03mf5ZSJVpcigPZ3tH5Vb66HIfCTKxatA4llc298= code.cloudfoundry.org/lager/v3 v3.0.3 h1:/UTmadZfIaKuT/whEinSxK1mzRfNu1uPfvjFfGqiwzM= code.cloudfoundry.org/lager/v3 v3.0.3/go.mod h1:Zn5q1SrIuuHjEUE7xerMKt3ztunrJQCZETAo7rV0CH8= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240531070121-45846c83d791 h1:yHi0SOEzBu92AIYWD6LiK5Mh5jtDAqKqAJyqQubYMyg= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240531070121-45846c83d791/go.mod h1:Ynx9d3+fKEc1XA6MVWuO99TWdmM1KUEYJHJjO2enHWs= -code.cloudfoundry.org/tlsconfig v0.0.0-20240530171334-2593348de0c6 h1:K5EpewdQg/ln6+GOV4Ef4783ZxJ1cho459mt9EsO58c= -code.cloudfoundry.org/tlsconfig v0.0.0-20240530171334-2593348de0c6/go.mod h1:Tc6/rvNklulglqxjDqqwapYFYpd4K2c92h98o9Lu5ss= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240604192159-0c056d8148bf h1:EmmABS4FL7hQNwPpyO4AEZ43EFFA8vOOq7njrt8qyHU= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240604192159-0c056d8148bf/go.mod h1:3g7KeOBzN+XZBvuRVkX41LBMRTG+HNgtA6DKJjFdrC0= +code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65 h1:0wXfAs/32wbpMKDGiV3XEHev1l9dLgIPjda+WRwJhCc= +code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65/go.mod h1:R4odSd3iXVYlIlesDHaQi9IZx2gXQryfvJmJSjPS/ug= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -696,7 +696,6 @@ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6Ni github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -859,11 +858,10 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= -github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= -github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -906,11 +904,8 @@ github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -993,10 +988,10 @@ github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= -github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= -github.com/prometheus/procfs v0.15.0 h1:A82kmvXJq2jTu5YUhSGNlYoxh85zLnKgPz4bMZgI5Ek= -github.com/prometheus/procfs v0.15.0/go.mod h1:Y0RJ/Y5g5wJpkTisOtqwDSo4HwhGmLB4VQSw2sQJLHk= +github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8= +github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1100,8 +1095,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1281,7 +1276,6 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1292,7 +1286,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1366,8 +1359,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1394,8 +1387,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1470,8 +1463,8 @@ golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= -golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 830df3cb69b9d06d39bb03afcb1c0c9927f93796 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 6 Jun 2024 15:53:07 +0200 Subject: [PATCH 105/186] Healthendpoint runs on same port than metricsforwarder --- Makefile | 11 +- src/autoscaler/Makefile | 4 + .../cmd/metricsforwarder/main.go | 23 ---- .../metricsforwarder/metricsforwarder_test.go | 119 ------------------ .../metricsforwarder/server/server.go | 52 ++++++-- .../server/server_suite_test.go | 13 +- .../metricsforwarder/server/server_test.go | 68 +++++++++- 7 files changed, 127 insertions(+), 163 deletions(-) diff --git a/Makefile b/Makefile index aefd25358e..32a0c4492c 100644 --- a/Makefile +++ b/Makefile @@ -127,11 +127,10 @@ target/scheduler_test_certs: .PHONY: test test-autoscaler test-scheduler test-changelog test-changeloglockcleaner test: test-autoscaler test-scheduler test-changelog test-changeloglockcleaner test-acceptance-unit test-autoscaler: check-db_type init-db test-certs - @echo " - using DBURL=${DBURL} OPTS=${OPTS}" - @make --directory='./src/autoscaler' test DBURL="${DBURL}" OPTS="${OPTS}" + @echo " - using DBURL=${DBURL}" + @make --directory='./src/autoscaler' test DBURL="${DBURL}" test-autoscaler-suite: check-db_type init-db test-certs - @echo " - using DBURL=${DBURL} TEST=${TEST} OPTS=${OPTS}" - @make --directory='./src/autoscaler' testsuite TEST=${TEST} DBURL="${DBURL}" OPTS="${OPTS}" + @make --directory='./src/autoscaler' testsuite TEST=${TEST} DBURL="${DBURL}" test-scheduler: check-db_type init-db test-certs @export DB_HOST=${DB_HOST}; \ cd src && mvn test --no-transfer-progress -Dspring.profiles.include=${db_type} && cd .. @@ -221,8 +220,8 @@ stop-db: check-db_type .PHONY: integration integration: build init-db test-certs - @echo " - using DBURL=${DBURL} OPTS=${OPTS}" - make --directory='./src/autoscaler' integration DBURL="${DBURL}" OPTS="${OPTS}" + @echo " - using DBURL=${DBURL}" + make --directory='./src/autoscaler' integration DBURL="${DBURL}" .PHONY:lint $(addprefix lint_,$(go_modules)) diff --git a/src/autoscaler/Makefile b/src/autoscaler/Makefile index 1d5eebc9be..f5dc764c0a 100644 --- a/src/autoscaler/Makefile +++ b/src/autoscaler/Makefile @@ -9,6 +9,9 @@ GO_DEPENDENCIES = $(shell find . -type f -name '*.go') PACKAGE_DIRS = $(shell go list './...' | grep --invert-match --regexp='/vendor/' \ | grep --invert-match --regexp='e2e') +DB_HOST ?= localhost +DBURL ?= "postgres://postgres:postgres@${DB_HOST}/autoscaler?sslmode=disable" + export GOWORK=off BUILDFLAGS := -ldflags '-linkmode=external' @@ -116,6 +119,7 @@ test: generate-fakes APP_AUTOSCALER_TEST_RUN='true' go run 'github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}' -p ${GINKGO_OPTS} --skip-package='integration' testsuite: generate-fakes + @echo " - using DBURL=${DBURL} TEST=${TEST}" APP_AUTOSCALER_TEST_RUN='true' go run 'github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}' -p ${GINKGO_OPTS} ${TEST} .PHONY: integration diff --git a/src/autoscaler/metricsforwarder/cmd/metricsforwarder/main.go b/src/autoscaler/metricsforwarder/cmd/metricsforwarder/main.go index a44263ed9a..511fbe4ede 100644 --- a/src/autoscaler/metricsforwarder/cmd/metricsforwarder/main.go +++ b/src/autoscaler/metricsforwarder/cmd/metricsforwarder/main.go @@ -4,7 +4,6 @@ import ( "flag" "fmt" "os" - "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cred_helper" @@ -19,7 +18,6 @@ import ( "code.cloudfoundry.org/clock" "code.cloudfoundry.org/lager/v3" "github.com/patrickmn/go-cache" - "github.com/prometheus/client_golang/prometheus" "github.com/tedsuo/ifrit" "github.com/tedsuo/ifrit/grouper" "github.com/tedsuo/ifrit/sigmon" @@ -71,12 +69,10 @@ func main() { allowedMetricCache := cache.New(conf.CacheTTL, conf.CacheCleanupInterval) customMetricsServer := createCustomMetricsServer(conf, logger, policyDb, credentialProvider, allowedMetricCache, httpStatusCollector) cacheUpdater := cacheUpdater(logger, mfClock, conf, policyDb, allowedMetricCache) - healthServer := createHealthServer(policyDb, credentialProvider, logger, conf, createPrometheusRegistry(policyDb, httpStatusCollector, logger)) members := grouper.Members{ {"cacheUpdater", cacheUpdater}, {"custom_metrics_server", customMetricsServer}, - {"health_server", healthServer}, } monitor := ifrit.Invoke(sigmon.New(grouper.NewOrdered(os.Interrupt, members))) @@ -91,15 +87,6 @@ func main() { logger.Info("exited") } -func createPrometheusRegistry(policyDB *sqldb.PolicySQLDB, httpStatusCollector healthendpoint.HTTPStatusCollector, logger lager.Logger) *prometheus.Registry { - promRegistry := prometheus.NewRegistry() - healthendpoint.RegisterCollectors(promRegistry, []prometheus.Collector{ - healthendpoint.NewDatabaseStatusCollector("autoscaler", "metricsforwarder", "policyDB", policyDB), - httpStatusCollector, - }, true, logger.Session("metricsforwarder-prometheus")) - return promRegistry -} - func cacheUpdater(logger lager.Logger, mfClock clock.Clock, conf *config.Config, policyDB *sqldb.PolicySQLDB, allowedMetricCache *cache.Cache) ifrit.RunFunc { policyManager := manager.NewPolicyManager(logger, mfClock, conf.PolicyPollerInterval, policyDB, *allowedMetricCache, conf.CacheTTL) cacheUpdater := ifrit.RunFunc(func(signals <-chan os.Signal, ready chan<- struct{}) error { @@ -121,13 +108,3 @@ func createCustomMetricsServer(conf *config.Config, logger lager.Logger, policyD } return httpServer } - -func createHealthServer(policyDB *sqldb.PolicySQLDB, credDb cred_helper.Credentials, logger lager.Logger, conf *config.Config, promRegistry *prometheus.Registry) ifrit.Runner { - checkers := []healthendpoint.Checker{healthendpoint.DbChecker(db.PolicyDb, policyDB), healthendpoint.DbChecker(db.StoredProcedureDb, credDb)} - healthServer, err := healthendpoint.NewServerWithBasicAuth(conf.Health, checkers, logger.Session("health-server"), promRegistry, time.Now) - if err != nil { - logger.Fatal("Failed to create health server", err) - os.Exit(1) - } - return healthServer -} diff --git a/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_test.go b/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_test.go index c9da9aaf03..ee9a596f2c 100644 --- a/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_test.go +++ b/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_test.go @@ -5,7 +5,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io" "net/http" "os" @@ -134,123 +133,5 @@ var _ = Describe("Metricsforwarder", func() { }) }) - - }) - - Describe("when Health server is ready to serve RESTful API", func() { - BeforeEach(func() { - - basicAuthConfig := cfg - basicAuthConfig.Health.HealthCheckUsername = "" - basicAuthConfig.Health.HealthCheckPassword = "" - runner.configPath = writeConfig(&basicAuthConfig).Name() - - runner.Start() - - }) - Context("when a request to query health comes", func() { - It("returns with a 200", func() { - rsp, err := healthHttpClient.Get(fmt.Sprintf("http://127.0.0.1:%d", healthport)) - Expect(err).NotTo(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) - raw, _ := io.ReadAll(rsp.Body) - healthData := string(raw) - Expect(healthData).To(ContainSubstring("autoscaler_metricsforwarder_concurrent_http_request")) - Expect(healthData).To(ContainSubstring("autoscaler_metricsforwarder_policyDB")) - Expect(healthData).To(ContainSubstring("go_goroutines")) - Expect(healthData).To(ContainSubstring("go_memstats_alloc_bytes")) - rsp.Body.Close() - - }) - }) - AfterEach(func() { - runner.Interrupt() - }) - }) - - Describe("when Health server is ready to serve RESTful API with basic Auth", func() { - BeforeEach(func() { - runner.Start() - }) - - Context("when username and password are incorrect for basic authentication during health check", func() { - It("should return 401", func() { - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) - - req.SetBasicAuth("wrongusername", "wrongpassword") - - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusUnauthorized)) - }) - }) - - Context("when username and password are correct for basic authentication during health check", func() { - It("should return 200 for /health", func() { - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) - - req.SetBasicAuth(cfg.Health.HealthCheckUsername, cfg.Health.HealthCheckPassword) - - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) - }) - It("should return 200 for /health/readiness", func() { - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health/readiness", healthport), nil) - Expect(err).NotTo(HaveOccurred()) - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) - body, err := io.ReadAll(rsp.Body) - Expect(err).ToNot(HaveOccurred()) - Expect(body).To(MatchJSON(`{"overall_status": "UP","checks": [ { "name":"policy_db", "type":"database", "status":"UP"}, { "name":"storedprocedure_db", "type":"database", "status":"UP"}]}`)) - }) - }) - - AfterEach(func() { - runner.Interrupt() - }) - }) - - Describe("when Health server is ready to serve RESTful API with basic Auth", func() { - BeforeEach(func() { - runner.Start() - }) - - Context("when username and password are incorrect for basic authentication during health check", func() { - It("should return 401", func() { - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) - - req.SetBasicAuth("wrongusername", "wrongpassword") - - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusUnauthorized)) - }) - }) - - Context("when username and password are correct for basic authentication during health check", func() { - It("should return 200", func() { - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", healthport), nil) - Expect(err).NotTo(HaveOccurred()) - - req.SetBasicAuth(cfg.Health.HealthCheckUsername, cfg.Health.HealthCheckPassword) - - rsp, err := healthHttpClient.Do(req) - Expect(err).ToNot(HaveOccurred()) - Expect(rsp.StatusCode).To(Equal(http.StatusOK)) - }) - }) - - AfterEach(func() { - runner.Interrupt() - }) }) }) diff --git a/src/autoscaler/metricsforwarder/server/server.go b/src/autoscaler/metricsforwarder/server/server.go index b27e1c251d..f83808e509 100644 --- a/src/autoscaler/metricsforwarder/server/server.go +++ b/src/autoscaler/metricsforwarder/server/server.go @@ -3,6 +3,7 @@ package server import ( "fmt" "os" + "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cred_helper" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db" @@ -17,31 +18,64 @@ import ( "go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux" "code.cloudfoundry.org/lager/v3" + "github.com/gorilla/mux" "github.com/patrickmn/go-cache" + "github.com/prometheus/client_golang/prometheus" "github.com/tedsuo/ifrit" ) -func NewServer(logger lager.Logger, conf *config.Config, policyDB db.PolicyDB, credentials cred_helper.Credentials, allowedMetricCache cache.Cache, httpStatusCollector healthendpoint.HTTPStatusCollector, rateLimiter ratelimiter.Limiter) (ifrit.Runner, error) { +func NewServer(logger lager.Logger, conf *config.Config, policyDb db.PolicyDB, credentials cred_helper.Credentials, allowedMetricCache cache.Cache, httpStatusCollector healthendpoint.HTTPStatusCollector, rateLimiter ratelimiter.Limiter) (ifrit.Runner, error) { metricForwarder, err := forwarder.NewMetricForwarder(logger, conf) if err != nil { logger.Error("failed-to-create-metricforwarder-server", err) os.Exit(1) } - mh := NewCustomMetricsHandler(logger, metricForwarder, policyDB, allowedMetricCache) + mh := NewCustomMetricsHandler(logger, metricForwarder, policyDb, allowedMetricCache) authenticator, err := auth.New(logger, credentials) if err != nil { return nil, fmt.Errorf("failed to add auth middleware: %w", err) } + httpStatusCollectMiddleware := healthendpoint.NewHTTPStatusCollectMiddleware(httpStatusCollector) rateLimiterMiddleware := ratelimiter.NewRateLimiterMiddleware("appid", rateLimiter, logger.Session("metricforwarder-ratelimiter-middleware")) - r := routes.MetricsForwarderRoutes() - r.Use(otelmux.Middleware("metricsforwarder")) - r.Use(rateLimiterMiddleware.CheckRateLimit) - r.Use(httpStatusCollectMiddleware.Collect) - r.Use(authenticator.Authenticate) - r.Get(routes.PostCustomMetricsRouteName).Handler(common.VarsFunc(mh.VerifyCredentialsAndPublishMetrics)) + mfRouter := routes.MetricsForwarderRoutes() + mfRouter.Use(otelmux.Middleware("metricsforwarder")) + mfRouter.Use(rateLimiterMiddleware.CheckRateLimit) + mfRouter.Use(httpStatusCollectMiddleware.Collect) + mfRouter.Use(authenticator.Authenticate) + mfRouter.Get(routes.PostCustomMetricsRouteName).Handler(common.VarsFunc(mh.VerifyCredentialsAndPublishMetrics)) + + healthRouter, _ := createHealthRouter(policyDb, credentials, logger, conf, httpStatusCollector) + + mainRouter := mux.NewRouter() + mainRouter.PathPrefix("/v1").Handler(mfRouter) + mainRouter.PathPrefix("/health").Handler(healthRouter) + mainRouter.PathPrefix("/").Handler(healthRouter) + + return helpers.NewHTTPServer(logger, conf.Server, mainRouter) +} + +func createHealthRouter(policyDb db.PolicyDB, credDb cred_helper.Credentials, logger lager.Logger, conf *config.Config, httpStatusCollector healthendpoint.HTTPStatusCollector) (*mux.Router, error) { + checkers := []healthendpoint.Checker{ + healthendpoint.DbChecker(db.PolicyDb, policyDb), + healthendpoint.DbChecker(db.StoredProcedureDb, credDb), + } + + gatherer := createPrometheusRegistry(policyDb, httpStatusCollector, logger) + healthRouter, err := healthendpoint.NewHealthRouter(conf.Health, checkers, logger.Session("health-server"), gatherer, time.Now) + if err != nil { + return nil, err + } + return healthRouter, nil +} - return helpers.NewHTTPServer(logger, conf.Server, r) +func createPrometheusRegistry(policyDb db.PolicyDB, httpStatusCollector healthendpoint.HTTPStatusCollector, logger lager.Logger) *prometheus.Registry { + promRegistry := prometheus.NewRegistry() + healthendpoint.RegisterCollectors(promRegistry, []prometheus.Collector{ + healthendpoint.NewDatabaseStatusCollector("autoscaler", "metricsforwarder", "policyDB", policyDb), + httpStatusCollector, + }, true, logger.Session("metricsforwarder-prometheus")) + return promRegistry } diff --git a/src/autoscaler/metricsforwarder/server/server_suite_test.go b/src/autoscaler/metricsforwarder/server/server_suite_test.go index 827209fd79..23dbdbcf31 100644 --- a/src/autoscaler/metricsforwarder/server/server_suite_test.go +++ b/src/autoscaler/metricsforwarder/server/server_suite_test.go @@ -7,6 +7,7 @@ import ( "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/healthendpoint" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/config" . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/metricsforwarder/server" @@ -23,6 +24,7 @@ import ( ) var ( + conf *config.Config serverProcess ifrit.Process serverUrl string policyDB *fakes.FakePolicyDB @@ -68,14 +70,21 @@ var _ = SynchronizedBeforeSuite(func() []byte { Level: "debug", } - conf := &config.Config{ + healthConfig := helpers.HealthConfig{ + ReadinessCheckEnabled: true, + HealthCheckUsername: "metricsforwarderhealthcheckuser", + HealthCheckPassword: "metricsforwarderhealthcheckpassword", + } + conf = &config.Config{ Server: serverConfig, Logging: loggerConfig, LoggregatorConfig: loggregatorConfig, + Health: healthConfig, } policyDB = &fakes.FakePolicyDB{} allowedMetricCache = *cache.New(10*time.Minute, -1) - httpStatusCollector := &fakes.FakeHTTPStatusCollector{} + httpStatusCollector := healthendpoint.NewHTTPStatusCollector("autoscaler", "metricsforwarder") + rateLimiter = &fakes.FakeLimiter{} fakeCredentials = &fakes.FakeCredentials{} diff --git a/src/autoscaler/metricsforwarder/server/server_test.go b/src/autoscaler/metricsforwarder/server/server_test.go index 193796f024..14059c0ce2 100644 --- a/src/autoscaler/metricsforwarder/server/server_test.go +++ b/src/autoscaler/metricsforwarder/server/server_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "errors" + "io" "net/http" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" @@ -22,7 +23,7 @@ var _ = Describe("CustomMetrics Server", func() { scalingPolicy *models.ScalingPolicy ) - Context("when a request to forward custom metrics comes", func() { + Context("POST /v1/apps/some-app-id/metrics", func() { BeforeEach(func() { scalingPolicy = &models.ScalingPolicy{ @@ -61,7 +62,7 @@ var _ = Describe("CustomMetrics Server", func() { }) }) - Context("when a request to forward custom metrics comes without Authorization header", func() { + When("A request to forward custom metrics comes without Authorization header", func() { BeforeEach(func() { body, err = json.Marshal(models.CustomMetric{Name: "queuelength", Value: 12, Unit: "unit", InstanceIndex: 123, AppGUID: "an-app-id"}) Expect(err).NotTo(HaveOccurred()) @@ -82,7 +83,7 @@ var _ = Describe("CustomMetrics Server", func() { }) }) - Context("when a request to forward custom metrics comes without 'Basic'", func() { + When("a request to forward custom metrics comes without 'Basic'", func() { BeforeEach(func() { body, err = json.Marshal(models.CustomMetric{Name: "queuelength", Value: 12, Unit: "unit", InstanceIndex: 123, AppGUID: "an-app-id"}) Expect(err).NotTo(HaveOccurred()) @@ -149,7 +150,7 @@ var _ = Describe("CustomMetrics Server", func() { }) }) - Context("when multiple requests to forward custom metrics comes beyond ratelimit", func() { + When("multiple requests to forward custom metrics comes beyond ratelimit", func() { BeforeEach(func() { rateLimiter.ExceedsLimitReturns(true) scalingPolicy = &models.ScalingPolicy{ @@ -180,6 +181,7 @@ var _ = Describe("CustomMetrics Server", func() { resp, err = client.Do(req) Expect(err).NotTo(HaveOccurred()) }) + AfterEach(func() { rateLimiter.ExceedsLimitReturns(false) }) @@ -191,4 +193,62 @@ var _ = Describe("CustomMetrics Server", func() { }) }) + When("when Health server is ready to serve RESTful API with basic Auth", func() { + + Context("when username and password are incorrect for basic authentication during health check", func() { + It("should return 401", func() { + client := &http.Client{} + req, err = http.NewRequest("GET", serverUrl+"/health", nil) + Expect(err).NotTo(HaveOccurred()) + req.SetBasicAuth("wrongusername", "wrongpassword") + rsp, err := client.Do(req) + Expect(err).ToNot(HaveOccurred()) + Expect(rsp.StatusCode).To(Equal(http.StatusUnauthorized)) + }) + }) + + Context("when username and password are correct for basic authentication during health check", func() { + + Context("when a request to query health comes", func() { + It("returns with a 200", func() { + client := &http.Client{} + req, err = http.NewRequest("GET", serverUrl, nil) + req.SetBasicAuth(conf.Health.HealthCheckUsername, conf.Health.HealthCheckPassword) + rsp, err := client.Do(req) + Expect(err).NotTo(HaveOccurred()) + Expect(rsp.StatusCode).To(Equal(http.StatusOK)) + raw, _ := io.ReadAll(rsp.Body) + healthData := string(raw) + Expect(healthData).To(ContainSubstring("autoscaler_metricsforwarder_concurrent_http_request")) + Expect(healthData).To(ContainSubstring("autoscaler_metricsforwarder_policyDB")) + Expect(healthData).To(ContainSubstring("go_goroutines")) + Expect(healthData).To(ContainSubstring("go_memstats_alloc_bytes")) + rsp.Body.Close() + + }) + }) + + It("should return 200 for /health", func() { + client := &http.Client{} + req, err = http.NewRequest("GET", serverUrl+"/health", nil) + Expect(err).NotTo(HaveOccurred()) + req.SetBasicAuth(conf.Health.HealthCheckUsername, conf.Health.HealthCheckPassword) + rsp, err := client.Do(req) + Expect(err).ToNot(HaveOccurred()) + Expect(rsp.StatusCode).To(Equal(http.StatusOK)) + }) + + It("should return 200 for /health/readiness", func() { + client := &http.Client{} + req, err = http.NewRequest("GET", serverUrl+"/health/readiness", nil) + Expect(err).NotTo(HaveOccurred()) + rsp, err := client.Do(req) + Expect(err).ToNot(HaveOccurred()) + Expect(rsp.StatusCode).To(Equal(http.StatusOK)) + body, err := io.ReadAll(rsp.Body) + Expect(err).ToNot(HaveOccurred()) + Expect(body).To(MatchJSON(`{"overall_status": "UP","checks": [ { "name":"policy_db", "type":"database", "status":"UP"}, { "name":"storedprocedure_db", "type":"database", "status":"UP"}]}`)) + }) + }) + }) }) From 426e8b78ed579ae317104a47855c7ee5e6eaaccc Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 6 Jun 2024 16:01:36 +0200 Subject: [PATCH 106/186] Fix the same port for route registrar healthendpoint in mf --- operations/use-cf-services.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/operations/use-cf-services.yml b/operations/use-cf-services.yml index 2ea51e42f3..2543f347e8 100644 --- a/operations/use-cf-services.yml +++ b/operations/use-cf-services.yml @@ -1,7 +1,14 @@ - - - type: replace path: /instance_groups/name=apiserver/jobs/name=golangapiserver/properties/autoscaler/apiserver/metrics_forwarder value: host: ((metricsforwarder_host)) mtls_host: ((metricsforwarder_host)) + +# Run healthendpoint on the same port of metricsforwarder +- type: remove + path: /instance_groups/name=metricsforwarder/jobs/name=metricsforwarder/properties/autoscaler/metricsforwarder/port + +# Set the same port for metricsforwarder and healthenpoint routes +- type: replace + path: /instance_groups/name=metricsforwarder/jobs/name=route_registrar/properties/route_registrar/routes=autoscaler_metricsforwarder_health/port + value: 6201 From c7478ea1e7024f3afac3b2d52a29a53a6b41d1ef Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 6 Jun 2024 16:29:50 +0200 Subject: [PATCH 107/186] Refactor server code --- .../metricsforwarder/server/server.go | 49 +++++++++++++------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/src/autoscaler/metricsforwarder/server/server.go b/src/autoscaler/metricsforwarder/server/server.go index f83808e509..582ee37b0e 100644 --- a/src/autoscaler/metricsforwarder/server/server.go +++ b/src/autoscaler/metricsforwarder/server/server.go @@ -2,7 +2,6 @@ package server import ( "fmt" - "os" "time" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/cred_helper" @@ -25,18 +24,37 @@ import ( ) func NewServer(logger lager.Logger, conf *config.Config, policyDb db.PolicyDB, credentials cred_helper.Credentials, allowedMetricCache cache.Cache, httpStatusCollector healthendpoint.HTTPStatusCollector, rateLimiter ratelimiter.Limiter) (ifrit.Runner, error) { - metricForwarder, err := forwarder.NewMetricForwarder(logger, conf) + metricForwarder, err := createMetricForwarder(logger, conf) if err != nil { - logger.Error("failed-to-create-metricforwarder-server", err) - os.Exit(1) + return nil, fmt.Errorf("failed to create metric forwarder: %w", err) } - mh := NewCustomMetricsHandler(logger, metricForwarder, policyDb, allowedMetricCache) + mh := NewCustomMetricsHandler(logger, *metricForwarder, policyDb, allowedMetricCache) authenticator, err := auth.New(logger, credentials) if err != nil { - return nil, fmt.Errorf("failed to add auth middleware: %w", err) + return nil, fmt.Errorf("failed to create auth middleware: %w", err) + } + + mfRouter := setupMetricForwarderRouter(logger, mh, authenticator, httpStatusCollector, rateLimiter) + healthRouter, err := createHealthRouter(policyDb, credentials, logger, conf, httpStatusCollector) + if err != nil { + return nil, fmt.Errorf("failed to create health router: %w", err) } + mainRouter := setupMainRouter(mfRouter, healthRouter) + return helpers.NewHTTPServer(logger, conf.Server, mainRouter) +} + +func createMetricForwarder(logger lager.Logger, conf *config.Config) (*forwarder.MetricForwarder, error) { + metricForwarder, err := forwarder.NewMetricForwarder(logger, conf) + if err != nil { + logger.Error("failed-to-create-metricforwarder-server", err) + return nil, err + } + return &metricForwarder, nil +} + +func setupMetricForwarderRouter(logger lager.Logger, mh *CustomMetricsHandler, authenticator *auth.Auth, httpStatusCollector healthendpoint.HTTPStatusCollector, rateLimiter ratelimiter.Limiter) *mux.Router { httpStatusCollectMiddleware := healthendpoint.NewHTTPStatusCollectMiddleware(httpStatusCollector) rateLimiterMiddleware := ratelimiter.NewRateLimiterMiddleware("appid", rateLimiter, logger.Session("metricforwarder-ratelimiter-middleware")) @@ -47,14 +65,7 @@ func NewServer(logger lager.Logger, conf *config.Config, policyDb db.PolicyDB, c mfRouter.Use(authenticator.Authenticate) mfRouter.Get(routes.PostCustomMetricsRouteName).Handler(common.VarsFunc(mh.VerifyCredentialsAndPublishMetrics)) - healthRouter, _ := createHealthRouter(policyDb, credentials, logger, conf, httpStatusCollector) - - mainRouter := mux.NewRouter() - mainRouter.PathPrefix("/v1").Handler(mfRouter) - mainRouter.PathPrefix("/health").Handler(healthRouter) - mainRouter.PathPrefix("/").Handler(healthRouter) - - return helpers.NewHTTPServer(logger, conf.Server, mainRouter) + return mfRouter } func createHealthRouter(policyDb db.PolicyDB, credDb cred_helper.Credentials, logger lager.Logger, conf *config.Config, httpStatusCollector healthendpoint.HTTPStatusCollector) (*mux.Router, error) { @@ -66,7 +77,7 @@ func createHealthRouter(policyDb db.PolicyDB, credDb cred_helper.Credentials, lo gatherer := createPrometheusRegistry(policyDb, httpStatusCollector, logger) healthRouter, err := healthendpoint.NewHealthRouter(conf.Health, checkers, logger.Session("health-server"), gatherer, time.Now) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create health router: %w", err) } return healthRouter, nil } @@ -79,3 +90,11 @@ func createPrometheusRegistry(policyDb db.PolicyDB, httpStatusCollector healthen }, true, logger.Session("metricsforwarder-prometheus")) return promRegistry } + +func setupMainRouter(mfRouter, healthRouter *mux.Router) *mux.Router { + mainRouter := mux.NewRouter() + mainRouter.PathPrefix("/v1").Handler(mfRouter) + mainRouter.PathPrefix("/health").Handler(healthRouter) + mainRouter.PathPrefix("/").Handler(healthRouter) + return mainRouter +} From a92bf521d10dd79f83bea9e4a43ce697446a7a97 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 6 Jun 2024 16:36:23 +0200 Subject: [PATCH 108/186] WIP: Refactor server --- src/autoscaler/healthendpoint/server.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/autoscaler/healthendpoint/server.go b/src/autoscaler/healthendpoint/server.go index 81d4d3186f..3635a0feaf 100644 --- a/src/autoscaler/healthendpoint/server.go +++ b/src/autoscaler/healthendpoint/server.go @@ -1,5 +1,29 @@ package healthendpoint +// package healthendpoint runs the health endpoint server for diferent autoscaler components +// +// The health endpoint server is used to expose the health +// and readiness check endpoints for the autoscaler components. +// +// It provides the following functionalities: +// - Expose the health and readiness check endpoints +// - Expose the prometheus metrics +// - Expose the pprof endpoints +// - Basic authentication for health and readiness check endpoints +// - Basic authentication for pprof endpoints +// - Basic authentication for prometheus metrics +// +// The healthendpoint package provide the following checkers: +// - NewHTTPChecker +// - NewTCPChecker +// - NewDatabaseChecker +// +// The health endpoint server is used by the following autoscaler components: +// - metricsforwarder +// - scalingengine +// - eventgenerator +// - scheduler + import ( "net/http" "net/http/pprof" From b18e3fd750ac9b9be1c288ed1d6522dcd7f47dec Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 6 Jun 2024 16:47:30 +0200 Subject: [PATCH 109/186] Refactor server test --- .../metricsforwarder/server/server_test.go | 129 ++++++++++-------- 1 file changed, 69 insertions(+), 60 deletions(-) diff --git a/src/autoscaler/metricsforwarder/server/server_test.go b/src/autoscaler/metricsforwarder/server/server_test.go index 14059c0ce2..ea7820b3b9 100644 --- a/src/autoscaler/metricsforwarder/server/server_test.go +++ b/src/autoscaler/metricsforwarder/server/server_test.go @@ -2,6 +2,7 @@ package server_test import ( "bytes" + "encoding/base64" "encoding/json" "errors" "io" @@ -14,6 +15,12 @@ import ( . "github.com/onsi/gomega" ) +// Helper function to create a basic auth string +func basicAuth(username, password string) string { + auth := username + ":" + password + return base64.StdEncoding.EncodeToString([]byte(auth)) +} + var _ = Describe("CustomMetrics Server", func() { var ( resp *http.Response @@ -21,11 +28,40 @@ var _ = Describe("CustomMetrics Server", func() { body []byte err error scalingPolicy *models.ScalingPolicy + client *http.Client + authHeader string ) - Context("POST /v1/apps/some-app-id/metrics", func() { - BeforeEach(func() { + // Helper function to create a new request + newRequest := func(method, url string, body []byte) (*http.Request, error) { + req, err := http.NewRequest(method, url, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Add("Content-Type", "application/json") + return req, nil + } + + // Helper function to set up a new client and request + setupRequest := func(method, url, authHeader string, body []byte) (*http.Client, *http.Request, error) { + client := &http.Client{} + req, err := newRequest(method, url, body) + if err != nil { + return nil, nil, err + } + if authHeader != "" { + req.Header.Add("Authorization", authHeader) + } + return client, req, nil + } + + BeforeEach(func() { + client = &http.Client{} + fakeCredentials.ValidateReturns(true, nil) + }) + When("POST /v1/apps/some-app-id/metrics", func() { + BeforeEach(func() { scalingPolicy = &models.ScalingPolicy{ InstanceMin: 1, InstanceMax: 6, @@ -38,25 +74,20 @@ var _ = Describe("CustomMetrics Server", func() { Adjustment: "+1"}}} policyDB.GetAppPolicyReturns(scalingPolicy, nil) customMetrics := []*models.CustomMetric{ - { - Name: "queuelength", Value: 12, Unit: "unit", InstanceIndex: 1, AppGUID: "an-app-id", - }, + {Name: "queuelength", Value: 12, Unit: "unit", InstanceIndex: 1, AppGUID: "an-app-id"}, } body, err = json.Marshal(models.MetricsConsumer{InstanceIndex: 0, CustomMetrics: customMetrics}) Expect(err).NotTo(HaveOccurred()) - fakeCredentials.ValidateReturns(true, nil) - - client := &http.Client{} - req, err = http.NewRequest("POST", serverUrl+"/v1/apps/an-app-id/metrics", bytes.NewReader(body)) - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Authorization", "Basic dXNlcm5hbWU6cGFzc3dvcmQ=") + authHeader = "Basic " + basicAuth("username", "Password") + client, req, err = setupRequest("POST", serverUrl+"/v1/apps/an-app-id/metrics", authHeader, body) + Expect(err).NotTo(HaveOccurred()) resp, err = client.Do(req) Expect(err).NotTo(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) }) It("returns status code 200", func() { - Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusOK)) resp.Body.Close() }) @@ -66,18 +97,14 @@ var _ = Describe("CustomMetrics Server", func() { BeforeEach(func() { body, err = json.Marshal(models.CustomMetric{Name: "queuelength", Value: 12, Unit: "unit", InstanceIndex: 123, AppGUID: "an-app-id"}) Expect(err).NotTo(HaveOccurred()) - client := &http.Client{} - req, err = http.NewRequest("POST", serverUrl+"/v1/apps/an-app-id/metrics", bytes.NewReader(body)) - req.Header.Add("Content-Type", "application/json") - - fakeCredentials.ValidateReturns(true, nil) + client, req, err = setupRequest("POST", serverUrl+"/v1/apps/an-app-id/metrics", "", body) + Expect(err).NotTo(HaveOccurred()) resp, err = client.Do(req) Expect(err).NotTo(HaveOccurred()) }) It("returns status code 401", func() { - Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) resp.Body.Close() }) @@ -87,70 +114,59 @@ var _ = Describe("CustomMetrics Server", func() { BeforeEach(func() { body, err = json.Marshal(models.CustomMetric{Name: "queuelength", Value: 12, Unit: "unit", InstanceIndex: 123, AppGUID: "an-app-id"}) Expect(err).NotTo(HaveOccurred()) - client := &http.Client{} - req, err = http.NewRequest("POST", serverUrl+"/v1/apps/san-app-id/metrics", bytes.NewReader(body)) - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Authorization", "dXNlcm5hbWU6cGFzc3dvcmQ=") - - fakeCredentials.ValidateReturns(true, nil) + authHeader = basicAuth("username", "password") + client, req, err = setupRequest("POST", serverUrl+"/v1/apps/san-app-id/metrics", authHeader, body) + Expect(err).NotTo(HaveOccurred()) resp, err = client.Do(req) Expect(err).NotTo(HaveOccurred()) }) It("returns status code 401", func() { - Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) resp.Body.Close() }) }) - Context("when a request to forward custom metrics comes with wrong user credentials", func() { + When("a request to forward custom metrics comes with wrong user credentials", func() { BeforeEach(func() { body, err = json.Marshal(models.CustomMetric{Name: "queuelength", Value: 12, Unit: "unit", InstanceIndex: 123, AppGUID: "an-app-id"}) Expect(err).NotTo(HaveOccurred()) fakeCredentials.ValidateReturns(false, errors.New("wrong credentials")) - client := &http.Client{} - req, err = http.NewRequest("POST", serverUrl+"/v1/apps/an-app-id/metrics", bytes.NewReader(body)) - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Authorization", "Basic M2YxZWY2MTJiMThlYTM5YmJlODRjZjUxMzY4MWYwYjc6YWYyNjk1Y2RmZDE0MzA4NThhMWY3MzJhYTI5NTQ2ZTk=") - + authHeader = "Basic " + basicAuth("invalidUsername", "invalidPassword") + client, req, err = setupRequest("POST", serverUrl+"/v1/apps/an-app-id/metrics", authHeader, body) + Expect(err).NotTo(HaveOccurred()) resp, err = client.Do(req) Expect(err).NotTo(HaveOccurred()) }) It("returns status code 401", func() { - Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) resp.Body.Close() }) }) - Context("when a request to forward custom metrics comes with unmatched metric types", func() { + When("a request to forward custom metrics comes with unmatched metric types", func() { BeforeEach(func() { body, err = json.Marshal(models.CustomMetric{Name: "queuelength", Value: 12, Unit: "unit", InstanceIndex: 123, AppGUID: "an-app-id"}) Expect(err).NotTo(HaveOccurred()) - fakeCredentials.ValidateReturns(true, nil) - - client := &http.Client{} - req, err = http.NewRequest("POST", serverUrl+"/v1/apps/an-app-id/metrics", bytes.NewReader(body)) - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Authorization", "Basic dXNlcm5hbWU6cGFzc3dvcmQ=") + authHeader = "Basic " + basicAuth("username", "password") + client, req, err = setupRequest("POST", serverUrl+"/v1/apps/an-app-id/metrics", authHeader, body) + Expect(err).NotTo(HaveOccurred()) resp, err = client.Do(req) Expect(err).NotTo(HaveOccurred()) }) It("returns status code 400", func() { - Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) resp.Body.Close() }) }) - When("multiple requests to forward custom metrics comes beyond ratelimit", func() { + When("multiple requests to forward custom metrics come beyond rate limit", func() { BeforeEach(func() { rateLimiter.ExceedsLimitReturns(true) scalingPolicy = &models.ScalingPolicy{ @@ -165,19 +181,14 @@ var _ = Describe("CustomMetrics Server", func() { Adjustment: "+1"}}} policyDB.GetAppPolicyReturns(scalingPolicy, nil) customMetrics := []*models.CustomMetric{ - { - Name: "queuelength", Value: 12, Unit: "unit", InstanceIndex: 1, AppGUID: "an-app-id", - }, + {Name: "queuelength", Value: 12, Unit: "unit", InstanceIndex: 1, AppGUID: "an-app-id"}, } body, err = json.Marshal(models.MetricsConsumer{InstanceIndex: 0, CustomMetrics: customMetrics}) Expect(err).NotTo(HaveOccurred()) - fakeCredentials.ValidateReturns(true, nil) - - client := &http.Client{} - req, err = http.NewRequest("POST", serverUrl+"/v1/apps/an-app-id/metrics", bytes.NewReader(body)) - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Authorization", "Basic dXNlcm5hbWU6cGFzc3dvcmQ=") + authHeader = "Basic " + basicAuth("username", "password") + client, req, err = setupRequest("POST", serverUrl+"/v1/apps/an-app-id/metrics", authHeader, body) + Expect(err).NotTo(HaveOccurred()) resp, err = client.Do(req) Expect(err).NotTo(HaveOccurred()) }) @@ -187,15 +198,13 @@ var _ = Describe("CustomMetrics Server", func() { }) It("returns status code 429", func() { - Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusTooManyRequests)) resp.Body.Close() }) }) - When("when Health server is ready to serve RESTful API with basic Auth", func() { - - Context("when username and password are incorrect for basic authentication during health check", func() { + When("the Health server is ready to serve RESTful API with basic Auth", func() { + When("username and password are incorrect for basic authentication during health check", func() { It("should return 401", func() { client := &http.Client{} req, err = http.NewRequest("GET", serverUrl+"/health", nil) @@ -207,12 +216,15 @@ var _ = Describe("CustomMetrics Server", func() { }) }) - Context("when username and password are correct for basic authentication during health check", func() { + When("username and password are correct for basic authentication during health check", func() { + BeforeEach(func() { + client = &http.Client{} + }) - Context("when a request to query health comes", func() { + When("a request to query health comes", func() { It("returns with a 200", func() { - client := &http.Client{} req, err = http.NewRequest("GET", serverUrl, nil) + Expect(err).NotTo(HaveOccurred()) req.SetBasicAuth(conf.Health.HealthCheckUsername, conf.Health.HealthCheckPassword) rsp, err := client.Do(req) Expect(err).NotTo(HaveOccurred()) @@ -224,12 +236,10 @@ var _ = Describe("CustomMetrics Server", func() { Expect(healthData).To(ContainSubstring("go_goroutines")) Expect(healthData).To(ContainSubstring("go_memstats_alloc_bytes")) rsp.Body.Close() - }) }) It("should return 200 for /health", func() { - client := &http.Client{} req, err = http.NewRequest("GET", serverUrl+"/health", nil) Expect(err).NotTo(HaveOccurred()) req.SetBasicAuth(conf.Health.HealthCheckUsername, conf.Health.HealthCheckPassword) @@ -239,7 +249,6 @@ var _ = Describe("CustomMetrics Server", func() { }) It("should return 200 for /health/readiness", func() { - client := &http.Client{} req, err = http.NewRequest("GET", serverUrl+"/health/readiness", nil) Expect(err).NotTo(HaveOccurred()) rsp, err := client.Do(req) From 3c392a5590550ca9b5ff51f3ccfca72e9816e2b9 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 6 Jun 2024 16:59:53 +0200 Subject: [PATCH 110/186] Adds make browse for ci makefile --- ci/Makefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/Makefile b/ci/Makefile index abf7810662..9d07aacc6b 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -16,12 +16,14 @@ delete-pipeline: set-target: @fly --target app-autoscaler-release login --team-name app-autoscaler --concourse-url https://concourse.app-runtime-interfaces.ci.cloudfoundry.org -.phony: lint +.PHONY: lint lint: shellcheck --external-sources -x -s bash autoscaler/**/**/*.sh -.phony: fix-lint +.PHONY: fix-lint fix-lint: shellcheck --external-sources -x -s bash -f diff autoscaler/**/**/*.sh - +.PHONY: browse +browse: + @open https://concourse.app-runtime-interfaces.ci.cloudfoundry.org From f928dbe8855cc4bab06ea6b43c412e0bea9b169c Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 6 Jun 2024 17:00:09 +0200 Subject: [PATCH 111/186] Fix spelling --- src/autoscaler/healthendpoint/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/healthendpoint/server.go b/src/autoscaler/healthendpoint/server.go index 3635a0feaf..4e4ca9272e 100644 --- a/src/autoscaler/healthendpoint/server.go +++ b/src/autoscaler/healthendpoint/server.go @@ -1,6 +1,6 @@ package healthendpoint -// package healthendpoint runs the health endpoint server for diferent autoscaler components +// package healthendpoint runs the health endpoint server for different autoscaler components // // The health endpoint server is used to expose the health // and readiness check endpoints for the autoscaler components. From d5ad37b2906567d81e67d1bdd47e6c928ac5853b Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 6 Jun 2024 17:04:42 +0200 Subject: [PATCH 112/186] Fix typo --- ci/autoscaler/scripts/vars.source.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/autoscaler/scripts/vars.source.sh b/ci/autoscaler/scripts/vars.source.sh index e09d018475..7033840f35 100644 --- a/ci/autoscaler/scripts/vars.source.sh +++ b/ci/autoscaler/scripts/vars.source.sh @@ -79,7 +79,7 @@ metricsforwarder_appname="${METRICSFORWARDER_APPNAME}" export METRICSFORWARDER_HOST="${METRICSFORWARDER_HOST:-"${METRICSFORWARDER_APPNAME}.${SYSTEM_DOMAIN}"}" debug "METRICSFORWARDER_HOST: ${METRICSFORWARDER_HOST}" -log "set up vars: METRICSFORWRDER_HOST=${METRICSFORWARDER_HOST}" +log "set up vars: METRICSFORWARDER_HOST=${METRICSFORWARDER_HOST}" metricsforwarder_host="${METRICSFORWARDER_HOST}" BBL_STATE_PATH="${BBL_STATE_PATH:-$( realpath -e "${root_dir}/../app-autoscaler-env-bbl-state/bbl-state" 2> /dev/null || echo "${root_dir}/../bbl-state/bbl-state" )}" From 740b23e1b14c0ac70fcf6cb31f4b5230dc086428 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 6 Jun 2024 17:11:02 +0200 Subject: [PATCH 113/186] Update devlock and gomod --- devbox.json | 79 ++++++++++++++++++++++--------------------- devbox.lock | 48 +++++++++++++------------- src/autoscaler/go.mod | 18 +++++----- src/autoscaler/go.sum | 32 +++++++++--------- 4 files changed, 89 insertions(+), 88 deletions(-) diff --git a/devbox.json b/devbox.json index 963a033274..1767886ad7 100644 --- a/devbox.json +++ b/devbox.json @@ -1,43 +1,44 @@ { - "packages": [ - "path:local-flake#bosh-bootloader", - "path:local-flake#app-autoscaler-cli-plugin", - "path:local-flake#log-cache-cli-plugin", - "path:local-flake#uaac", - "delve@latest", - "gh@latest", - "go-tools@latest", - "gopls@latest", - "nodejs@latest", - "nodePackages.yaml-language-server@latest", - "rubocop@latest", - "rubyPackages.solargraph@latest", - "swagger-cli@latest", - "actionlint@1.6.27", - "direnv@2.34.0", - "gnumake@4.4", - "maven@3.8.6", - "pre-commit@latest", - "bosh-cli@7.3.1", - "golangci-lint@1.57.2", - "yq-go@4.43.1", - "which@latest", - "jq@latest", - "fly@7.10.0", - "python@latest", - "cloudfoundry-cli@8.7.10", - "shellcheck@0.10.0", - "go@1.21.5", - "ruby@3.3.1", - "bundix@latest", - "oha@latest", - "credhub-cli@2.9.29", - "gum@0.13.0", - "act@latest", - "google-cloud-sdk@latest", - "ginkgo@2.19.0", - "temurin-bin-17@latest" - ], + "packages": { + "path:local-flake#bosh-bootloader": "", + "path:local-flake#app-autoscaler-cli-plugin": "", + "path:local-flake#log-cache-cli-plugin": "", + "path:local-flake#cloud-mta-build-tool": "", + "path:local-flake#uaac": "", + "credhub-cli": "2.9.29", + "delve": "latest", + "gh": "latest", + "go-tools": "latest", + "gopls": "latest", + "nodejs": "latest", + "nodePackages.yaml-language-server": "latest", + "rubocop": "latest", + "rubyPackages.solargraph": "latest", + "swagger-cli": "latest", + "actionlint": "1.6.27", + "direnv": "2.34.0", + "gnumake": "4.4", + "maven": "3.8.6", + "pre-commit": "latest", + "bosh-cli": "7.3.1", + "golangci-lint": "1.57.2", + "yq-go": "4.43.1", + "which": "latest", + "jq": "latest", + "fly": "7.10.0", + "python": "latest", + "cloudfoundry-cli": "8.7.10", + "shellcheck": "0.10.0", + "ruby": "3.3.1", + "ginkgo": "2.19.0", + "bundix": "latest", + "oha": "latest", + "gum": "0.13.0", + "act": "0.2.63", + "google-cloud-sdk": "latest", + "go": "1.21.6", + "temurin-bin-17": "latest" + }, "shell": { "init_hook": [ "cf install-plugin -f $(which app-autoscaler-cli-plugin)", diff --git a/devbox.lock b/devbox.lock index 6f2399a947..e2840f1999 100644 --- a/devbox.lock +++ b/devbox.lock @@ -1,51 +1,51 @@ { "lockfile_version": "1", "packages": { - "act@latest": { - "last_modified": "2024-05-29T10:04:41Z", - "resolved": "github:NixOS/nixpkgs/ac82a513e55582291805d6f09d35b6d8b60637a1#act", + "act@0.2.63": { + "last_modified": "2024-06-03T07:19:07Z", + "resolved": "github:NixOS/nixpkgs/4a4ecb0ab415c9fccfb005567a215e6a9564cdf5#act", "source": "devbox-search", - "version": "0.2.62", + "version": "0.2.63", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/anliky2rlgzcicrhn1yyv9q4nkz7p8wp-act-0.2.62", + "path": "/nix/store/fhn4kf6qa88l2pl0n202b4mplyc5pcyf-act-0.2.63", "default": true } ], - "store_path": "/nix/store/anliky2rlgzcicrhn1yyv9q4nkz7p8wp-act-0.2.62" + "store_path": "/nix/store/fhn4kf6qa88l2pl0n202b4mplyc5pcyf-act-0.2.63" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/jz4lqgv3amm5vg2kd9qxcw43vzrbcnxv-act-0.2.62", + "path": "/nix/store/5pmgbyh36x428ivx8dg2rq3rvaicjb66-act-0.2.63", "default": true } ], - "store_path": "/nix/store/jz4lqgv3amm5vg2kd9qxcw43vzrbcnxv-act-0.2.62" + "store_path": "/nix/store/5pmgbyh36x428ivx8dg2rq3rvaicjb66-act-0.2.63" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/g4zs1xipwgxx1prvi7bpvymibhiq4cm5-act-0.2.62", + "path": "/nix/store/iwjzs1r05xrdk4xsw6a2c33436s08x18-act-0.2.63", "default": true } ], - "store_path": "/nix/store/g4zs1xipwgxx1prvi7bpvymibhiq4cm5-act-0.2.62" + "store_path": "/nix/store/iwjzs1r05xrdk4xsw6a2c33436s08x18-act-0.2.63" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/xl5x5p5axgrqr9h6p2a3idpmcgmj9407-act-0.2.62", + "path": "/nix/store/4d5qhzbrwm9k9yxnwlkwjn8740mh2hlz-act-0.2.63", "default": true } ], - "store_path": "/nix/store/xl5x5p5axgrqr9h6p2a3idpmcgmj9407-act-0.2.62" + "store_path": "/nix/store/4d5qhzbrwm9k9yxnwlkwjn8740mh2hlz-act-0.2.63" } } }, @@ -485,51 +485,51 @@ } } }, - "go@1.21.5": { - "last_modified": "2024-01-14T03:55:27Z", - "resolved": "github:NixOS/nixpkgs/dd5621df6dcb90122b50da5ec31c411a0de3e538#go", + "go@1.21.6": { + "last_modified": "2024-02-10T18:15:24Z", + "resolved": "github:NixOS/nixpkgs/10b813040df67c4039086db0f6eaf65c536886c6#go", "source": "devbox-search", - "version": "1.21.5", + "version": "1.21.6", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/z6p2j8shdwi74kbm86jwdh03vxq91l0q-go-1.21.5", + "path": "/nix/store/44g95npjwgjyrxm1sx54g22w5sfmkz1i-go-1.21.6", "default": true } ], - "store_path": "/nix/store/z6p2j8shdwi74kbm86jwdh03vxq91l0q-go-1.21.5" + "store_path": "/nix/store/44g95npjwgjyrxm1sx54g22w5sfmkz1i-go-1.21.6" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/1p00q366rk2j7w9lnflrl7xaks7wfq5d-go-1.21.5", + "path": "/nix/store/bq5scmlrr463qwx4yfbid9p3w7009r57-go-1.21.6", "default": true } ], - "store_path": "/nix/store/1p00q366rk2j7w9lnflrl7xaks7wfq5d-go-1.21.5" + "store_path": "/nix/store/bq5scmlrr463qwx4yfbid9p3w7009r57-go-1.21.6" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/brv7d6mlrclkzywf1vaf35wqhq4c0c82-go-1.21.5", + "path": "/nix/store/f0mzci3szxjf0cws1a1vlls6d4zbd8s4-go-1.21.6", "default": true } ], - "store_path": "/nix/store/brv7d6mlrclkzywf1vaf35wqhq4c0c82-go-1.21.5" + "store_path": "/nix/store/f0mzci3szxjf0cws1a1vlls6d4zbd8s4-go-1.21.6" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/rccpgf2bnh58hhh4908p78i1rln9gzvp-go-1.21.5", + "path": "/nix/store/zg65r8ys8y5865lcwmmybrq5gn30n1az-go-1.21.6", "default": true } ], - "store_path": "/nix/store/rccpgf2bnh58hhh4908p78i1rln9gzvp-go-1.21.5" + "store_path": "/nix/store/zg65r8ys8y5865lcwmmybrq5gn30n1az-go-1.21.6" } } }, diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 56f13c5f21..6bc479c1a7 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -1,6 +1,6 @@ module code.cloudfoundry.org/app-autoscaler/src/autoscaler -go 1.21.5 +go 1.21.6 require ( code.cloudfoundry.org/cfhttp/v2 v2.1.0 @@ -8,7 +8,7 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240604192159-0c056d8148bf + code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240605163137-482a07ab6dbc code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65 dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 @@ -46,15 +46,15 @@ require ( go.opentelemetry.io/otel/trace v1.26.0 golang.org/x/crypto v0.24.0 golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc - golang.org/x/net v0.25.0 + golang.org/x/net v0.26.0 golang.org/x/time v0.5.0 google.golang.org/grpc v1.64.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - code.cloudfoundry.org/go-diodes v0.0.0-20240515174142-71582f284718 // indirect - code.cloudfoundry.org/go-metric-registry v0.0.0-20240523160243-6c152ef80e25 // indirect + code.cloudfoundry.org/go-diodes v0.0.0-20240604201846-c756bfed2ed3 // indirect + code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -96,13 +96,13 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/mod v0.17.0 // indirect + golang.org/x/mod v0.18.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect + golang.org/x/tools v0.22.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 707272e824..e76131babf 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -596,18 +596,18 @@ code.cloudfoundry.org/cfhttp/v2 v2.1.0 h1:HbQ5H2R+HEKG/rcB6Gk3okeC3h2fAC4PPnLQoM code.cloudfoundry.org/cfhttp/v2 v2.1.0/go.mod h1:k9R36Y/9dUc9OsX4dfDuEjHZ7Q00ttklKQj6HD6h6+U= code.cloudfoundry.org/clock v1.1.0 h1:XLzC6W3Ah/Y7ht1rmZ6+QfPdt1iGWEAAtIZXgiaj57c= code.cloudfoundry.org/clock v1.1.0/go.mod h1:yA3fxddT9RINQL2XHS7PS+OXxKCGhfrZmlNUCIM6AKo= -code.cloudfoundry.org/go-diodes v0.0.0-20240515174142-71582f284718 h1:6wazSuHaJjatGy8pvchSy6L+4M67WPfjgK9yh7cJLMs= -code.cloudfoundry.org/go-diodes v0.0.0-20240515174142-71582f284718/go.mod h1:eVHabU/rcpC5ocoIAXvnaySkdH6+PgORRVlY5l3SDys= +code.cloudfoundry.org/go-diodes v0.0.0-20240604201846-c756bfed2ed3 h1:4WCYwJmqSfV7ChDohsJB8Z0aDVklIE+n8OTBJxpif0c= +code.cloudfoundry.org/go-diodes v0.0.0-20240604201846-c756bfed2ed3/go.mod h1:8O5g1DEzJU9ktEmykKPhY4mZOM/dBENWVHKVInuuch8= code.cloudfoundry.org/go-log-cache/v2 v2.0.7 h1:yR/JjQ/RscO1n4xVAT9HDYcpx5ET/3Cq2/RhpJml6ZU= code.cloudfoundry.org/go-log-cache/v2 v2.0.7/go.mod h1:6KQe2FeeaqRheD5vCvpyTa80YoJojB/r21E54mT97Mc= code.cloudfoundry.org/go-loggregator/v9 v9.2.1 h1:S6Lgg5UJbhh2bt2TGQxs6R00CF8PrUA3GFPYDxy56Fk= code.cloudfoundry.org/go-loggregator/v9 v9.2.1/go.mod h1:FTFFruqGeOhVCDFvyLgl8EV8YW63NNwRzLhxJcporu8= -code.cloudfoundry.org/go-metric-registry v0.0.0-20240523160243-6c152ef80e25 h1:p1RGKEtKpZJ4CtjLoQudjEpvMLi+l2MZCyheWthmaNg= -code.cloudfoundry.org/go-metric-registry v0.0.0-20240523160243-6c152ef80e25/go.mod h1:C+P03mf5ZSJVpcigPZ3tH5Vb66HIfCTKxatA4llc298= +code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a h1:XpebbxgIBBy7SrwIGW+gREZuAtnJ9PHWC4Y+k7yje2I= +code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a/go.mod h1:/Be8VtLiCeMUoYdUzFtmW8GGkk89HAy3zD79KUXzbhs= code.cloudfoundry.org/lager/v3 v3.0.3 h1:/UTmadZfIaKuT/whEinSxK1mzRfNu1uPfvjFfGqiwzM= code.cloudfoundry.org/lager/v3 v3.0.3/go.mod h1:Zn5q1SrIuuHjEUE7xerMKt3ztunrJQCZETAo7rV0CH8= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240604192159-0c056d8148bf h1:EmmABS4FL7hQNwPpyO4AEZ43EFFA8vOOq7njrt8qyHU= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240604192159-0c056d8148bf/go.mod h1:3g7KeOBzN+XZBvuRVkX41LBMRTG+HNgtA6DKJjFdrC0= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240605163137-482a07ab6dbc h1:KzDejtGhap8QSBkGMGVQ+u9HupyXgBRsr8PQlWLPq6g= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240605163137-482a07ab6dbc/go.mod h1:wvherEAITeIr/OoqKfSZa8ngKJiglUjfgfmCSW9OGUc= code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65 h1:0wXfAs/32wbpMKDGiV3XEHev1l9dLgIPjda+WRwJhCc= code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65/go.mod h1:R4odSd3iXVYlIlesDHaQi9IZx2gXQryfvJmJSjPS/ug= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= @@ -1159,8 +1159,8 @@ golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1224,8 +1224,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1463,8 +1463,8 @@ golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1677,10 +1677,10 @@ google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= -google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= From 497b9521ab88ac3846079f6cacc3b63aaa820b13 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 11 Jun 2024 11:06:54 +0200 Subject: [PATCH 114/186] Fix dependency for job --- ci/autoscaler/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index 4bf86bf2dd..ebcb712838 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -418,7 +418,7 @@ jobs: - in_parallel: - get: bbl-state - get: app-autoscaler-release - # passed: [unit-tests, integration-tests] + passed: [unit-tests, integration-tests] trigger: true - get: ci - task: make-prerelease From cbfd05eee414e2d59fd39d51869a1cd1a5cb50bd Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 11 Jun 2024 11:24:09 +0200 Subject: [PATCH 115/186] Refactor deploy apps --- ci/autoscaler/scripts/common.sh | 28 +++++++++++++++-- ci/autoscaler/scripts/deploy-apps.sh | 46 ++++------------------------ 2 files changed, 32 insertions(+), 42 deletions(-) diff --git a/ci/autoscaler/scripts/common.sh b/ci/autoscaler/scripts/common.sh index b4e41ca8d7..f8934be498 100644 --- a/ci/autoscaler/scripts/common.sh +++ b/ci/autoscaler/scripts/common.sh @@ -31,8 +31,8 @@ function bosh_login(){ function cf_login(){ step "login to cf" cf api "https://api.${system_domain}" --skip-ssl-validation - CF_ADMIN_PASSWORD=$(credhub get -n /bosh-autoscaler/cf/cf_admin_password -q) - cf auth admin "$CF_ADMIN_PASSWORD" + cf_admin_password="$(credhub get --quiet --name='/bosh-autoscaler/cf/cf_admin_password')" + cf auth admin "$cf_admin_password" } function cleanup_acceptance_run(){ @@ -91,3 +91,27 @@ function unset_vars() { unset NAME_PREFIX unset GINKGO_OPTS } + +function find_or_create_org(){ + local org_name="$1" + if ! cf orgs | grep -q "${org_name}"; then + cf create-org "${org_name}" + fi + cf target -o "${org_name}" +} + +function find_or_create_space(){ + local space_name="$1" + if ! cf spaces | grep -q "${space_name}"; then + cf create-space "${space_name}" + fi + cf target -s "${space_name}" +} + +function cf_target(){ + local org_name="$1" + local space_name="$2" + + find_or_create_org "${org_name}" + find_or_create_space "${space_name}" +} diff --git a/ci/autoscaler/scripts/deploy-apps.sh b/ci/autoscaler/scripts/deploy-apps.sh index 782a482317..074cf6d508 100755 --- a/ci/autoscaler/scripts/deploy-apps.sh +++ b/ci/autoscaler/scripts/deploy-apps.sh @@ -2,54 +2,20 @@ # shellcheck disable=SC2086,SC2034,SC2155 set -euo pipefail script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${script_dir}/common.sh" source "${script_dir}/vars.source.sh" -pushd "${bbl_state_path}" > /dev/null - eval "$(bbl print-env)" -popd > /dev/null -function find_or_create_org(){ - local org_name="$1" - if ! cf orgs | grep -q "${org_name}"; then - cf create-org "${org_name}" - fi - cf target -o "${org_name}" -} - -function find_or_create_space(){ - local space_name="$1" - if ! cf spaces | grep -q "${space_name}"; then - cf create-space "${space_name}" - fi - cf target -s "${space_name}" -} - -function cf_login(){ - cf api "https://api.${system_domain}" --skip-ssl-validation - cf_admin_password="$(credhub get -n /bosh-autoscaler/cf/cf_admin_password -q)" - cf auth admin "${cf_admin_password}" -} - -function cf_target(){ - local org_name="$1" - local space_name="$2" - - find_or_create_org "${org_name}" - find_or_create_space "${space_name}" -} function deploy() { + log "Deploying autoscaler apps for bosh deployment '${deployment_name}' " pushd "${autoscaler_dir}/src/autoscaler/metricsforwarder" > /dev/null log "Deploying autoscaler apps" make cf-push popd > /dev/null } - -log "Deploying autoscaler apps for bosh deployment '${deployment_name}' " - -pushd "${autoscaler_dir}" > /dev/null - cf_login - cf_target "${autoscaler_org}" "${autoscaler_space}" - deploy -popd > /dev/null +bosh_login +cf_login +cf_target "${autoscaler_org}" "${autoscaler_space}" +deploy From 2682116596573cd5874fcfe1cd51e2ccd77560d8 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 11 Jun 2024 11:30:21 +0200 Subject: [PATCH 116/186] Fix greps --- ci/autoscaler/scripts/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/autoscaler/scripts/common.sh b/ci/autoscaler/scripts/common.sh index f8934be498..c0e609c247 100644 --- a/ci/autoscaler/scripts/common.sh +++ b/ci/autoscaler/scripts/common.sh @@ -94,7 +94,7 @@ function unset_vars() { function find_or_create_org(){ local org_name="$1" - if ! cf orgs | grep -q "${org_name}"; then + if ! cf orgs | grep --quiet --regexp="${org_name}"; then cf create-org "${org_name}" fi cf target -o "${org_name}" @@ -102,7 +102,7 @@ function find_or_create_org(){ function find_or_create_space(){ local space_name="$1" - if ! cf spaces | grep -q "${space_name}"; then + if ! cf spaces | grep --quiet --regexp="${space_name}"; then cf create-space "${space_name}" fi cf target -s "${space_name}" From 62751d4d430ba76d942f521e07a1950def6b158a Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 11 Jun 2024 11:33:13 +0200 Subject: [PATCH 117/186] Rename ops file --- ci/autoscaler/scripts/deploy-autoscaler.sh | 2 +- ...{remove-postgres-tls.yml => disable-postgres-tls-config.yml} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename operations/{remove-postgres-tls.yml => disable-postgres-tls-config.yml} (100%) diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index 3e11dc951a..93604d8249 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -21,7 +21,7 @@ ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\ ${autoscaler_dir}/operations/remove-metricsgateway.yml\ ${autoscaler_dir}/operations/enable-log-cache-via-uaa.yml\ ${autoscaler_dir}/operations/enable-metricsforwarder-via-syslog-agent.yml\ - ${autoscaler_dir}/operations/remove-postgres-tls.yml\ + ${autoscaler_dir}/operations/disable-postgres-tls-config.yml\ ${autoscaler_dir}/operations/enable-scheduler-logging.yml"} diff --git a/operations/remove-postgres-tls.yml b/operations/disable-postgres-tls-config.yml similarity index 100% rename from operations/remove-postgres-tls.yml rename to operations/disable-postgres-tls-config.yml From 6b60fa43c53916650831269d5ca624470127df10 Mon Sep 17 00:00:00 2001 From: bonzofenix <317403+bonzofenix@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:34:44 +0200 Subject: [PATCH 118/186] Update src/acceptance/assets/app/go_app/internal/app/custom_metrics.go Co-authored-by: Arsalan Khan --- src/acceptance/assets/app/go_app/internal/app/custom_metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go b/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go index 82dfdc079c..143b343fbe 100644 --- a/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go +++ b/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go @@ -42,7 +42,7 @@ func handleCustomMetricsEndpoint(logger logr.Logger, customMetricTest CustomMetr var err error if metricName = c.Param("name"); metricName == "" { - logger.Error(err, "empty metric name") + logger.Error(nil, "empty metric name") Error(c, http.StatusBadRequest, "empty metric name") return } From 851ab76165dfc9d1143da5c8514447c37b60c999 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 11 Jun 2024 11:35:54 +0200 Subject: [PATCH 119/186] fix missing ops file reference --- ci/autoscaler/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index ebcb712838..6e58e23823 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -56,7 +56,7 @@ anchors: operations/remove-metricsserver.yml operations/remove-metricsgateway.yml operations/enable-scheduler-logging.yml - operations/remove-postgres-tls.yml + operations/disable-postgres-tls-config.yml operations/use-cf-services.yml From a7d76ebdf0e7dd32f3c3151d0441612b9577b4ee Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 11 Jun 2024 11:48:54 +0200 Subject: [PATCH 120/186] Refactor server test --- .../metricsforwarder/server/server_test.go | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/autoscaler/metricsforwarder/server/server_test.go b/src/autoscaler/metricsforwarder/server/server_test.go index ea7820b3b9..8c3d121249 100644 --- a/src/autoscaler/metricsforwarder/server/server_test.go +++ b/src/autoscaler/metricsforwarder/server/server_test.go @@ -21,6 +21,29 @@ func basicAuth(username, password string) string { return base64.StdEncoding.EncodeToString([]byte(auth)) } +// Helper function to create a new request +func newRequest(method, url string, body []byte) (*http.Request, error) { + req, err := http.NewRequest(method, url, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Add("Content-Type", "application/json") + return req, nil +} + +// Helper function to set up a new client and request +func setupRequest(method, url, authHeader string, body []byte) (*http.Client, *http.Request, error) { + client := &http.Client{} + req, err := newRequest(method, url, body) + if err != nil { + return nil, nil, err + } + if authHeader != "" { + req.Header.Add("Authorization", authHeader) + } + return client, req, nil +} + var _ = Describe("CustomMetrics Server", func() { var ( resp *http.Response @@ -32,29 +55,6 @@ var _ = Describe("CustomMetrics Server", func() { authHeader string ) - // Helper function to create a new request - newRequest := func(method, url string, body []byte) (*http.Request, error) { - req, err := http.NewRequest(method, url, bytes.NewReader(body)) - if err != nil { - return nil, err - } - req.Header.Add("Content-Type", "application/json") - return req, nil - } - - // Helper function to set up a new client and request - setupRequest := func(method, url, authHeader string, body []byte) (*http.Client, *http.Request, error) { - client := &http.Client{} - req, err := newRequest(method, url, body) - if err != nil { - return nil, nil, err - } - if authHeader != "" { - req.Header.Add("Authorization", authHeader) - } - return client, req, nil - } - BeforeEach(func() { client = &http.Client{} fakeCredentials.ValidateReturns(true, nil) From 800b4375d05fc105d0f98ce3e587365999923c75 Mon Sep 17 00:00:00 2001 From: bonzofenix <317403+bonzofenix@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:49:43 +0200 Subject: [PATCH 121/186] Update Makefile Co-authored-by: joergdw --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 32a0c4492c..2fbf544c14 100644 --- a/Makefile +++ b/Makefile @@ -221,7 +221,7 @@ stop-db: check-db_type .PHONY: integration integration: build init-db test-certs @echo " - using DBURL=${DBURL}" - make --directory='./src/autoscaler' integration DBURL="${DBURL}" + @make --directory='./src/autoscaler' integration DBURL="${DBURL}" .PHONY:lint $(addprefix lint_,$(go_modules)) From fa5786043db81211608cae56a04da4850c987be9 Mon Sep 17 00:00:00 2001 From: bonzofenix <317403+bonzofenix@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:50:13 +0200 Subject: [PATCH 122/186] Update ci/Makefile Co-authored-by: joergdw --- ci/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/Makefile b/ci/Makefile index 9d07aacc6b..5aef99a645 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -18,7 +18,7 @@ set-target: .PHONY: lint lint: - shellcheck --external-sources -x -s bash autoscaler/**/**/*.sh + shellcheck --external-sources --shell='bash' autoscaler/**/**/*.sh .PHONY: fix-lint fix-lint: From f0bdbd97d209dfcf2d3cd65d0df8c99617b888a7 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 11 Jun 2024 11:56:23 +0200 Subject: [PATCH 123/186] Fix ci makefile linting for scripts --- ci/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/Makefile b/ci/Makefile index 5aef99a645..f30922761a 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -18,11 +18,11 @@ set-target: .PHONY: lint lint: - shellcheck --external-sources --shell='bash' autoscaler/**/**/*.sh + @shellcheck --external-sources --shell='bash' **/scripts/*.sh .PHONY: fix-lint fix-lint: - shellcheck --external-sources -x -s bash -f diff autoscaler/**/**/*.sh + @shellcheck --external-sources --shell='bash' -f diff **/scripts/*.sh .PHONY: browse browse: From 4f58aac5bccce94f19ea811496432806471518dc Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 11 Jun 2024 11:58:57 +0200 Subject: [PATCH 124/186] Remove comment --- ci/autoscaler/scripts/vars.source.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/ci/autoscaler/scripts/vars.source.sh b/ci/autoscaler/scripts/vars.source.sh index 7033840f35..d4d2dd07a9 100644 --- a/ci/autoscaler/scripts/vars.source.sh +++ b/ci/autoscaler/scripts/vars.source.sh @@ -1,7 +1,4 @@ #!/bin/bash -# Source this file please. -# Moved to ci/ *DO NOT MODIFY MANUALLY* - # NOTE: to turn on debug use DEBUG=true # shellcheck disable=SC2155,SC2034 # From 17a279c41da050142623a5a566313c92c7d32993 Mon Sep 17 00:00:00 2001 From: bonzofenix <317403+bonzofenix@users.noreply.github.com> Date: Tue, 11 Jun 2024 12:02:34 +0200 Subject: [PATCH 125/186] Update ci/autoscaler/tasks/update-sdk/update_java_package.sh Co-authored-by: joergdw --- ci/autoscaler/tasks/update-sdk/update_java_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/autoscaler/tasks/update-sdk/update_java_package.sh b/ci/autoscaler/tasks/update-sdk/update_java_package.sh index f1b9d4ddf8..05261097a5 100755 --- a/ci/autoscaler/tasks/update-sdk/update_java_package.sh +++ b/ci/autoscaler/tasks/update-sdk/update_java_package.sh @@ -8,7 +8,7 @@ source "${script_dir}/vars.source.sh" source "${script_dir}/vendor_package.sh" java_dir=${JAVA_DIR:-"${autoscaler_dir}/../java-release"} -java_dir=$(realpath -e "${java_dir}") +java_dir=$(realpath --canonicalize-existing "${java_dir}") export java_dir # shellcheck disable=SC2154 From 4c6c8c4f9579548158bbf5ec7b19e0e5345fc116 Mon Sep 17 00:00:00 2001 From: bonzofenix <317403+bonzofenix@users.noreply.github.com> Date: Tue, 11 Jun 2024 12:03:35 +0200 Subject: [PATCH 126/186] Update ci/scripts/delete-pipeline.sh Co-authored-by: joergdw --- ci/scripts/delete-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/delete-pipeline.sh b/ci/scripts/delete-pipeline.sh index 4347e50d4e..c63261f883 100755 --- a/ci/scripts/delete-pipeline.sh +++ b/ci/scripts/delete-pipeline.sh @@ -5,7 +5,7 @@ target="app-autoscaler-release" function delete-pipeline(){ - payload=$(fly -t "$target" pipelines --json) + payload=$(fly --target= "$target" pipelines --json) pipelines=$(echo "$payload" | jq ".[] |.name" -r | sort) # ignore shellcheck warning From 84b491f32ba1e0855395960bec9b64e428b70f40 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 11 Jun 2024 12:02:04 +0200 Subject: [PATCH 127/186] Change docker repository path for deploy apps task --- ci/autoscaler/tasks/deploy-apps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/autoscaler/tasks/deploy-apps.yml b/ci/autoscaler/tasks/deploy-apps.yml index 858df217e9..b2af432e65 100755 --- a/ci/autoscaler/tasks/deploy-apps.yml +++ b/ci/autoscaler/tasks/deploy-apps.yml @@ -4,7 +4,7 @@ platform: linux image_resource: type: registry-image source: - repository: docker.io/bonzofenix/app-autoscaler-release-tools + repository: ghcr.io/cloudfoundry/app-autoscaler-release-tools tag: latest From b837d95ea6f2f821643044c1eef6e7d65c2ca47c Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 11 Jun 2024 12:51:13 +0200 Subject: [PATCH 128/186] Refactor metricsforwarder makefile --- src/autoscaler/metricsforwarder/Makefile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/autoscaler/metricsforwarder/Makefile b/src/autoscaler/metricsforwarder/Makefile index 1870176ba2..fe10a104aa 100644 --- a/src/autoscaler/metricsforwarder/Makefile +++ b/src/autoscaler/metricsforwarder/Makefile @@ -1,6 +1,8 @@ PR_NUMBER ?= $(shell gh pr view --json number --jq '.number') DEPLOYMENT_NAME ?= autoscaler-$(PR_NUMBER) METIRCSFORWARDER_VM := $(shell bosh -d $(DEPLOYMENT_NAME) vms --json | jq '.Tables | .[] | .Rows | .[] | select(.instance|test("metricsforwarder")) | .instance') +POSTGRES_IP := $(shell bosh -d ${DEPLOYMENT_NAME} vms --json | jq -r '.Tables | .[] | .Rows | .[] | select(.instance|test("postgres")) | .ips' ) +LOG_CACHE_IP := $(shell bosh -d cf vms --json | jq -r '.Tables | .[] | .Rows | .[] | select(.instance|test("log-cache")) | .ips' ) METRICSFORWARDER_APPNAME ?= "metricsforwarder" METRICSFORWARDER_HOSTNAME ?= $(METRICSFORWARDER_APPNAME) SYSTEM_DOMAIN ?= "autoscaler.app-runtime-interfaces.ci.cloudfoundry.org" @@ -18,13 +20,9 @@ fetch-config: start-metricsforwarder-vm # how to define variables in deployment name mkdir -p assets/certs/policy_db assets/certs/storedprocedure_db assets/certs/syslog_client - $(eval POSTGRES_IP := $(shell bosh -d ${DEPLOYMENT_NAME} vms --json | jq -r '.Tables | .[] | .Rows | .[] | select(.instance|test("postgres")) | .ips' )) - echo "Postgres IP: $(POSTGRES_IP)" - - $(eval LOG_CACHE_IP := $(shell bosh -d cf vms --json | jq -r '.Tables | .[] | .Rows | .[] | select(.instance|test("log-cache")) | .ips' )) + echo "POSTGRES IP: $(POSTGRES_IP)" echo "LOG_CACHE IP: $(LOG_CACHE_IP)" - @echo "Pulling metricforwarder config from $(METIRCSFORWARDER_VM)..." bosh -d $(DEPLOYMENT_NAME) scp $(METIRCSFORWARDER_VM):/var/vcap/jobs/metricsforwarder/config/metricsforwarder.yml assets/metricsforwarder.yml From 1c201295d0ceaffcd293d81b2ced34c3b2e7c051 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 11 Jun 2024 13:07:43 +0200 Subject: [PATCH 129/186] Fix config issue with error handling --- src/autoscaler/metricsforwarder/config/config.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/autoscaler/metricsforwarder/config/config.go b/src/autoscaler/metricsforwarder/config/config.go index 217d43d42e..e8a964f4f0 100644 --- a/src/autoscaler/metricsforwarder/config/config.go +++ b/src/autoscaler/metricsforwarder/config/config.go @@ -90,6 +90,9 @@ func LoadConfig(reader io.Reader) (*Config, error) { dec := yaml.NewDecoder(reader) dec.KnownFields(true) err := dec.Decode(conf) + if err != nil { + return nil, err + } if os.Getenv("PORT") != "" { port := os.Getenv("PORT") @@ -100,10 +103,6 @@ func LoadConfig(reader io.Reader) (*Config, error) { conf.Server.Port = portNumber } - if err != nil { - return nil, err - } - return conf, nil } From a55ee4eeda13ece0d410697275536f1886d1708c Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 11 Jun 2024 13:08:15 +0200 Subject: [PATCH 130/186] Does not disable postgres tls by default --- ci/autoscaler/scripts/deploy-autoscaler.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index 93604d8249..0af6d13d97 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -21,7 +21,6 @@ ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\ ${autoscaler_dir}/operations/remove-metricsgateway.yml\ ${autoscaler_dir}/operations/enable-log-cache-via-uaa.yml\ ${autoscaler_dir}/operations/enable-metricsforwarder-via-syslog-agent.yml\ - ${autoscaler_dir}/operations/disable-postgres-tls-config.yml\ ${autoscaler_dir}/operations/enable-scheduler-logging.yml"} From d2079813078712c4b7d0182d11b5046013de74e7 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 11 Jun 2024 13:47:21 +0200 Subject: [PATCH 131/186] Rebase go mods to main --- devbox.json | 8 ++++---- src/autoscaler/go.mod | 10 +++++----- src/autoscaler/go.sum | 24 ++++++++++++------------ 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/devbox.json b/devbox.json index 1767886ad7..232a677b98 100644 --- a/devbox.json +++ b/devbox.json @@ -15,7 +15,6 @@ "rubocop": "latest", "rubyPackages.solargraph": "latest", "swagger-cli": "latest", - "actionlint": "1.6.27", "direnv": "2.34.0", "gnumake": "4.4", "maven": "3.8.6", @@ -29,15 +28,16 @@ "python": "latest", "cloudfoundry-cli": "8.7.10", "shellcheck": "0.10.0", - "ruby": "3.3.1", "ginkgo": "2.19.0", "bundix": "latest", "oha": "latest", "gum": "0.13.0", "act": "0.2.63", - "google-cloud-sdk": "latest", "go": "1.21.6", - "temurin-bin-17": "latest" + "actionlint": "1.7.1", + "google-cloud-sdk": "latest", + "temurin-bin-17": "latest", + "ruby": "latest" }, "shell": { "init_hook": [ diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 6bc479c1a7..8d3a801a27 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,8 +8,8 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240605163137-482a07ab6dbc - code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65 + code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240607070054-fde7ff8f79b7 + code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/go-chi/chi/v5 v5.0.12 @@ -19,7 +19,7 @@ require ( github.com/go-sql-driver/mysql v1.8.1 github.com/golang/protobuf v1.5.4 github.com/gorilla/mux v1.8.1 - github.com/gorilla/websocket v1.5.1 + github.com/gorilla/websocket v1.5.2 github.com/hashicorp/go-retryablehttp v0.7.7 github.com/jackc/pgx/v5 v5.6.0 github.com/jmoiron/sqlx v1.4.0 @@ -33,7 +33,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.1 github.com/rubyist/circuitbreaker v2.2.1+incompatible - github.com/steinfletcher/apitest v1.5.15 + github.com/steinfletcher/apitest v1.5.16 github.com/stretchr/testify v1.9.0 github.com/tedsuo/ifrit v0.0.0-20230516164442-7862c310ad26 github.com/uptrace/opentelemetry-go-extra/otelsql v0.2.4 @@ -45,7 +45,7 @@ require ( go.opentelemetry.io/otel/sdk v1.26.0 go.opentelemetry.io/otel/trace v1.26.0 golang.org/x/crypto v0.24.0 - golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc + golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 golang.org/x/net v0.26.0 golang.org/x/time v0.5.0 google.golang.org/grpc v1.64.0 diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index e76131babf..2ee0538170 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -606,10 +606,10 @@ code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a h1:X code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a/go.mod h1:/Be8VtLiCeMUoYdUzFtmW8GGkk89HAy3zD79KUXzbhs= code.cloudfoundry.org/lager/v3 v3.0.3 h1:/UTmadZfIaKuT/whEinSxK1mzRfNu1uPfvjFfGqiwzM= code.cloudfoundry.org/lager/v3 v3.0.3/go.mod h1:Zn5q1SrIuuHjEUE7xerMKt3ztunrJQCZETAo7rV0CH8= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240605163137-482a07ab6dbc h1:KzDejtGhap8QSBkGMGVQ+u9HupyXgBRsr8PQlWLPq6g= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240605163137-482a07ab6dbc/go.mod h1:wvherEAITeIr/OoqKfSZa8ngKJiglUjfgfmCSW9OGUc= -code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65 h1:0wXfAs/32wbpMKDGiV3XEHev1l9dLgIPjda+WRwJhCc= -code.cloudfoundry.org/tlsconfig v0.0.0-20240604172015-5be76e21ea65/go.mod h1:R4odSd3iXVYlIlesDHaQi9IZx2gXQryfvJmJSjPS/ug= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240607070054-fde7ff8f79b7 h1:2NXdiGqiv6XaAKpEAJBlzOkWjI2FWw/dXlSiyClCCVI= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240607070054-fde7ff8f79b7/go.mod h1:djkW8FecJqemEchtA9hUzp0QsB3V7MG3xSXADV3JL3g= +code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea h1:PjyR5Os5IVETrZkgATzSNl4R++/h+r6uQchSJ3dZsgc= +code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea/go.mod h1:RwJBA7HrvmyPNY8pRzAwlJ56WFmqJiPjxywZpxCZdWQ= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -849,8 +849,8 @@ github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gorilla/websocket v1.5.2 h1:qoW6V1GT3aZxybsbC6oLnailWnB+qTMVwMreOso9XUw= +github.com/gorilla/websocket v1.5.2/go.mod h1:0n9H61RBAcf5/38py2MCYbxzPIY9rOkpvvMT24Rqs30= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= @@ -1013,8 +1013,8 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/square/certstrap v1.3.0 h1:N9P0ZRA+DjT8pq5fGDj0z3FjafRKnBDypP0QHpMlaAk= github.com/square/certstrap v1.3.0/go.mod h1:wGZo9eE1B7WX2GKBn0htJ+B3OuRl2UsdCFySNooy9hU= -github.com/steinfletcher/apitest v1.5.15 h1:AAdTN0yMbf0VMH/PMt9uB2I7jljepO6i+5uhm1PjH3c= -github.com/steinfletcher/apitest v1.5.15/go.mod h1:mF+KnYaIkuHM0C4JgGzkIIOJAEjo+EA5tTjJ+bHXnQc= +github.com/steinfletcher/apitest v1.5.16 h1:J/ZoBmhgdzH4qfxPSw9kaXRBgzy3OsCoKh1gcc1h2zM= +github.com/steinfletcher/apitest v1.5.16/go.mod h1:mF+KnYaIkuHM0C4JgGzkIIOJAEjo+EA5tTjJ+bHXnQc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -1077,8 +1077,8 @@ go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZu go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -go.step.sm/crypto v0.46.0 h1:cuVZMpDbmEsUX+atC24+VineQr4gO+zO46MxbIVai4Y= -go.step.sm/crypto v0.46.0/go.mod h1:hcr0oTS2vGRTGSZxoVYxE+RRcsd4xP3rpqt3wPwYGqc= +go.step.sm/crypto v0.47.0 h1:LWxiKWiN0Y/A5+dq+fTIAvFYAL8oe3PQmCurjtn6ZBU= +go.step.sm/crypto v0.47.0/go.mod h1:0NMEfYrFfV5jqs8aJY5wRqIShBV8y/fyDLTseyv5xhY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -1112,8 +1112,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc h1:O9NuF4s+E/PvMIy+9IUZB9znFwUIXEWSstNjek6VpVg= -golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM= +golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= From 316032d64fdde4aa5c7e7adcdff7cdbd205a8033 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:30:18 +0200 Subject: [PATCH 132/186] remove forgotten metricsserver/metricsgateway occurences in ops file --- operations/connect_to_postgres_with_certs.yml | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/operations/connect_to_postgres_with_certs.yml b/operations/connect_to_postgres_with_certs.yml index 5f73193c51..23fd439797 100644 --- a/operations/connect_to_postgres_with_certs.yml +++ b/operations/connect_to_postgres_with_certs.yml @@ -73,26 +73,6 @@ - type: replace path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/policy_db/tls/private_key value: ((postgres_client.private_key)) -- type: replace - path: /instance_groups/name=metricsserver/jobs/name=metricsserver/properties/autoscaler/instancemetrics_db/tls/certificate - value: ((postgres_client.certificate)) -- type: replace - path: /instance_groups/name=metricsserver/jobs/name=metricsserver/properties/autoscaler/instancemetrics_db/tls/private_key - value: ((postgres_client.private_key)) - -- type: replace - path: /instance_groups/name=metricsserver/jobs/name=metricsserver/properties/autoscaler/policy_db/tls/certificate - value: ((postgres_client.certificate)) -- type: replace - path: /instance_groups/name=metricsserver/jobs/name=metricsserver/properties/autoscaler/policy_db/tls/private_key - value: ((postgres_client.private_key)) - -- type: replace - path: /instance_groups/name=metricsgateway/jobs/name=metricsgateway/properties/autoscaler/policy_db/tls/certificate - value: ((postgres_client.certificate)) -- type: replace - path: /instance_groups/name=metricsgateway/jobs/name=metricsgateway/properties/autoscaler/policy_db/tls/private_key - value: ((postgres_client.private_key)) - type: replace path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/policy_db/tls/certificate From eb6b7f2d2b63db2ba5a9d977b04038bb50721b1e Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:35:50 +0200 Subject: [PATCH 133/186] remove info that operator uses appinstancemetrics table that is no longer there --- docs/components-db-usage.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/components-db-usage.svg b/docs/components-db-usage.svg index 5654e6e6ec..5d0a7c4c08 100644 --- a/docs/components-db-usage.svg +++ b/docs/components-db-usage.svg @@ -1,4 +1,4 @@ -
GolangAPi Server
GolangAPi Server
policy_db
policy_db
binding_db
binding_db
Storedprocedure_db
Storedproce...
EventGenerator
EventGenerator
policy_db
policy_db
app_metrics_db
app_metric...
Operator
Operator
instance_metrics_db
instance_m...
scaling_engine_db
scaling...
app_metrics_db
app_metrics...
Scalingengine
Scalingengine
policy_db
policy_db
scheduler_db
schedule...
scalingengine_db
scalingen...
scheduler
scheduler
policy_db
policy_db
scheduler_db
scheduler_...
Autoscaler Components and Database Usage
Autoscaler Components and Database Usage
lock_db
lock_dbText is not SVG - cannot display \ No newline at end of file +
GolangAPi Server
GolangAPi Server
policy_db
policy_db
binding_db
binding_db
Storedprocedure_db
Storedproce...
EventGenerator
EventGenerator
policy_db
policy_db
app_metrics_db
app_metric...
Operator
Operator
scaling_engine_db
scaling...
app_metrics_db
app_metrics...
Scalingengine
Scalingengine
policy_db
policy_db
scheduler_db
schedule...
scalingengine_db
scalingen...
scheduler
scheduler
policy_db
policy_db
scheduler_db
scheduler_...
Autoscaler Components and Database Usage
Autoscaler Components and Database Usage
lock_db
lock_dbText is not SVG - cannot display \ No newline at end of file From 82125bb3aa8cca77ea46b7cdaf71fc3572c68cba Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 11 Jun 2024 17:56:12 +0200 Subject: [PATCH 134/186] WIP --- ci/autoscaler/scripts/deploy-autoscaler.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index 0af6d13d97..0ff8d53cc9 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -1,6 +1,7 @@ #! /usr/bin/env bash # shellcheck disable=SC2086,SC2034,SC2155 set -euo pipefail +set -x script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source "${script_dir}/vars.source.sh" From e3325dd7bb9bd3d371e448a08f16f6540f54635a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 22:57:12 +0000 Subject: [PATCH 135/186] fix(deps): update code.cloudfoundry.org/loggregator-agent-release/src digest to 8dca185 --- src/autoscaler/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 8d3a801a27..926961c2fc 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,7 +8,7 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240607070054-fde7ff8f79b7 + code.cloudfoundry.org/loggregator-agent-release/src 8dca1858f233 code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 From 87e8d8074be4d47a3278fb48d58b89894d82743b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 12 Jun 2024 00:03:59 +0100 Subject: [PATCH 136/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/autoscaler/go.mod | 6 +++--- src/autoscaler/go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 926961c2fc..6696413ea4 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,7 +8,7 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src 8dca1858f233 + code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240611070037-8dca1858f233 code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 @@ -101,8 +101,8 @@ require ( golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/tools v0.22.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 2ee0538170..e7873a36dd 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -606,8 +606,8 @@ code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a h1:X code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a/go.mod h1:/Be8VtLiCeMUoYdUzFtmW8GGkk89HAy3zD79KUXzbhs= code.cloudfoundry.org/lager/v3 v3.0.3 h1:/UTmadZfIaKuT/whEinSxK1mzRfNu1uPfvjFfGqiwzM= code.cloudfoundry.org/lager/v3 v3.0.3/go.mod h1:Zn5q1SrIuuHjEUE7xerMKt3ztunrJQCZETAo7rV0CH8= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240607070054-fde7ff8f79b7 h1:2NXdiGqiv6XaAKpEAJBlzOkWjI2FWw/dXlSiyClCCVI= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240607070054-fde7ff8f79b7/go.mod h1:djkW8FecJqemEchtA9hUzp0QsB3V7MG3xSXADV3JL3g= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240611070037-8dca1858f233 h1:WtTkTZV2NQxnUnsrW+VfziN3WMMbv0RNqwA930wkDgA= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240611070037-8dca1858f233/go.mod h1:gU3Si7csL/NVYz+6ppIHX8mx2C6HlZ1WPJ9uVKVfaqk= code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea h1:PjyR5Os5IVETrZkgATzSNl4R++/h+r6uQchSJ3dZsgc= code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea/go.mod h1:RwJBA7HrvmyPNY8pRzAwlJ56WFmqJiPjxywZpxCZdWQ= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= @@ -1677,10 +1677,10 @@ google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 h1:QW9+G6Fir4VcRXVH8x3LilNAb6cxBGLa6+GM4hRwexE= +google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3/go.mod h1:kdrSS/OiLkPrNUpzD4aHgCq2rVuC/YRxok32HXZ4vRE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 h1:9Xyg6I9IWQZhRVfCWjKK+l6kI0jHcPesVlMnT//aHNo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= From cbb91a255d301feb6f6348105f986ea4f2a20ae8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 22:57:19 +0000 Subject: [PATCH 137/186] chore(deps): update dependency golangci-lint to v1.59.1 --- .tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index 67a361b72d..d5555a39a7 100644 --- a/.tool-versions +++ b/.tool-versions @@ -8,7 +8,7 @@ direnv 2.34.0 gcloud 473.0.0 ginkgo 2.19.0 golang 1.21.6 -golangci-lint 1.57.2 +golangci-lint 1.59.1 java temurin-21.0.3+9.0.LTS make 4.4 maven 3.8.6 From c759931408fe057629ee3a2203aac47c5a60769b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 23:04:24 +0000 Subject: [PATCH 138/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20scripts/asdf2devbox.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- devbox.json | 2 +- devbox.lock | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/devbox.json b/devbox.json index 232a677b98..c303404bb1 100644 --- a/devbox.json +++ b/devbox.json @@ -20,7 +20,6 @@ "maven": "3.8.6", "pre-commit": "latest", "bosh-cli": "7.3.1", - "golangci-lint": "1.57.2", "yq-go": "4.43.1", "which": "latest", "jq": "latest", @@ -36,6 +35,7 @@ "go": "1.21.6", "actionlint": "1.7.1", "google-cloud-sdk": "latest", + "golangci-lint": "latest", "temurin-bin-17": "latest", "ruby": "latest" }, diff --git a/devbox.lock b/devbox.lock index e2840f1999..7f16feed54 100644 --- a/devbox.lock +++ b/devbox.lock @@ -533,51 +533,51 @@ } } }, - "golangci-lint@1.57.2": { - "last_modified": "2024-04-19T17:36:04-04:00", - "resolved": "github:NixOS/nixpkgs/92d295f588631b0db2da509f381b4fb1e74173c5#golangci-lint", + "golangci-lint@latest": { + "last_modified": "2024-05-29T10:04:41Z", + "resolved": "github:NixOS/nixpkgs/ac82a513e55582291805d6f09d35b6d8b60637a1#golangci-lint", "source": "devbox-search", - "version": "1.57.2", + "version": "1.59.0", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/v65kn6pxwkj281pyx3ns5d7rlp0sp762-golangci-lint-1.57.2", + "path": "/nix/store/3krci1kc4n89bg67yzc4kghsyrfwm7z9-golangci-lint-1.59.0", "default": true } ], - "store_path": "/nix/store/v65kn6pxwkj281pyx3ns5d7rlp0sp762-golangci-lint-1.57.2" + "store_path": "/nix/store/3krci1kc4n89bg67yzc4kghsyrfwm7z9-golangci-lint-1.59.0" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/8gn8xyjpw24670wixdy2hrqpy91fjgv5-golangci-lint-1.57.2", + "path": "/nix/store/6x2xik7aic148qrkihvr5phc2iifn669-golangci-lint-1.59.0", "default": true } ], - "store_path": "/nix/store/8gn8xyjpw24670wixdy2hrqpy91fjgv5-golangci-lint-1.57.2" + "store_path": "/nix/store/6x2xik7aic148qrkihvr5phc2iifn669-golangci-lint-1.59.0" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/ws2rl2l8p4zjqx0zqwm1r57qqjwbzxly-golangci-lint-1.57.2", + "path": "/nix/store/dgcdvx2n9dx1b73kz9d8gvx0bzqj4m0a-golangci-lint-1.59.0", "default": true } ], - "store_path": "/nix/store/ws2rl2l8p4zjqx0zqwm1r57qqjwbzxly-golangci-lint-1.57.2" + "store_path": "/nix/store/dgcdvx2n9dx1b73kz9d8gvx0bzqj4m0a-golangci-lint-1.59.0" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/n2ng9hhfwvr480wvv45837hqsivaaqdp-golangci-lint-1.57.2", + "path": "/nix/store/vbd636wrhqrnixnnyxdjp49mpv64hgfx-golangci-lint-1.59.0", "default": true } ], - "store_path": "/nix/store/n2ng9hhfwvr480wvv45837hqsivaaqdp-golangci-lint-1.57.2" + "store_path": "/nix/store/vbd636wrhqrnixnnyxdjp49mpv64hgfx-golangci-lint-1.59.0" } } }, From 1c2864eafb51796326cf5806ed389b89f1f63ac5 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Wed, 12 Jun 2024 11:10:11 +0200 Subject: [PATCH 139/186] ignore gosec findings for test-code --- src/autoscaler/api/cmd/api/api_test.go | 1 + .../eventgenerator/cmd/eventgenerator/eventgenerator_test.go | 1 + .../cmd/metricsforwarder/metricsforwarder_test.go | 1 + .../metricsgateway/cmd/metricsgateway/metricsgateway_test.go | 1 + .../metricsserver/cmd/metricsserver/metricsserver_test.go | 1 + src/autoscaler/operator/cmd/operator/operator_test.go | 1 + .../scalingengine/cmd/scalingengine/scalingengine_test.go | 1 + 7 files changed, 7 insertions(+) diff --git a/src/autoscaler/api/cmd/api/api_test.go b/src/autoscaler/api/cmd/api/api_test.go index c73fda4e3b..4a2b35689b 100644 --- a/src/autoscaler/api/cmd/api/api_test.go +++ b/src/autoscaler/api/cmd/api/api_test.go @@ -51,6 +51,7 @@ var _ = Describe("Api", func() { badfile, err := os.CreateTemp("", "bad-ap-config") Expect(err).NotTo(HaveOccurred()) runner.configPath = badfile.Name() + // #nosec G306 err = os.WriteFile(runner.configPath, []byte("bogus"), os.ModePerm) Expect(err).NotTo(HaveOccurred()) runner.Start() diff --git a/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go b/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go index 21e07bcde5..35692bd6b1 100644 --- a/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go +++ b/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go @@ -76,6 +76,7 @@ var _ = Describe("Eventgenerator", func() { badfile, err := os.CreateTemp("", "bad-mc-config") Expect(err).NotTo(HaveOccurred()) runner.configPath = badfile.Name() + // #nosec G306 err = os.WriteFile(runner.configPath, []byte("bogus"), os.ModePerm) Expect(err).NotTo(HaveOccurred()) runner.Start() diff --git a/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_test.go b/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_test.go index c9da9aaf03..9d5cf1903d 100644 --- a/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_test.go +++ b/src/autoscaler/metricsforwarder/cmd/metricsforwarder/metricsforwarder_test.go @@ -49,6 +49,7 @@ var _ = Describe("Metricsforwarder", func() { badfile, err := os.CreateTemp("", "bad-mf-config") Expect(err).NotTo(HaveOccurred()) runner.configPath = badfile.Name() + // #nosec G306 err = os.WriteFile(runner.configPath, []byte("bogus"), os.ModePerm) Expect(err).NotTo(HaveOccurred()) runner.Start() diff --git a/src/autoscaler/metricsgateway/cmd/metricsgateway/metricsgateway_test.go b/src/autoscaler/metricsgateway/cmd/metricsgateway/metricsgateway_test.go index 00ca8d4fba..eb5826b45a 100644 --- a/src/autoscaler/metricsgateway/cmd/metricsgateway/metricsgateway_test.go +++ b/src/autoscaler/metricsgateway/cmd/metricsgateway/metricsgateway_test.go @@ -43,6 +43,7 @@ var _ = Describe("Metricsgateway", func() { badfile, err := os.CreateTemp("", "bad-mc-config") Expect(err).NotTo(HaveOccurred()) runner.configPath = badfile.Name() + // #nosec G306 err = os.WriteFile(runner.configPath, []byte("bogus"), os.ModePerm) Expect(err).NotTo(HaveOccurred()) diff --git a/src/autoscaler/metricsserver/cmd/metricsserver/metricsserver_test.go b/src/autoscaler/metricsserver/cmd/metricsserver/metricsserver_test.go index eab57b378a..921185ad5a 100644 --- a/src/autoscaler/metricsserver/cmd/metricsserver/metricsserver_test.go +++ b/src/autoscaler/metricsserver/cmd/metricsserver/metricsserver_test.go @@ -47,6 +47,7 @@ var _ = Describe("MetricsServer", func() { badfile, err := os.CreateTemp("", "bad-ms-config") Expect(err).NotTo(HaveOccurred()) runner.configPath = badfile.Name() + // #nosec G306 err = os.WriteFile(runner.configPath, []byte("bogus"), os.ModePerm) Expect(err).NotTo(HaveOccurred()) runner.Start() diff --git a/src/autoscaler/operator/cmd/operator/operator_test.go b/src/autoscaler/operator/cmd/operator/operator_test.go index 4a13cda514..dfb79ee002 100644 --- a/src/autoscaler/operator/cmd/operator/operator_test.go +++ b/src/autoscaler/operator/cmd/operator/operator_test.go @@ -55,6 +55,7 @@ var _ = Describe("Operator", Serial, func() { badfile, err := os.CreateTemp("", "bad-pr-config") Expect(err).NotTo(HaveOccurred()) runner.configPath = badfile.Name() + // #nosec G306 err = os.WriteFile(runner.configPath, []byte("bogus"), os.ModePerm) Expect(err).NotTo(HaveOccurred()) runner.Start() diff --git a/src/autoscaler/scalingengine/cmd/scalingengine/scalingengine_test.go b/src/autoscaler/scalingengine/cmd/scalingengine/scalingengine_test.go index 2da28f1adc..9c19c16ecc 100644 --- a/src/autoscaler/scalingengine/cmd/scalingengine/scalingengine_test.go +++ b/src/autoscaler/scalingengine/cmd/scalingengine/scalingengine_test.go @@ -108,6 +108,7 @@ var _ = Describe("Main", func() { badfile, err := os.CreateTemp("", "bad-engine-config") Expect(err).NotTo(HaveOccurred()) runner.configPath = badfile.Name() + // #nosec G306 err = os.WriteFile(runner.configPath, []byte("bogus"), os.ModePerm) Expect(err).NotTo(HaveOccurred()) }) From b7c34a8bbb2e693cc7f5187ef95e85a2a26fa6c1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 12 Jun 2024 11:54:07 +0000 Subject: [PATCH 140/186] chore(deps): update dependency org.apache.maven.plugins:maven-checkstyle-plugin to v3.4.0 --- src/scheduler/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scheduler/pom.xml b/src/scheduler/pom.xml index 16b46b22c1..cffea4cc39 100644 --- a/src/scheduler/pom.xml +++ b/src/scheduler/pom.xml @@ -173,7 +173,7 @@ org.apache.maven.plugins maven-checkstyle-plugin - 3.3.1 + 3.4.0 checkstyle.config.path=${basedir}/../../style-guide ${basedir}/../../style-guide/google_checks.xml From ab162c8408097c4b6643149025baeb2a6fc86047 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 12 Jun 2024 10:25:29 +0200 Subject: [PATCH 141/186] Fix opts file --- ci/autoscaler/scripts/common.sh | 1 + ci/autoscaler/scripts/deploy-apps.sh | 3 +-- ci/autoscaler/tasks/deploy-apps.yml | 2 +- operations/use-cf-services.yml | 6 +----- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ci/autoscaler/scripts/common.sh b/ci/autoscaler/scripts/common.sh index c0e609c247..919ebd8695 100644 --- a/ci/autoscaler/scripts/common.sh +++ b/ci/autoscaler/scripts/common.sh @@ -15,6 +15,7 @@ function retry(){ [ "${retries}" -lt "${max_retries}" ] || { echo "ERROR: Command '$*' failed after ${max_retries} attempts"; return 1; } } + function bosh_login(){ step "bosh login" if [[ ! -d ${bbl_state_path} ]]; then diff --git a/ci/autoscaler/scripts/deploy-apps.sh b/ci/autoscaler/scripts/deploy-apps.sh index 074cf6d508..057da79c24 100755 --- a/ci/autoscaler/scripts/deploy-apps.sh +++ b/ci/autoscaler/scripts/deploy-apps.sh @@ -1,12 +1,11 @@ #! /usr/bin/env bash # shellcheck disable=SC2086,SC2034,SC2155 set -euo pipefail + script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source "${script_dir}/common.sh" source "${script_dir}/vars.source.sh" - - function deploy() { log "Deploying autoscaler apps for bosh deployment '${deployment_name}' " pushd "${autoscaler_dir}/src/autoscaler/metricsforwarder" > /dev/null diff --git a/ci/autoscaler/tasks/deploy-apps.yml b/ci/autoscaler/tasks/deploy-apps.yml index b2af432e65..d656afd8d1 100755 --- a/ci/autoscaler/tasks/deploy-apps.yml +++ b/ci/autoscaler/tasks/deploy-apps.yml @@ -5,7 +5,7 @@ image_resource: type: registry-image source: repository: ghcr.io/cloudfoundry/app-autoscaler-release-tools - tag: latest + tag: main params: diff --git a/operations/use-cf-services.yml b/operations/use-cf-services.yml index 2543f347e8..6564e1b628 100644 --- a/operations/use-cf-services.yml +++ b/operations/use-cf-services.yml @@ -4,11 +4,7 @@ host: ((metricsforwarder_host)) mtls_host: ((metricsforwarder_host)) -# Run healthendpoint on the same port of metricsforwarder -- type: remove - path: /instance_groups/name=metricsforwarder/jobs/name=metricsforwarder/properties/autoscaler/metricsforwarder/port - # Set the same port for metricsforwarder and healthenpoint routes - type: replace - path: /instance_groups/name=metricsforwarder/jobs/name=route_registrar/properties/route_registrar/routes=autoscaler_metricsforwarder_health/port + path: /instance_groups/name=metricsforwarder/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_metricsforwarder_health/port value: 6201 From 33d262ccc0f56d9379bd42d47e4d6e4bb9abb791 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Wed, 12 Jun 2024 16:38:12 +0200 Subject: [PATCH 142/186] Remove unused debugging --- ci/autoscaler/scripts/deploy-autoscaler.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index 0ff8d53cc9..145feef398 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -1,7 +1,7 @@ #! /usr/bin/env bash # shellcheck disable=SC2086,SC2034,SC2155 set -euo pipefail -set -x + script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source "${script_dir}/vars.source.sh" From 8d77715f7cd398af8729892fa776297366a92841 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 13 Jun 2024 11:00:47 +0200 Subject: [PATCH 143/186] Set shell options for error handling and pipe behavior in script. --- ci/scripts/delete-pipeline.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/scripts/delete-pipeline.sh b/ci/scripts/delete-pipeline.sh index c63261f883..fbc1965fec 100755 --- a/ci/scripts/delete-pipeline.sh +++ b/ci/scripts/delete-pipeline.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash # shellcheck disable=SC2086 # +# +set -eu -o pipefail target="app-autoscaler-release" From 727d28d98115db8f516439e7917e4a63c3730231 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 13 Jun 2024 11:05:39 +0200 Subject: [PATCH 144/186] Refactor error handling for empty metric name check --- .../assets/app/go_app/internal/app/custom_metrics.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go b/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go index 143b343fbe..b295a7c096 100644 --- a/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go +++ b/src/acceptance/assets/app/go_app/internal/app/custom_metrics.go @@ -42,7 +42,8 @@ func handleCustomMetricsEndpoint(logger logr.Logger, customMetricTest CustomMetr var err error if metricName = c.Param("name"); metricName == "" { - logger.Error(nil, "empty metric name") + err = fmt.Errorf("empty metric name") + logger.Error(err, err.Error()) Error(c, http.StatusBadRequest, "empty metric name") return } From b4f8cf56da6de69fc49ebd680506018ed1c1fc20 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Thu, 13 Jun 2024 14:16:45 +0200 Subject: [PATCH 145/186] Fix issues health work --- templates/app-autoscaler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/app-autoscaler.yml b/templates/app-autoscaler.yml index 5efa2bb159..e5721c5b12 100644 --- a/templates/app-autoscaler.yml +++ b/templates/app-autoscaler.yml @@ -675,7 +675,7 @@ instance_groups: - *metricsforwarder_mtls_public_domain - name: autoscaler_metricsforwarder_health registration_interval: 20s - port: *metricsforwarderHealthPort + port: *metricsforwarderServerPort tags: component: autoscaler_metricsforwarder_health uris: From 21b00a8e7e72e2716ff9fdd496d0768039f9a27f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 13:13:59 +0000 Subject: [PATCH 146/186] chore(deps): update actions/checkout action to v4.1.7 --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index bd5c98a1f7..2bd31b9fa1 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -27,7 +27,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: persist-credentials: false From 7ed4a0424ad116f745bd39bc1670132b128d2946 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 13:13:53 +0000 Subject: [PATCH 147/186] fix(deps): update code.cloudfoundry.org/loggregator-agent-release/src digest to 8479c7b --- src/autoscaler/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 6696413ea4..2ef2d76237 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,7 +8,7 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240611070037-8dca1858f233 + code.cloudfoundry.org/loggregator-agent-release/src 8479c7b33126 code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 From 1a0bc0b35ddb24e02d49c22e7981433420242470 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 14:21:19 +0100 Subject: [PATCH 148/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/autoscaler/go.mod | 6 +++--- src/autoscaler/go.sum | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 2ef2d76237..1477bc934d 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,7 +8,7 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src 8479c7b33126 + code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240613070107-8479c7b33126 code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 @@ -79,7 +79,7 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect - github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/openzipkin/zipkin-go v0.4.2 // indirect @@ -103,6 +103,6 @@ require ( golang.org/x/tools v0.22.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect - google.golang.org/protobuf v1.34.1 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index e7873a36dd..ec1de4d2de 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -606,8 +606,8 @@ code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a h1:X code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a/go.mod h1:/Be8VtLiCeMUoYdUzFtmW8GGkk89HAy3zD79KUXzbhs= code.cloudfoundry.org/lager/v3 v3.0.3 h1:/UTmadZfIaKuT/whEinSxK1mzRfNu1uPfvjFfGqiwzM= code.cloudfoundry.org/lager/v3 v3.0.3/go.mod h1:Zn5q1SrIuuHjEUE7xerMKt3ztunrJQCZETAo7rV0CH8= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240611070037-8dca1858f233 h1:WtTkTZV2NQxnUnsrW+VfziN3WMMbv0RNqwA930wkDgA= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240611070037-8dca1858f233/go.mod h1:gU3Si7csL/NVYz+6ppIHX8mx2C6HlZ1WPJ9uVKVfaqk= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240613070107-8479c7b33126 h1:VZNZFuhLddGbwPTiSBIc4ooEkMJD/gn7EpGgU1u002U= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240613070107-8479c7b33126/go.mod h1:lAijiGcfNgqAQDN5mk4YmWYMN92Bq7DucH1IAjW3Htc= code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea h1:PjyR5Os5IVETrZkgATzSNl4R++/h+r6uQchSJ3dZsgc= code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea/go.mod h1:RwJBA7HrvmyPNY8pRzAwlJ56WFmqJiPjxywZpxCZdWQ= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= @@ -886,8 +886,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -1077,8 +1077,8 @@ go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZu go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -go.step.sm/crypto v0.47.0 h1:LWxiKWiN0Y/A5+dq+fTIAvFYAL8oe3PQmCurjtn6ZBU= -go.step.sm/crypto v0.47.0/go.mod h1:0NMEfYrFfV5jqs8aJY5wRqIShBV8y/fyDLTseyv5xhY= +go.step.sm/crypto v0.47.1 h1:XvqgWLA1OTJXkmkmD6QSDZrmGKP4flv3PEoau60htcU= +go.step.sm/crypto v0.47.1/go.mod h1:0fz8+Am8oIwfOJgr9HHf7MwTa7Gffliv35VxDrQqU0Y= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -1744,8 +1744,8 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From 1d10a274e71a91853f3950feb4f55206bf98595d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 18:21:05 +0000 Subject: [PATCH 149/186] fix(deps): update code.cloudfoundry.org/tlsconfig digest to 075d5b1 --- src/autoscaler/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 1477bc934d..cc391451f6 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -9,7 +9,7 @@ require ( code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240613070107-8479c7b33126 - code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea + code.cloudfoundry.org/tlsconfig 075d5b187a0d dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/go-chi/chi/v5 v5.0.12 From 8b6c22742407c1d1e83d2f943d17e03810c2ddcd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 19:27:56 +0100 Subject: [PATCH 150/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/autoscaler/go.mod | 2 +- src/autoscaler/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index cc391451f6..3ccb101216 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -9,7 +9,7 @@ require ( code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240613070107-8479c7b33126 - code.cloudfoundry.org/tlsconfig 075d5b187a0d + code.cloudfoundry.org/tlsconfig v0.0.0-20240613173017-075d5b187a0d dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/go-chi/chi/v5 v5.0.12 diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index ec1de4d2de..1a28d1e5a7 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -608,8 +608,8 @@ code.cloudfoundry.org/lager/v3 v3.0.3 h1:/UTmadZfIaKuT/whEinSxK1mzRfNu1uPfvjFfGq code.cloudfoundry.org/lager/v3 v3.0.3/go.mod h1:Zn5q1SrIuuHjEUE7xerMKt3ztunrJQCZETAo7rV0CH8= code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240613070107-8479c7b33126 h1:VZNZFuhLddGbwPTiSBIc4ooEkMJD/gn7EpGgU1u002U= code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240613070107-8479c7b33126/go.mod h1:lAijiGcfNgqAQDN5mk4YmWYMN92Bq7DucH1IAjW3Htc= -code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea h1:PjyR5Os5IVETrZkgATzSNl4R++/h+r6uQchSJ3dZsgc= -code.cloudfoundry.org/tlsconfig v0.0.0-20240606172222-82aa02bc07ea/go.mod h1:RwJBA7HrvmyPNY8pRzAwlJ56WFmqJiPjxywZpxCZdWQ= +code.cloudfoundry.org/tlsconfig v0.0.0-20240613173017-075d5b187a0d h1:nedO/GTZuf/FlRvrAuXWX9FdBBEd1hWXlhn1bp6li68= +code.cloudfoundry.org/tlsconfig v0.0.0-20240613173017-075d5b187a0d/go.mod h1:qDYY1KwaNSWV7beH9J/IJR5erYD3bwO4zs+nDbpgbf8= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= From 97da6da7376cfbdb6a95877c5cf54c2561b2bf23 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 21:52:45 +0000 Subject: [PATCH 151/186] chore(deps): update dependency ruby to v3.3.3 --- .tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index d5555a39a7..edf1bbddd0 100644 --- a/.tool-versions +++ b/.tool-versions @@ -12,7 +12,7 @@ golangci-lint 1.59.1 java temurin-21.0.3+9.0.LTS make 4.4 maven 3.8.6 -ruby 3.3.2 +ruby 3.3.3 shellcheck 0.10.0 yq 4.43.1 gum 0.13.0 From 55ac84fef8dd60f7b001a2a2b646c9a9274369a5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 14 Jun 2024 01:37:18 +0000 Subject: [PATCH 152/186] fix(deps): update golang.org/x/exp digest to 7f521ea --- src/autoscaler/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 3ccb101216..51a72ed426 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -45,7 +45,7 @@ require ( go.opentelemetry.io/otel/sdk v1.26.0 go.opentelemetry.io/otel/trace v1.26.0 golang.org/x/crypto v0.24.0 - golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 + golang.org/x/exp 7f521ea00fb8 golang.org/x/net v0.26.0 golang.org/x/time v0.5.0 google.golang.org/grpc v1.64.0 From bb777e4234772711ed0f1f9c819a4fbb876aea1f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 14 Jun 2024 02:44:03 +0100 Subject: [PATCH 153/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/autoscaler/go.mod | 2 +- src/autoscaler/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 51a72ed426..6710fd7b23 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -45,7 +45,7 @@ require ( go.opentelemetry.io/otel/sdk v1.26.0 go.opentelemetry.io/otel/trace v1.26.0 golang.org/x/crypto v0.24.0 - golang.org/x/exp 7f521ea00fb8 + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 golang.org/x/net v0.26.0 golang.org/x/time v0.5.0 google.golang.org/grpc v1.64.0 diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 1a28d1e5a7..3e963207a2 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -1112,8 +1112,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM= -golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= From 4cfefefb01c717a033ea301e04e437cdf9db5326 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 14 Jun 2024 03:58:09 +0000 Subject: [PATCH 154/186] chore(deps): update github/codeql-action action to v3.25.10 --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 2bd31b9fa1..09ba21e548 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -54,6 +54,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # v3.25.8 + uses: github/codeql-action/upload-sarif@23acc5c183826b7a8a97bce3cecc52db901f8251 # v3.25.10 with: sarif_file: results.sarif From f68e2246e56d7ab1297e5af29bf644fa1d55a6da Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:13:00 +0200 Subject: [PATCH 155/186] replace direct credhub references with variables in manifest template --- ci/autoscaler/scripts/deploy-autoscaler.sh | 11 +++++++---- templates/app-autoscaler.yml | 12 ++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index 2dcfc864af..a851de0502 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -91,10 +91,13 @@ function create_manifest(){ -v admin_password="$(credhub get -n /bosh-autoscaler/cf/cf_admin_password -q)" \ -v cf_client_id=autoscaler_client_id \ -v cf_client_secret=autoscaler_client_secret \ - -v eventgenerator_uaa_client_id=firehose_exporter \ - -v eventgenerator_uaa_client_secret="$(credhub get -n /bosh-autoscaler/cf/uaa_clients_firehose_exporter_secret --quiet)"\ - -v eventgenerator_uaa_skip_ssl_validation=true \ - -v skip_ssl_validation=true \ + -v log_cache_syslog_tls_ca="$(credhub get -n /bosh-autoscaler/cf/log_cache_syslog_tls --key ca --quiet)"\ + -v syslog_agent_log_cache_tls_certificate="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key certificate --quiet)"\ + -v syslog_agent_log_cache_tls_key="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key private_key --quiet)"\ + -v metricscollector_ca_cert="$(credhub get -n /bosh-autoscaler/cf/log_cache --key ca --quiet)"\ + -v metricscollector_client_cert="$(credhub get -n /bosh-autoscaler/cf/log_cache --key certificate --quiet)"\ + -v metricscollector_client_key="$(credhub get -n /bosh-autoscaler/cf/log_cache --key private_key --quiet)"\ + -v skip_ssl_validation=true \ > "${tmp_manifest_file}" # shellcheck disable=SC2064 diff --git a/templates/app-autoscaler.yml b/templates/app-autoscaler.yml index 0814be9d7e..5225bcab4d 100644 --- a/templates/app-autoscaler.yml +++ b/templates/app-autoscaler.yml @@ -446,9 +446,9 @@ instance_groups: client_key: ((!scalingengine_client_cert.private_key)) host: *scalingengine_domain metricscollector: - ca_cert: ((/bosh-autoscaler/cf/log_cache.ca)) - client_cert: ((/bosh-autoscaler/cf/log_cache.certificate)) - client_key: ((/bosh-autoscaler/cf/log_cache.private_key)) + ca_cert: ((metricscollector_ca_cert)) + client_cert: ((metricscollector_client_cert)) + client_key: ((metricscollector_client_key)) port: 8080 host: logcache - name: route_registrar @@ -532,9 +532,9 @@ instance_groups: # reusing these certificates here is a workaround so that we don't need to generate own ones. # the problem is that when we generate own certificates (see variables section of app-autoscaler.yml), # we have no possibility to reuse the CA from CF to issue new certificates. - ca: ((/bosh-autoscaler/cf/log_cache_syslog_tls.ca)) - cert: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.certificate)) - key: ((/bosh-autoscaler/cf/syslog_agent_log_cache_tls.private_key)) + ca: ((log_cache_syslog_tls_ca)) + cert: ((syslog_agent_log_cache_tls_certificate)) + key: ((syslog_agent_log_cache_tls_key)) metrics: # connection for metric scrapers, here are dummy values configured since the /metrics endpoint can't be disabled via configuration ca_cert: ((!loggr_syslog_binding_cache_metrics.ca)) cert: ((!loggr_syslog_binding_cache_metrics.certificate)) From 478d7d47fee715c28dea4bfdc6e0b1d0c8fdea87 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:15:39 +0200 Subject: [PATCH 156/186] remove example operations --- example/.gitkeep | 0 example/operation/add-extra-plan.yml | 12 - example/operation/alternative-vm-type.yml | 20 -- example/operation/cf-mysql-db.yml | 146 -------- .../disable-basicauth-on-health-endpoints.yml | 65 ---- .../enable-name-based-deployments.yml | 328 ------------------ example/operation/enable-nats-tls.yml | 69 ---- example/operation/external-db.yml | 134 ------- .../instance-identity-cert-from-cf.yml | 21 -- .../operation/postgres-persistent-disk.yml | 4 - example/operation/set-release-version.yml | 7 - 11 files changed, 806 deletions(-) delete mode 100644 example/.gitkeep delete mode 100644 example/operation/add-extra-plan.yml delete mode 100644 example/operation/alternative-vm-type.yml delete mode 100644 example/operation/cf-mysql-db.yml delete mode 100644 example/operation/disable-basicauth-on-health-endpoints.yml delete mode 100644 example/operation/enable-name-based-deployments.yml delete mode 100644 example/operation/enable-nats-tls.yml delete mode 100644 example/operation/external-db.yml delete mode 100644 example/operation/instance-identity-cert-from-cf.yml delete mode 100644 example/operation/postgres-persistent-disk.yml delete mode 100644 example/operation/set-release-version.yml diff --git a/example/.gitkeep b/example/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/example/operation/add-extra-plan.yml b/example/operation/add-extra-plan.yml deleted file mode 100644 index d10e0ec13f..0000000000 --- a/example/operation/add-extra-plan.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/apiserver/broker/server/catalog/services/0/plans/- - value: - id: acceptance-standard - name: acceptance-standard - plan_updateable: false - description: This is the standard service plan for the Auto-Scaling service. - -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/apiserver/broker/server/catalog/services/0/plans/0/plan_updateable? - value: true diff --git a/example/operation/alternative-vm-type.yml b/example/operation/alternative-vm-type.yml deleted file mode 100644 index 742f3a256d..0000000000 --- a/example/operation/alternative-vm-type.yml +++ /dev/null @@ -1,20 +0,0 @@ - -- type: replace - path: /instance_groups/name=asactors/vm_type - value: ((alternative_vm_type)) - -- type: replace - path: /instance_groups/name=asmetrics/vm_type - value: ((alternative_vm_type)) - -- type: replace - path: /instance_groups/name=asnozzle/vm_type - value: ((alternative_vm_type)) - -- type: replace - path: /instance_groups/name=asapi/vm_type - value: ((alternative_vm_type)) - -- type: replace - path: /instance_groups/name=postgres_autoscaler/vm_type - value: ((alternative_vm_type)) diff --git a/example/operation/cf-mysql-db.yml b/example/operation/cf-mysql-db.yml deleted file mode 100644 index 9cbb8e0b94..0000000000 --- a/example/operation/cf-mysql-db.yml +++ /dev/null @@ -1,146 +0,0 @@ - -- type: remove - path: /instance_groups/name=postgres_autoscaler - -- type: replace - path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/- - value: - domain: sql-db.service.cf.internal - targets: - - deployment: cf - domain: bosh - instance_group: database - network: default - query: '*' - -# asactors/scalingengine -- type: replace - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/scalingengine_db - value: &external_database - sslmode: &sslmode "false" - tls: &database_tls - ca: "((mysql_server_certificate.ca))" - databases: - - name: autoscaler - tag: default - address: sql-db.service.cf.internal - db_scheme: mysql - port: 3306 - roles: - - name: autoscaler - password: ((autoscaler_database_password)) - tag: default - -- type: replace - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/scalingengine_db_connection_config - value: &databaseConnectionConfig - max_open_connections: 100 - max_idle_connections: 10 - connection_max_lifetime: 60s - -- type: replace - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/scheduler_db - value: *external_database -- type: replace - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/scheduler_db_connection_config - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig - -# asactors/scheduler -- type: replace - path: /instance_groups/name=asactors/jobs/name=scheduler/properties/autoscaler/policy_db - value: *external_database - -- type: replace - path: /instance_groups/name=asactors/jobs/name=scheduler/properties/autoscaler/scheduler_db - value: *external_database - -#asactors/operator -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/appmetrics_db - value: *external_database -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/appmetrics_db_connection_config - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/instancemetrics_db - value: *external_database -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/instancemetrics_db_connection_config - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/scalingengine_db - value: *external_database -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/scalingengine_db_connection_config - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/lock_db - value: *external_database -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/lock_db_connection_config - value: *databaseConnectionConfig - -# asmetrics/eventgenerator -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/appmetrics_db - value: *external_database -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/appmetrics_db_connection_config - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig - -# asapi/golangapiserver -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/binding_db - value: *external_database -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/binding_db_connection_config - value: *databaseConnectionConfig - -# asapi/metricsforwarder -- type: replace - path: /instance_groups/name=asapi/jobs/name=metricsforwarder/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asapi/jobs/name=metricsforwarder/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig - - - - - - - - - - - diff --git a/example/operation/disable-basicauth-on-health-endpoints.yml b/example/operation/disable-basicauth-on-health-endpoints.yml deleted file mode 100644 index d079d28174..0000000000 --- a/example/operation/disable-basicauth-on-health-endpoints.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- -- type: remove - path: /variables/name=autoscaler_scheduler_health_password - -- type: remove - path: /variables/name=autoscaler_eventgenerator_health_password - -- type: remove - path: /variables/name=autoscaler_metricsforwarder_health_password - -- type: remove - path: /variables/name=autoscaler_operator_health_password - -- type: remove - path: /variables/name=autoscaler_scalingengine_health_password - -# asactors -- type: remove - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/scalingengine/health/username - -- type: remove - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/scalingengine/health/password - -- type: remove - path: /instance_groups/name=asactors/jobs/name=scheduler/properties/autoscaler/scheduler/health/basicAuthEnabled - -- type: remove - path: /instance_groups/name=asactors/jobs/name=scheduler/properties/autoscaler/scheduler/health/username - -- type: remove - path: /instance_groups/name=asactors/jobs/name=scheduler/properties/autoscaler/scheduler/health/password - -- type: remove - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/operator/health/username - -- type: remove - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/operator/health/password - -#asmetrics -- type: remove - path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/eventgenerator/health/username - -- type: remove - path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/eventgenerator/health/password - -# asapi -- type: remove - path: /instance_groups/name=asapi/jobs/name=metricsforwarder/properties/autoscaler/metricsforwarder/health/username - -- type: remove - path: /instance_groups/name=asapi/jobs/name=metricsforwarder/properties/autoscaler/metricsforwarder/health/password - -# route_registrars - -- type: remove - path: /instance_groups/name=asactors/jobs/name=route_registrar - -- type: remove - path: /instance_groups/name=asmetrics/jobs/name=route_registrar - -- type: remove - path: /instance_groups/name=asnozzle/jobs/name=route_registrar - -- type: remove - path: /instance_groups/name=asapi/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_metricsforwarder_health diff --git a/example/operation/enable-name-based-deployments.yml b/example/operation/enable-name-based-deployments.yml deleted file mode 100644 index 9b450e8f4f..0000000000 --- a/example/operation/enable-name-based-deployments.yml +++ /dev/null @@ -1,328 +0,0 @@ -# This ops file is used to change the deployment ID of the bosh deployment to enable multiple deployments -# with different dns names (independent name spaces) -# Parameters: -# deployment_name: Unique deployment name. -# Anchors here is to make the ops files simpler using anchors for replacement instead of multiple commands in the ops file. ---- - -- type: replace - path: /name - value: ((deployment_name)) - anchors: - domains: - postgres: &postgres_domain ((deployment_name)).autoscalerpostgres.service.cf.internal - api: &api_domain ((deployment_name)).apiserver.service.cf.internal - scheduler: &scheduler_domain ((deployment_name)).autoscalerscheduler.service.cf.internal - servicebroker: &servicebroker_domain ((deployment_name)).servicebroker.service.cf.internal - eventgenerator: &eventgenerator_domain ((deployment_name)).eventgenerator.service.cf.internal - scalingengine: &scalingengine_domain ((deployment_name)).scalingengine.service.cf.internal - public_domains: - metricsforwarder: &metricsforwarder_public_domain ((deployment_name))metrics.((system_domain)) - metricsforwarder_mtls: &metricsforwarder_mtls_public_domain ((deployment_name))-metricsforwarder-mtls.((system_domain)) - servicebroker: &servicebroker_public_domain ((deployment_name))servicebroker.((system_domain)) - -# Addons -- type: replace - path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties - value: - aliases: - - domain: *postgres_domain - targets: - - query: '*' - instance_group: postgres_autoscaler - deployment: ((deployment_name)) - network: default - domain: bosh - - domain: *api_domain - targets: - - query: '*' - instance_group: asapi - deployment: ((deployment_name)) - network: default - domain: bosh - - domain: *scheduler_domain - targets: - - query: '*' - instance_group: asactors - deployment: ((deployment_name)) - network: default - domain: bosh - - domain: *servicebroker_domain - targets: - - query: '*' - instance_group: asapi - deployment: ((deployment_name)) - network: default - domain: bosh - - domain: *eventgenerator_domain - targets: - - query: '*' - instance_group: asmetrics - deployment: ((deployment_name)) - network: default - domain: bosh - - domain: *scalingengine_domain - targets: - - query: '*' - instance_group: asactors - deployment: ((deployment_name)) - network: default - domain: bosh - # Cf internal names - - domain: nats.service.cf.internal - targets: - - deployment: cf - domain: bosh - instance_group: nats - network: default - query: '*' - - domain: reverse-log-proxy.service.cf.internal - targets: - - query: '*' - instance_group: log-api - deployment: cf - network: default - domain: bosh - - domain: _.nats.service.cf.internal - targets: - - deployment: cf - domain: bosh - instance_group: nats - network: default - query: _ - - domain: logcache - targets: - - deployment: cf - domain: bosh - instance_group: log-cache - network: default - query: '*' - - domain: log-cache.service.cf.internal - targets: - - deployment: cf - domain: bosh - instance_group: log-cache - network: default - query: '*' - - -# Set the name of the database so it does not use the default name -- type: replace - path: /instance_groups/name=postgres_autoscaler/jobs/name=postgres/properties/databases/address? - value: *postgres_domain - -########### -# asactors - scalingengine -- type: replace - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/scalingengine_db/address? - value: *postgres_domain - -- type: replace - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/policy_db/address? - value: *postgres_domain - -- type: replace - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/scheduler_db/address? - value: *postgres_domain - -########### -# asactors - scheduler -- type: replace - path: /instance_groups/name=asactors/jobs/name=scheduler/properties/autoscaler/scheduler/scaling_engine/host? - value: *scalingengine_domain - -- type: replace - path: /instance_groups/name=asactors/jobs/name=scheduler/properties/autoscaler/scheduler_db/address? - value: *postgres_domain - -- type: replace - path: /instance_groups/name=asactors/jobs/name=scheduler/properties/autoscaler/policy_db/address? - value: *postgres_domain - -########### -# asactors - operator -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/policy_db/address? - value: *postgres_domain - -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/instancemetrics_db/address? - value: *postgres_domain - -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/appmetrics_db/address? - value: *postgres_domain - -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/scalingengine_db/address? - value: *postgres_domain - -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/operator/scaling_engine/host? - value: *scalingengine_domain - -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/scheduler.host? - value: *scheduler_domain - -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/lock_db/address? - value: *postgres_domain - - -########### -# asmetrics - eventgenerator -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/eventgenerator/scaling_engine/host - value: *scalingengine_domain - -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/appmetrics_db/address? - value: *postgres_domain - -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/policy_db/address? - value: *postgres_domain - -########### -# asapi - golangapiserver -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/binding_db/address? - value: *postgres_domain - -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/policy_db/address? - value: *postgres_domain - -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/apiserver/scheduler/host? - value: *scheduler_domain - -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/apiserver/scaling_engine/host? - value: *scalingengine_domain - -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/apiserver/event_generator/host? - value: *eventgenerator_domain - -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/apiserver/broker/server/catalog/services/id=autoscaler-guid/name - value: ((deployment_name)) - -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/apiserver/metrics_forwarder/host? - value: *metricsforwarder_public_domain - -## values originally in instance-identity-cert-from-cf.yml -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/apiserver/metrics_forwarder/mtls_host? - value: *metricsforwarder_mtls_public_domain - -- type: replace - path: /instance_groups/name=asapi/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_metrics_forwarder_mtls/uris - value: [ *metricsforwarder_mtls_public_domain ] - -########### -# asapi - metricsforwarder -- type: replace - path: /instance_groups/name=asapi/jobs/name=metricsforwarder/properties/autoscaler/policy_db/address? - value: *postgres_domain - -# route_registrar external/Public dns entries -## asapi -- type: replace - path: /instance_groups/name=asapi/jobs/name=route_registrar/properties/route_registrar/routes/name=api_server/uris - value: [ ((deployment_name)).((system_domain)) ] - -- type: replace - path: /instance_groups/name=asapi/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_service_broker/uris - value: [ *servicebroker_public_domain ] - -- type: replace - path: /instance_groups/name=asapi/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_metrics_forwarder/uris - value: [ *metricsforwarder_public_domain ] - -- type: replace - path: /instance_groups/name=asapi/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_metricsforwarder_health/uris - value: [ ((deployment_name))-metricsforwarder.((system_domain)) ] - -## asmetrics - -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_eventgenerator_health/uris - value: [ ((deployment_name))-eventgenerator.((system_domain)) ] - -## asactors - -- type: replace - path: /instance_groups/name=asactors/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_scalingengine_health/uris - value: [ ((deployment_name))-scalingengine.((system_domain)) ] - -- type: replace - path: /instance_groups/name=asactors/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_operator_health/uris - value: [ ((deployment_name))-operator.((system_domain)) ] - -- type: replace - path: /instance_groups/name=asactors/jobs/name=route_registrar/properties/route_registrar/routes/name=autoscaler_scheduler_health/uris - value: [ ((deployment_name))-scheduler.((system_domain)) ] - - -# variables -# - scalingengine_server -- type: replace - path: /variables/name=scalingengine_server/options/common_name - value: *scalingengine_domain -- type: replace - path: /variables/name=scalingengine_server/options/alternative_names - value: [*scalingengine_domain] - -# - eventgenerator_server -- type: replace - path: /variables/name=eventgenerator_server/options/common_name - value: *eventgenerator_domain -- type: replace - path: /variables/name=eventgenerator_server/options/alternative_names - value: [*eventgenerator_domain] - -# - apiserver_server -- type: replace - path: /variables/name=apiserver_server/options/common_name - value: *api_domain -- type: replace - path: /variables/name=apiserver_server/options/alternative_names - value: [*api_domain] - -# - servicebroker_server -- type: replace - path: /variables/name=servicebroker_server/options/common_name - value: *servicebroker_domain -- type: replace - path: /variables/name=servicebroker_server/options/alternative_names - value: [*servicebroker_domain] - -# - servicebroker_server -- type: replace - path: /variables/name=servicebroker_server/options/common_name - value: *servicebroker_domain -- type: replace - path: /variables/name=servicebroker_server/options/alternative_names - value: [*servicebroker_domain] - -# - scheduler_server -- type: replace - path: /variables/name=scheduler_server/options/common_name - value: *scheduler_domain -- type: replace - path: /variables/name=scheduler_server/options/alternative_names - value: [*scheduler_domain] - -# - postgres_server -- type: replace - path: /variables/name=postgres_server/options/common_name - value: *postgres_domain -- type: replace - path: /variables/name=postgres_server/options/alternative_names - value: [*postgres_domain] - - diff --git a/example/operation/enable-nats-tls.yml b/example/operation/enable-nats-tls.yml deleted file mode 100644 index a7f26957d5..0000000000 --- a/example/operation/enable-nats-tls.yml +++ /dev/null @@ -1,69 +0,0 @@ ---- -# routing api from certs -- type: replace - path: /instance_groups/name=asactors/jobs/name=route_registrar/properties/nats? - value: - tls: - enabled: true - client_cert: ((/bosh-autoscaler/cf/nats_client_cert.certificate)) - client_key: ((/bosh-autoscaler/cf/nats_client_cert.private_key)) - -- type: remove - path: /instance_groups/name=asactors/jobs/name=route_registrar/consumes/nats - -- type: replace - path: /instance_groups/name=asactors/jobs/name=route_registrar/consumes/nats-tls? - value: - from: nats-tls - deployment: cf - -- type: replace - path: /instance_groups/name=asapi/jobs/name=route_registrar/properties/nats? - value: - tls: - enabled: true - client_cert: ((/bosh-autoscaler/cf/nats_client_cert.certificate)) - client_key: ((/bosh-autoscaler/cf/nats_client_cert.private_key)) - -- type: remove - path: /instance_groups/name=asapi/jobs/name=route_registrar/consumes/nats - -- type: replace - path: /instance_groups/name=asapi/jobs/name=route_registrar/consumes/nats-tls? - value: - from: nats-tls - deployment: cf - -- type: replace - path: /instance_groups/name=asnozzle/jobs/name=route_registrar/properties/nats? - value: - tls: - enabled: true - client_cert: ((/bosh-autoscaler/cf/nats_client_cert.certificate)) - client_key: ((/bosh-autoscaler/cf/nats_client_cert.private_key)) - -- type: remove - path: /instance_groups/name=asnozzle/jobs/name=route_registrar/consumes/nats - -- type: replace - path: /instance_groups/name=asnozzle/jobs/name=route_registrar/consumes/nats-tls? - value: - from: nats-tls - deployment: cf - -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=route_registrar/properties/nats? - value: - tls: - enabled: true - client_cert: ((/bosh-autoscaler/cf/nats_client_cert.certificate)) - client_key: ((/bosh-autoscaler/cf/nats_client_cert.private_key)) - -- type: remove - path: /instance_groups/name=asmetrics/jobs/name=route_registrar/consumes/nats - -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=route_registrar/consumes/nats-tls? - value: - from: nats-tls - deployment: cf diff --git a/example/operation/external-db.yml b/example/operation/external-db.yml deleted file mode 100644 index 8359349097..0000000000 --- a/example/operation/external-db.yml +++ /dev/null @@ -1,134 +0,0 @@ - -- type: remove - path: /instance_groups/name=postgres_autoscaler - -# asactors/scalingengine -- type: replace - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/scalingengine_db - value: &external_database - sslmode: &sslmode ((database.sslmode)) - tls: &database_tls - ca: ((database.tls.ca)) - databases: - - name: ((database.name)) - tag: default - address: ((database.host)) - db_scheme: ((database.scheme)) - port: ((database.port)) - sslmode: ((database.sslmode)) - roles: - - name: ((database.username)) - password: ((database.password)) - tag: default - -- type: replace - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/scalingengine_db_connection_config - value: &databaseConnectionConfig - max_open_connections: 100 - max_idle_connections: 10 - connection_max_lifetime: 60s - -- type: replace - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/scheduler_db - value: *external_database -- type: replace - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/scheduler_db_connection_config - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig - -# asactors/scheduler -- type: replace - path: /instance_groups/name=asactors/jobs/name=scheduler/properties/autoscaler/policy_db - value: *external_database - -- type: replace - path: /instance_groups/name=asactors/jobs/name=scheduler/properties/autoscaler/scheduler_db - value: *external_database - -#asactors/operator -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/appmetrics_db - value: *external_database -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/appmetrics_db_connection_config - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/instancemetrics_db - value: *external_database -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/instancemetrics_db_connection_config - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/scalingengine_db - value: *external_database -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/scalingengine_db_connection_config - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/lock_db - value: *external_database -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/lock_db_connection_config - value: *databaseConnectionConfig - -# asmetrics/eventgenerator -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/appmetrics_db - value: *external_database -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/appmetrics_db_connection_config - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig - -# asapi/golangapiserver -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/binding_db - value: *external_database -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/binding_db_connection_config - value: *databaseConnectionConfig -# asapi/metricsforwarder -- type: replace - path: /instance_groups/name=asapi/jobs/name=metricsforwarder/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asapi/jobs/name=metricsforwarder/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig - - - - - - - - - - diff --git a/example/operation/instance-identity-cert-from-cf.yml b/example/operation/instance-identity-cert-from-cf.yml deleted file mode 100644 index d6c1bad56c..0000000000 --- a/example/operation/instance-identity-cert-from-cf.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -- type: replace - path: /instance_groups/name=asapi/jobs/name=metricsforwarder/properties/autoscaler/metricsforwarder/metricshandler? - value: - tls: - ca_cert: ((/bosh-autoscaler/cf/diego_instance_identity_ca.certificate)) - -- type: replace - path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/apiserver/metrics_forwarder/mtls_host? - value: autoscaler-metricsforwarder-mtls.((system_domain)) - -- type: replace - path: /instance_groups/name=asapi/jobs/name=route_registrar/properties/route_registrar/routes/- - value: - name: autoscaler_metrics_forwarder_mtls - registration_interval: 20s - port: 6201 - tags: - component: autoscaler_metrics_forwarder_mtls - uris: - - autoscaler-metricsforwarder-mtls.((system_domain)) diff --git a/example/operation/postgres-persistent-disk.yml b/example/operation/postgres-persistent-disk.yml deleted file mode 100644 index 34402d5050..0000000000 --- a/example/operation/postgres-persistent-disk.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- type: replace - path: /instance_groups/name=postgres_autoscaler/persistent_disk_type? - value: 10GB diff --git a/example/operation/set-release-version.yml b/example/operation/set-release-version.yml deleted file mode 100644 index 303c5267f0..0000000000 --- a/example/operation/set-release-version.yml +++ /dev/null @@ -1,7 +0,0 @@ -# This can be used to set the release version of the autoscaler and is primarily used in our CI to modify the release version in our dev pipeline. -# Parameters: -# app_autoscaler_version: Version numbering of the release to deploy. ---- -- type: replace - path: /releases/name=app-autoscaler/version? - value: ((app_autoscaler_version)) \ No newline at end of file From d8894de6fb0942cca8b20c8a183a321e942443af Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:17:59 +0200 Subject: [PATCH 157/186] revert "disable requirement of succeeded jobs for testing" --- ci/autoscaler/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index 78e788e264..268836f767 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -215,7 +215,7 @@ jobs: - in_parallel: - get: bbl-state - get: app-autoscaler-release - # passed: [unit-tests, integration-tests] disabled this here just for testing + passed: [unit-tests, integration-tests] trigger: true - get: ci - task: make-prerelease @@ -268,7 +268,7 @@ jobs: - in_parallel: - get: bbl-state - get: app-autoscaler-release - # passed: [unit-tests, integration-tests] disabled this here just for testing + passed: [unit-tests, integration-tests] trigger: true - get: ci - task: make-prerelease From d9c98eb9847839dbfbddb0323dfd4d9ab6bf8883 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:20:32 +0200 Subject: [PATCH 158/186] delete test related to metricsserver --- spec/jobs/eventgenerator/eventgenerator_spec.rb | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/spec/jobs/eventgenerator/eventgenerator_spec.rb b/spec/jobs/eventgenerator/eventgenerator_spec.rb index 9627477cfd..e868cb2cd5 100644 --- a/spec/jobs/eventgenerator/eventgenerator_spec.rb +++ b/spec/jobs/eventgenerator/eventgenerator_spec.rb @@ -52,22 +52,6 @@ ) end - describe "when using log-cache via grpc" do - before do - properties["autoscaler"]["eventgenerator"] = { - "metricscollector" => { - "host" => "logcache", - "port" => "8080" - } - } - end - - it "should not add https protocol to metric_collector_url" do - expect(rendered_template["metricCollector"]["metric_collector_url"]) - .not_to include("http") - end - end - describe "when using log-cache via https/uaa" do before do properties["autoscaler"]["eventgenerator"] = { From fc345a124953440f941bdf398f3374f8d8ec9c43 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:25:35 +0200 Subject: [PATCH 159/186] restore accidentally deleted diagram --- docs/images/autoscaler.svg | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 docs/images/autoscaler.svg diff --git a/docs/images/autoscaler.svg b/docs/images/autoscaler.svg deleted file mode 100644 index 53c8910909..0000000000 --- a/docs/images/autoscaler.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file From 7b3449181980467cfd362d99239557d7fcfdf902 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:37:33 +0200 Subject: [PATCH 160/186] remove leftover occurences of instancemetrics_db --- operations/cf-mysql-db.yml | 32 +------------------------------- operations/external-db.yml | 30 ------------------------------ spec/fixtures/operator.yml | 18 +----------------- spec/fixtures/scalingengine.yml | 18 +----------------- src/autoscaler/db/db.go | 9 --------- src/autoscaler/generate-fakes.go | 1 - templates/app-autoscaler.yml | 1 - 7 files changed, 3 insertions(+), 106 deletions(-) diff --git a/operations/cf-mysql-db.yml b/operations/cf-mysql-db.yml index 441574e742..40e8b4ee45 100644 --- a/operations/cf-mysql-db.yml +++ b/operations/cf-mysql-db.yml @@ -29,7 +29,7 @@ roles: - name: autoscaler password: ((autoscaler_database_password)) - tag: default + tag: default - type: replace path: /instance_groups/name=asactors/jobs/name=scalingengine/properties/autoscaler/scalingengine_db_connection_config @@ -76,13 +76,6 @@ path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/appmetrics_db_connection_config value: *databaseConnectionConfig -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/instancemetrics_db - value: *external_database -- type: replace - path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/instancemetrics_db_connection_config - value: *databaseConnectionConfig - - type: replace path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/scalingengine_db value: *external_database @@ -97,21 +90,6 @@ path: /instance_groups/name=asactors/jobs/name=operator/properties/autoscaler/lock_db_connection_config value: *databaseConnectionConfig -# asmetrics/metricsserver -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/instancemetrics_db - value: *external_database -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/instancemetrics_db_connection_config - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asmetrics/jobs/name=metricsserver/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig - # asmetrics/eventgenerator - type: replace path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/appmetrics_db @@ -127,14 +105,6 @@ path: /instance_groups/name=asmetrics/jobs/name=eventgenerator/properties/autoscaler/policy_db_connection_config value: *databaseConnectionConfig -# asnozzle/metricsgateway -- type: replace - path: /instance_groups/name=asnozzle/jobs/name=metricsgateway/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=asnozzle/jobs/name=metricsgateway/properties/autoscaler/policy_db_connection_config - value: *databaseConnectionConfig - # asapi/golangapiserver - type: replace path: /instance_groups/name=asapi/jobs/name=golangapiserver/properties/autoscaler/policy_db diff --git a/operations/external-db.yml b/operations/external-db.yml index 0fd366ce98..7e2a43ae6a 100644 --- a/operations/external-db.yml +++ b/operations/external-db.yml @@ -65,13 +65,6 @@ path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/appmetrics_db_connection_config? value: *databaseConnectionConfig -- type: replace - path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/instancemetrics_db - value: *external_database -- type: replace - path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/instancemetrics_db_connection_config? - value: *databaseConnectionConfig - - type: replace path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/scalingengine_db value: *external_database @@ -86,21 +79,6 @@ path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/lock_db_connection_config? value: *databaseConnectionConfig -# metricsserver/metricsserver -- type: replace - path: /instance_groups/name=metricsserver/jobs/name=metricsserver/properties/autoscaler/instancemetrics_db - value: *external_database -- type: replace - path: /instance_groups/name=metricsserver/jobs/name=metricsserver/properties/autoscaler/instancemetrics_db_connection_config? - value: *databaseConnectionConfig - -- type: replace - path: /instance_groups/name=metricsserver/jobs/name=metricsserver/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=metricsserver/jobs/name=metricsserver/properties/autoscaler/policy_db_connection_config? - value: *databaseConnectionConfig - # eventgenerator/eventgenerator - type: replace path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/appmetrics_db @@ -123,14 +101,6 @@ path: /instance_groups/name=eventgenerator/jobs/name=eventgenerator/properties/autoscaler/lock_db_connection_config? value: *databaseConnectionConfig -# metricsgateway/metricsgateway -- type: replace - path: /instance_groups/name=metricsgateway/jobs/name=metricsgateway/properties/autoscaler/policy_db - value: *external_database -- type: replace - path: /instance_groups/name=metricsgateway/jobs/name=metricsgateway/properties/autoscaler/policy_db_connection_config? - value: *databaseConnectionConfig - # apiserver/golangapiserver - type: replace path: /instance_groups/name=apiserver/jobs/name=golangapiserver/properties/autoscaler/policy_db diff --git a/spec/fixtures/operator.yml b/spec/fixtures/operator.yml index 8f85c4c253..cbcf24ca7e 100644 --- a/spec/fixtures/operator.yml +++ b/spec/fixtures/operator.yml @@ -14,22 +14,6 @@ autoscaler: ca: BEGIN---CA---END certificate: BEGIN---CERT---END private_key: BEGIN---KEY---END - instancemetrics_db: - address: 10.11.137.101 - databases: - - name: foo - tag: default - db_scheme: postgres - port: 5432 - tls.ca: aa - roles: - - name: foo - password: default - tag: default - tls: - ca: BEGIN---CA---END - certificate: BEGIN---CERT---END - private_key: BEGIN---KEY---END appmetrics_db: address: 10.11.137.101 databases: @@ -81,4 +65,4 @@ autoscaler: client_id: client_id secret: uaa_secret uaa_api: https://login.cf.domain/uaa - grant_type: ALLOW_ALL \ No newline at end of file + grant_type: ALLOW_ALL diff --git a/spec/fixtures/scalingengine.yml b/spec/fixtures/scalingengine.yml index f8d171de06..cd400dd448 100644 --- a/spec/fixtures/scalingengine.yml +++ b/spec/fixtures/scalingengine.yml @@ -14,22 +14,6 @@ autoscaler: ca: BEGIN---CA---END certificate: BEGIN---CERT---END private_key: BEGIN---KEY---END - instancemetrics_db: - address: 10.11.137.101 - databases: - - name: foo - tag: default - db_scheme: postgres - port: 5432 - tls.ca: aa - roles: - - name: foo - password: default - tag: default - tls: - ca: BEGIN---CA---END - certificate: BEGIN---CERT---END - private_key: BEGIN---KEY---END appmetrics_db: address: 10.11.137.101 databases: @@ -94,4 +78,4 @@ autoscaler: auth_endpoint: https://login.cf.domain client_id: "client_id" secret: "uaa_secret" - uaa_api: https://login.cf.domain/uaa \ No newline at end of file + uaa_api: https://login.cf.domain/uaa diff --git a/src/autoscaler/db/db.go b/src/autoscaler/db/db.go index 5fbe95550e..34afd8a8a0 100644 --- a/src/autoscaler/db/db.go +++ b/src/autoscaler/db/db.go @@ -43,15 +43,6 @@ type DatabaseConfig struct { ConnectionMaxIdleTime time.Duration `yaml:"connection_max_idletime"` } -type InstanceMetricsDB interface { - healthendpoint.DatabaseStatus - RetrieveInstanceMetrics(appid string, instanceIndex int, name string, start int64, end int64, orderType OrderType) ([]*models.AppInstanceMetric, error) - SaveMetric(metric *models.AppInstanceMetric) error - SaveMetricsInBulk(metrics []*models.AppInstanceMetric) error - PruneInstanceMetrics(ctx context.Context, before int64) error - io.Closer -} - type PolicyDB interface { healthendpoint.DatabaseStatus healthendpoint.Pinger diff --git a/src/autoscaler/generate-fakes.go b/src/autoscaler/generate-fakes.go index 4342d78076..2b0c2bdf23 100644 --- a/src/autoscaler/generate-fakes.go +++ b/src/autoscaler/generate-fakes.go @@ -9,7 +9,6 @@ package fakes //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_scalingengine.go ./scalingengine ScalingEngine //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_binding_db.go ./db BindingDB //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_app_metric_db.go ./db AppMetricDB -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_instancemetrics_db.go ./db InstanceMetricsDB //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_credentials.go ./cred_helper Credentials //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_storedprocedure_db.go ./db StoredProcedureDB //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_metric_forwarder.go ./metricsforwarder/forwarder MetricForwarder diff --git a/templates/app-autoscaler.yml b/templates/app-autoscaler.yml index 5225bcab4d..f907b20738 100644 --- a/templates/app-autoscaler.yml +++ b/templates/app-autoscaler.yml @@ -367,7 +367,6 @@ instance_groups: properties: autoscaler: instancemetrics_db: *database - instancemetrics_db_connection_config: *databaseConnectionConfig appmetrics_db: *database appmetrics_db_connection_config: *databaseConnectionConfig scalingengine_db: *database From 0df11712cf844a225a2264b0498bbb68e971c952 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:39:01 +0200 Subject: [PATCH 161/186] remove leftover arrow to metricsserver url --- docs/images/eventgenerator.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/images/eventgenerator.svg b/docs/images/eventgenerator.svg index 43a6548fa7..2f8248cdb6 100644 --- a/docs/images/eventgenerator.svg +++ b/docs/images/eventgenerator.svg @@ -1,4 +1,4 @@ -
SaveMetricToCache(...)
SaveMetricToCache(...)
Aggregator
Aggregator

(PolicyDB, appMetricsDB)
 (PolicyDB, appMetricsDB)
AppManager
AppManager
MetricPoller
MetricPoller
1..config.MetricPollerCount
1..config.MetricPollerCount
MetricPoller
MetricPoller
MetricPoller
MetricPoller
The aggregator does not aggregate but only say what and how much will be aggregated. The real aggregation is performed by the MetricPoller and send to the Aggregator.
The aggregator does not aggrega...
Start(): Listen to the instructions in form of *models.AppMonitor.

For each of these metric-collection-instructions, poll the log-cache and do some form of aggregation. Send the result to the Aggregator via the channel “appMetricChan”.
Start(): Listen to the instruct...
Server
Server
eventgen_handler
eventgen_handler
Server
Server
Start(): Caches and refresh app policies managed by this node every conf.Aggregator.PolicyPollerInterval duration internal. It shards policies by node.
Start(): Caches and refresh app...
AppManager.QueryAppMetrics(...)
AppManager.QueryAppMetrics(...)
AppMetrics[]
AppMetrics[]
conf.aggregatorInterval
conf.aggre...
Autoscaler - Eventgenerator
Autoscaler - Eventgenerator
Generator
Generator
1..config.EvaluatorCount
1..config.EvaluatorCount
TriggersChan:
 *[]Triggers
TriggersChan:...
evaluator
evaluator
evaluator
evaluator
evaluator
evaluatoruuu
AppEvaluationManager
AppEvaluationManager
ScalingEngine
ScalingEngine
POST https
POST https
doEvaluate():
- Create any missing circuite
 breaker for the appPolicies
retrived by AppManager.
- writes on triggers channel
an updated list per app of all
existing rules on that app
policy. This happens 
 every 60s by default.
%3CmxGraphModel%3E%3Croot%3E%3CmxCell%20id%3D%220%22%2F%3E%3CmxCell%20id%3D%221%22%20parent%3D%220%22%2F%3E%3CmxCell%20id%3D%222%22%20value%3D%22%22%20style%3D%22group%22%20vertex%3D%221%22%20connectable%3D%220%22%20parent%3D%221%22%3E%3CmxGeometry%20x%3D%2210%22%20y%3D%2275%22%20width%3D%22230%22%20height%3D%2289.99999999999999%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%223%22%20value%3D%22%22%20style%3D%22verticalLabelPosition%3Dbottom%3Bsketch%3D0%3Baspect%3Dfixed%3Bhtml%3D1%3BverticalAlign%3Dtop%3BstrokeColor%3Dnone%3Balign%3Dcenter%3BoutlineConnect%3D0%3Bshape%3Dmxgraph.citrix.process%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20y%3D%2221.67%22%20width%3D%2226.84%22%20height%3D%2232%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%224%22%20value%3D%22Start()%3A%20Caches%20and%20refresh%20app%20%26lt%3Bbr%26gt%3Bpolicies%20managed%20by%20this%20%26lt%3Bbr%26gt%3BeventGenerator%20node%20every%20x%20time%20%26lt%3Bbr%26gt%3Binternal.%20It%20shards%20policies%20by%20%26lt%3Bbr%26gt%3BeventGenerator%26amp%3Bnbsp%3Bnode.%26lt%3Bbr%26gt%3B%26amp%3Bnbsp%3B%22%20style%3D%22text%3Bhtml%3D1%3Balign%3Dleft%3BverticalAlign%3Dmiddle%3Bresizable%3D0%3Bpoints%3D%5B%5D%3Bautosize%3D1%3BstrokeColor%3Dnone%3BfillColor%3Dnone%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20x%3D%2229.999999999999943%22%20width%3D%22200%22%20height%3D%2290%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3C%2Froot%3E%3C%2FmxGraphModel%3ECra
doEvaluate():...
doEvaluate():
- sets default break duration if 
duration is missing for specific 
triggers
%3CmxGraphModel%3E%3Croot%3E%3CmxCell%20id%3D%220%22%2F%3E%3CmxCell%20id%3D%221%22%20parent%3D%220%22%2F%3E%3CmxCell%20id%3D%222%22%20value%3D%22%22%20style%3D%22group%22%20vertex%3D%221%22%20connectable%3D%220%22%20parent%3D%221%22%3E%3CmxGeometry%20x%3D%2210%22%20y%3D%2275%22%20width%3D%22230%22%20height%3D%2289.99999999999999%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%223%22%20value%3D%22%22%20style%3D%22verticalLabelPosition%3Dbottom%3Bsketch%3D0%3Baspect%3Dfixed%3Bhtml%3D1%3BverticalAlign%3Dtop%3BstrokeColor%3Dnone%3Balign%3Dcenter%3BoutlineConnect%3D0%3Bshape%3Dmxgraph.citrix.process%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20y%3D%2221.67%22%20width%3D%2226.84%22%20height%3D%2232%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%224%22%20value%3D%22Start()%3A%20Caches%20and%20refresh%20app%20%26lt%3Bbr%26gt%3Bpolicies%20managed%20by%20this%20%26lt%3Bbr%26gt%3BeventGenerator%20node%20every%20x%20time%20%26lt%3Bbr%26gt%3Binternal.%20It%20shards%20policies%20by%20%26lt%3Bbr%26gt%3BeventGenerator%26amp%3Bnbsp%3Bnode.%26lt%3Bbr%26gt%3B%26amp%3Bnbsp%3B%22%20style%3D%22text%3Bhtml%3D1%3Balign%3Dleft%3BverticalAlign%3Dmiddle%3Bresizable%3D0%3Bpoints%3D%5B%5D%3Bautosize%3D1%3BstrokeColor%3Dnone%3BfillColor%3Dnone%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20x%3D%2229.999999999999943%22%20width%3D%22200%22%20height%3D%2290%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3C%2Froot%3E%3C%2FmxGraphModel%3ECra
doEvaluate():...
GET /v1/apps/{appid}/metric_histories/{metrictype}
GET /v1/apps/{appid}/metric_histories/{metrictype}
Aggregator
Aggregator
Start():
Pulls all policies via AppManager and turns them into AppMonitor rules that get streamed to the a MetricPoller.
Caches appMetrics and periodically saves them in Bulk via AppManager 

Start():...
appMonitorChan:
*models.AppMonitor
appMonitorChan:...
AppManager.QueryAppMetrics(...)
AppManager.QueryAppMetrics(...)
GET "/v1/apps/{appid}/aggregated_metric_histories/{metrictype}"
GET "/v1/apps/{appid}/aggregated_metric_histories/{metrictype}"
Instance metrics 
Instance metrics 
App Metrics
App Metrics
apiserver
apiserver
log-cache
log-cache
LogCacheClient
LogCacheClient
via HTTPS or gRPC
via HTTPS or gRPCText is not SVG - cannot display \ No newline at end of file +
SaveMetricToCache(...)
SaveMetricToCache(...)
Aggregator
Aggregator

(PolicyDB, appMetricsDB)
 (PolicyDB, appMetricsDB)
AppManager
AppManager
MetricPoller
MetricPoller
1..config.MetricPollerCount
1..config.MetricPollerCount
MetricPoller
MetricPoller
MetricPoller
MetricPoller
The aggregator does not aggregate but only say what and how much will be aggregated. The real aggregation is performed by the MetricPoller and send to the Aggregator.
The aggregator does not aggrega...
Start(): Listen to the instructions in form of *models.AppMonitor.

For each of these metric-collection-instructions, poll the log-cache and do some form of aggregation. Send the result to the Aggregator via the channel “appMetricChan”.
Start(): Listen to the instruct...
Server
Server
eventgen_handler
eventgen_handler
Server
Server
Start(): Caches and refresh app policies managed by this node every conf.Aggregator.PolicyPollerInterval duration internal. It shards policies by node.
Start(): Caches and refresh app...
AppManager.QueryAppMetrics(...)
AppManager.QueryAppMetrics(...)
AppMetrics[]
AppMetrics[]
conf.aggregatorInterval
conf.aggre...
Autoscaler - Eventgenerator
Autoscaler - Eventgenerator
Generator
Generator
1..config.EvaluatorCount
1..config.EvaluatorCount
TriggersChan:
 *[]Triggers
TriggersChan:...
evaluator
evaluator
evaluator
evaluator
evaluator
evaluatoruuu
AppEvaluationManager
AppEvaluationManager
ScalingEngine
ScalingEngine
POST https
POST https
doEvaluate():
- Create any missing circuite
 breaker for the appPolicies
retrived by AppManager.
- writes on triggers channel
an updated list per app of all
existing rules on that app
policy. This happens 
 every 60s by default.
%3CmxGraphModel%3E%3Croot%3E%3CmxCell%20id%3D%220%22%2F%3E%3CmxCell%20id%3D%221%22%20parent%3D%220%22%2F%3E%3CmxCell%20id%3D%222%22%20value%3D%22%22%20style%3D%22group%22%20vertex%3D%221%22%20connectable%3D%220%22%20parent%3D%221%22%3E%3CmxGeometry%20x%3D%2210%22%20y%3D%2275%22%20width%3D%22230%22%20height%3D%2289.99999999999999%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%223%22%20value%3D%22%22%20style%3D%22verticalLabelPosition%3Dbottom%3Bsketch%3D0%3Baspect%3Dfixed%3Bhtml%3D1%3BverticalAlign%3Dtop%3BstrokeColor%3Dnone%3Balign%3Dcenter%3BoutlineConnect%3D0%3Bshape%3Dmxgraph.citrix.process%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20y%3D%2221.67%22%20width%3D%2226.84%22%20height%3D%2232%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%224%22%20value%3D%22Start()%3A%20Caches%20and%20refresh%20app%20%26lt%3Bbr%26gt%3Bpolicies%20managed%20by%20this%20%26lt%3Bbr%26gt%3BeventGenerator%20node%20every%20x%20time%20%26lt%3Bbr%26gt%3Binternal.%20It%20shards%20policies%20by%20%26lt%3Bbr%26gt%3BeventGenerator%26amp%3Bnbsp%3Bnode.%26lt%3Bbr%26gt%3B%26amp%3Bnbsp%3B%22%20style%3D%22text%3Bhtml%3D1%3Balign%3Dleft%3BverticalAlign%3Dmiddle%3Bresizable%3D0%3Bpoints%3D%5B%5D%3Bautosize%3D1%3BstrokeColor%3Dnone%3BfillColor%3Dnone%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20x%3D%2229.999999999999943%22%20width%3D%22200%22%20height%3D%2290%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3C%2Froot%3E%3C%2FmxGraphModel%3ECra
doEvaluate():...
doEvaluate():
- sets default break duration if 
duration is missing for specific 
triggers
%3CmxGraphModel%3E%3Croot%3E%3CmxCell%20id%3D%220%22%2F%3E%3CmxCell%20id%3D%221%22%20parent%3D%220%22%2F%3E%3CmxCell%20id%3D%222%22%20value%3D%22%22%20style%3D%22group%22%20vertex%3D%221%22%20connectable%3D%220%22%20parent%3D%221%22%3E%3CmxGeometry%20x%3D%2210%22%20y%3D%2275%22%20width%3D%22230%22%20height%3D%2289.99999999999999%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%223%22%20value%3D%22%22%20style%3D%22verticalLabelPosition%3Dbottom%3Bsketch%3D0%3Baspect%3Dfixed%3Bhtml%3D1%3BverticalAlign%3Dtop%3BstrokeColor%3Dnone%3Balign%3Dcenter%3BoutlineConnect%3D0%3Bshape%3Dmxgraph.citrix.process%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20y%3D%2221.67%22%20width%3D%2226.84%22%20height%3D%2232%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3CmxCell%20id%3D%224%22%20value%3D%22Start()%3A%20Caches%20and%20refresh%20app%20%26lt%3Bbr%26gt%3Bpolicies%20managed%20by%20this%20%26lt%3Bbr%26gt%3BeventGenerator%20node%20every%20x%20time%20%26lt%3Bbr%26gt%3Binternal.%20It%20shards%20policies%20by%20%26lt%3Bbr%26gt%3BeventGenerator%26amp%3Bnbsp%3Bnode.%26lt%3Bbr%26gt%3B%26amp%3Bnbsp%3B%22%20style%3D%22text%3Bhtml%3D1%3Balign%3Dleft%3BverticalAlign%3Dmiddle%3Bresizable%3D0%3Bpoints%3D%5B%5D%3Bautosize%3D1%3BstrokeColor%3Dnone%3BfillColor%3Dnone%3B%22%20vertex%3D%221%22%20parent%3D%222%22%3E%3CmxGeometry%20x%3D%2229.999999999999943%22%20width%3D%22200%22%20height%3D%2290%22%20as%3D%22geometry%22%2F%3E%3C%2FmxCell%3E%3C%2Froot%3E%3C%2FmxGraphModel%3ECra
doEvaluate():...
Aggregator
Aggregator
Start():
Pulls all policies via AppManager and turns them into AppMonitor rules that get streamed to the a MetricPoller.
Caches appMetrics and periodically saves them in Bulk via AppManager 

Start():...
appMonitorChan:
*models.AppMonitor
appMonitorChan:...
AppManager.QueryAppMetrics(...)
AppManager.QueryAppMetrics(...)
GET "/v1/apps/{appid}/aggregated_metric_histories/{metrictype}"
GET "/v1/apps/{appid}/aggregated_metric_histories/{metrictype}"
Instance metrics 
Instance metrics 
App Metrics
App Metrics
apiserver
apiserver
log-cache
log-cache
LogCacheClient
LogCacheClient
via HTTPS or gRPC
via HTTPS or gRPCText is not SVG - cannot display \ No newline at end of file From 716204bc98866fa7f857154799b999b1b0553a2d Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:06:20 +0200 Subject: [PATCH 162/186] move liquibase file that deletes the appinstancemetrics table to the eventgenerator to be able to get rid of the src/metricsserver/ folder completely, apply changeset that deletes appinstancemetrics table during the pre-start of the eventgenerator --- jobs/eventgenerator/templates/bpm-pre-start.erb | 10 ++++++++-- .../db/metricscollector.db.changelog.yml | 0 2 files changed, 8 insertions(+), 2 deletions(-) rename src/autoscaler/{metricsserver => eventgenerator}/db/metricscollector.db.changelog.yml (100%) diff --git a/jobs/eventgenerator/templates/bpm-pre-start.erb b/jobs/eventgenerator/templates/bpm-pre-start.erb index d947219509..cb7e3b1887 100644 --- a/jobs/eventgenerator/templates/bpm-pre-start.erb +++ b/jobs/eventgenerator/templates/bpm-pre-start.erb @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash <% def p_arr(property) @@ -53,10 +53,16 @@ DBURI="$USER:$PASSWORD@tcp($HOST:$PORT)/$DBNAME?tls=$SSLMODE" detect_dns $HOST $PORT /var/vcap/packages/changeloglockcleaner/changeloglockcleaner -d "$DBURI" -e <%= p("autoscaler.changeloglock_timeout_seconds") %> -<% if p('autoscaler.appmetrics_db.db_scheme') == "postgres" %> +<% if p('autoscaler.appmetrics_db.db_scheme') == "postgres" %> retry java -cp "/var/vcap/packages/db/target/lib/*:/var/vcap/packages/eventgenerator" liquibase.integration.commandline.Main --url "$JDBCDBURL" --username=$USER --password=$PASSWORD \ --driver=org.postgresql.Driver --changeLogFile=dataaggregator.db.changelog.yml update + +retry java -cp "/var/vcap/packages/db/target/lib/*:/var/vcap/packages/eventgenerator" liquibase.integration.commandline.Main --url "$JDBCDBURL" --username=$USER --password=$PASSWORD \ +--driver=org.postgresql.Driver --changeLogFile=metricscollector.db.changelog.yml update <% else %> retry java -cp "/var/vcap/packages/db/target/lib/*:/var/vcap/packages/eventgenerator" liquibase.integration.commandline.Main --url "$JDBCDBURL" --username=$USER --password=$PASSWORD \ --driver=com.mysql.cj.jdbc.Driver --changeLogFile=dataaggregator.db.changelog.yml update + +retry java -cp "/var/vcap/packages/db/target/lib/*:/var/vcap/packages/eventgenerator" liquibase.integration.commandline.Main --url "$JDBCDBURL" --username=$USER --password=$PASSWORD \ +--driver=com.mysql.cj.jdbc.Driver --changeLogFile=metricscollector.db.changelog.yml update <% end %> diff --git a/src/autoscaler/metricsserver/db/metricscollector.db.changelog.yml b/src/autoscaler/eventgenerator/db/metricscollector.db.changelog.yml similarity index 100% rename from src/autoscaler/metricsserver/db/metricscollector.db.changelog.yml rename to src/autoscaler/eventgenerator/db/metricscollector.db.changelog.yml From 7fb45eb9ad775397f5f03f03dcde96973913a292 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:10:46 +0200 Subject: [PATCH 163/186] bring back the autoscaler-diagram --- docs/images/autoscaler.svg | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 docs/images/autoscaler.svg diff --git a/docs/images/autoscaler.svg b/docs/images/autoscaler.svg new file mode 100644 index 0000000000..fffad64b63 --- /dev/null +++ b/docs/images/autoscaler.svg @@ -0,0 +1,4 @@ + + + +
Log Cache
Log Cache
retrive envelops 
via Grpc
retrive envelops...
eventGenerator
eventGenerator
syslog
syslog
MetricsForwarder
MetricsForwarder
CF Deployment
CF Deployment
App Autoscaler
App Autosc...
scalingEngine
scalingEngine
ApiServer
ApiServerText is not SVG - cannot display \ No newline at end of file From 3fdc401b107e1b48e419677e037262ff3cc8d0ae Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:32:52 +0200 Subject: [PATCH 164/186] use mTLS instead of UAA for log cache connection --- ci/autoscaler/scripts/deploy-autoscaler.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index a851de0502..90fd69c565 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -15,7 +15,6 @@ ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\ ${autoscaler_dir}/operations/enable-nats-tls.yml\ ${autoscaler_dir}/operations/add-extra-plan.yml\ ${autoscaler_dir}/operations/set-release-version.yml\ - ${autoscaler_dir}/operations/enable-log-cache-via-uaa.yml\ ${autoscaler_dir}/operations/enable-metricsforwarder-via-syslog-agent.yml\ ${autoscaler_dir}/operations/enable-scheduler-logging.yml"} From 16f04bab716722592a41e2398c999083f87d9de5 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 14 Jun 2024 16:01:20 +0200 Subject: [PATCH 165/186] return err on .Start() and expose a function to retrieve url via .URL() --- .../aggregator/metric_poller_test.go | 5 ++-- .../eventgenerator_suite_test.go | 26 +++++++++---------- ...cache_eventgenerator_scalingengine_test.go | 2 +- .../integration/integration_suite_test.go | 4 +-- src/autoscaler/testhelpers/log_cache_mock.go | 9 ++++--- 5 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go b/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go index b1a0f90fa1..0f80f18500 100644 --- a/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go +++ b/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go @@ -185,11 +185,12 @@ var _ = Describe("MetricPoller", func() { }, }, }, nil) - mockLogCacheEndpoint := mockLogCache.Start(3000 + GinkgoParallelProcess()) + err = mockLogCache.Start(3000 + GinkgoParallelProcess()) + Expect(err).ToNot(HaveOccurred()) metricClient = NewMetricClientFactory().GetMetricClient(logger, &config.Config{ MetricCollector: config.MetricCollectorConfig{ - MetricCollectorURL: mockLogCacheEndpoint, + MetricCollectorURL: mockLogCache.URL(), TLSClientCerts: models.TLSCerts{ KeyFile: filepath.Join(testCertDir, "log-cache.key"), CertFile: filepath.Join(testCertDir, "log-cache.crt"), diff --git a/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_suite_test.go b/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_suite_test.go index 4daa6613b3..c53cd7057a 100644 --- a/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_suite_test.go +++ b/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_suite_test.go @@ -35,17 +35,16 @@ var ( metricType = "a-metric-type" metricUnit = "a-metric-unit" - regPath = regexp.MustCompile(`^/v1/apps/.*/scale$`) - configFile *os.File - conf config.Config - egPort int - healthport int - httpClient *http.Client - healthHttpClient *http.Client - mockLogCache *testhelpers.MockLogCache - mockLogCacheEndpoint string - mockScalingEngine *ghttp.Server - breachDurationSecs = 10 + regPath = regexp.MustCompile(`^/v1/apps/.*/scale$`) + configFile *os.File + conf config.Config + egPort int + healthport int + httpClient *http.Client + healthHttpClient *http.Client + mockLogCache *testhelpers.MockLogCache + mockScalingEngine *ghttp.Server + breachDurationSecs = 10 scalingResult = &models.AppScalingResult{ AppId: testAppId, @@ -225,7 +224,8 @@ func initHttpEndPoints() { }, }, }, nil) - mockLogCacheEndpoint = mockLogCache.Start(10000 + GinkgoParallelProcess()) + err = mockLogCache.Start(10000 + GinkgoParallelProcess()) + Expect(err).ToNot(HaveOccurred()) mockScalingEngine = ghttp.NewUnstartedServer() mockScalingEngine.HTTPTestServer.TLS = testhelpers.ServerTlsConfig("scalingengine") @@ -293,7 +293,7 @@ func initConfig() { }, }, MetricCollector: config.MetricCollectorConfig{ - MetricCollectorURL: mockLogCacheEndpoint, + MetricCollectorURL: mockLogCache.URL(), TLSClientCerts: models.TLSCerts{ KeyFile: filepath.Join(testCertDir, "eventgenerator.key"), CertFile: filepath.Join(testCertDir, "eventgenerator.crt"), diff --git a/src/autoscaler/integration/integration_logcache_eventgenerator_scalingengine_test.go b/src/autoscaler/integration/integration_logcache_eventgenerator_scalingengine_test.go index 93c2117cf7..73e46d3afc 100644 --- a/src/autoscaler/integration/integration_logcache_eventgenerator_scalingengine_test.go +++ b/src/autoscaler/integration/integration_logcache_eventgenerator_scalingengine_test.go @@ -25,7 +25,7 @@ var _ = Describe("Integration_Eventgenerator_Scalingengine", func() { }) JustBeforeEach(func() { - eventGeneratorConfPath = components.PrepareEventGeneratorConfig(dbUrl, components.Ports[EventGenerator], fmt.Sprintf("127.0.0.1:%d", mockLogCachePort), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[ScalingEngine]), aggregatorExecuteInterval, policyPollerInterval, saveInterval, evaluationManagerInterval, defaultHttpClientTimeout, tmpDir) + eventGeneratorConfPath = components.PrepareEventGeneratorConfig(dbUrl, components.Ports[EventGenerator], mockLogCache.URL(), fmt.Sprintf("https://127.0.0.1:%d", components.Ports[ScalingEngine]), aggregatorExecuteInterval, policyPollerInterval, saveInterval, evaluationManagerInterval, defaultHttpClientTimeout, tmpDir) scalingEngineConfPath = components.PrepareScalingEngineConfig(dbUrl, components.Ports[ScalingEngine], fakeCCNOAAUAA.URL(), defaultHttpClientTimeout, tmpDir) startEventGenerator() diff --git a/src/autoscaler/integration/integration_suite_test.go b/src/autoscaler/integration/integration_suite_test.go index 004c94cecf..450475a561 100644 --- a/src/autoscaler/integration/integration_suite_test.go +++ b/src/autoscaler/integration/integration_suite_test.go @@ -55,7 +55,6 @@ var ( testUserScope = []string{"cloud_controller.read", "cloud_controller.write", "password.write", "openid", "network.admin", "network.write", "uaa.user"} processMap = map[string]ifrit.Process{} mockLogCache = &MockLogCache{} - mockLogCachePort = 20000 + GinkgoParallelProcess() defaultHttpClientTimeout = 10 * time.Second @@ -202,7 +201,8 @@ func startMockLogCache() { Expect(err).ToNot(HaveOccurred()) mockLogCache = NewMockLogCache(tlsConfig) - mockLogCache.Start(mockLogCachePort) + err = mockLogCache.Start(20000 + GinkgoParallelProcess()) + Expect(err).ToNot(HaveOccurred()) } func stopGolangApiServer() { diff --git a/src/autoscaler/testhelpers/log_cache_mock.go b/src/autoscaler/testhelpers/log_cache_mock.go index 613594ebab..4c4ccf18de 100644 --- a/src/autoscaler/testhelpers/log_cache_mock.go +++ b/src/autoscaler/testhelpers/log_cache_mock.go @@ -5,7 +5,6 @@ import ( "crypto/x509" "errors" "fmt" - "log" "net" "os" "strings" @@ -49,11 +48,11 @@ func NewMockLogCache(tlsConfig *tls.Config) *MockLogCache { } } -func (m *MockLogCache) Start(port int) string { +func (m *MockLogCache) Start(port int) error { var err error m.lis, err = net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) if err != nil { - log.Fatalf("failed to listen: %v", err) + return err } var srv *grpc.Server @@ -69,6 +68,10 @@ func (m *MockLogCache) Start(port int) string { //nolint:errcheck go srv.Serve(m.lis) + return nil +} + +func (m *MockLogCache) URL() string { return m.lis.Addr().String() } From 3331a67a0d310c7b80241594a7bcf31ae40e338c Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Wed, 12 Jun 2024 21:34:13 +0200 Subject: [PATCH 166/186] replace custom LogCacheClient with a simple LogCacheFetcher that no longer takes care of also building the actual logcache.Client --- packages/eventgenerator/spec | 2 +- .../aggregator/metric_poller.go | 10 +- .../aggregator/metric_poller_test.go | 16 +- .../eventgenerator/client/log_cache_client.go | 315 ---------- .../client/log_cache_client_test.go | 539 ------------------ .../client/metric_client_factory.go | 59 -- .../client/metric_client_factory_test.go | 115 ---- .../cmd/eventgenerator/eventgenerator_test.go | 8 - .../eventgenerator/cmd/eventgenerator/main.go | 15 +- .../eventgenerator/metric/fetcher.go | 205 +++++++ .../eventgenerator/metric/fetcher_factory.go | 60 ++ .../metric/fetcher_factory_test.go | 133 +++++ .../eventgenerator/metric/fetcher_test.go | 313 ++++++++++ .../metric_suite_test.go} | 6 +- src/autoscaler/generate-fakes.go | 7 +- src/autoscaler/models/uaa_creds.go | 4 +- 16 files changed, 745 insertions(+), 1062 deletions(-) delete mode 100644 src/autoscaler/eventgenerator/client/log_cache_client.go delete mode 100644 src/autoscaler/eventgenerator/client/log_cache_client_test.go delete mode 100644 src/autoscaler/eventgenerator/client/metric_client_factory.go delete mode 100644 src/autoscaler/eventgenerator/client/metric_client_factory_test.go create mode 100644 src/autoscaler/eventgenerator/metric/fetcher.go create mode 100644 src/autoscaler/eventgenerator/metric/fetcher_factory.go create mode 100644 src/autoscaler/eventgenerator/metric/fetcher_factory_test.go create mode 100644 src/autoscaler/eventgenerator/metric/fetcher_test.go rename src/autoscaler/eventgenerator/{client/client_suite_test.go => metric/metric_suite_test.go} (56%) diff --git a/packages/eventgenerator/spec b/packages/eventgenerator/spec index 65a681767f..ec2fbd3681 100644 --- a/packages/eventgenerator/spec +++ b/packages/eventgenerator/spec @@ -14,10 +14,10 @@ files: - autoscaler/db/sqldb/* # gosub - autoscaler/envelopeprocessor/* # gosub - autoscaler/eventgenerator/aggregator/* # gosub -- autoscaler/eventgenerator/client/* # gosub - autoscaler/eventgenerator/cmd/eventgenerator/* # gosub - autoscaler/eventgenerator/config/* # gosub - autoscaler/eventgenerator/generator/* # gosub +- autoscaler/eventgenerator/metric/* # gosub - autoscaler/eventgenerator/server/* # gosub - autoscaler/healthendpoint/* # gosub - autoscaler/helpers/* # gosub diff --git a/src/autoscaler/eventgenerator/aggregator/metric_poller.go b/src/autoscaler/eventgenerator/aggregator/metric_poller.go index b1f5140dc1..4f723c34cd 100644 --- a/src/autoscaler/eventgenerator/aggregator/metric_poller.go +++ b/src/autoscaler/eventgenerator/aggregator/metric_poller.go @@ -6,7 +6,7 @@ import ( "strconv" "time" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/client" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/metric" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" "code.cloudfoundry.org/lager/v3" @@ -15,16 +15,16 @@ import ( type MetricPoller struct { logger lager.Logger doneChan chan bool - metricClient client.MetricClient + metricClient metric.Fetcher appMonitorsChan chan *models.AppMonitor appMetricChan chan *models.AppMetric } -func NewMetricPoller(logger lager.Logger, metricClient client.MetricClient, appMonitorsChan chan *models.AppMonitor, appMetricChan chan *models.AppMetric) *MetricPoller { +func NewMetricPoller(logger lager.Logger, metricFetcher metric.Fetcher, appMonitorsChan chan *models.AppMonitor, appMetricChan chan *models.AppMetric) *MetricPoller { return &MetricPoller{ logger: logger.Session("MetricPoller"), appMonitorsChan: appMonitorsChan, - metricClient: metricClient, + metricClient: metricFetcher, doneChan: make(chan bool), appMetricChan: appMetricChan, } @@ -62,7 +62,7 @@ func (m *MetricPoller) retrieveMetric(appMonitor *models.AppMonitor) error { endTime := time.Now() startTime := endTime.Add(0 - statWindow) - metrics, err := m.metricClient.GetMetrics(appId, metricType, startTime, endTime) + metrics, err := m.metricClient.FetchMetrics(appId, metricType, startTime, endTime) if err != nil { return fmt.Errorf("retrieveMetric Failed: %w", err) } diff --git a/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go b/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go index 0f80f18500..396b9f46eb 100644 --- a/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go +++ b/src/autoscaler/eventgenerator/aggregator/metric_poller_test.go @@ -7,8 +7,8 @@ import ( "time" . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/aggregator" - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/client" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/config" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/metric" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" rpc "code.cloudfoundry.org/go-log-cache/v2/rpc/logcache_v1" @@ -29,7 +29,7 @@ var _ = Describe("MetricPoller", func() { appMonitorsChan chan *models.AppMonitor appMetricChan chan *models.AppMetric metricPoller *MetricPoller - metricClient MetricClient + metricFetcher metric.Fetcher mockLogCache *testhelpers.MockLogCache appMonitor *models.AppMonitor ) @@ -51,13 +51,14 @@ var _ = Describe("MetricPoller", func() { Context("When metric-collector is not running", func() { BeforeEach(func() { - metricClient = NewMetricClientFactory().GetMetricClient(logger, &config.Config{ + metricFetcher, err := metric.NewLogCacheFetcherFactory(metric.StandardLogCacheFetcherCreator).CreateFetcher(logger, config.Config{ MetricCollector: config.MetricCollectorConfig{ MetricCollectorURL: "this.endpoint.does.not.exist:1234", }, }) + Expect(err).ToNot(HaveOccurred()) - metricPoller = NewMetricPoller(logger, metricClient, appMonitorsChan, appMetricChan) + metricPoller = NewMetricPoller(logger, metricFetcher, appMonitorsChan, appMetricChan) metricPoller.Start() Expect(appMonitorsChan).Should(BeSent(appMonitor)) @@ -188,7 +189,7 @@ var _ = Describe("MetricPoller", func() { err = mockLogCache.Start(3000 + GinkgoParallelProcess()) Expect(err).ToNot(HaveOccurred()) - metricClient = NewMetricClientFactory().GetMetricClient(logger, &config.Config{ + metricFetcher, err = metric.NewLogCacheFetcherFactory(metric.StandardLogCacheFetcherCreator).CreateFetcher(logger, config.Config{ MetricCollector: config.MetricCollectorConfig{ MetricCollectorURL: mockLogCache.URL(), TLSClientCerts: models.TLSCerts{ @@ -198,10 +199,11 @@ var _ = Describe("MetricPoller", func() { }, }, }) + Expect(err).ToNot(HaveOccurred()) }) JustBeforeEach(func() { - metricPoller = NewMetricPoller(logger, metricClient, appMonitorsChan, appMetricChan) + metricPoller = NewMetricPoller(logger, metricFetcher, appMonitorsChan, appMetricChan) metricPoller.Start() Expect(appMonitorsChan).Should(BeSent(appMonitor)) @@ -265,7 +267,7 @@ var _ = Describe("MetricPoller", func() { Context("Stop", func() { BeforeEach(func() { - metricPoller = NewMetricPoller(logger, metricClient, appMonitorsChan, appMetricChan) + metricPoller = NewMetricPoller(logger, metricFetcher, appMonitorsChan, appMetricChan) metricPoller.Start() metricPoller.Stop() Eventually(logger.Buffer).Should(Say("stopped")) diff --git a/src/autoscaler/eventgenerator/client/log_cache_client.go b/src/autoscaler/eventgenerator/client/log_cache_client.go deleted file mode 100644 index 83ffef3f33..0000000000 --- a/src/autoscaler/eventgenerator/client/log_cache_client.go +++ /dev/null @@ -1,315 +0,0 @@ -package client - -import ( - "context" - "crypto/tls" - "fmt" - "math" - "net/http" - "net/url" - "strconv" - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/envelopeprocessor" - gogrpc "google.golang.org/grpc" - - "google.golang.org/grpc/credentials" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - logcache "code.cloudfoundry.org/go-log-cache/v2" - "code.cloudfoundry.org/go-log-cache/v2/rpc/logcache_v1" - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "code.cloudfoundry.org/lager/v3" -) - -type LogCacheClient struct { - logger lager.Logger - Client LogCacheClientReader - - now func() time.Time - envelopeProcessor envelopeprocessor.EnvelopeProcessor - goLogCache GoLogCache - TLSConfig *tls.Config - uaaCreds models.UAACreds - url string - collectionInterval time.Duration - - grpc GRPC -} - -type LogCacheClientReader interface { - Read(ctx context.Context, sourceID string, start time.Time, opts ...logcache.ReadOption) ([]*loggregator_v2.Envelope, error) - PromQL(ctx context.Context, query string, opts ...logcache.PromQLOption) (*logcache_v1.PromQL_InstantQueryResult, error) -} - -type GRPCOptions interface { - WithTransportCredentials(creds credentials.TransportCredentials) gogrpc.DialOption -} - -type GRPC struct { - WithTransportCredentials func(creds credentials.TransportCredentials) gogrpc.DialOption -} - -type GoLogCacheClient interface { - NewClient(addr string, opts ...logcache.ClientOption) *logcache.Client - WithViaGRPC(opts ...gogrpc.DialOption) logcache.ClientOption - WithHTTPClient(h logcache.HTTPClient) logcache.ClientOption - NewOauth2HTTPClient(oauth2Addr, client, clientSecret string, opts ...logcache.Oauth2Option) *logcache.Oauth2HTTPClient - WithOauth2HTTPClient(client logcache.HTTPClient) logcache.Oauth2Option -} - -type GoLogCache struct { - NewClient func(addr string, opts ...logcache.ClientOption) *logcache.Client - WithViaGRPC func(opts ...gogrpc.DialOption) logcache.ClientOption - WithHTTPClient func(h logcache.HTTPClient) logcache.ClientOption - NewOauth2HTTPClient func(oauth2Addr string, client string, clientSecret string, opts ...logcache.Oauth2Option) *logcache.Oauth2HTTPClient - WithOauth2HTTPClient func(client logcache.HTTPClient) logcache.Oauth2Option -} - -type LogCacheClientCreator interface { - NewLogCacheClient(logger lager.Logger, getTime func() time.Time, collectionInterval time.Duration, envelopeProcessor envelopeprocessor.EnvelopeProcessor, url string) MetricClient -} - -func NewLogCacheClient(logger lager.Logger, getTime func() time.Time, collectionInterval time.Duration, envelopeProcessor envelopeprocessor.EnvelopeProcessor, url string) *LogCacheClient { - var c = &LogCacheClient{ - logger: logger.Session("LogCacheClient"), - - envelopeProcessor: envelopeProcessor, - now: getTime, - url: url, - goLogCache: GoLogCache{ - NewClient: logcache.NewClient, - WithViaGRPC: logcache.WithViaGRPC, - WithHTTPClient: logcache.WithHTTPClient, - NewOauth2HTTPClient: logcache.NewOauth2HTTPClient, - WithOauth2HTTPClient: logcache.WithOauth2HTTPClient, - }, - collectionInterval: collectionInterval, - - grpc: GRPC{ - WithTransportCredentials: gogrpc.WithTransportCredentials, - }, - } - return c -} - -func (c *LogCacheClient) emptyAppInstanceMetrics(appId string, name string, unit string, now time.Time) ([]models.AppInstanceMetric, error) { - return []models.AppInstanceMetric{ - { - AppId: appId, - InstanceIndex: 0, - Name: name, - Unit: unit, - Value: "0", - CollectedAt: now.UnixNano(), - Timestamp: now.UnixNano(), - }, - }, nil -} - -func (c *LogCacheClient) getMetricsPromQLAPI(appId string, metricType string) ([]models.AppInstanceMetric, error) { - logger := c.logger.Session("getMetricsPromQLAPI", lager.Data{"appId": appId, "metricType": metricType}) - collectionInterval := fmt.Sprintf("%.0f", c.CollectionInterval().Seconds()) - now := time.Now() - - query := "" - metricTypeUnit := "" - if metricType == models.MetricNameThroughput { - query = fmt.Sprintf("sum by (instance_id) (count_over_time(http{source_id='%s',peer_type='Client'}[%ss])) / %s", appId, collectionInterval, collectionInterval) - metricTypeUnit = models.UnitRPS - } - - if metricType == models.MetricNameResponseTime { - query = fmt.Sprintf("avg by (instance_id) (max_over_time(http{source_id='%s',peer_type='Client'}[%ss])) / (1000 * 1000)", appId, collectionInterval) - metricTypeUnit = models.UnitMilliseconds - } - - logger.Debug("query-promql-api", lager.Data{"query": query}) - result, err := c.Client.PromQL(context.Background(), query, logcache.WithPromQLTime(now)) - if err != nil { - return []models.AppInstanceMetric{}, fmt.Errorf("failed getting PromQL result (metricType: %s, appId: %s, collectionInterval: %s, query: %s, time: %s): %w", metricType, appId, collectionInterval, query, now.String(), err) - } - logger.Info("received-promql-api-result", lager.Data{"result": result, "query": query}) - - // safeguard: the query ensures that we get a vector but let's double-check - vector := result.GetVector() - if vector == nil { - return []models.AppInstanceMetric{}, fmt.Errorf("result does not contain a vector") - } - - // return empty metrics if there are no samples, this usually happens in case there were no recent http-requests towards the application - if len(vector.GetSamples()) <= 0 { - return c.emptyAppInstanceMetrics(appId, metricType, metricTypeUnit, now) - } - - // convert result into autoscaler metric model - var metrics []models.AppInstanceMetric - for _, sample := range vector.GetSamples() { - // safeguard: metric label instance_id should be always there but let's double-check - instanceIdStr, ok := sample.GetMetric()["instance_id"] - if !ok { - return []models.AppInstanceMetric{}, fmt.Errorf("sample does not contain instance_id: %w", err) - } - - instanceIdUInt, err := strconv.ParseUint(instanceIdStr, 10, 32) - if err != nil { - return []models.AppInstanceMetric{}, fmt.Errorf("could not convert instance_id to uint32: %w", err) - } - - // safeguard: the query ensures that we get a point but let's double-check - point := sample.GetPoint() - if point == nil { - return []models.AppInstanceMetric{}, fmt.Errorf("sample does not contain a point") - } - - instanceId := uint32(instanceIdUInt) - valueWithoutDecimalsRoundedToCeiling := fmt.Sprintf("%.0f", math.Ceil(point.GetValue())) - - metrics = append(metrics, models.AppInstanceMetric{ - AppId: appId, - InstanceIndex: instanceId, - Name: metricType, - Unit: metricTypeUnit, - Value: valueWithoutDecimalsRoundedToCeiling, - CollectedAt: now.UnixNano(), - Timestamp: now.UnixNano(), - }) - } - return metrics, nil -} - -func (c *LogCacheClient) getMetricsRestAPI(appId string, metricType string, startTime time.Time, endTime time.Time) ([]models.AppInstanceMetric, error) { - filters := logCacheFiltersFor(endTime, metricType) - logger := c.logger.Session("getMetricsRestAPI", lager.Data{"appId": appId, "metricType": metricType, "startTime": startTime, "endTime": endTime, "filters": valuesFrom(filters)}) - logger.Debug("query-rest-api-with-filters") - - envelopes, err := c.Client.Read(context.Background(), appId, startTime, filters...) - if err != nil { - return []models.AppInstanceMetric{}, fmt.Errorf("fail to Read %s metric from %s GoLogCache client: %w", logcache_v1.EnvelopeType_GAUGE, appId, err) - } - logger.Info("received-rest-api-result", lager.Data{"envelopes": envelopes}) - - collectedAt := c.now().UnixNano() - metrics, err := c.envelopeProcessor.GetGaugeMetrics(envelopes, collectedAt) - - return filter(metrics, metricType), err -} - -func (c *LogCacheClient) GetMetrics(appId string, metricType string, startTime time.Time, endTime time.Time) ([]models.AppInstanceMetric, error) { - c.logger.Debug("GetMetrics") - - // the log-cache REST API only return max. 1000 envelopes: https://github.com/cloudfoundry/log-cache-release/tree/main/src#get-apiv1readsource-id. - // receiving a limited set of envelopes breaks throughput and responsetime, because all envelopes are required to calculate these metric types properly. - // pagination via `start_time` and `end_time` could be done but is very error-prone. - // using the PromQL API also has an advantage over REST API because it shifts the metric aggregations to log-cache. - if metricType == models.MetricNameThroughput || metricType == models.MetricNameResponseTime { - c.logger.Debug("get-metrics-via-promql-api", lager.Data{"metricType": metricType}) - return c.getMetricsPromQLAPI(appId, metricType) - } - - c.logger.Debug("get-metrics-via-rest-api", lager.Data{"metricType": metricType}) - return c.getMetricsRestAPI(appId, metricType, startTime, endTime) -} - -func (c *LogCacheClient) SetTLSConfig(tlsConfig *tls.Config) { - c.TLSConfig = tlsConfig -} - -func (c *LogCacheClient) GetTlsConfig() *tls.Config { - return c.TLSConfig -} - -func (c *LogCacheClient) SetUAACreds(uaaCreds models.UAACreds) { - c.uaaCreds = uaaCreds -} - -func (c *LogCacheClient) GetUAACreds() models.UAACreds { - return c.uaaCreds -} - -func (c *LogCacheClient) GetUrl() string { - return c.url -} - -func (c *LogCacheClient) SetGoLogCache(goLogCache GoLogCache) { - c.goLogCache = goLogCache -} - -func (c *LogCacheClient) SetGRPC(grpc GRPC) { - c.grpc = grpc -} - -func (c *LogCacheClient) Configure() { - var opts []logcache.ClientOption - - if c.uaaCreds.IsEmpty() { - opts = append(opts, c.goLogCache.WithViaGRPC(c.grpc.WithTransportCredentials(credentials.NewTLS(c.TLSConfig)))) - } else { - oauth2HTTPOpts := c.goLogCache.WithOauth2HTTPClient(c.getUaaHttpClient()) - oauth2HTTPClient := c.goLogCache.NewOauth2HTTPClient(c.uaaCreds.URL, c.uaaCreds.ClientID, c.uaaCreds.ClientSecret, oauth2HTTPOpts) - opts = append(opts, c.goLogCache.WithHTTPClient(oauth2HTTPClient)) - } - - c.Client = c.goLogCache.NewClient(c.url, opts...) -} - -func (c *LogCacheClient) GetUaaTlsConfig() *tls.Config { - //nolint:gosec - return &tls.Config{InsecureSkipVerify: c.uaaCreds.SkipSSLValidation} -} - -func (c *LogCacheClient) CollectionInterval() time.Duration { - return c.collectionInterval -} - -func valuesFrom(filters []logcache.ReadOption) url.Values { - values := url.Values{} - for _, f := range filters { - f(nil, values) - } - return values -} - -func filter(metrics []models.AppInstanceMetric, metricType string) []models.AppInstanceMetric { - var result []models.AppInstanceMetric - for i := range metrics { - if metrics[i].Name == metricType { - result = append(result, metrics[i]) - } - } - - return result -} - -func logCacheFiltersFor(endTime time.Time, metricType string) (readOptions []logcache.ReadOption) { - readOptions = append(readOptions, logcache.WithEndTime(endTime)) - readOptions = append(readOptions, logcache.WithEnvelopeTypes(logcache_v1.EnvelopeType_GAUGE)) - - switch metricType { - case models.MetricNameMemoryUtil: - readOptions = append(readOptions, logcache.WithNameFilter("memory|memory_quota")) - case models.MetricNameMemoryUsed: - readOptions = append(readOptions, logcache.WithNameFilter("memory")) - case models.MetricNameCPU: - readOptions = append(readOptions, logcache.WithNameFilter("cpu")) - case models.MetricNameCPUUtil: - readOptions = append(readOptions, logcache.WithNameFilter("cpu_entitlement")) - case models.MetricNameDisk: - readOptions = append(readOptions, logcache.WithNameFilter("disk")) - case models.MetricNameDiskUtil: - readOptions = append(readOptions, logcache.WithNameFilter("disk|disk_quota")) - default: - readOptions = append(readOptions, logcache.WithNameFilter(metricType)) - } - - return readOptions -} - -func (c *LogCacheClient) getUaaHttpClient() logcache.HTTPClient { - return &http.Client{ - Timeout: 5 * time.Second, - Transport: &http.Transport{ - TLSClientConfig: c.GetUaaTlsConfig(), - }, - } -} diff --git a/src/autoscaler/eventgenerator/client/log_cache_client_test.go b/src/autoscaler/eventgenerator/client/log_cache_client_test.go deleted file mode 100644 index d84acc1db8..0000000000 --- a/src/autoscaler/eventgenerator/client/log_cache_client_test.go +++ /dev/null @@ -1,539 +0,0 @@ -package client_test - -import ( - "context" - "crypto/tls" - "errors" - "net/http" - "net/url" - "path/filepath" - "reflect" - "strconv" - "time" - - "code.cloudfoundry.org/go-log-cache/v2/rpc/logcache_v1" - gogrpc "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - - logcache "code.cloudfoundry.org/go-log-cache/v2" - - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/client" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" - "code.cloudfoundry.org/lager/v3/lagertest" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("LogCacheClient", func() { - var ( - fakeEnvelopeProcessor *fakes.FakeEnvelopeProcessor - fakeGoLogCacheReader *fakes.FakeLogCacheClientReader - fakeGoLogCacheClient *fakes.FakeGoLogCacheClient - fakeGRPC *fakes.FakeGRPCOptions - appId string - logger *lagertest.TestLogger - logCacheClient *LogCacheClient - envelopes []*loggregator_v2.Envelope - metrics []models.AppInstanceMetric - startTime time.Time - endTime time.Time - collectedAt time.Time - logCacheClientReadError error - expectedClientOption logcache.ClientOption - ) - - BeforeEach(func() { - fakeEnvelopeProcessor = &fakes.FakeEnvelopeProcessor{} - fakeGoLogCacheReader = &fakes.FakeLogCacheClientReader{} - fakeGoLogCacheClient = &fakes.FakeGoLogCacheClient{} - fakeGRPC = &fakes.FakeGRPCOptions{} - - logCacheClientReadError = nil - logger = lagertest.NewTestLogger("MetricPoller-test") - startTime = time.Now() - endTime = startTime.Add(-60 * time.Second) - collectedAt = startTime.Add(10 * time.Millisecond) - appId = "some-app-id" - envelopes = []*loggregator_v2.Envelope{ - {SourceId: "some-id"}, - } - - metrics = []models.AppInstanceMetric{ - {AppId: "some-id"}, - } - - logCacheClient = NewLogCacheClient(logger, func() time.Time { return collectedAt }, 40*time.Second, fakeEnvelopeProcessor, "") - }) - - JustBeforeEach(func() { - fakeGoLogCacheReader.ReadReturns(envelopes, logCacheClientReadError) - fakeEnvelopeProcessor.GetGaugeMetricsReturnsOnCall(0, metrics, nil) - fakeEnvelopeProcessor.GetGaugeMetricsReturnsOnCall(1, nil, errors.New("some error")) - - fakeGoLogCacheClient.WithViaGRPCReturns(expectedClientOption) - - goLogCache := GoLogCache{ - NewClient: fakeGoLogCacheClient.NewClient, - WithViaGRPC: fakeGoLogCacheClient.WithViaGRPC, - - WithHTTPClient: fakeGoLogCacheClient.WithHTTPClient, - NewOauth2HTTPClient: fakeGoLogCacheClient.NewOauth2HTTPClient, - WithOauth2HTTPClient: fakeGoLogCacheClient.WithOauth2HTTPClient, - } - - logCacheClient.SetGoLogCache(goLogCache) - logCacheClient.Configure() - }) - - Context("NewLogCacheClient", func() { - var expectedAddrs string - - BeforeEach(func() { - expectedAddrs = "logcache:8080" - logCacheClient = NewLogCacheClient(logger, func() time.Time { return collectedAt }, 40*time.Second, fakeEnvelopeProcessor, expectedAddrs) - }) - - Context("when consuming log cache via grpc/mtls", func() { - var ( - expectedTransportCredential credentials.TransportCredentials - expectedDialOpt gogrpc.DialOption - caCertFilePath string - certFilePath string - keyFilePath string - ) - - BeforeEach(func() { - caCertFilePath = filepath.Join(testCertDir, "autoscaler-ca.crt") - certFilePath = filepath.Join(testCertDir, "eventgenerator.crt") - keyFilePath = filepath.Join(testCertDir, "eventgenerator.key") - - expectedTlSCerts := &models.TLSCerts{KeyFile: keyFilePath, CertFile: certFilePath, CACertFile: caCertFilePath} - expectedTLSConfig, err := expectedTlSCerts.CreateClientConfig() - logCacheClient.SetTLSConfig(expectedTLSConfig) - expectedTransportCredential = credentials.NewTLS(expectedTLSConfig) - expectedDialOpt = gogrpc.WithTransportCredentials(expectedTransportCredential) - expectedClientOption = logcache.WithViaGRPC(expectedDialOpt) - fakeGRPC.WithTransportCredentialsReturns(expectedDialOpt) - Expect(err).NotTo(HaveOccurred()) - - grpc := GRPC{ - WithTransportCredentials: fakeGRPC.WithTransportCredentials, - } - logCacheClient.SetGRPC(grpc) - }) - - JustBeforeEach(func() { - fakeGoLogCacheClient.WithViaGRPCReturns(expectedClientOption) - }) - - It("Should setup correct tls configurations", func() { - actualAddrs, actualClientOptions := fakeGoLogCacheClient.NewClientArgsForCall(0) - - By("Creating the go log cache client with the correct params") - Expect(actualAddrs).To(Equal(expectedAddrs)) - Expect(actualClientOptions).NotTo(BeEmpty()) - Expect(reflect.ValueOf(actualClientOptions[0]).Pointer()).To(Equal(reflect.ValueOf(expectedClientOption).Pointer())) - - By("Configuring GRPC client options to the logcache client") - actualGRPCDialOpts := fakeGoLogCacheClient.WithViaGRPCArgsForCall(0) - Expect(actualGRPCDialOpts).NotTo(BeEmpty()) - Expect(reflect.ValueOf(actualGRPCDialOpts[0]).Pointer()).To(Equal(reflect.ValueOf(expectedDialOpt).Pointer())) - - By("Sending the right transport credentials to the logcache client") - Expect(fakeGRPC.WithTransportCredentialsCallCount()).To(Equal(1)) - actualTransportCredentials := fakeGRPC.WithTransportCredentialsArgsForCall(0) - Expect(actualTransportCredentials).To(Equal(expectedTransportCredential)) - }) - }) - - Describe("when consuming log cache via uaa/oauth", func() { - var ( - uaaCreds models.UAACreds - expectedHTTPClient *http.Client - expectedOauth2HTTPClient *logcache.Oauth2HTTPClient - expectedOauth2HTTPClientOpt logcache.Oauth2Option - ) - - BeforeEach(func() { - uaaCreds = models.UAACreds{ - URL: "https:some-uaa", - ClientID: "some-id", - ClientSecret: "some-secret", - } - - expectedHTTPClient = &http.Client{ - Timeout: 5 * time.Second, - Transport: &http.Transport{ - //nolint:gosec - TLSClientConfig: &tls.Config{InsecureSkipVerify: uaaCreds.SkipSSLValidation}, - }, - } - logCacheClient.SetUAACreds(uaaCreds) - expectedOauth2HTTPClient = &logcache.Oauth2HTTPClient{} - expectedOauth2HTTPClientOpt = logcache.WithOauth2HTTPClient(expectedHTTPClient) - fakeGoLogCacheClient.NewOauth2HTTPClientReturns(expectedOauth2HTTPClient) - fakeGoLogCacheClient.WithOauth2HTTPClientReturns(expectedOauth2HTTPClientOpt) - fakeGoLogCacheClient.WithHTTPClientReturns(expectedClientOption) - }) - - Describe("when skip_ssl_validation is enabled", func() { - BeforeEach(func() { - uaaCreds.SkipSSLValidation = true - }) - - It("Should create a LogCacheClient Clientvia OauthHTTP", func() { - _, _, _, actualNewOauth2HTTPClientOpts := fakeGoLogCacheClient.NewOauth2HTTPClientArgsForCall(0) - Expect(reflect.ValueOf(actualNewOauth2HTTPClientOpts[0]).Pointer()).Should(Equal(reflect.ValueOf(expectedOauth2HTTPClientOpt).Pointer())) - actualHttpClient := fakeGoLogCacheClient.WithOauth2HTTPClientArgsForCall(0) - Expect(actualHttpClient).To(Equal(expectedHTTPClient)) - }) - }) - - It("Should create a LogCacheClient via OauthHTTP", func() { - By("Sending the right argument when creating the Oauth2HTTPClient") - Expect(fakeGoLogCacheClient.NewOauth2HTTPClientCallCount()).To(Equal(1)) - uaaURL, uaaClientID, uaaClientSecret, oauthOpts := fakeGoLogCacheClient.NewOauth2HTTPClientArgsForCall(0) - Expect(uaaURL).NotTo(BeNil()) - Expect(uaaClientID).NotTo(BeNil()) - Expect(uaaClientSecret).NotTo(BeNil()) - Expect(oauthOpts).NotTo(BeEmpty()) - - By("Calling logcache.NewClient with an Oauth2HTTPClient as an option") - Expect(fakeGoLogCacheClient.NewClientCallCount()).To(Equal(1)) - actualLogCacheAddrs, actualClientOptions := fakeGoLogCacheClient.NewClientArgsForCall(0) - Expect(actualLogCacheAddrs).To(Equal(expectedAddrs)) - Expect(fakeGoLogCacheClient.WithHTTPClientCallCount()).To(Equal(1)) - actualHTTPClient := fakeGoLogCacheClient.WithHTTPClientArgsForCall(0) - Expect(actualHTTPClient).NotTo(BeNil()) - Expect(actualHTTPClient).To(Equal(expectedOauth2HTTPClient)) - Expect(&actualClientOptions).NotTo(Equal(expectedClientOption)) - }) - }) - }) - - Describe("CollectionInterval", func() { - It("returns correct collection interval", func() { - logCacheClient = NewLogCacheClient(logger, func() time.Time { return time.Now() }, 40*time.Second, fakeEnvelopeProcessor, "url") - Expect(logCacheClient.CollectionInterval()).To(Equal(40 * time.Second)) - }) - }) - - Context("GetMetrics", func() { - JustBeforeEach(func() { - logCacheClient.Client = fakeGoLogCacheReader - }) - - Describe("when log cache returns error on read", func() { - BeforeEach(func() { - logCacheClientReadError = errors.New("some Read error") - }) - - It("return error", func() { - _, err := logCacheClient.GetMetrics(appId, models.MetricNameMemoryUtil, startTime, endTime) - Expect(err).To(HaveOccurred()) - }) - }) - - When("errors occur reading via PromQL API", func() { - When("PromQL call fails", func() { - It("returns an error", func() { - fakeGoLogCacheReader.PromQLReturns(nil, errors.New("fail")) - _, err := logCacheClient.GetMetrics("app-id", "throughput", startTime, endTime) - Expect(err).To(HaveOccurred()) - }) - }) - - When("PromQL result is not a vector", func() { - It("returns an error", func() { - fakeGoLogCacheReader.PromQLReturns(nil, nil) - _, err := logCacheClient.GetMetrics("app-id", "throughput", startTime, endTime) - Expect(err).To(HaveOccurred()) - }) - }) - - When("sample does not contain instance_id", func() { - It("returns an error", func() { - fakeGoLogCacheReader.PromQLReturns(&logcache_v1.PromQL_InstantQueryResult{ - Result: &logcache_v1.PromQL_InstantQueryResult_Vector{ - Vector: &logcache_v1.PromQL_Vector{ - Samples: []*logcache_v1.PromQL_Sample{ - { - Metric: map[string]string{ - // "instance_id": "0", is missing here - }, - }, - }, - }, - }, - }, nil) - _, err := logCacheClient.GetMetrics("app-id", "throughput", startTime, endTime) - Expect(err).To(HaveOccurred()) - }) - }) - - When("instance_id can not be parsed to uint", func() { - It("returns an error", func() { - fakeGoLogCacheReader.PromQLReturns(&logcache_v1.PromQL_InstantQueryResult{ - Result: &logcache_v1.PromQL_InstantQueryResult_Vector{ - Vector: &logcache_v1.PromQL_Vector{ - Samples: []*logcache_v1.PromQL_Sample{ - { - Metric: map[string]string{ - "instance_id": "iam-no-uint", - }, - }, - }, - }, - }, - }, nil) - _, err := logCacheClient.GetMetrics("app-id", "throughput", startTime, endTime) - Expect(err).To(HaveOccurred()) - }) - }) - - When("sample does not contain a point", func() { - It("returns an error", func() { - fakeGoLogCacheReader.PromQLReturns(&logcache_v1.PromQL_InstantQueryResult{ - Result: &logcache_v1.PromQL_InstantQueryResult_Vector{ - Vector: &logcache_v1.PromQL_Vector{ - Samples: []*logcache_v1.PromQL_Sample{ - { - Metric: map[string]string{ - "instance_id": "0", - }, - }, - }, - }, - }, - }, nil) - _, err := logCacheClient.GetMetrics("app-id", "throughput", startTime, endTime) - Expect(err).To(HaveOccurred()) - }) - }) - }) - - When("reading throughput metrics", func() { - When("PromQL API returns a vector with no samples", func() { - It("returns empty metrics", func() { - fakeGoLogCacheReader.PromQLReturns(&logcache_v1.PromQL_InstantQueryResult{ - Result: &logcache_v1.PromQL_InstantQueryResult_Vector{ - Vector: &logcache_v1.PromQL_Vector{}, - }, - }, nil) - - metrics, err := logCacheClient.GetMetrics("app-id", "throughput", startTime, endTime) - - Expect(err).To(Not(HaveOccurred())) - Expect(metrics).To(HaveLen(1)) - - Expect(metrics[0].AppId).To(Equal("app-id")) - Expect(metrics[0].InstanceIndex).To(Equal(uint32(0))) - Expect(metrics[0].Name).To(Equal("throughput")) - Expect(metrics[0].Unit).To(Equal("rps")) - Expect(metrics[0].Value).To(Equal("0")) - }) - }) - - When("promql api returns a vector with samples", func() { - It("returns metrics", func() { - fakeGoLogCacheReader.PromQLReturns(&logcache_v1.PromQL_InstantQueryResult{ - Result: &logcache_v1.PromQL_InstantQueryResult_Vector{ - Vector: &logcache_v1.PromQL_Vector{ - Samples: []*logcache_v1.PromQL_Sample{ - { - Metric: map[string]string{ - "instance_id": "0", - }, - Point: &logcache_v1.PromQL_Point{ - Value: 123, - }, - }, - { - Metric: map[string]string{ - "instance_id": "1", - }, - Point: &logcache_v1.PromQL_Point{ - Value: 321, - }, - }, - }, - }, - }, - }, nil) - - metrics, err := logCacheClient.GetMetrics("app-id", "throughput", startTime, endTime) - - _, query, _ := fakeGoLogCacheReader.PromQLArgsForCall(0) - Expect(query).To(Equal("sum by (instance_id) (count_over_time(http{source_id='app-id',peer_type='Client'}[40s])) / 40")) - - Expect(err).To(Not(HaveOccurred())) - Expect(metrics).To(HaveLen(2)) - - Expect(metrics[0].AppId).To(Equal("app-id")) - Expect(metrics[0].InstanceIndex).To(Equal(uint32(0))) - Expect(metrics[0].Name).To(Equal("throughput")) - Expect(metrics[0].Unit).To(Equal("rps")) - Expect(metrics[0].Value).To(Equal("123")) - - Expect(metrics[1].AppId).To(Equal("app-id")) - Expect(metrics[1].InstanceIndex).To(Equal(uint32(1))) - Expect(metrics[1].Name).To(Equal("throughput")) - Expect(metrics[1].Unit).To(Equal("rps")) - Expect(metrics[1].Value).To(Equal("321")) - }) - }) - }) - - When("reading responsetime metrics", func() { - When("PromQL API returns a vector with no samples", func() { - It("returns empty metrics", func() { - fakeGoLogCacheReader.PromQLReturns(&logcache_v1.PromQL_InstantQueryResult{ - Result: &logcache_v1.PromQL_InstantQueryResult_Vector{ - Vector: &logcache_v1.PromQL_Vector{}, - }, - }, nil) - - metrics, err := logCacheClient.GetMetrics("app-id", "responsetime", startTime, endTime) - - Expect(err).To(Not(HaveOccurred())) - Expect(metrics).To(HaveLen(1)) - - Expect(metrics[0].AppId).To(Equal("app-id")) - Expect(metrics[0].InstanceIndex).To(Equal(uint32(0))) - Expect(metrics[0].Name).To(Equal("responsetime")) - Expect(metrics[0].Unit).To(Equal("ms")) - Expect(metrics[0].Value).To(Equal("0")) - }) - }) - - When("promql api returns a vector with samples", func() { - It("returns metrics", func() { - fakeGoLogCacheReader.PromQLReturns(&logcache_v1.PromQL_InstantQueryResult{ - Result: &logcache_v1.PromQL_InstantQueryResult_Vector{ - Vector: &logcache_v1.PromQL_Vector{ - Samples: []*logcache_v1.PromQL_Sample{ - { - Metric: map[string]string{ - "instance_id": "0", - }, - Point: &logcache_v1.PromQL_Point{ - Value: 200, - }, - }, - { - Metric: map[string]string{ - "instance_id": "1", - }, - Point: &logcache_v1.PromQL_Point{ - Value: 300, - }, - }, - }, - }, - }, - }, nil) - - metrics, err := logCacheClient.GetMetrics("app-id", "responsetime", startTime, endTime) - - _, query, _ := fakeGoLogCacheReader.PromQLArgsForCall(0) - Expect(query).To(Equal("avg by (instance_id) (max_over_time(http{source_id='app-id',peer_type='Client'}[40s])) / (1000 * 1000)")) - - Expect(err).To(Not(HaveOccurred())) - Expect(metrics).To(HaveLen(2)) - - Expect(metrics[0].AppId).To(Equal("app-id")) - Expect(metrics[0].InstanceIndex).To(Equal(uint32(0))) - Expect(metrics[0].Name).To(Equal("responsetime")) - Expect(metrics[0].Unit).To(Equal("ms")) - Expect(metrics[0].Value).To(Equal("200")) - - Expect(metrics[1].AppId).To(Equal("app-id")) - Expect(metrics[1].InstanceIndex).To(Equal(uint32(1))) - Expect(metrics[1].Name).To(Equal("responsetime")) - Expect(metrics[1].Unit).To(Equal("ms")) - Expect(metrics[1].Value).To(Equal("300")) - }) - }) - }) - - DescribeTable("GetMetrics for Gauge Metrics", - func(autoscalerMetricType string, requiredFilters []string) { - metrics = []models.AppInstanceMetric{ - { - AppId: "some-id", - Name: autoscalerMetricType, - }, - } - fakeEnvelopeProcessor.GetGaugeMetricsReturnsOnCall(0, metrics, nil) - actualMetrics, err := logCacheClient.GetMetrics(appId, autoscalerMetricType, startTime, endTime) - Expect(err).NotTo(HaveOccurred()) - Expect(actualMetrics).To(Equal(metrics)) - - By("Sends the right arguments to log-cache-client") - actualContext, actualAppId, actualStartTime, readOptions := fakeGoLogCacheReader.ReadArgsForCall(0) - Expect(actualContext).To(Equal(context.Background())) - Expect(actualAppId).To(Equal(appId)) - Expect(actualStartTime).To(Equal(startTime)) - - Expect(valuesFrom(readOptions[0])["end_time"][0]).To(Equal(strconv.FormatInt(int64(endTime.UnixNano()), 10))) - - Expect(len(readOptions)).To(Equal(3), "filters by envelope type and metric names based on the requested metric type sent to GetMetrics") - Expect(valuesFrom(readOptions[1])["envelope_types"][0]).To(Equal("GAUGE")) - - // after starTime and envelopeType we filter the metric names - Expect(valuesFrom(readOptions[2])["name_filter"][0]).To(Equal(requiredFilters[2])) - - By("Sends the right arguments to the gauge processor") - actualEnvelopes, actualCurrentTimestamp := fakeEnvelopeProcessor.GetGaugeMetricsArgsForCall(0) - Expect(actualEnvelopes).To(Equal(envelopes)) - Expect(actualCurrentTimestamp).To(Equal(collectedAt.UnixNano())) - }, - Entry("When metric type is memoryutil", models.MetricNameMemoryUtil, []string{"endtime", "envelope_type", "memory|memory_quota"}), - Entry("When metric type is memoryused", models.MetricNameMemoryUsed, []string{"endtime", "envelope_type", "memory"}), - Entry("When metric type is cpu", models.MetricNameCPU, []string{"endtime", "envelope_type", "cpu"}), - Entry("When metric type is cpuutil", models.MetricNameCPUUtil, []string{"endtime", "envelope_type", "cpu_entitlement"}), - Entry("When metric type is disk", models.MetricNameDisk, []string{"endtime", "envelope_type", "disk"}), - Entry("When metric type is diskutil", models.MetricNameDiskUtil, []string{"endtime", "envelope_type", "disk|disk_quota"}), - Entry("When metric type is CustomMetrics", "a-custom-metric", []string{"endtime", "envelope_type", "a-custom-metric"}), - ) - - Describe("when quering 1 metrics", func() { - - BeforeEach(func() { - metrics = nil - metrics = append(metrics, models.AppInstanceMetric{ - AppId: appId, - InstanceIndex: 0, - Name: models.MetricNameMemoryUsed, - Unit: models.UnitMegaBytes, - Value: "21", - Timestamp: 1111, - }) - metrics = append(metrics, models.AppInstanceMetric{ - AppId: appId, - InstanceIndex: 0, - Name: models.MetricNameMemoryUtil, - Unit: models.UnitPercentage, - Value: "51", - Timestamp: 1111, - }) - }) - - It("should retrieve requested metrics only", func() { - actualMetrics, err := logCacheClient.GetMetrics(appId, models.MetricNameMemoryUsed, startTime, endTime) - Expect(err).NotTo(HaveOccurred()) - Expect(len(actualMetrics)).To(Equal(1)) - Expect(actualMetrics[0]).To(Equal(metrics[0])) - }) - }) - }) - -}) - -func valuesFrom(option logcache.ReadOption) url.Values { - values := url.Values{} - option(nil, values) - return values -} diff --git a/src/autoscaler/eventgenerator/client/metric_client_factory.go b/src/autoscaler/eventgenerator/client/metric_client_factory.go deleted file mode 100644 index 3a3307d307..0000000000 --- a/src/autoscaler/eventgenerator/client/metric_client_factory.go +++ /dev/null @@ -1,59 +0,0 @@ -package client - -import ( - "time" - - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/envelopeprocessor" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/config" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - "code.cloudfoundry.org/lager/v3" -) - -type GetMetricFunc func(appId string, metricType string, startTime time.Time, endTime time.Time) ([]models.AppInstanceMetric, error) - -type MetricClient interface { - GetMetrics(appId string, metricType string, startTime time.Time, endTime time.Time) ([]models.AppInstanceMetric, error) -} - -var NewProcessor = envelopeprocessor.NewProcessor - -type MetricClientFactory struct { -} - -func NewMetricClientFactory() *MetricClientFactory { - return &MetricClientFactory{} -} - -func (mc *MetricClientFactory) GetMetricClient(logger lager.Logger, conf *config.Config) MetricClient { - return mc.createLogCacheMetricClient(logger, conf) -} - -func (mc *MetricClientFactory) createLogCacheMetricClient(logger lager.Logger, conf *config.Config) MetricClient { - envelopeProcessor := NewProcessor(logger) - c := NewLogCacheClient(logger, time.Now, conf.Aggregator.AggregatorExecuteInterval, envelopeProcessor, conf.MetricCollector.MetricCollectorURL) - - if hasUAACreds(conf) { - uaaCreds := models.UAACreds{ - URL: conf.MetricCollector.UAACreds.URL, - ClientID: conf.MetricCollector.UAACreds.ClientID, - ClientSecret: conf.MetricCollector.UAACreds.ClientSecret, - SkipSSLValidation: conf.MetricCollector.UAACreds.SkipSSLValidation, - } - c.SetUAACreds(uaaCreds) - } else { - tlsCerts := &models.TLSCerts{ - KeyFile: conf.MetricCollector.TLSClientCerts.KeyFile, - CertFile: conf.MetricCollector.TLSClientCerts.CertFile, - CACertFile: conf.MetricCollector.TLSClientCerts.CACertFile, - } - tlsConfig, _ := tlsCerts.CreateClientConfig() - c.SetTLSConfig(tlsConfig) - } - c.Configure() - return c -} - -func hasUAACreds(conf *config.Config) bool { - return conf.MetricCollector.UAACreds.URL != "" && conf.MetricCollector.UAACreds.ClientSecret != "" && - conf.MetricCollector.UAACreds.ClientID != "" -} diff --git a/src/autoscaler/eventgenerator/client/metric_client_factory_test.go b/src/autoscaler/eventgenerator/client/metric_client_factory_test.go deleted file mode 100644 index bf0dc1295d..0000000000 --- a/src/autoscaler/eventgenerator/client/metric_client_factory_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package client_test - -import ( - "path/filepath" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - . "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/client" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/config" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" - logcache "code.cloudfoundry.org/go-log-cache/v2" - "code.cloudfoundry.org/lager/v3/lagertest" -) - -const testCertDir = "../../../../test-certs" - -var _ = Describe("MetricClientFactory", func() { - var ( - conf config.Config - metricClient MetricClient - metricClientFactory *MetricClientFactory - fakeEnvelopeProcessorCreator fakes.FakeEnvelopeProcessorCreator - fakeGoLogCacheClient fakes.FakeGoLogCacheClient - expectedTLSLogCacheClient logcache.Client - expectedHTTPClient logcache.HTTPClient - expectedOauth2HTTPClientOpt logcache.Oauth2Option - logger *lagertest.TestLogger - expectedMetricCollectorURL string - tlsCerts models.TLSCerts - uaaCreds models.UAACreds - caCertFilePath string - certFilePath string - keyFilePath string - ) - - BeforeEach(func() { - caCertFilePath = filepath.Join(testCertDir, "autoscaler-ca.crt") - certFilePath = filepath.Join(testCertDir, "eventgenerator.crt") - keyFilePath = filepath.Join(testCertDir, "eventgenerator.key") - fakeGoLogCacheClient = fakes.FakeGoLogCacheClient{} - metricClientFactory = NewMetricClientFactory() - NewProcessor = fakeEnvelopeProcessorCreator.NewProcessor - fakeGoLogCacheClient.NewClientReturns(&expectedTLSLogCacheClient) - expectedOauth2HTTPClientOpt = logcache.WithOauth2HTTPClient(expectedHTTPClient) - fakeGoLogCacheClient.WithOauth2HTTPClientReturns(expectedOauth2HTTPClientOpt) - }) - - JustBeforeEach(func() { - conf = config.Config{ - Aggregator: config.AggregatorConfig{ - AggregatorExecuteInterval: 51 * time.Second, - }, - MetricCollector: config.MetricCollectorConfig{ - MetricCollectorURL: expectedMetricCollectorURL, - TLSClientCerts: tlsCerts, - UAACreds: uaaCreds, - }, - } - - logger = lagertest.NewTestLogger("LogCache") - metricClient = metricClientFactory.GetMetricClient(logger, &conf) - }) - Describe("GetMetricClient", func() { - BeforeEach(func() { - expectedMetricCollectorURL = "some-log-cache-url:8080" - tlsCerts = models.TLSCerts{ - KeyFile: keyFilePath, - CertFile: certFilePath, - CACertFile: caCertFilePath, - } - }) - - It("Should create a LogCacheClient", func() { - Expect(metricClient).To(BeAssignableToTypeOf(&LogCacheClient{})) - actualURL := metricClient.(*LogCacheClient).GetUrl() - Expect(actualURL).To(Equal(expectedMetricCollectorURL)) - }) - - Describe("when uaa client and secret are not provided", func() { - BeforeEach(func() { - uaaCreds = models.UAACreds{} - }) - - It("Should set TLSConfig from config opts", func() { - expectedTlSCreds := &models.TLSCerts{KeyFile: keyFilePath, CertFile: certFilePath, CACertFile: caCertFilePath} - expectedTLSConfig, err := expectedTlSCreds.CreateClientConfig() - Expect(err).NotTo(HaveOccurred()) - actualTLSConfig := metricClient.(*LogCacheClient).GetTlsConfig() - Expect(actualTLSConfig.Certificates).To(Equal(expectedTLSConfig.Certificates)) - }) - }) - - Describe("when uaa client and secret are provided", func() { - BeforeEach(func() { - uaaCreds = models.UAACreds{ - URL: "log-cache.some-url.com", - ClientID: "some-id", - ClientSecret: "some-secret", - } - }) - - It("should set uaa creds from config", func() { - actualUAACreds := metricClient.(*LogCacheClient).GetUAACreds() - Expect(actualUAACreds).To(Equal(uaaCreds)) - }) - }) - - It("Should set AggregatorExecuteInterval as collectionInterval on LogCacheClient", func() { - Expect(metricClient.(*LogCacheClient).CollectionInterval()).To(Equal(conf.Aggregator.AggregatorExecuteInterval)) - }) - }) -}) diff --git a/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go b/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go index d3ba1f5a56..b9d1a86a9b 100644 --- a/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go +++ b/src/autoscaler/eventgenerator/cmd/eventgenerator/eventgenerator_test.go @@ -40,14 +40,6 @@ var _ = Describe("Eventgenerator", func() { Eventually(func() bool { return mockLogCache.ReadRequestsCount() >= 1 }, 5*time.Second).Should(BeTrue()) Eventually(func() bool { return len(mockScalingEngine.ReceivedRequests()) >= 1 }, time.Duration(2*breachDurationSecs)*time.Second).Should(BeTrue()) }) - - It("Should create a LogCacheClient", func() { - Eventually(runner.Session.Buffer(), 2).Should(Say("eventgenerator.LogCacheClient.GetMetrics")) - }) - - It("Should initialized an envelopeProcessor", func() { - Eventually(runner.Session.Buffer(), 2).Should(Say("eventgenerator.EnvelopeProcessor.GetGaugeMetrics")) - }) }) Context("with a missing config file", func() { diff --git a/src/autoscaler/eventgenerator/cmd/eventgenerator/main.go b/src/autoscaler/eventgenerator/cmd/eventgenerator/main.go index d8e4dbef87..d16ec0196b 100644 --- a/src/autoscaler/eventgenerator/cmd/eventgenerator/main.go +++ b/src/autoscaler/eventgenerator/cmd/eventgenerator/main.go @@ -6,9 +6,9 @@ import ( "code.cloudfoundry.org/app-autoscaler/src/autoscaler/db/sqldb" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/aggregator" - "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/client" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/config" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/generator" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/metric" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/server" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/healthendpoint" "code.cloudfoundry.org/app-autoscaler/src/autoscaler/helpers" @@ -82,8 +82,15 @@ func main() { appMonitorsChan := make(chan *models.AppMonitor, conf.Aggregator.AppMonitorChannelSize) appMetricChan := make(chan *models.AppMetric, conf.Aggregator.AppMetricChannelSize) - metricClient := client.NewMetricClientFactory().GetMetricClient(logger, conf) - metricPollers, err := createMetricPollers(logger, conf, appMonitorsChan, appMetricChan, metricClient) + + fetcherFactory := metric.NewLogCacheFetcherFactory(metric.StandardLogCacheFetcherCreator) + metricFetcher, err := fetcherFactory.CreateFetcher(logger, *conf) + if err != nil { + logger.Error("failed to create metric fetcher", err) + os.Exit(1) + } + + metricPollers, err := createMetricPollers(logger, conf, appMonitorsChan, appMetricChan, metricFetcher) if err != nil { logger.Error("failed to create MetricPoller", err) os.Exit(1) @@ -190,7 +197,7 @@ func createEvaluators(logger lager.Logger, conf *config.Config, triggersChan cha return evaluators, nil } -func createMetricPollers(logger lager.Logger, conf *config.Config, appMonitorsChan chan *models.AppMonitor, appMetricChan chan *models.AppMetric, metricClient client.MetricClient) ([]*aggregator.MetricPoller, error) { +func createMetricPollers(logger lager.Logger, conf *config.Config, appMonitorsChan chan *models.AppMonitor, appMetricChan chan *models.AppMetric, metricClient metric.Fetcher) ([]*aggregator.MetricPoller, error) { pollers := make([]*aggregator.MetricPoller, conf.Aggregator.MetricPollerCount) for i := 0; i < len(pollers); i++ { pollers[i] = aggregator.NewMetricPoller(logger, metricClient, appMonitorsChan, appMetricChan) diff --git a/src/autoscaler/eventgenerator/metric/fetcher.go b/src/autoscaler/eventgenerator/metric/fetcher.go new file mode 100644 index 0000000000..e895299313 --- /dev/null +++ b/src/autoscaler/eventgenerator/metric/fetcher.go @@ -0,0 +1,205 @@ +package metric + +import ( + "context" + "fmt" + "math" + "net/url" + "strconv" + "time" + + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/envelopeprocessor" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" + logcache "code.cloudfoundry.org/go-log-cache/v2" + "code.cloudfoundry.org/go-log-cache/v2/rpc/logcache_v1" + "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" + "code.cloudfoundry.org/lager/v3" +) + +type Fetcher interface { + FetchMetrics(appId string, metricType string, startTime time.Time, endTime time.Time) ([]models.AppInstanceMetric, error) +} + +type LogCacheFetcherCreator interface { + NewLogCacheFetcher(logger lager.Logger, client LogCacheClient, envelopeProcessor envelopeprocessor.EnvelopeProcessor, collectionInterval time.Duration) Fetcher +} + +type LogCacheClient interface { + Read(ctx context.Context, sourceID string, start time.Time, opts ...logcache.ReadOption) ([]*loggregator_v2.Envelope, error) + PromQL(ctx context.Context, query string, opts ...logcache.PromQLOption) (*logcache_v1.PromQL_InstantQueryResult, error) +} + +var StandardLogCacheFetcherCreator = &logCacheFetcherCreator{} + +type logCacheFetcherCreator struct{} + +type logCacheFetcher struct { + logger lager.Logger + logCacheClient LogCacheClient + envelopeProcessor envelopeprocessor.EnvelopeProcessor + collectionInterval time.Duration +} + +func (s *logCacheFetcherCreator) NewLogCacheFetcher(logger lager.Logger, client LogCacheClient, envelopeProcessor envelopeprocessor.EnvelopeProcessor, collectionInterval time.Duration) Fetcher { + return &logCacheFetcher{ + logCacheClient: client, + logger: logger, + envelopeProcessor: envelopeProcessor, + collectionInterval: collectionInterval, + } +} + +func (l *logCacheFetcher) FetchMetrics(appId string, metricType string, startTime time.Time, endTime time.Time) ([]models.AppInstanceMetric, error) { + // the log-cache REST API only return max. 1000 envelopes: https://github.com/cloudfoundry/log-cache-release/tree/main/src#get-apiv1readsource-id. + // receiving a limited set of envelopes breaks throughput and responsetime, because all envelopes are required to calculate these metric types properly. + // pagination via `start_time` and `end_time` could be done but is very error-prone. + // using the PromQL API also has an advantage over REST API because it shifts the metric aggregations to log-cache. + if metricType == models.MetricNameThroughput || metricType == models.MetricNameResponseTime { + return l.getMetricsPromQLAPI(appId, metricType, l.collectionInterval) + } + + return l.getMetricsRestAPI(appId, metricType, startTime, endTime) +} + +func (l *logCacheFetcher) getMetricsPromQLAPI(appId string, metricType string, collectionInterval time.Duration) ([]models.AppInstanceMetric, error) { + l.logger.Info("get-metric-promql-api", lager.Data{"appId": appId, "metricType": metricType}) + collectionIntervalSeconds := fmt.Sprintf("%.0f", collectionInterval.Seconds()) + now := time.Now() + + query := "" + metricTypeUnit := "" + if metricType == models.MetricNameThroughput { + query = fmt.Sprintf("sum by (instance_id) (count_over_time(http{source_id='%s',peer_type='Client'}[%ss])) / %s", appId, collectionIntervalSeconds, collectionIntervalSeconds) + metricTypeUnit = models.UnitRPS + } + + if metricType == models.MetricNameResponseTime { + query = fmt.Sprintf("avg by (instance_id) (max_over_time(http{source_id='%s',peer_type='Client'}[%ss])) / (1000 * 1000)", appId, collectionIntervalSeconds) + metricTypeUnit = models.UnitMilliseconds + } + + l.logger.Info("query-promql-api", lager.Data{"query": query}) + result, err := l.logCacheClient.PromQL(context.Background(), query, logcache.WithPromQLTime(now)) + if err != nil { + return []models.AppInstanceMetric{}, fmt.Errorf("failed getting PromQL result (metricType: %s, appId: %s, collectionInterval: %s, query: %s, time: %s): %w", metricType, appId, collectionIntervalSeconds, query, now.String(), err) + } + l.logger.Info("received-promql-api-result", lager.Data{"result": result, "query": query}) + + // safeguard: the query ensures that we get a vector but let's double-check + vector := result.GetVector() + if vector == nil { + return []models.AppInstanceMetric{}, fmt.Errorf("result does not contain a vector") + } + + // return empty metric if there are no samples, this usually happens in case there were no recent http-requests towards the application + if len(vector.GetSamples()) <= 0 { + return l.emptyAppInstanceMetrics(appId, metricType, metricTypeUnit, now) + } + + // convert result into autoscaler metric model + var metrics []models.AppInstanceMetric + for _, sample := range vector.GetSamples() { + // safeguard: metric label instance_id should be always there but let's double-check + instanceIdStr, ok := sample.GetMetric()["instance_id"] + if !ok { + return []models.AppInstanceMetric{}, fmt.Errorf("sample does not contain instance_id: %w", err) + } + + instanceIdUInt, err := strconv.ParseUint(instanceIdStr, 10, 32) + if err != nil { + return []models.AppInstanceMetric{}, fmt.Errorf("could not convert instance_id to uint32: %w", err) + } + + // safeguard: the query ensures that we get a point but let's double-check + point := sample.GetPoint() + if point == nil { + return []models.AppInstanceMetric{}, fmt.Errorf("sample does not contain a point") + } + + instanceId := uint32(instanceIdUInt) + valueWithoutDecimalsRoundedToCeiling := fmt.Sprintf("%.0f", math.Ceil(point.GetValue())) + + metrics = append(metrics, models.AppInstanceMetric{ + AppId: appId, + InstanceIndex: instanceId, + Name: metricType, + Unit: metricTypeUnit, + Value: valueWithoutDecimalsRoundedToCeiling, + CollectedAt: now.UnixNano(), + Timestamp: now.UnixNano(), + }) + } + return metrics, nil +} + +func (l *logCacheFetcher) emptyAppInstanceMetrics(appId string, name string, unit string, now time.Time) ([]models.AppInstanceMetric, error) { + return []models.AppInstanceMetric{ + { + AppId: appId, + InstanceIndex: 0, + Name: name, + Unit: unit, + Value: "0", + CollectedAt: now.UnixNano(), + Timestamp: now.UnixNano(), + }, + }, nil +} + +func (l *logCacheFetcher) getMetricsRestAPI(appId string, metricType string, startTime time.Time, endTime time.Time) ([]models.AppInstanceMetric, error) { + filters := l.readOptions(endTime, metricType) + + l.logger.Info("query-rest-api-with-filters", lager.Data{"appId": appId, "metricType": metricType, "startTime": startTime, "endTime": endTime, "filters": l.valuesFrom(filters)}) + envelopes, err := l.logCacheClient.Read(context.Background(), appId, startTime, filters...) + if err != nil { + return []models.AppInstanceMetric{}, fmt.Errorf("fail to Read %s metric from %s GoLogCache client: %w", logcache_v1.EnvelopeType_GAUGE, appId, err) + } + l.logger.Info("received-rest-api-result", lager.Data{"envelopes": envelopes}) + + metrics, err := l.envelopeProcessor.GetGaugeMetrics(envelopes, time.Now().UnixNano()) + + return l.filter(metrics, metricType), err +} + +func (l *logCacheFetcher) readOptions(endTime time.Time, metricType string) (readOptions []logcache.ReadOption) { + readOptions = append(readOptions, logcache.WithEndTime(endTime)) + readOptions = append(readOptions, logcache.WithEnvelopeTypes(logcache_v1.EnvelopeType_GAUGE)) + + switch metricType { + case models.MetricNameMemoryUtil: + readOptions = append(readOptions, logcache.WithNameFilter("memory|memory_quota")) + case models.MetricNameMemoryUsed: + readOptions = append(readOptions, logcache.WithNameFilter("memory")) + case models.MetricNameCPU: + readOptions = append(readOptions, logcache.WithNameFilter("cpu")) + case models.MetricNameCPUUtil: + readOptions = append(readOptions, logcache.WithNameFilter("cpu_entitlement")) + case models.MetricNameDisk: + readOptions = append(readOptions, logcache.WithNameFilter("disk")) + case models.MetricNameDiskUtil: + readOptions = append(readOptions, logcache.WithNameFilter("disk|disk_quota")) + default: + readOptions = append(readOptions, logcache.WithNameFilter(metricType)) + } + + return readOptions +} + +func (l *logCacheFetcher) filter(metrics []models.AppInstanceMetric, metricType string) []models.AppInstanceMetric { + var result []models.AppInstanceMetric + for i := range metrics { + if metrics[i].Name == metricType { + result = append(result, metrics[i]) + } + } + + return result +} + +func (l *logCacheFetcher) valuesFrom(filters []logcache.ReadOption) url.Values { + values := url.Values{} + for _, f := range filters { + f(nil, values) + } + return values +} diff --git a/src/autoscaler/eventgenerator/metric/fetcher_factory.go b/src/autoscaler/eventgenerator/metric/fetcher_factory.go new file mode 100644 index 0000000000..cec264b62e --- /dev/null +++ b/src/autoscaler/eventgenerator/metric/fetcher_factory.go @@ -0,0 +1,60 @@ +package metric + +import ( + "crypto/tls" + "net/http" + "time" + + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/envelopeprocessor" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/config" + logcache "code.cloudfoundry.org/go-log-cache/v2" + "code.cloudfoundry.org/lager/v3" + gogrpc "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +type FetcherFactory interface { + CreateFetcher(logger lager.Logger, conf config.Config) (Fetcher, error) +} + +type logCacheFetcherFactory struct { + fetcherCreator LogCacheFetcherCreator +} + +func NewLogCacheFetcherFactory(fetcherCreator LogCacheFetcherCreator) FetcherFactory { + return &logCacheFetcherFactory{ + fetcherCreator: fetcherCreator, + } +} + +func (l *logCacheFetcherFactory) CreateFetcher(logger lager.Logger, conf config.Config) (Fetcher, error) { + var options []logcache.ClientOption + + metricsCollectorConfig := conf.MetricCollector + uaaCredsConfig := metricsCollectorConfig.UAACreds + if uaaCredsConfig.IsNotEmpty() { + oauth2HTTPClient := logcache.NewOauth2HTTPClient(uaaCredsConfig.URL, uaaCredsConfig.ClientID, uaaCredsConfig.ClientSecret, logcache.WithOauth2HTTPClient(&http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + // #nosec G402 + InsecureSkipVerify: uaaCredsConfig.SkipSSLValidation, + }, + }, + })) + options = append(options, logcache.WithHTTPClient(oauth2HTTPClient)) + } else { + tlsConfig, err := metricsCollectorConfig.TLSClientCerts.CreateClientConfig() + if err != nil { + return nil, err + } + options = append(options, logcache.WithViaGRPC(gogrpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)))) + } + + return l.fetcherCreator.NewLogCacheFetcher( + logger, + logcache.NewClient(metricsCollectorConfig.MetricCollectorURL, options...), + envelopeprocessor.NewProcessor(logger), + conf.Aggregator.AggregatorExecuteInterval, + ), nil +} diff --git a/src/autoscaler/eventgenerator/metric/fetcher_factory_test.go b/src/autoscaler/eventgenerator/metric/fetcher_factory_test.go new file mode 100644 index 0000000000..49c0a09975 --- /dev/null +++ b/src/autoscaler/eventgenerator/metric/fetcher_factory_test.go @@ -0,0 +1,133 @@ +package metric_test + +import ( + "crypto/tls" + "net/http" + "path/filepath" + "reflect" + "time" + "unsafe" + + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/config" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/metric" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/testhelpers" + logcache "code.cloudfoundry.org/go-log-cache/v2" + "code.cloudfoundry.org/lager/v3/lagertest" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("logCacheFetcherFactory", func() { + + var ( + testLogger *lagertest.TestLogger + conf config.Config + + mockLogCacheMetricFetcherCreator *fakes.FakeLogCacheFetcherCreator + mockMetricFetcher *fakes.FakeFetcher + metricFetcherFactory metric.FetcherFactory + ) + + BeforeEach(func() { + testLogger = lagertest.NewTestLogger("testLogger") + + mockLogCacheMetricFetcherCreator = &fakes.FakeLogCacheFetcherCreator{} + mockMetricFetcher = &fakes.FakeFetcher{} + + metricFetcherFactory = metric.NewLogCacheFetcherFactory(mockLogCacheMetricFetcherCreator) + }) + + Describe("CreateFetcher", func() { + When("UAACreds are configured", func() { + BeforeEach(func() { + conf = config.Config{ + + Aggregator: config.AggregatorConfig{ + AggregatorExecuteInterval: 40 * time.Second, + }, + MetricCollector: config.MetricCollectorConfig{ + MetricCollectorURL: "foo", + UAACreds: models.UAACreds{ + URL: "foo", + ClientSecret: "foo", + ClientID: "foo", + SkipSSLValidation: true, + }, + }, + } + }) + + It("creates a log cache client that uses an HTTP-client", func() { + expectedLogCacheClient := logcache.NewClient( + conf.MetricCollector.MetricCollectorURL, + logcache.WithHTTPClient( + logcache.NewOauth2HTTPClient( + conf.MetricCollector.UAACreds.URL, + conf.MetricCollector.UAACreds.ClientID, + conf.MetricCollector.UAACreds.ClientSecret, + logcache.WithOauth2HTTPClient(&http.Client{ + Timeout: 5 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + // #nosec G402 + InsecureSkipVerify: conf.MetricCollector.UAACreds.SkipSSLValidation, + }, + }, + }), + ), + ), + ) + mockLogCacheMetricFetcherCreator.NewLogCacheFetcherReturns(mockMetricFetcher) + + metricFetcher, err := metricFetcherFactory.CreateFetcher(testLogger, conf) + + Expect(err).ToNot(HaveOccurred()) + Expect(metricFetcher).To(Equal(mockMetricFetcher)) + Expect(mockLogCacheMetricFetcherCreator.NewLogCacheFetcherCallCount()).To(Equal(1)) + logger, logCacheClient, envelopeProcessor, collectionInterval := mockLogCacheMetricFetcherCreator.NewLogCacheFetcherArgsForCall(0) + Expect(logger).To(Equal(testLogger)) + Expect(logCacheClient).To(Equal(expectedLogCacheClient)) + Expect(envelopeProcessor).ToNot(BeNil()) + Expect(collectionInterval).To(Equal(conf.Aggregator.AggregatorExecuteInterval)) + }) + }) + + When("no UAACreds are configured but TLSClientCerts are there", func() { + BeforeEach(func() { + testCertDir := testhelpers.TestCertFolder() + conf = config.Config{ + MetricCollector: config.MetricCollectorConfig{ + MetricCollectorURL: "foo", + TLSClientCerts: models.TLSCerts{ + CACertFile: filepath.Join(testCertDir, "autoscaler-ca.crt"), + CertFile: filepath.Join(testCertDir, "eventgenerator.crt"), + KeyFile: filepath.Join(testCertDir, "eventgenerator.key"), + }, + }, + } + }) + + It("creates a log cache client that uses a GRPC-client", func() { + mockLogCacheMetricFetcherCreator.NewLogCacheFetcherReturns(mockMetricFetcher) + + metricFetcher, err := metricFetcherFactory.CreateFetcher(testLogger, conf) + + Expect(err).ToNot(HaveOccurred()) + Expect(metricFetcher).To(Equal(mockMetricFetcher)) + Expect(mockLogCacheMetricFetcherCreator.NewLogCacheFetcherCallCount()).To(Equal(1)) + logger, logCacheClient, envelopeProcessor, collectionInterval := mockLogCacheMetricFetcherCreator.NewLogCacheFetcherArgsForCall(0) + Expect(logger).To(Equal(testLogger)) + Expect(getUnexportedField("grpcClient", logCacheClient)).ToNot(BeNil()) + Expect(envelopeProcessor).ToNot(BeNil()) + Expect(collectionInterval).To(Equal(conf.Aggregator.AggregatorExecuteInterval)) + }) + }) + }) +}) + +func getUnexportedField(name string, client metric.LogCacheClient) interface{} { + field := reflect.ValueOf(client).Elem().FieldByName(name) + return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface() +} diff --git a/src/autoscaler/eventgenerator/metric/fetcher_test.go b/src/autoscaler/eventgenerator/metric/fetcher_test.go new file mode 100644 index 0000000000..7a4361f188 --- /dev/null +++ b/src/autoscaler/eventgenerator/metric/fetcher_test.go @@ -0,0 +1,313 @@ +package metric_test + +import ( + "context" + "errors" + "fmt" + "net/url" + "time" + + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/eventgenerator/metric" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/fakes" + "code.cloudfoundry.org/app-autoscaler/src/autoscaler/models" + logcache "code.cloudfoundry.org/go-log-cache/v2" + "code.cloudfoundry.org/go-log-cache/v2/rpc/logcache_v1" + "code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2" + "code.cloudfoundry.org/lager/v3/lagertest" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("logCacheFetcher", func() { + + var ( + testLogger *lagertest.TestLogger + + mockLogCacheClient *fakes.FakeLogCacheClient + mockEnvelopeProcess *fakes.FakeEnvelopeProcessor + + collectionInterval time.Duration + + metricFetcher metric.Fetcher + ) + + BeforeEach(func() { + testLogger = lagertest.NewTestLogger("testLogger") + + mockLogCacheClient = &fakes.FakeLogCacheClient{} + mockEnvelopeProcess = &fakes.FakeEnvelopeProcessor{} + + collectionInterval = 40 * time.Second + + metricFetcher = metric.StandardLogCacheFetcherCreator.NewLogCacheFetcher(testLogger, mockLogCacheClient, mockEnvelopeProcess, collectionInterval) + }) + + Describe("FetchMetrics", func() { + + When("reading metric from PromQL API", func() { + + When("PromQL call fails", func() { + It("returns an error", func() { + mockLogCacheClient.PromQLReturns(nil, errors.New("fail")) + _, err := metricFetcher.FetchMetrics("app-id", "throughput", time.Now(), time.Now()) + Expect(err).To(HaveOccurred()) + }) + }) + + When("PromQL result is not a vector", func() { + It("returns an error", func() { + mockLogCacheClient.PromQLReturns(nil, nil) + _, err := metricFetcher.FetchMetrics("app-id", "throughput", time.Now(), time.Now()) + Expect(err).To(HaveOccurred()) + }) + }) + + When("vector does not contain samples", func() { + It("returns empty metric", func() { + mockLogCacheClient.PromQLReturns(&logcache_v1.PromQL_InstantQueryResult{ + Result: &logcache_v1.PromQL_InstantQueryResult_Vector{ + Vector: &logcache_v1.PromQL_Vector{ + Samples: nil, + }, + }, + }, nil) + + metrics, err := metricFetcher.FetchMetrics("app-id", "throughput", time.Now(), time.Now()) + + Expect(err).ToNot(HaveOccurred()) + Expect(metrics).To(HaveLen(1)) + Expect(metrics[0].AppId).To(Equal("app-id")) + Expect(metrics[0].InstanceIndex).To(Equal(uint32(0))) + Expect(metrics[0].Name).To(Equal("throughput")) + Expect(metrics[0].Unit).To(Equal("rps")) + Expect(metrics[0].Value).To(Equal("0")) + Expect(time.Unix(0, metrics[0].CollectedAt)).To(BeTemporally("~", time.Now(), time.Second)) + Expect(time.Unix(0, metrics[0].Timestamp)).To(BeTemporally("~", time.Now(), time.Second)) + }) + }) + + When("sample does not contain instance_id", func() { + It("returns an error", func() { + mockLogCacheClient.PromQLReturns(&logcache_v1.PromQL_InstantQueryResult{ + Result: &logcache_v1.PromQL_InstantQueryResult_Vector{ + Vector: &logcache_v1.PromQL_Vector{ + Samples: []*logcache_v1.PromQL_Sample{ + { + Metric: map[string]string{ + // "instance_id": "0", is missing here + }, + }, + }, + }, + }, + }, nil) + _, err := metricFetcher.FetchMetrics("app-id", "throughput", time.Now(), time.Now()) + Expect(err).To(HaveOccurred()) + }) + }) + + When("instance_id can not be parsed to uint", func() { + It("returns an error", func() { + mockLogCacheClient.PromQLReturns(&logcache_v1.PromQL_InstantQueryResult{ + Result: &logcache_v1.PromQL_InstantQueryResult_Vector{ + Vector: &logcache_v1.PromQL_Vector{ + Samples: []*logcache_v1.PromQL_Sample{ + { + Metric: map[string]string{ + "instance_id": "iam-no-uint", + }, + }, + }, + }, + }, + }, nil) + _, err := metricFetcher.FetchMetrics("app-id", "throughput", time.Now(), time.Now()) + Expect(err).To(HaveOccurred()) + }) + }) + + When("sample does not contain a point", func() { + It("returns an error", func() { + mockLogCacheClient.PromQLReturns(&logcache_v1.PromQL_InstantQueryResult{ + Result: &logcache_v1.PromQL_InstantQueryResult_Vector{ + Vector: &logcache_v1.PromQL_Vector{ + Samples: []*logcache_v1.PromQL_Sample{ + { + Metric: map[string]string{ + "instance_id": "0", + }, + }, + }, + }, + }, + }, nil) + _, err := metricFetcher.FetchMetrics("app-id", "throughput", time.Now(), time.Now()) + Expect(err).To(HaveOccurred()) + }) + }) + + When("reading metric for metric_type responsetime", func() { + It("should succeed", func() { + mockLogCacheClient.PromQLReturns(&logcache_v1.PromQL_InstantQueryResult{ + Result: &logcache_v1.PromQL_InstantQueryResult_Vector{ + Vector: &logcache_v1.PromQL_Vector{ + Samples: []*logcache_v1.PromQL_Sample{ + { + Metric: map[string]string{ + "instance_id": "0", + }, + Point: &logcache_v1.PromQL_Point{ + Value: 200, + }, + }, + { + Metric: map[string]string{ + "instance_id": "1", + }, + Point: &logcache_v1.PromQL_Point{ + Value: 300, + }, + }, + }, + }, + }, + }, nil) + + metrics, err := metricFetcher.FetchMetrics("app-id", "responsetime", time.Now(), time.Now()) + + Expect(err).ToNot(HaveOccurred()) + Expect(metrics).To(HaveLen(2)) + Expect(metrics[0].AppId).To(Equal("app-id")) + Expect(metrics[0].InstanceIndex).To(Equal(uint32(0))) + Expect(metrics[0].Name).To(Equal("responsetime")) + Expect(metrics[0].Unit).To(Equal("ms")) + Expect(metrics[0].Value).To(Equal("200")) + + Expect(metrics[1].AppId).To(Equal("app-id")) + Expect(metrics[1].InstanceIndex).To(Equal(uint32(1))) + Expect(metrics[1].Name).To(Equal("responsetime")) + Expect(metrics[1].Unit).To(Equal("ms")) + Expect(metrics[1].Value).To(Equal("300")) + + _, query, _ := mockLogCacheClient.PromQLArgsForCall(0) + Expect(query).To(Equal("avg by (instance_id) (max_over_time(http{source_id='app-id',peer_type='Client'}[40s])) / (1000 * 1000)")) + }) + }) + + When("reading metric for metric_type throughput", func() { + It("should succeed", func() { + mockLogCacheClient.PromQLReturns(&logcache_v1.PromQL_InstantQueryResult{ + Result: &logcache_v1.PromQL_InstantQueryResult_Vector{ + Vector: &logcache_v1.PromQL_Vector{ + Samples: []*logcache_v1.PromQL_Sample{ + { + Metric: map[string]string{ + "instance_id": "0", + }, + Point: &logcache_v1.PromQL_Point{ + Value: 123, + }, + }, + { + Metric: map[string]string{ + "instance_id": "1", + }, + Point: &logcache_v1.PromQL_Point{ + Value: 321, + }, + }, + }, + }, + }, + }, nil) + + metrics, err := metricFetcher.FetchMetrics("app-id", "throughput", time.Now(), time.Now()) + + Expect(err).To(Not(HaveOccurred())) + Expect(metrics).To(HaveLen(2)) + + Expect(metrics[0].AppId).To(Equal("app-id")) + Expect(metrics[0].InstanceIndex).To(Equal(uint32(0))) + Expect(metrics[0].Name).To(Equal("throughput")) + Expect(metrics[0].Unit).To(Equal("rps")) + Expect(metrics[0].Value).To(Equal("123")) + + Expect(metrics[1].AppId).To(Equal("app-id")) + Expect(metrics[1].InstanceIndex).To(Equal(uint32(1))) + Expect(metrics[1].Name).To(Equal("throughput")) + Expect(metrics[1].Unit).To(Equal("rps")) + Expect(metrics[1].Value).To(Equal("321")) + + _, query, _ := mockLogCacheClient.PromQLArgsForCall(0) + Expect(query).To(Equal("sum by (instance_id) (count_over_time(http{source_id='app-id',peer_type='Client'}[40s])) / 40")) + }) + }) + }) + + When("reading metric from REST API", func() { + When("log cache returns error", func() { + BeforeEach(func() { + mockLogCacheClient.ReadReturns(nil, errors.New("error")) + }) + + It("return error", func() { + _, err := metricFetcher.FetchMetrics("app-id", models.MetricNameMemoryUtil, time.Now(), time.Now()) + Expect(err).To(HaveOccurred()) + }) + }) + + DescribeTable("via specific metric_type that causes a read from the REST API", + func(metricType string, nameFilter string) { + expectedMetrics := []models.AppInstanceMetric{ + { + AppId: "app-id", + Name: metricType, + }, + } + mockEnvelopeProcess.GetGaugeMetricsReturns(expectedMetrics, nil) + mockLogCacheClient.ReadReturns([]*loggregator_v2.Envelope{ + { + SourceId: "app-id", + }, + }, nil) + expectedStartTime := time.Now() + expectedEndTime := time.Now() + + actualMetrics, err := metricFetcher.FetchMetrics("app-id", metricType, expectedStartTime, expectedEndTime) + + Expect(err).NotTo(HaveOccurred()) + Expect(actualMetrics).To(Equal(expectedMetrics)) + + actualContext, actualAppId, actualStartTime, readOptions := mockLogCacheClient.ReadArgsForCall(0) + Expect(actualContext).To(Equal(context.Background())) + Expect(actualAppId).To(Equal("app-id")) + Expect(actualStartTime).To(Equal(expectedStartTime)) + + Expect(len(readOptions)).To(Equal(3)) + Expect(valuesFrom(readOptions[0])["end_time"][0]).To(Equal(fmt.Sprintf("%d", expectedEndTime.UnixNano()))) + Expect(valuesFrom(readOptions[1])["envelope_types"][0]).To(Equal("GAUGE")) + Expect(valuesFrom(readOptions[2])["name_filter"][0]).To(Equal(nameFilter)) + + actualEnvelopes, actualCurrentTimestamp := mockEnvelopeProcess.GetGaugeMetricsArgsForCall(0) + Expect(actualEnvelopes).To(Equal([]*loggregator_v2.Envelope{ + {SourceId: "app-id"}, + })) + Expect(time.Unix(0, actualCurrentTimestamp)).Should(BeTemporally("~", time.Now(), time.Second)) + }, + Entry("metric type memoryutil", models.MetricNameMemoryUtil, "memory|memory_quota"), + Entry("metric type memoryused", models.MetricNameMemoryUsed, "memory"), + Entry("metric type cpu", models.MetricNameCPU, "cpu"), + Entry("metric type cpuutil", models.MetricNameCPUUtil, "cpu_entitlement"), + Entry("metric type disk", models.MetricNameDisk, "disk"), + Entry("metric type diskutil", models.MetricNameDiskUtil, "disk|disk_quota"), + Entry("metric type CustomMetrics", "a-custom-metric", "a-custom-metric"), + ) + }) + }) +}) + +func valuesFrom(option logcache.ReadOption) url.Values { + values := url.Values{} + option(nil, values) + return values +} diff --git a/src/autoscaler/eventgenerator/client/client_suite_test.go b/src/autoscaler/eventgenerator/metric/metric_suite_test.go similarity index 56% rename from src/autoscaler/eventgenerator/client/client_suite_test.go rename to src/autoscaler/eventgenerator/metric/metric_suite_test.go index 79e004c4a1..92aca5cbfd 100644 --- a/src/autoscaler/eventgenerator/client/client_suite_test.go +++ b/src/autoscaler/eventgenerator/metric/metric_suite_test.go @@ -1,4 +1,4 @@ -package client_test +package metric_test import ( "testing" @@ -7,7 +7,7 @@ import ( . "github.com/onsi/gomega" ) -func TestClient(t *testing.T) { +func TestMetricsSuite(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Client Suite") + RunSpecs(t, "Metrics Suite") } diff --git a/src/autoscaler/generate-fakes.go b/src/autoscaler/generate-fakes.go index 2b0c2bdf23..b14bcd09f4 100644 --- a/src/autoscaler/generate-fakes.go +++ b/src/autoscaler/generate-fakes.go @@ -13,11 +13,7 @@ package fakes //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_storedprocedure_db.go ./db StoredProcedureDB //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_metric_forwarder.go ./metricsforwarder/forwarder MetricForwarder //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_plan_checker.go ./api/plancheck PlanChecker -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_log_cache_client.go ./eventgenerator/client LogCacheClientReader //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_envelope_processor.go ./envelopeprocessor EnvelopeProcessor -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_log_cache_creator.go ./eventgenerator/client LogCacheClientCreator -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_go_log_cache_client.go ./eventgenerator/client GoLogCacheClient -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_grpc.go ./eventgenerator/client GRPCOptions //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_envelope_processor_creator.go ./envelopeprocessor EnvelopeProcessorCreator //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_plan_checker.go ./api/plancheck PlanChecker //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_ratelimiter.go ./ratelimiter Limiter @@ -25,3 +21,6 @@ package fakes //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_database_status.go ./healthendpoint DatabaseStatus //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_operator.go ./operator Operator //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_sychronizer.go ./scalingengine/schedule ActiveScheduleSychronizer +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_log_cache_fetcher_creator.go ./eventgenerator/metric LogCacheFetcherCreator +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_fetcher.go ./eventgenerator/metric Fetcher +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o ./fakes/fake_log_cache_client.go ./eventgenerator/metric LogCacheClient diff --git a/src/autoscaler/models/uaa_creds.go b/src/autoscaler/models/uaa_creds.go index af3a368a7b..46ea6d7872 100644 --- a/src/autoscaler/models/uaa_creds.go +++ b/src/autoscaler/models/uaa_creds.go @@ -7,6 +7,6 @@ type UAACreds struct { SkipSSLValidation bool `yaml:"skip_ssl_validation" json:"skipSSLValidation"` } -func (c UAACreds) IsEmpty() bool { - return c.URL == "" +func (c UAACreds) IsNotEmpty() bool { + return c.URL != "" } From 4d34f66e002da8cc63f940a9bab8569b9120d50a Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 14 Jun 2024 16:35:21 +0200 Subject: [PATCH 167/186] copy liquibasefile during packaging --- packages/eventgenerator/packaging | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/eventgenerator/packaging b/packages/eventgenerator/packaging index f24cdb44c3..6cc02c9803 100644 --- a/packages/eventgenerator/packaging +++ b/packages/eventgenerator/packaging @@ -11,3 +11,4 @@ popd cp -a ${BOSH_COMPILE_TARGET}/autoscaler/build/eventgenerator ${BOSH_INSTALL_TARGET} cp -a ${BOSH_COMPILE_TARGET}/autoscaler/eventgenerator/db/dataaggregator.db.changelog.yml ${BOSH_INSTALL_TARGET} +cp -a ${BOSH_COMPILE_TARGET}/autoscaler/eventgenerator/db/metricscollector.db.changelog.yml ${BOSH_INSTALL_TARGET} From 3f4e6df991a3f41b4c8dffc7ba87f82f1caacc72 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 14 Jun 2024 17:14:30 +0200 Subject: [PATCH 168/186] remove merge leftovers --- ci/autoscaler/pipeline.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index 2ca9c94901..f5a86e2a07 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -28,13 +28,9 @@ anchors: operations/instance-identity-cert-from-cf.yml operations/add-postgres-variables.yml operations/enable-nats-tls.yml - operations/loggregator-certs-from-cf.yml operations/add-extra-plan.yml operations/set-release-version.yml - operations/configure-log-cache-and-forward-metrics-via-mtls.yml operations/enable-metricsforwarder-via-syslog-agent.yml - operations/remove-metricsserver.yml - operations/remove-metricsgateway.yml operations/enable-scheduler-logging.yml operations/disable-postgres-tls-config.yml operations/use-cf-services.yml From 147829665aaeaf401a11515b02a18c3b596bf09f Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Fri, 14 Jun 2024 20:44:35 +0200 Subject: [PATCH 169/186] improve readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 26c63ef229..fe1210d875 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ through ### Prerequisites * [Docker](https://www.docker.com/products/docker-desktop/) to spin up the required databases -* [devbox](https://github.com/jetify-com/devbox) to start a shell with all required tools -* clone of https://github.com/cloudfoundry/app-autoscaler-env-bbl-state next to the clone of this repo to access systems -* [direnv](https://direnv.net/) to spin up the shell properly before running the make-targets +* [devbox](https://github.com/jetify-com/devbox) to start a shell with all required tools (see [devbox.json](/devbox.json)) +* A clone of [cloudfoundry/app-autoscaler-env-bbl-state](https://github.com/cloudfoundry/app-autoscaler-env-bbl-state) next to the clone of this repo in order to access dev-systems +* [direnv](https://direnv.net/) to automatically spin up the devbox shell before running the make targets (see [.envrc](/.envrc)) ### Make Targets From c5e6d59065d24d1808a34921a182a8fccfe93455 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 14 Jun 2024 23:25:35 +0000 Subject: [PATCH 170/186] fix(deps): update code.cloudfoundry.org/loggregator-agent-release/src digest to 69a1db0 --- src/autoscaler/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 6710fd7b23..03009e6f75 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,7 +8,7 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240613070107-8479c7b33126 + code.cloudfoundry.org/loggregator-agent-release/src 69a1db0ef542 code.cloudfoundry.org/tlsconfig v0.0.0-20240613173017-075d5b187a0d dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 From 49393a00cd33f919096e16222fb19795e6688de6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 15 Jun 2024 00:32:25 +0100 Subject: [PATCH 171/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/autoscaler/go.mod | 2 +- src/autoscaler/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index 03009e6f75..30cae99d6a 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,7 +8,7 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src 69a1db0ef542 + code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240614070214-69a1db0ef542 code.cloudfoundry.org/tlsconfig v0.0.0-20240613173017-075d5b187a0d dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 3e963207a2..8aef20b3cf 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -606,8 +606,8 @@ code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a h1:X code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a/go.mod h1:/Be8VtLiCeMUoYdUzFtmW8GGkk89HAy3zD79KUXzbhs= code.cloudfoundry.org/lager/v3 v3.0.3 h1:/UTmadZfIaKuT/whEinSxK1mzRfNu1uPfvjFfGqiwzM= code.cloudfoundry.org/lager/v3 v3.0.3/go.mod h1:Zn5q1SrIuuHjEUE7xerMKt3ztunrJQCZETAo7rV0CH8= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240613070107-8479c7b33126 h1:VZNZFuhLddGbwPTiSBIc4ooEkMJD/gn7EpGgU1u002U= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240613070107-8479c7b33126/go.mod h1:lAijiGcfNgqAQDN5mk4YmWYMN92Bq7DucH1IAjW3Htc= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240614070214-69a1db0ef542 h1:OxdgOGWm4jaUHeFVIbsScBc1BkLMTHa5rjBt/yfZrlk= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240614070214-69a1db0ef542/go.mod h1:/Dzry/KNUjigc8cvCd5SwSqRM2StxctoNBRLRiLbta4= code.cloudfoundry.org/tlsconfig v0.0.0-20240613173017-075d5b187a0d h1:nedO/GTZuf/FlRvrAuXWX9FdBBEd1hWXlhn1bp6li68= code.cloudfoundry.org/tlsconfig v0.0.0-20240613173017-075d5b187a0d/go.mod h1:qDYY1KwaNSWV7beH9J/IJR5erYD3bwO4zs+nDbpgbf8= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= From 0f8bef571f73f5ad728d99dca86e36a935e7ff96 Mon Sep 17 00:00:00 2001 From: Alan Moran Date: Tue, 18 Jun 2024 15:14:13 +0200 Subject: [PATCH 172/186] Adds missing variable to deploy-autoscaler script --- ci/autoscaler/scripts/deploy-autoscaler.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/autoscaler/scripts/deploy-autoscaler.sh b/ci/autoscaler/scripts/deploy-autoscaler.sh index 90fd69c565..5b15eb3008 100755 --- a/ci/autoscaler/scripts/deploy-autoscaler.sh +++ b/ci/autoscaler/scripts/deploy-autoscaler.sh @@ -95,6 +95,7 @@ function create_manifest(){ -v syslog_agent_log_cache_tls_key="$(credhub get -n /bosh-autoscaler/cf/syslog_agent_log_cache_tls --key private_key --quiet)"\ -v metricscollector_ca_cert="$(credhub get -n /bosh-autoscaler/cf/log_cache --key ca --quiet)"\ -v metricscollector_client_cert="$(credhub get -n /bosh-autoscaler/cf/log_cache --key certificate --quiet)"\ + -v metricsforwarder_host="${metricsforwarder_host}"\ -v metricscollector_client_key="$(credhub get -n /bosh-autoscaler/cf/log_cache --key private_key --quiet)"\ -v skip_ssl_validation=true \ > "${tmp_manifest_file}" From 536f3d02c8908319452413b679904b967d2191ef Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Tue, 18 Jun 2024 15:29:17 +0200 Subject: [PATCH 173/186] fix(ci): fix `fly` param value assignment so that `make delete-pipeline` works (#3031) --- ci/scripts/delete-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/delete-pipeline.sh b/ci/scripts/delete-pipeline.sh index fbc1965fec..3d41924e38 100755 --- a/ci/scripts/delete-pipeline.sh +++ b/ci/scripts/delete-pipeline.sh @@ -7,7 +7,7 @@ set -eu -o pipefail target="app-autoscaler-release" function delete-pipeline(){ - payload=$(fly --target= "$target" pipelines --json) + payload=$(fly --target="$target" pipelines --json) pipelines=$(echo "$payload" | jq ".[] |.name" -r | sort) # ignore shellcheck warning From 2853daa60855165da91096163091019cff324cf5 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Tue, 18 Jun 2024 17:15:19 +0200 Subject: [PATCH 174/186] fix(readme): drop HTML-tags to fix rendering (#3032) --- README.md | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index fe1210d875..820093f5bb 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ The Application Autoscaler provides the capability to adjust the computation res through * dynamic scaling based on application performance metrics +* dynamic scaling based on custom metrics * scheduled scaling based on time ## Local Development @@ -17,23 +18,23 @@ through ### Make Targets -| Category | Description | Target | -|-------------------|----------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| -| mock | generate mocks | `make generate-fakes` | -| unit-tests | run against PostgreSQL | `make test` | -| unit-tests | run against specific PostgreSQL version | 
make clean #Only if you're changing versions to refresh the running docker image
make test POSTGRES_TAG=x.y
| -| unit-tests | run against MySQL | `make test db_type=mysql` | -| unit-tests | run against specific MySQL version | 
make clean #Only if you're changing versions to refresh the running docker image
make test db_type=mysql MYSQL_TAG=x.y
| -| integration-tests | run against PostgreSQL | `make integration` | -| integration-tests | run against specific PostgreSQL version | 
make clean #Only if you're changing versions to refresh the running docker image
make integration POSTGRES_TAG=x.y
| -| integration-tests | run against MySQL | `make integration db_type=mysql` | -| integration-tests | run against specific MySQL version | 
make clean #Only if you're changing versions to refresh the running docker image
make integration db_type=mysql MYSQL_TAG=x.y
| -| acceptance-tests | run acceptance-tests, see [AutoScaler UAT guide](src/acceptance/README.md) for details | `make acceptance-tests` | -| lint | check code style | `make lint` | -| lint | check code style and apply auto-fixes | `OPTS=--fix RUBOCOP_OPTS=-A make lint` | -| build | compile project | `make build` | -| deploy | deploy Application Autoscaler and register the service broker in CF | `make deploy-autoscaler` | -| cleanup | remove build artifacts | `make clean` | +| Target | Description | +|--------------------------------------------------------------------------|----------------------------------------------------------------------------------------| +| `make generate-fakes` | generate mocks | +| `make test` | run unit-tests against PostgreSQL | +| `make clean && make test POSTGRES_TAG=x.y` | run unit-tests against specific PostgreSQL version | +| `make test db_type=mysql` | run unit-tests against MySQL | +| `make clean && make test db_type=mysql MYSQL_TAG=x.y` | run unit-tests against specific MySQL version | +| `make integration` | run integration-tests against PostgreSQL | +| `make clean && make integration POSTGRES_TAG=x.y` | run integration-tests against specific PostgreSQL version | +| `make integration db_type=mysql` | run integration-tests against MySQL | +| `make clean && make integration db_type=mysql MYSQL_TAG=x.y` | run integration-tests against specific MySQL version | +| `make acceptance-tests` | run acceptance-tests, see [AutoScaler UAT guide](src/acceptance/README.md) for details | +| `make lint` | check code style | +| `OPTS=--fix RUBOCOP_OPTS=-A make lint` | check code style and apply auto-fixes | +| `make build` | compile project | +| `make deploy-autoscaler` | deploy Application Autoscaler and register the service broker in CF | +| `make clean` | remove build artifacts | ## Use Application Autoscaler Service From 5178562922a457ae8d02445ff9b1c99de3934187 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 19 Jun 2024 09:06:15 +0000 Subject: [PATCH 175/186] chore(deps): update dependency gruntwork-io/terragrunt to v0.59.4 --- ci/terragrunt/.tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/terragrunt/.tool-versions b/ci/terragrunt/.tool-versions index c50c3d9ae8..780f781f6d 100644 --- a/ci/terragrunt/.tool-versions +++ b/ci/terragrunt/.tool-versions @@ -3,4 +3,4 @@ helm 3.14.4 python 3.12.4 terraform 1.8.5 terraform-lsp -terragrunt 0.58.15 +terragrunt 0.59.4 From 8cf76285c9babb5f7f994d1ad1886cc89a17fcb6 Mon Sep 17 00:00:00 2001 From: app-autoscaler-ci-bot Date: Wed, 19 Jun 2024 12:05:52 +0100 Subject: [PATCH 176/186] Updated release version to 14.0.0 in golangapiserver --- jobs/golangapiserver/spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/golangapiserver/spec b/jobs/golangapiserver/spec index 4cce6c9f1e..364c12e268 100644 --- a/jobs/golangapiserver/spec +++ b/jobs/golangapiserver/spec @@ -92,7 +92,7 @@ properties: default: "Autoscaler" autoscaler.apiserver.info.build: description: "Autoscaler application build number" - default: "13.2.0" + default: "14.0.0" autoscaler.apiserver.info.support_url: description: "Autoscaler application support page URL" default: "https://github.com/cloudfoundry/app-autoscaler-release" From b3496a4965d36f7c74250572695583e0148060b8 Mon Sep 17 00:00:00 2001 From: app-autoscaler-ci-bot Date: Wed, 19 Jun 2024 12:08:42 +0100 Subject: [PATCH 177/186] created release v14.0.0 --- .final_builds/jobs/eventgenerator/index.yml | 4 + .final_builds/jobs/golangapiserver/index.yml | 4 + .final_builds/jobs/operator/index.yml | 4 + .final_builds/packages/db/index.yml | 4 + .../packages/eventgenerator/index.yml | 4 + .../packages/golangapiserver/index.yml | 4 + .../packages/metricsforwarder/index.yml | 4 + .final_builds/packages/operator/index.yml | 4 + .../packages/scalingengine/index.yml | 4 + .final_builds/packages/scheduler/index.yml | 4 + .../app-autoscaler/app-autoscaler-14.0.0.yml | 132 ++++++++++++++++++ releases/app-autoscaler/index.yml | 2 + 12 files changed, 174 insertions(+) create mode 100644 releases/app-autoscaler/app-autoscaler-14.0.0.yml diff --git a/.final_builds/jobs/eventgenerator/index.yml b/.final_builds/jobs/eventgenerator/index.yml index bb70aea85e..b85aee425d 100644 --- a/.final_builds/jobs/eventgenerator/index.yml +++ b/.final_builds/jobs/eventgenerator/index.yml @@ -35,6 +35,10 @@ builds: version: 75fb3afde771fea3e53a17a75a49cd555c05ba61c68052aed740105ea75631b5 blobstore_id: cd205543-2581-442a-5d8a-73cf495c17eb sha1: sha256:2f1774e3059ac144123d8d04a269e4b6949a00b855fefb86a92b67ca1861ccbd + 7b661855f25b17690b0bea56557a4553ee64c3b559866d73bede9c2aab451bee: + version: 7b661855f25b17690b0bea56557a4553ee64c3b559866d73bede9c2aab451bee + blobstore_id: 10be7f0a-01c5-4a63-66f4-8ea6adb66bc6 + sha1: sha256:94a8ce91eec09122ef2fd2e0f773d0df8adad0355671c7081e5c8c5cd9c424e2 b2d2b8153753c2db2a166a851c62b51e7cb1a3b6f9c05c48ab3461299a8c4f23: version: b2d2b8153753c2db2a166a851c62b51e7cb1a3b6f9c05c48ab3461299a8c4f23 blobstore_id: 74a42a2d-cd6c-4886-63f8-383558bfb3ff diff --git a/.final_builds/jobs/golangapiserver/index.yml b/.final_builds/jobs/golangapiserver/index.yml index 0164f3a677..acdf0b06e0 100644 --- a/.final_builds/jobs/golangapiserver/index.yml +++ b/.final_builds/jobs/golangapiserver/index.yml @@ -39,6 +39,10 @@ builds: version: 1e58708446ecef136cedf26d28e2f599b763780e3b46a7f5bed376e17b912ce9 blobstore_id: eedac5f9-0e7a-4129-676c-71e74a277869 sha1: sha256:d8f006d0c2754e7c8601cdc18cf9b0065b69dfa4108ff78fd361bb91f4c1a7aa + 1ef290f6ad6cd69dc5a7e6ae14e3c00f6a210f4235bc255392a39470c7b772fe: + version: 1ef290f6ad6cd69dc5a7e6ae14e3c00f6a210f4235bc255392a39470c7b772fe + blobstore_id: 24ad6ed3-278c-4ff4-4e34-a9a66038c607 + sha1: sha256:b997aa0e97b486979e5dea88eb250e450b95491d75bc7829f5d91478bf836a22 2a19e043c61898b209dc7edd3da20d2ad541c475eba188d2fb794fcd8521ad5d: version: 2a19e043c61898b209dc7edd3da20d2ad541c475eba188d2fb794fcd8521ad5d blobstore_id: 82acc62a-2df8-4a50-4e5b-02194316e9e8 diff --git a/.final_builds/jobs/operator/index.yml b/.final_builds/jobs/operator/index.yml index a59eb13727..89ce4b21ab 100644 --- a/.final_builds/jobs/operator/index.yml +++ b/.final_builds/jobs/operator/index.yml @@ -55,4 +55,8 @@ builds: version: dd776a1fadf9f040a854c9a72adbbbcfbeecf26385cda00eb8763d0385e3105d blobstore_id: 832c40af-413a-4893-4024-18adcaaf9f8b sha1: sha256:8e7f1276de22f49114219ff73d0c2b8f6471a0cda08997a195423de02ab33677 + e5cd03d5bb9ec7b6b08c65936e14f0d6fb9b4b7acdd1c8352388270a86bd8241: + version: e5cd03d5bb9ec7b6b08c65936e14f0d6fb9b4b7acdd1c8352388270a86bd8241 + blobstore_id: ea51caa8-1305-4a4e-4fe7-35022c7353dc + sha1: sha256:cf741475455a0e51d4c5ab7d1b020917665eb7db6d56624ab3f07ddf74e4bb8a format-version: "2" diff --git a/.final_builds/packages/db/index.yml b/.final_builds/packages/db/index.yml index 9ee0602fdd..ed043cc11b 100644 --- a/.final_builds/packages/db/index.yml +++ b/.final_builds/packages/db/index.yml @@ -231,6 +231,10 @@ builds: version: b549921b71cee8fe2e3947d5df9a488c6825bac834a363d67e859c38bf09daca blobstore_id: 2552f3d5-43bc-4a86-4002-60639d7efb5b sha1: sha256:7f4e438ba0084dcd3fad0cf229ac1566a44e661280b9e795f6d536225d3ae30f + b99ed9f99c38d3a8e2b59ed2231038519f6ddacc8680c664e02f155c4271e70b: + version: b99ed9f99c38d3a8e2b59ed2231038519f6ddacc8680c664e02f155c4271e70b + blobstore_id: dd1a4fe4-0eb1-47e5-53e7-223629148fdf + sha1: sha256:c48d23e191829d2f8e070c951cd3964cd7bf2872958660ec31dc8670ae80b01d be1ac9ad010d1b1c260d9369677ec2b99979397f0d770a5ae1bf4bb171ad11b1: version: be1ac9ad010d1b1c260d9369677ec2b99979397f0d770a5ae1bf4bb171ad11b1 blobstore_id: 432ad1cb-4e16-4c94-53aa-ebde49c06f65 diff --git a/.final_builds/packages/eventgenerator/index.yml b/.final_builds/packages/eventgenerator/index.yml index 6e47f46411..c9a8a0c228 100644 --- a/.final_builds/packages/eventgenerator/index.yml +++ b/.final_builds/packages/eventgenerator/index.yml @@ -147,6 +147,10 @@ builds: version: aef63e2c8962850918e1c669e78bf84f7e4530aacfeab999d80287ccc0257d65 blobstore_id: ceff281d-72c2-4965-661a-cd463cf8500c sha1: sha256:0343133130a374487b4d780b51fedcd3c37344bc7b073d58d76c8d9c3b6b7b60 + b32343e0a594b4391cae796e7a5c8f722b4f88aa0555419aa63c2e0e0c14c0eb: + version: b32343e0a594b4391cae796e7a5c8f722b4f88aa0555419aa63c2e0e0c14c0eb + blobstore_id: a10a75dd-b1ef-403f-5086-2972cc9256f2 + sha1: sha256:3eda902537b59c9ac34bf7adaf9c632692ca3be0402221287e6e9dfcb6fbcc8c b3e52bb1730d04898d270f89c38519d006e4e3ce126c6029baae4b0efc05f5f9: version: b3e52bb1730d04898d270f89c38519d006e4e3ce126c6029baae4b0efc05f5f9 blobstore_id: 133c6ab5-952d-4006-703f-ee5a6a98f264 diff --git a/.final_builds/packages/golangapiserver/index.yml b/.final_builds/packages/golangapiserver/index.yml index 3da2f01dac..20794fcd70 100644 --- a/.final_builds/packages/golangapiserver/index.yml +++ b/.final_builds/packages/golangapiserver/index.yml @@ -131,6 +131,10 @@ builds: version: 6cb2b8c268d472421b72dc6e8e96464ad3f049dfd598904d1861c2655a7aa46f blobstore_id: 02d38106-a8ef-4d5a-72d7-a1a3b61100f5 sha1: sha256:86a0aa1f4d27bf6f68f961afd99cd14b38c3020591619ef88d87e0b457d1ce1d + 6d11de094fc9cc17fe7ce88cfb210b3be832f78d594bcdb3b106c04ba8573ae5: + version: 6d11de094fc9cc17fe7ce88cfb210b3be832f78d594bcdb3b106c04ba8573ae5 + blobstore_id: 024aba04-3209-4706-48c9-c94d2999af36 + sha1: sha256:859414dedf4e6027513a9b6abe5df09f7771bef9912072b0cdde911983e4d95e 7e91ae765cc76dc8151ae5bd0a31a09299bfe8d93f49c7358b87349ca4ebdccc: version: 7e91ae765cc76dc8151ae5bd0a31a09299bfe8d93f49c7358b87349ca4ebdccc blobstore_id: 3d801a3e-3304-4060-7713-b7c2e0b261ad diff --git a/.final_builds/packages/metricsforwarder/index.yml b/.final_builds/packages/metricsforwarder/index.yml index cb639ccdef..1c3c3f8ab1 100644 --- a/.final_builds/packages/metricsforwarder/index.yml +++ b/.final_builds/packages/metricsforwarder/index.yml @@ -1,4 +1,8 @@ builds: + 03b26f8298ed058f48adde5220c62f949ee42198a2591037dae76ca2a8da6307: + version: 03b26f8298ed058f48adde5220c62f949ee42198a2591037dae76ca2a8da6307 + blobstore_id: e12dd3fb-3400-4d43-6c01-35300f97e71b + sha1: sha256:2df5a3e3c08a1f43116befbfe2ccb5027e7f2eb0411b56f4d612f86d2a2af418 0542dc3eba60381c948b11c9febff5f704760d028592e9531b7cea746dd72d8c: version: 0542dc3eba60381c948b11c9febff5f704760d028592e9531b7cea746dd72d8c blobstore_id: 202f1fc2-3dd8-422b-5a3e-34c9fe9529fc diff --git a/.final_builds/packages/operator/index.yml b/.final_builds/packages/operator/index.yml index 5399661818..4e4ec3cd9d 100644 --- a/.final_builds/packages/operator/index.yml +++ b/.final_builds/packages/operator/index.yml @@ -99,6 +99,10 @@ builds: version: 76c6e188c76a5747387b02bfcd7dba5bf90a2edd2a448f758d439e1290cf18b9 blobstore_id: c6e0c911-3fbc-46aa-6bbb-b1a04fd6c375 sha1: sha256:7fb7a2d04014f6f6b36c252828b31b51b931c6df1fdf4a756f2aceaf79512743 + 7ed28368d76e84b62decb11dac26989981c423b6fdfb008094187a0d19c7cc6c: + version: 7ed28368d76e84b62decb11dac26989981c423b6fdfb008094187a0d19c7cc6c + blobstore_id: 1b516657-c678-4285-4b68-18a3a5965f10 + sha1: sha256:7fc361281933a528281ad8c27e2b47fbd7a412cacec16dda4f4718155536ef0b 82dc56fb404573f1b6686be869dce4ee0c23cb8c8f8987debe212da692c94943: version: 82dc56fb404573f1b6686be869dce4ee0c23cb8c8f8987debe212da692c94943 blobstore_id: 4bcd4933-aa63-4775-761c-1c8e9346b5c5 diff --git a/.final_builds/packages/scalingengine/index.yml b/.final_builds/packages/scalingengine/index.yml index f24e27d9ff..857d4bb482 100644 --- a/.final_builds/packages/scalingengine/index.yml +++ b/.final_builds/packages/scalingengine/index.yml @@ -1,4 +1,8 @@ builds: + 00c2ff355536fbc5cefb65c50742639fbae9083568bfa9fa73f50d19e53c0f05: + version: 00c2ff355536fbc5cefb65c50742639fbae9083568bfa9fa73f50d19e53c0f05 + blobstore_id: eaeeede3-7ed9-4358-6298-0b1fc9ed94ee + sha1: sha256:adc6501586a5cac8d0d342d4ea0dd39ddd586713d6d876b9a455a63c06c839f7 0478828722f4ff9ec3e9982d076d5b8c249fddee5d8be324d36ba7d64bfa94ec: version: 0478828722f4ff9ec3e9982d076d5b8c249fddee5d8be324d36ba7d64bfa94ec blobstore_id: 4e7bd5e2-e548-4b20-4dd7-39b458b24aa2 diff --git a/.final_builds/packages/scheduler/index.yml b/.final_builds/packages/scheduler/index.yml index 563675b1f6..c1de3fd80d 100644 --- a/.final_builds/packages/scheduler/index.yml +++ b/.final_builds/packages/scheduler/index.yml @@ -39,6 +39,10 @@ builds: version: 2822fa6cdea9b724918c4ab953f61831a4428c29449c7f968c678573ddb201ba blobstore_id: 2dfb40c5-9dcc-43b8-7d8d-902ba779516a sha1: sha256:42c7fdb273030dd5c473e1398cb686efa8eabb0c2ebefbe50ae4cda577ad4363 + 28d5dfcc9358e2423327b1f6ad262524a5cc0dd11fbfb858211ba1bce19bce31: + version: 28d5dfcc9358e2423327b1f6ad262524a5cc0dd11fbfb858211ba1bce19bce31 + blobstore_id: 64855ade-3de8-4b74-522d-069328ee8869 + sha1: sha256:a06c08002487b24478157509bffe613d523c682c67cd3b7fda355ae2592ed5fa 28da6c388de4601cd008df2975886ff503ff4893b254410b913127cae2642dd2: version: 28da6c388de4601cd008df2975886ff503ff4893b254410b913127cae2642dd2 blobstore_id: a68956b1-ee47-4d6d-7470-067da53667a9 diff --git a/releases/app-autoscaler/app-autoscaler-14.0.0.yml b/releases/app-autoscaler/app-autoscaler-14.0.0.yml new file mode 100644 index 0000000000..1dc42177fe --- /dev/null +++ b/releases/app-autoscaler/app-autoscaler-14.0.0.yml @@ -0,0 +1,132 @@ +name: app-autoscaler +version: 14.0.0 +commit_hash: 8cf76285c +uncommitted_changes: false +jobs: +- name: eventgenerator + version: 7b661855f25b17690b0bea56557a4553ee64c3b559866d73bede9c2aab451bee + fingerprint: 7b661855f25b17690b0bea56557a4553ee64c3b559866d73bede9c2aab451bee + sha1: sha256:94a8ce91eec09122ef2fd2e0f773d0df8adad0355671c7081e5c8c5cd9c424e2 + packages: + - openjdk-17 + - db + - eventgenerator + - common + - changeloglockcleaner +- name: golangapiserver + version: 1ef290f6ad6cd69dc5a7e6ae14e3c00f6a210f4235bc255392a39470c7b772fe + fingerprint: 1ef290f6ad6cd69dc5a7e6ae14e3c00f6a210f4235bc255392a39470c7b772fe + sha1: sha256:b997aa0e97b486979e5dea88eb250e450b95491d75bc7829f5d91478bf836a22 + packages: + - golang-1-linux + - golangapiserver + - db + - openjdk-17 + - common + - changeloglockcleaner +- name: metricsforwarder + version: 8d0b8d5a1f17f01079172163d6a78147cde52d4b0c81ea2771b3af1f5bd0e649 + fingerprint: 8d0b8d5a1f17f01079172163d6a78147cde52d4b0c81ea2771b3af1f5bd0e649 + sha1: sha256:07bcd92da7f7f66257110d6e1f91d9c624dbb7937b9bd11c9b26784317c8c641 + packages: + - db + - metricsforwarder + - common +- name: operator + version: e5cd03d5bb9ec7b6b08c65936e14f0d6fb9b4b7acdd1c8352388270a86bd8241 + fingerprint: e5cd03d5bb9ec7b6b08c65936e14f0d6fb9b4b7acdd1c8352388270a86bd8241 + sha1: sha256:cf741475455a0e51d4c5ab7d1b020917665eb7db6d56624ab3f07ddf74e4bb8a + packages: + - operator + - common + - db + - openjdk-17 + - changeloglockcleaner +- name: scalingengine + version: fbd7ee3f304cf1ac2ec8fdc10e67f003d50d949ab25a58df838f0497ac503f5f + fingerprint: fbd7ee3f304cf1ac2ec8fdc10e67f003d50d949ab25a58df838f0497ac503f5f + sha1: sha256:d55638d6a6f8ebc26ea9fd29a5aa3dd83db59be9aa3f0fad2a4d3dda3721941f + packages: + - openjdk-17 + - db + - scalingengine + - common + - changeloglockcleaner +- name: scheduler + version: 81741f07889700443355701c5afdaa04c492acf69d5f89b94dc32b4b997bc288 + fingerprint: 81741f07889700443355701c5afdaa04c492acf69d5f89b94dc32b4b997bc288 + sha1: sha256:57067b1b7c6884b17db77bc691cce8de2a71db5ca5076d619d158a9be608e514 + packages: + - scheduler + - db + - openjdk-17 + - common + - changeloglockcleaner +packages: +- name: changeloglockcleaner + version: ce9be5cce207009a889a305a1dc96774bd60fe319dbe7c97ce4b1f3b5248f654 + fingerprint: ce9be5cce207009a889a305a1dc96774bd60fe319dbe7c97ce4b1f3b5248f654 + sha1: sha256:34b6f5db7aad87bae7bf13308b2fdd46cf31cccb2ae69079fe992317d96095b6 + dependencies: + - golang-1-linux +- name: common + version: 9d7726aa454227e3087cd4a04425acae72d883a5bb19d314930ab58e83f04d0e + fingerprint: 9d7726aa454227e3087cd4a04425acae72d883a5bb19d314930ab58e83f04d0e + sha1: sha256:979ff40c6e44031bec55344aa4a522cac3d265ab23deaea0f7bc7541aa2b4124 + dependencies: [] +- name: db + version: b99ed9f99c38d3a8e2b59ed2231038519f6ddacc8680c664e02f155c4271e70b + fingerprint: b99ed9f99c38d3a8e2b59ed2231038519f6ddacc8680c664e02f155c4271e70b + sha1: sha256:c48d23e191829d2f8e070c951cd3964cd7bf2872958660ec31dc8670ae80b01d + dependencies: + - openjdk-17 +- name: eventgenerator + version: b32343e0a594b4391cae796e7a5c8f722b4f88aa0555419aa63c2e0e0c14c0eb + fingerprint: b32343e0a594b4391cae796e7a5c8f722b4f88aa0555419aa63c2e0e0c14c0eb + sha1: sha256:3eda902537b59c9ac34bf7adaf9c632692ca3be0402221287e6e9dfcb6fbcc8c + dependencies: + - golang-1-linux +- name: golang-1-linux + version: c2342901fca75f4c7ec3f32e6a757e923089c6c50d8eb3effd2c25eac1009e31 + fingerprint: c2342901fca75f4c7ec3f32e6a757e923089c6c50d8eb3effd2c25eac1009e31 + sha1: sha256:d354b084ed879adc3218f89b9109b474a68c3eafb5c0ea204c4b77963e4736b0 + dependencies: [] +- name: golangapiserver + version: 6d11de094fc9cc17fe7ce88cfb210b3be832f78d594bcdb3b106c04ba8573ae5 + fingerprint: 6d11de094fc9cc17fe7ce88cfb210b3be832f78d594bcdb3b106c04ba8573ae5 + sha1: sha256:859414dedf4e6027513a9b6abe5df09f7771bef9912072b0cdde911983e4d95e + dependencies: + - golang-1-linux +- name: metricsforwarder + version: 03b26f8298ed058f48adde5220c62f949ee42198a2591037dae76ca2a8da6307 + fingerprint: 03b26f8298ed058f48adde5220c62f949ee42198a2591037dae76ca2a8da6307 + sha1: sha256:2df5a3e3c08a1f43116befbfe2ccb5027e7f2eb0411b56f4d612f86d2a2af418 + dependencies: + - golang-1-linux +- name: openjdk-17 + version: be13f95495365f8879a75d9237320cd587774f6cbc087020bd6508fb8a82184a + fingerprint: be13f95495365f8879a75d9237320cd587774f6cbc087020bd6508fb8a82184a + sha1: sha256:99e2ef396973d7bbbfddffad6968b419627c20bb426117f8c3115cad5b206227 + dependencies: [] +- name: operator + version: 7ed28368d76e84b62decb11dac26989981c423b6fdfb008094187a0d19c7cc6c + fingerprint: 7ed28368d76e84b62decb11dac26989981c423b6fdfb008094187a0d19c7cc6c + sha1: sha256:7fc361281933a528281ad8c27e2b47fbd7a412cacec16dda4f4718155536ef0b + dependencies: + - golang-1-linux +- name: scalingengine + version: 00c2ff355536fbc5cefb65c50742639fbae9083568bfa9fa73f50d19e53c0f05 + fingerprint: 00c2ff355536fbc5cefb65c50742639fbae9083568bfa9fa73f50d19e53c0f05 + sha1: sha256:adc6501586a5cac8d0d342d4ea0dd39ddd586713d6d876b9a455a63c06c839f7 + dependencies: + - golang-1-linux +- name: scheduler + version: 28d5dfcc9358e2423327b1f6ad262524a5cc0dd11fbfb858211ba1bce19bce31 + fingerprint: 28d5dfcc9358e2423327b1f6ad262524a5cc0dd11fbfb858211ba1bce19bce31 + sha1: sha256:a06c08002487b24478157509bffe613d523c682c67cd3b7fda355ae2592ed5fa + dependencies: + - openjdk-17 +license: + version: 15eb386a0db48f190d29bd28058f5d0d6605f5acde7c6cfc6f014d57747e41f9 + fingerprint: 15eb386a0db48f190d29bd28058f5d0d6605f5acde7c6cfc6f014d57747e41f9 + sha1: sha256:6c7a834097b96efc340787fbe5876c1e366de0235def5e2b93fa016c8655535c diff --git a/releases/app-autoscaler/index.yml b/releases/app-autoscaler/index.yml index a6967c59ba..138a79484a 100644 --- a/releases/app-autoscaler/index.yml +++ b/releases/app-autoscaler/index.yml @@ -93,6 +93,8 @@ builds: version: 3.3.0 a1a7dbf8-e630-4307-4415-f160d6f43293: version: 12.1.0 + a3229c1a-d235-4525-543b-f23ecaa4c1f3: + version: 14.0.0 a35bf8d5-02f9-423a-4947-9cce70c4bc97: version: 11.5.0 a67232ea-d7cf-410c-73d1-106a7086b60c: From e3d8c00997947e20a04cbd3ba9cc245684255a4d Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Wed, 19 Jun 2024 13:30:31 +0200 Subject: [PATCH 178/186] fix(ci): make `acceptance-log-cache-syslog` run with a `metricsforwarder` sending metrics to the `loggr-syslog-agent` (#3034) --- ci/autoscaler/pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/autoscaler/pipeline.yml b/ci/autoscaler/pipeline.yml index f5a86e2a07..48ddfcdf08 100644 --- a/ci/autoscaler/pipeline.yml +++ b/ci/autoscaler/pipeline.yml @@ -19,7 +19,6 @@ anchors: operations/enable-nats-tls.yml operations/add-extra-plan.yml operations/set-release-version.yml - operations/enable-metricsforwarder-via-syslog-agent.yml operations/enable-scheduler-logging.yml app-autoscaler-ops-files-log-cache-syslog-cf: &app-autoscaler-ops-files-log-cache-syslog-cf From 0a313e9b7a59991709b979b17af865d765210193 Mon Sep 17 00:00:00 2001 From: geigerj0 <112163019+geigerj0@users.noreply.github.com> Date: Wed, 19 Jun 2024 16:29:13 +0200 Subject: [PATCH 179/186] fix(metrics-architecture): remove configuration referencing the dead table `appinstancemetrics` (#3035) --- jobs/operator/spec | 32 ------------------- operations/connect_to_postgres_with_certs.yml | 6 ---- operations/disable-postgres-tls-config.yml | 4 --- templates/app-autoscaler.yml | 1 - 4 files changed, 43 deletions(-) diff --git a/jobs/operator/spec b/jobs/operator/spec index ba529df29d..26c101cfd6 100644 --- a/jobs/operator/spec +++ b/jobs/operator/spec @@ -90,32 +90,6 @@ properties: default: disable description: "sslmode to connect to postgres server" - - autoscaler.instancemetrics_db.address: - description: "IP address on which the instancemetricsdb server will listen" - default: "autoscalerpostgres.service.cf.internal" - autoscaler.instancemetrics_db.databases: - description: "The list of databases used in instancemetricsdb database including name" - autoscaler.instancemetrics_db.db_scheme: - description: "Database scheme to be used to access instancemetricsdb" - default: postgres - autoscaler.instancemetrics_db.port: - description: "Port on which the instancemetricsdb server will listen" - autoscaler.instancemetrics_db.roles: - description: "The list of database roles used in instancemetricsdb database including name/password" - autoscaler.instancemetrics_db.tls.ca: - default: '' - description: 'PEM-encoded ca certificate for TLS database server' - autoscaler.instancemetrics_db.tls.certificate: - default: '' - description: 'PEM-encoded certificate for TLS database client' - autoscaler.instancemetrics_db.tls.private_key: - default: '' - description: 'PEM-encoded key for tls database client' - autoscaler.instancemetrics_db.sslmode: - default: disable - description: "sslmode to connect to postgres server" - autoscaler.appmetrics_db.address: description: "IP address on which the appmetricsdb server will listen" default: "autoscalerpostgres.service.cf.internal" @@ -181,12 +155,6 @@ properties: autoscaler.appmetrics_db_connection_config.connection_max_lifetime: default: 60s - autoscaler.instancemetrics_db_connection_config.max_open_connections: - default: 10 - autoscaler.instancemetrics_db_connection_config.max_idle_connections: - default: 1 - autoscaler.instancemetrics_db_connection_config.connection_max_lifetime: - default: 60s autoscaler.scalingengine_db_connection_config.max_open_connections: default: 10 diff --git a/operations/connect_to_postgres_with_certs.yml b/operations/connect_to_postgres_with_certs.yml index 23fd439797..65770615f4 100644 --- a/operations/connect_to_postgres_with_certs.yml +++ b/operations/connect_to_postgres_with_certs.yml @@ -43,12 +43,6 @@ path: /instance_groups/name=scheduler/jobs/name=scheduler/properties/autoscaler/policy_db/tls/private_key value: ((postgres_client.private_key)) -- type: replace - path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/instancemetrics_db/tls/certificate - value: ((postgres_client.certificate)) -- type: replace - path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/instancemetrics_db/tls/private_key - value: ((postgres_client.private_key)) - type: replace path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/appmetrics_db/tls/certificate value: ((postgres_client.certificate)) diff --git a/operations/disable-postgres-tls-config.yml b/operations/disable-postgres-tls-config.yml index 56a402ae71..2544303d58 100644 --- a/operations/disable-postgres-tls-config.yml +++ b/operations/disable-postgres-tls-config.yml @@ -45,10 +45,6 @@ path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/appmetrics_db/sslmode value: disable -- type: replace - path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/instancemetrics_db/sslmode - value: disable - - type: replace path: /instance_groups/name=operator/jobs/name=operator/properties/autoscaler/lock_db/sslmode value: disable diff --git a/templates/app-autoscaler.yml b/templates/app-autoscaler.yml index a1a0e3aeb3..64bd2a35de 100644 --- a/templates/app-autoscaler.yml +++ b/templates/app-autoscaler.yml @@ -366,7 +366,6 @@ instance_groups: release: app-autoscaler properties: autoscaler: - instancemetrics_db: *database appmetrics_db: *database appmetrics_db_connection_config: *databaseConnectionConfig scalingengine_db: *database From d92daceb697956fa2a5df9db059e5eae10955316 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 19 Jun 2024 22:05:05 +0000 Subject: [PATCH 180/186] fix(deps): update code.cloudfoundry.org/loggregator-agent-release/src digest to bc3ba16 --- src/autoscaler/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index ee3b05dbdf..b55d1a826e 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,7 +8,7 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240614070214-69a1db0ef542 + code.cloudfoundry.org/loggregator-agent-release/src bc3ba1627d10 code.cloudfoundry.org/tlsconfig v0.0.0-20240613173017-075d5b187a0d dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 From c4aa362ae91cec50b3e28beb87f79a2117ee0298 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 19 Jun 2024 23:10:59 +0100 Subject: [PATCH 181/186] =?UTF-8?q?=F0=9F=A4=96=F0=9F=A6=BE=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=20go=20mod=20tidy=20&=20make=20package-specs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/autoscaler/go.mod | 12 ++++++------ src/autoscaler/go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/autoscaler/go.mod b/src/autoscaler/go.mod index b55d1a826e..c611a43420 100644 --- a/src/autoscaler/go.mod +++ b/src/autoscaler/go.mod @@ -8,11 +8,11 @@ require ( code.cloudfoundry.org/go-log-cache/v2 v2.0.7 code.cloudfoundry.org/go-loggregator/v9 v9.2.1 code.cloudfoundry.org/lager/v3 v3.0.3 - code.cloudfoundry.org/loggregator-agent-release/src bc3ba1627d10 + code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240619070148-bc3ba1627d10 code.cloudfoundry.org/tlsconfig v0.0.0-20240613173017-075d5b187a0d dario.cat/mergo v1.0.0 github.com/cenkalti/backoff/v4 v4.3.0 - github.com/go-chi/chi/v5 v5.0.12 + github.com/go-chi/chi/v5 v5.0.13 github.com/go-faster/errors v0.7.1 github.com/go-faster/jx v1.1.0 github.com/go-logr/logr v1.4.2 @@ -70,7 +70,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect + github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -89,7 +89,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.54.0 // indirect + github.com/valyala/fasthttp v1.55.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect go.uber.org/multierr v1.11.0 // indirect @@ -99,8 +99,8 @@ require ( golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/tools v0.22.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/src/autoscaler/go.sum b/src/autoscaler/go.sum index 82efd7c09f..bd3eb78cdd 100644 --- a/src/autoscaler/go.sum +++ b/src/autoscaler/go.sum @@ -606,8 +606,8 @@ code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a h1:X code.cloudfoundry.org/go-metric-registry v0.0.0-20240604201903-7cef498efb7a/go.mod h1:/Be8VtLiCeMUoYdUzFtmW8GGkk89HAy3zD79KUXzbhs= code.cloudfoundry.org/lager/v3 v3.0.3 h1:/UTmadZfIaKuT/whEinSxK1mzRfNu1uPfvjFfGqiwzM= code.cloudfoundry.org/lager/v3 v3.0.3/go.mod h1:Zn5q1SrIuuHjEUE7xerMKt3ztunrJQCZETAo7rV0CH8= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240614070214-69a1db0ef542 h1:OxdgOGWm4jaUHeFVIbsScBc1BkLMTHa5rjBt/yfZrlk= -code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240614070214-69a1db0ef542/go.mod h1:/Dzry/KNUjigc8cvCd5SwSqRM2StxctoNBRLRiLbta4= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240619070148-bc3ba1627d10 h1:81sz0CddodU9V1vb2OFu+n2toJzHTtIKJ9jgls0kXeQ= +code.cloudfoundry.org/loggregator-agent-release/src v0.0.0-20240619070148-bc3ba1627d10/go.mod h1:Fyt7rmEuDWn0PTdjF3D/+4mi68q6pvixnExg81Jih0s= code.cloudfoundry.org/tlsconfig v0.0.0-20240613173017-075d5b187a0d h1:nedO/GTZuf/FlRvrAuXWX9FdBBEd1hWXlhn1bp6li68= code.cloudfoundry.org/tlsconfig v0.0.0-20240613173017-075d5b187a0d/go.mod h1:qDYY1KwaNSWV7beH9J/IJR5erYD3bwO4zs+nDbpgbf8= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= @@ -708,8 +708,8 @@ github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwV github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= -github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/chi/v5 v5.0.13 h1:JlH2F2M8qnwl0N1+JFFzlX9TlKJYas3aPXdiuTmJL+w= +github.com/go-chi/chi/v5 v5.0.13/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= github.com/go-faster/jx v1.1.0 h1:ZsW3wD+snOdmTDy9eIVgQdjUpXRRV4rqW8NS3t+20bg= @@ -819,8 +819,8 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20240528025155-186aa0362fba h1:ql1qNgCyOB7iAEk8JTNM+zJrgIbnyCKX/wdlyPufP5g= -github.com/google/pprof v0.0.0-20240528025155-186aa0362fba/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8 h1:ASJ/LAqdCHOyMYI+dwNxn7Rd8FscNkMyTr1KZU1JI/M= +github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -1034,8 +1034,8 @@ github.com/uptrace/opentelemetry-go-extra/otelsqlx v0.2.4 h1:Pt/+CUTRusJb471SBXw github.com/uptrace/opentelemetry-go-extra/otelsqlx v0.2.4/go.mod h1:kQgNoghy4K/wguxbOd/u0OJw/Y0maNPc7PF4JpEGeUc= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.54.0 h1:cCL+ZZR3z3HPLMVfEYVUMtJqVaui0+gu7Lx63unHwS0= -github.com/valyala/fasthttp v1.54.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM= +github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8= +github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -1673,10 +1673,10 @@ google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 h1:QW9+G6Fir4VcRXVH8x3LilNAb6cxBGLa6+GM4hRwexE= -google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3/go.mod h1:kdrSS/OiLkPrNUpzD4aHgCq2rVuC/YRxok32HXZ4vRE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 h1:9Xyg6I9IWQZhRVfCWjKK+l6kI0jHcPesVlMnT//aHNo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 h1:Di6ANFilr+S60a4S61ZM00vLdw0IrQOSMS2/6mrnOU0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= From c456b28030790eb77c5796d25dde26e6dbc5c005 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 20 Jun 2024 01:29:58 +0000 Subject: [PATCH 182/186] chore(deps): update dependency helm to v3.15.2 --- ci/terragrunt/.tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/terragrunt/.tool-versions b/ci/terragrunt/.tool-versions index 780f781f6d..100283551c 100644 --- a/ci/terragrunt/.tool-versions +++ b/ci/terragrunt/.tool-versions @@ -1,5 +1,5 @@ gcloud 473.0.0 -helm 3.14.4 +helm 3.15.2 python 3.12.4 terraform 1.8.5 terraform-lsp From b250a41b42887901b9c3673318810455af2188e2 Mon Sep 17 00:00:00 2001 From: app-autoscaler-ci-bot Date: Thu, 20 Jun 2024 10:21:46 +0100 Subject: [PATCH 183/186] Updated release version to 14.0.1 in golangapiserver --- jobs/golangapiserver/spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/golangapiserver/spec b/jobs/golangapiserver/spec index 364c12e268..50ab2e0866 100644 --- a/jobs/golangapiserver/spec +++ b/jobs/golangapiserver/spec @@ -92,7 +92,7 @@ properties: default: "Autoscaler" autoscaler.apiserver.info.build: description: "Autoscaler application build number" - default: "14.0.0" + default: "14.0.1" autoscaler.apiserver.info.support_url: description: "Autoscaler application support page URL" default: "https://github.com/cloudfoundry/app-autoscaler-release" From 6f1f75245db7ac105ab82bf21fa4645686210e7f Mon Sep 17 00:00:00 2001 From: app-autoscaler-ci-bot Date: Thu, 20 Jun 2024 10:24:26 +0100 Subject: [PATCH 184/186] created release v14.0.1 --- .final_builds/jobs/golangapiserver/index.yml | 4 + .final_builds/jobs/operator/index.yml | 4 + .final_builds/packages/db/index.yml | 4 + .../packages/eventgenerator/index.yml | 4 + .../packages/golangapiserver/index.yml | 4 + .../packages/metricsforwarder/index.yml | 4 + .final_builds/packages/operator/index.yml | 4 + .../packages/scalingengine/index.yml | 4 + .final_builds/packages/scheduler/index.yml | 4 + .../app-autoscaler/app-autoscaler-14.0.1.yml | 132 ++++++++++++++++++ releases/app-autoscaler/index.yml | 2 + 11 files changed, 170 insertions(+) create mode 100644 releases/app-autoscaler/app-autoscaler-14.0.1.yml diff --git a/.final_builds/jobs/golangapiserver/index.yml b/.final_builds/jobs/golangapiserver/index.yml index acdf0b06e0..4d7606dff5 100644 --- a/.final_builds/jobs/golangapiserver/index.yml +++ b/.final_builds/jobs/golangapiserver/index.yml @@ -107,6 +107,10 @@ builds: version: 4a9ce8fbb2ded4fac86ba5dd6bd269cc04f52d2892cfbfb7132c0178dfbd451c blobstore_id: 028d812c-1898-4649-7e7d-1f94b4dc769e sha1: sha256:be0069b4bff0981b7a02ca5f4e3c800db4bfddbc860a3254c8485f756ab19b09 + 4c8e296d6a7df60f7a9151eaa01ecc4eb901c454546a2e5c36052b01040fa6e1: + version: 4c8e296d6a7df60f7a9151eaa01ecc4eb901c454546a2e5c36052b01040fa6e1 + blobstore_id: 8641ce8c-ab0d-41f5-5df0-da913b2ab63e + sha1: sha256:7cdd0ab027359cce51ddd7d695cae16a03030d3d7c433cc83079b2cea8a48706 4ccf30be7b4dffa60bd581d242d4a8fc6c41a1164a748eb83bc3ad6cf874a6cd: version: 4ccf30be7b4dffa60bd581d242d4a8fc6c41a1164a748eb83bc3ad6cf874a6cd blobstore_id: e734eb74-4f3c-4fa2-6da6-ba933ab67c9e diff --git a/.final_builds/jobs/operator/index.yml b/.final_builds/jobs/operator/index.yml index 89ce4b21ab..4c2af8ec80 100644 --- a/.final_builds/jobs/operator/index.yml +++ b/.final_builds/jobs/operator/index.yml @@ -7,6 +7,10 @@ builds: version: 11ffdb19acd96f1099e42bdce522013ee7440519f746be68d99d7b1c37581375 blobstore_id: a5fff270-32e0-43d2-62b0-0d66fb31f91b sha1: sha256:3502913539aebf8f2e0b3a117ca5dc10c69bb87ead1d4939d538e1c869361bc1 + 19a54e6a1bbede2148206bd8daffe1be0b40310b2e3f418fb6a3eebd5578c082: + version: 19a54e6a1bbede2148206bd8daffe1be0b40310b2e3f418fb6a3eebd5578c082 + blobstore_id: a77acbfb-0348-43e3-7f6b-50074b00c16a + sha1: sha256:c2ca2e91780a5c7824669eb3a3c398bcc510962d9a131d3b05438febffb930ea 37ed2810cdc5da55689535ac33119289636017767cfbbe7bc43084519748af73: version: 37ed2810cdc5da55689535ac33119289636017767cfbbe7bc43084519748af73 blobstore_id: 28465f4e-6ce3-4257-6a86-2bbec7a5a06f diff --git a/.final_builds/packages/db/index.yml b/.final_builds/packages/db/index.yml index ed043cc11b..6be251afbc 100644 --- a/.final_builds/packages/db/index.yml +++ b/.final_builds/packages/db/index.yml @@ -219,6 +219,10 @@ builds: version: b2e5d5e4315b3163a5f439f914dd9ba6950102ff635f5536b101665dc7c5db00 blobstore_id: 3aabdb38-1c6e-4ea4-7738-a3ddb26db3c3 sha1: sha256:6a02a648fc892b1ca4c880b1b9bb24a7c4e22a26a91686d45061c3a64fed14f3 + b38b0ec45e3475b88ca55cb888bd1ba63e9e2e1966cbdd408839ad26fd1315f3: + version: b38b0ec45e3475b88ca55cb888bd1ba63e9e2e1966cbdd408839ad26fd1315f3 + blobstore_id: e3dcc2a4-32bc-402c-5a4f-ccb2a24eae10 + sha1: sha256:d173175d643c3eb7dfdb5bddb0e7b03db42b0aefb53c186a9a61828797e0048b b430cc03d4fde448fb58cfd57877702dcbbe33ca8ce75886f97e4ff7568ee049: version: b430cc03d4fde448fb58cfd57877702dcbbe33ca8ce75886f97e4ff7568ee049 blobstore_id: 9f06d2bb-ff99-4249-44c6-9274d5b7fc9e diff --git a/.final_builds/packages/eventgenerator/index.yml b/.final_builds/packages/eventgenerator/index.yml index c9a8a0c228..73ebc79988 100644 --- a/.final_builds/packages/eventgenerator/index.yml +++ b/.final_builds/packages/eventgenerator/index.yml @@ -39,6 +39,10 @@ builds: version: 298683c55dd78040d5ec82a75328db7e843b7fb16d7fb39f8d867d53bbdd990e blobstore_id: d3fa5ac1-4585-47e5-5112-66b077b9f959 sha1: sha256:150059acba82889cd9a203fd023afe99569e712844493e3da22d0cc1dec75770 + 30bc1297f2f12bf07f941a8e86a5643217c60d96e997a6332748101a8a74724f: + version: 30bc1297f2f12bf07f941a8e86a5643217c60d96e997a6332748101a8a74724f + blobstore_id: 8e268a53-8bee-486a-6307-b8bbe5d3a958 + sha1: sha256:fab2a857b5e7456ae7fe671baaa7b847007328e381c510fdf54f5f960653b287 35a2a85e9de3dfd2802ff6123bf2ff897bf3d144166798f7aa48e5f328c0e28b: version: 35a2a85e9de3dfd2802ff6123bf2ff897bf3d144166798f7aa48e5f328c0e28b blobstore_id: 731cab9e-4ec7-4d5b-49a0-68ab8d592eeb diff --git a/.final_builds/packages/golangapiserver/index.yml b/.final_builds/packages/golangapiserver/index.yml index 20794fcd70..fc92426c78 100644 --- a/.final_builds/packages/golangapiserver/index.yml +++ b/.final_builds/packages/golangapiserver/index.yml @@ -119,6 +119,10 @@ builds: version: 5f7bb9858fb910603ce6880d7c0b68dca1f02e8f3ff7bc0a4f0df70fd96bf595 blobstore_id: 1c918292-11ce-4906-4945-d966347358e7 sha1: sha256:a431ffb3978a83cd64f8742aa5de178c744f31507bfe37c005ae04c5cfff5fd5 + 60fdcf1d84a1ad871a9c7daf81001d5678ffefdbcf761269620fe890d0e1b581: + version: 60fdcf1d84a1ad871a9c7daf81001d5678ffefdbcf761269620fe890d0e1b581 + blobstore_id: 16277e55-a629-4801-561c-a606026dff33 + sha1: sha256:18ab0784ba25d7f495ae10be2567f467a7ae488f3748fa3a392f71457586fd49 63980b4a9db2847ef2fb8de2954fbc3e1bdc7b9f5a5c80b4520866cb3bb0650d: version: 63980b4a9db2847ef2fb8de2954fbc3e1bdc7b9f5a5c80b4520866cb3bb0650d blobstore_id: 26202192-eed7-44bc-6f2b-f0cc9fd63a73 diff --git a/.final_builds/packages/metricsforwarder/index.yml b/.final_builds/packages/metricsforwarder/index.yml index 1c3c3f8ab1..383da2cbd9 100644 --- a/.final_builds/packages/metricsforwarder/index.yml +++ b/.final_builds/packages/metricsforwarder/index.yml @@ -27,6 +27,10 @@ builds: version: 195d67a5de0d0fbcdb287818172d0ec97193e4ce7c0adbfbc076132db997cf89 blobstore_id: d499f779-e7fe-49fe-4309-cfa1453491e6 sha1: sha256:ff279a16c8d05d686eb25f927fd857e995d1b304069c3b25f6d5f260b73105f9 + 1a67b6ca84beabe244e88fc7c3d75b6a47655b77c84048cd003df927ce3ed2f0: + version: 1a67b6ca84beabe244e88fc7c3d75b6a47655b77c84048cd003df927ce3ed2f0 + blobstore_id: 4053be03-097f-4596-5f14-2f04b3b48dea + sha1: sha256:41cdb0232be0d776757f33b05d3719df12b9640d05dc43295721ef83a6e74656 1ec6c3375e03c34d6315807d28cd76bc7b9926e18647626b4a8778e881d9c299: version: 1ec6c3375e03c34d6315807d28cd76bc7b9926e18647626b4a8778e881d9c299 blobstore_id: 89b51fdb-ceca-4e98-6f3e-aef3775d5660 diff --git a/.final_builds/packages/operator/index.yml b/.final_builds/packages/operator/index.yml index 4e4ec3cd9d..97d6c5972e 100644 --- a/.final_builds/packages/operator/index.yml +++ b/.final_builds/packages/operator/index.yml @@ -231,6 +231,10 @@ builds: version: ec043e053ae90ac40cb49941c287bad90611a31f79ffc1e16e95a280a7545716 blobstore_id: af736404-ce2e-43db-5415-52a0d810a070 sha1: sha256:161b542c93ec638ee6e8f47cbb0f549201dd780c487d3c0cc45c4f3723b9d4a1 + ed7253824a453cf1c37bcd1f7a3c5bbdf318f1cf30c08b347f96ae2706096000: + version: ed7253824a453cf1c37bcd1f7a3c5bbdf318f1cf30c08b347f96ae2706096000 + blobstore_id: db52c8c4-27be-467a-4d0c-50ee3a97a718 + sha1: sha256:9f2bdaf75d77638c46c6758381108d7a8867c5bfc4c7b99e61a0cdbe616eeb58 ef8dcd44423c6bdb034ad1bf98f05f64eb645c7970791ffe159e4ede6cf06258: version: ef8dcd44423c6bdb034ad1bf98f05f64eb645c7970791ffe159e4ede6cf06258 blobstore_id: ebae4e08-bf93-4971-76ab-6154a49d822e diff --git a/.final_builds/packages/scalingengine/index.yml b/.final_builds/packages/scalingengine/index.yml index 857d4bb482..93d9b0d1fb 100644 --- a/.final_builds/packages/scalingengine/index.yml +++ b/.final_builds/packages/scalingengine/index.yml @@ -191,6 +191,10 @@ builds: version: bf11eb376a9c7cd7c3cc080aa4c5775037b1c1aabac7baa71dccc12114d1188d blobstore_id: 07ccafd9-b856-4b67-63ad-86844f23ae9c sha1: sha256:04247fe78f2ed2c6f637850bd16e4aff3c4a141c02cc2d62c325171a6a99987e + c0b68b4084cdfad00a1e95cd43899c19b27ae6210c29b2041a26dd64c6e8d73f: + version: c0b68b4084cdfad00a1e95cd43899c19b27ae6210c29b2041a26dd64c6e8d73f + blobstore_id: 525a05d1-362a-4b0a-4a2d-537c1ea466c0 + sha1: sha256:8bf0f9ca1702785e44e68c12f3a16e71602d17ea4abbad6dcf07384d4bc1026f c58e892724785912f4745cff635bd3b08b79d32965d57dafe3c5c09ce11c610f: version: c58e892724785912f4745cff635bd3b08b79d32965d57dafe3c5c09ce11c610f blobstore_id: 3da66345-235c-49c5-6996-814e8eb7abc9 diff --git a/.final_builds/packages/scheduler/index.yml b/.final_builds/packages/scheduler/index.yml index c1de3fd80d..fd14bc195d 100644 --- a/.final_builds/packages/scheduler/index.yml +++ b/.final_builds/packages/scheduler/index.yml @@ -247,6 +247,10 @@ builds: version: a9d9716f069279b349904f186bf73662fc50ecd5a8811bfec5d605ba5c769715 blobstore_id: 87f06c25-411c-41df-5835-82037089f2e2 sha1: sha256:878c5a530ca5ad8ad70dbb0f6efa8dba8a2b639ab45638f25c61c697210304f1 + b211db77c5d759e5a584e8b24c992b961684a082f66cc9e143d8724fc91b37bd: + version: b211db77c5d759e5a584e8b24c992b961684a082f66cc9e143d8724fc91b37bd + blobstore_id: 0ae29e2b-8639-4840-63c7-9dad81b2f5b7 + sha1: sha256:a1c3c49ff5712aa21a3e01eea028bd20c2baedb90e6142b052f2c34084e767be b2ff3441dda717be36cd3d34b61232e8af2871b6de279d41884cdba0b3cefc44: version: b2ff3441dda717be36cd3d34b61232e8af2871b6de279d41884cdba0b3cefc44 blobstore_id: b5613518-f0ec-4384-5981-5054d689c79f diff --git a/releases/app-autoscaler/app-autoscaler-14.0.1.yml b/releases/app-autoscaler/app-autoscaler-14.0.1.yml new file mode 100644 index 0000000000..f5248f52ef --- /dev/null +++ b/releases/app-autoscaler/app-autoscaler-14.0.1.yml @@ -0,0 +1,132 @@ +name: app-autoscaler +version: 14.0.1 +commit_hash: b250a41b4 +uncommitted_changes: false +jobs: +- name: eventgenerator + version: 7b661855f25b17690b0bea56557a4553ee64c3b559866d73bede9c2aab451bee + fingerprint: 7b661855f25b17690b0bea56557a4553ee64c3b559866d73bede9c2aab451bee + sha1: sha256:94a8ce91eec09122ef2fd2e0f773d0df8adad0355671c7081e5c8c5cd9c424e2 + packages: + - openjdk-17 + - db + - eventgenerator + - common + - changeloglockcleaner +- name: golangapiserver + version: 4c8e296d6a7df60f7a9151eaa01ecc4eb901c454546a2e5c36052b01040fa6e1 + fingerprint: 4c8e296d6a7df60f7a9151eaa01ecc4eb901c454546a2e5c36052b01040fa6e1 + sha1: sha256:7cdd0ab027359cce51ddd7d695cae16a03030d3d7c433cc83079b2cea8a48706 + packages: + - golang-1-linux + - golangapiserver + - db + - openjdk-17 + - common + - changeloglockcleaner +- name: metricsforwarder + version: 8d0b8d5a1f17f01079172163d6a78147cde52d4b0c81ea2771b3af1f5bd0e649 + fingerprint: 8d0b8d5a1f17f01079172163d6a78147cde52d4b0c81ea2771b3af1f5bd0e649 + sha1: sha256:07bcd92da7f7f66257110d6e1f91d9c624dbb7937b9bd11c9b26784317c8c641 + packages: + - db + - metricsforwarder + - common +- name: operator + version: 19a54e6a1bbede2148206bd8daffe1be0b40310b2e3f418fb6a3eebd5578c082 + fingerprint: 19a54e6a1bbede2148206bd8daffe1be0b40310b2e3f418fb6a3eebd5578c082 + sha1: sha256:c2ca2e91780a5c7824669eb3a3c398bcc510962d9a131d3b05438febffb930ea + packages: + - operator + - common + - db + - openjdk-17 + - changeloglockcleaner +- name: scalingengine + version: fbd7ee3f304cf1ac2ec8fdc10e67f003d50d949ab25a58df838f0497ac503f5f + fingerprint: fbd7ee3f304cf1ac2ec8fdc10e67f003d50d949ab25a58df838f0497ac503f5f + sha1: sha256:d55638d6a6f8ebc26ea9fd29a5aa3dd83db59be9aa3f0fad2a4d3dda3721941f + packages: + - openjdk-17 + - db + - scalingengine + - common + - changeloglockcleaner +- name: scheduler + version: 81741f07889700443355701c5afdaa04c492acf69d5f89b94dc32b4b997bc288 + fingerprint: 81741f07889700443355701c5afdaa04c492acf69d5f89b94dc32b4b997bc288 + sha1: sha256:57067b1b7c6884b17db77bc691cce8de2a71db5ca5076d619d158a9be608e514 + packages: + - scheduler + - db + - openjdk-17 + - common + - changeloglockcleaner +packages: +- name: changeloglockcleaner + version: ce9be5cce207009a889a305a1dc96774bd60fe319dbe7c97ce4b1f3b5248f654 + fingerprint: ce9be5cce207009a889a305a1dc96774bd60fe319dbe7c97ce4b1f3b5248f654 + sha1: sha256:34b6f5db7aad87bae7bf13308b2fdd46cf31cccb2ae69079fe992317d96095b6 + dependencies: + - golang-1-linux +- name: common + version: 9d7726aa454227e3087cd4a04425acae72d883a5bb19d314930ab58e83f04d0e + fingerprint: 9d7726aa454227e3087cd4a04425acae72d883a5bb19d314930ab58e83f04d0e + sha1: sha256:979ff40c6e44031bec55344aa4a522cac3d265ab23deaea0f7bc7541aa2b4124 + dependencies: [] +- name: db + version: b38b0ec45e3475b88ca55cb888bd1ba63e9e2e1966cbdd408839ad26fd1315f3 + fingerprint: b38b0ec45e3475b88ca55cb888bd1ba63e9e2e1966cbdd408839ad26fd1315f3 + sha1: sha256:d173175d643c3eb7dfdb5bddb0e7b03db42b0aefb53c186a9a61828797e0048b + dependencies: + - openjdk-17 +- name: eventgenerator + version: 30bc1297f2f12bf07f941a8e86a5643217c60d96e997a6332748101a8a74724f + fingerprint: 30bc1297f2f12bf07f941a8e86a5643217c60d96e997a6332748101a8a74724f + sha1: sha256:fab2a857b5e7456ae7fe671baaa7b847007328e381c510fdf54f5f960653b287 + dependencies: + - golang-1-linux +- name: golang-1-linux + version: c2342901fca75f4c7ec3f32e6a757e923089c6c50d8eb3effd2c25eac1009e31 + fingerprint: c2342901fca75f4c7ec3f32e6a757e923089c6c50d8eb3effd2c25eac1009e31 + sha1: sha256:d354b084ed879adc3218f89b9109b474a68c3eafb5c0ea204c4b77963e4736b0 + dependencies: [] +- name: golangapiserver + version: 60fdcf1d84a1ad871a9c7daf81001d5678ffefdbcf761269620fe890d0e1b581 + fingerprint: 60fdcf1d84a1ad871a9c7daf81001d5678ffefdbcf761269620fe890d0e1b581 + sha1: sha256:18ab0784ba25d7f495ae10be2567f467a7ae488f3748fa3a392f71457586fd49 + dependencies: + - golang-1-linux +- name: metricsforwarder + version: 1a67b6ca84beabe244e88fc7c3d75b6a47655b77c84048cd003df927ce3ed2f0 + fingerprint: 1a67b6ca84beabe244e88fc7c3d75b6a47655b77c84048cd003df927ce3ed2f0 + sha1: sha256:41cdb0232be0d776757f33b05d3719df12b9640d05dc43295721ef83a6e74656 + dependencies: + - golang-1-linux +- name: openjdk-17 + version: be13f95495365f8879a75d9237320cd587774f6cbc087020bd6508fb8a82184a + fingerprint: be13f95495365f8879a75d9237320cd587774f6cbc087020bd6508fb8a82184a + sha1: sha256:99e2ef396973d7bbbfddffad6968b419627c20bb426117f8c3115cad5b206227 + dependencies: [] +- name: operator + version: ed7253824a453cf1c37bcd1f7a3c5bbdf318f1cf30c08b347f96ae2706096000 + fingerprint: ed7253824a453cf1c37bcd1f7a3c5bbdf318f1cf30c08b347f96ae2706096000 + sha1: sha256:9f2bdaf75d77638c46c6758381108d7a8867c5bfc4c7b99e61a0cdbe616eeb58 + dependencies: + - golang-1-linux +- name: scalingengine + version: c0b68b4084cdfad00a1e95cd43899c19b27ae6210c29b2041a26dd64c6e8d73f + fingerprint: c0b68b4084cdfad00a1e95cd43899c19b27ae6210c29b2041a26dd64c6e8d73f + sha1: sha256:8bf0f9ca1702785e44e68c12f3a16e71602d17ea4abbad6dcf07384d4bc1026f + dependencies: + - golang-1-linux +- name: scheduler + version: b211db77c5d759e5a584e8b24c992b961684a082f66cc9e143d8724fc91b37bd + fingerprint: b211db77c5d759e5a584e8b24c992b961684a082f66cc9e143d8724fc91b37bd + sha1: sha256:a1c3c49ff5712aa21a3e01eea028bd20c2baedb90e6142b052f2c34084e767be + dependencies: + - openjdk-17 +license: + version: 15eb386a0db48f190d29bd28058f5d0d6605f5acde7c6cfc6f014d57747e41f9 + fingerprint: 15eb386a0db48f190d29bd28058f5d0d6605f5acde7c6cfc6f014d57747e41f9 + sha1: sha256:6c7a834097b96efc340787fbe5876c1e366de0235def5e2b93fa016c8655535c diff --git a/releases/app-autoscaler/index.yml b/releases/app-autoscaler/index.yml index 138a79484a..0657f07b1e 100644 --- a/releases/app-autoscaler/index.yml +++ b/releases/app-autoscaler/index.yml @@ -19,6 +19,8 @@ builds: version: 5.0.5 20b8ceb8-2d84-42ce-4646-75fbc4a65d56: version: 5.2.6 + 26492884-8950-4ade-4145-b9a075ce9db3: + version: 14.0.1 27f0ebca-978f-4b46-6c63-29e459b0494a: version: 6.0.0 28082e04-3605-4919-7a2c-3a2939c7e47d: From fd4681416d275f170226f3232bc4b7f89ddc5bfd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 21 Jun 2024 02:20:45 +0000 Subject: [PATCH 185/186] chore(deps): update dependency credhub-cli to v2.9.33 --- ci/dockerfiles/autoscaler-tools/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/dockerfiles/autoscaler-tools/Dockerfile b/ci/dockerfiles/autoscaler-tools/Dockerfile index ed4ec976a8..a82ab5615b 100644 --- a/ci/dockerfiles/autoscaler-tools/Dockerfile +++ b/ci/dockerfiles/autoscaler-tools/Dockerfile @@ -79,7 +79,7 @@ RUN wget -q https://github.com/cloudfoundry/bosh-bootloader/releases/download/${ # Install credhub # renovate: datasource=github-releases depName=credhub-cli lookupName=cloudfoundry/credhub-cli -ARG CREDHUB_VERSION=2.9.31 +ARG CREDHUB_VERSION=2.9.33 RUN wget -q https://github.com/cloudfoundry/credhub-cli/releases/download/${CREDHUB_VERSION}/credhub-linux-amd64-${CREDHUB_VERSION}.tgz && \ tar xvfz credhub-linux-amd64-${CREDHUB_VERSION}.tgz && \ mv credhub /usr/local/bin/credhub &&\ From 9246818352cab78bad8ce996aec14a9d51c55317 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 20 Jun 2024 11:10:20 +0000 Subject: [PATCH 186/186] chore(deps): update dependency org.apache.maven:maven-core to v3.9.8 --- ci/dockerfiles/autoscaler-tools/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/dockerfiles/autoscaler-tools/Dockerfile b/ci/dockerfiles/autoscaler-tools/Dockerfile index a82ab5615b..fd2f5cd782 100644 --- a/ci/dockerfiles/autoscaler-tools/Dockerfile +++ b/ci/dockerfiles/autoscaler-tools/Dockerfile @@ -107,7 +107,7 @@ RUN wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download # get maven # renovate: datasource=maven depName=org.apache.maven:maven-core -ARG MAVEN_VERSION=3.9.7 +ARG MAVEN_VERSION=3.9.8 ENV MAVEN_HOME /opt/maven RUN wget --no-verbose -O /tmp/apache-maven-${MAVEN_VERSION}.tar.gz http://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz && \ tar xzf /tmp/apache-maven-${MAVEN_VERSION}.tar.gz -C /opt/ && \