From a6dd61bdcf778cf351737362a2b069690eccb165 Mon Sep 17 00:00:00 2001 From: Eamonn Faherty Date: Mon, 16 Dec 2024 09:27:28 -0500 Subject: [PATCH] Optimising S3 Parameter usage to reduce calls to S3 --- docs/requirements.txt | 2 +- poetry.lock | 24 ++++++------- pyproject.toml | 2 +- .../complete_generator.py | 6 ++-- .../generators/s3_parameter_handler.py | 19 +++++++++- servicecatalog_puppet/constants.py | 1 + .../waluigi/task_mixins/io_mixin.py | 12 +++++-- .../dependencies/resources_factory.py | 3 ++ .../workflow/dependencies/task_factory.py | 9 +++++ .../workflow/s3/get_s3_object_task.py | 36 +++++++++++++++++++ .../workflow/s3/get_s3_parameter_task.py | 33 ++++++++--------- .../workflow/ssm/get_ssm_parameter_task.py | 2 +- setup.py | 2 +- 13 files changed, 109 insertions(+), 42 deletions(-) create mode 100644 servicecatalog_puppet/workflow/s3/get_s3_object_task.py diff --git a/docs/requirements.txt b/docs/requirements.txt index 769b020fd..5fd0bec43 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ sphinx==2.0.0 recommonmark==0.5.0 -setuptools==65.5.1 +setuptools==70.0.0 aws-service-catalog-puppet diff --git a/poetry.lock b/poetry.lock index afb1d96ff..b71285781 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1200,23 +1200,23 @@ files = [ [[package]] name = "tornado" -version = "6.4.1" +version = "6.4.2" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:163b0aafc8e23d8cdc3c9dfb24c5368af84a81e3364745ccb4427669bf84aec8"}, - {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6d5ce3437e18a2b66fbadb183c1d3364fb03f2be71299e7d10dbeeb69f4b2a14"}, - {file = "tornado-6.4.1-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e20b9113cd7293f164dc46fffb13535266e713cdb87bd2d15ddb336e96cfc4"}, - {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ae50a504a740365267b2a8d1a90c9fbc86b780a39170feca9bcc1787ff80842"}, - {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:613bf4ddf5c7a95509218b149b555621497a6cc0d46ac341b30bd9ec19eac7f3"}, - {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25486eb223babe3eed4b8aecbac33b37e3dd6d776bc730ca14e1bf93888b979f"}, - {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:454db8a7ecfcf2ff6042dde58404164d969b6f5d58b926da15e6b23817950fc4"}, - {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a02a08cc7a9314b006f653ce40483b9b3c12cda222d6a46d4ac63bb6c9057698"}, - {file = "tornado-6.4.1-cp38-abi3-win32.whl", hash = "sha256:d9a566c40b89757c9aa8e6f032bcdb8ca8795d7c1a9762910c722b1635c9de4d"}, - {file = "tornado-6.4.1-cp38-abi3-win_amd64.whl", hash = "sha256:b24b8982ed444378d7f21d563f4180a2de31ced9d8d84443907a0a64da2072e7"}, - {file = "tornado-6.4.1.tar.gz", hash = "sha256:92d3ab53183d8c50f8204a51e6f91d18a15d5ef261e84d452800d4ff6fc504e9"}, + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a017d239bd1bb0919f72af256a970624241f070496635784d9bf0db640d3fec"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c36e62ce8f63409301537222faffcef7dfc5284f27eec227389f2ad11b09d946"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca9eb02196e789c9cb5c3c7c0f04fb447dc2adffd95265b2c7223a8a615ccbf"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:304463bd0772442ff4d0f5149c6f1c2135a1fae045adf070821c6cdc76980634"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:c82c46813ba483a385ab2a99caeaedf92585a1f90defb5693351fa7e4ea0bf73"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:932d195ca9015956fa502c6b56af9eb06106140d844a335590c1ec7f5277d10c"}, + {file = "tornado-6.4.2-cp38-abi3-win32.whl", hash = "sha256:2876cef82e6c5978fde1e0d5b1f919d756968d5b4282418f3146b79b58556482"}, + {file = "tornado-6.4.2-cp38-abi3-win_amd64.whl", hash = "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38"}, + {file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index ed08f7db4..7668e713f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ [tool.poetry] name = "aws-service-catalog-puppet" -version = "0.254.1" +version = "0.254.2" description = "Making it easier to deploy ServiceCatalog products" classifiers = ["Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Natural Language :: English"] homepage = "https://service-catalog-tools-workshop.com/" diff --git a/servicecatalog_puppet/commands/task_reference_helpers/complete_generator.py b/servicecatalog_puppet/commands/task_reference_helpers/complete_generator.py index c1963f141..ee1bc7c30 100644 --- a/servicecatalog_puppet/commands/task_reference_helpers/complete_generator.py +++ b/servicecatalog_puppet/commands/task_reference_helpers/complete_generator.py @@ -15,15 +15,15 @@ from servicecatalog_puppet.commands.task_reference_helpers.generators.boto3_parameter_handler import ( boto3_parameter_handler, ) +from servicecatalog_puppet.commands.task_reference_helpers.generators.s3_parameter_handler import ( + s3_parameter_handler, +) from servicecatalog_puppet.commands.task_reference_helpers.generators.ssm_outputs_handler import ( ssm_outputs_handler, ) from servicecatalog_puppet.commands.task_reference_helpers.generators.ssm_parameter_handler import ( ssm_parameter_handler, ) -from servicecatalog_puppet.commands.task_reference_helpers.generators.s3_parameter_handler import ( - s3_parameter_handler, -) from servicecatalog_puppet.waluigi.shared_tasks.task_topological_generations_without_scheduler_unit_test import ( dependency_task_reference, ) diff --git a/servicecatalog_puppet/commands/task_reference_helpers/generators/s3_parameter_handler.py b/servicecatalog_puppet/commands/task_reference_helpers/generators/s3_parameter_handler.py index 009a8d60b..824be0dbb 100644 --- a/servicecatalog_puppet/commands/task_reference_helpers/generators/s3_parameter_handler.py +++ b/servicecatalog_puppet/commands/task_reference_helpers/generators/s3_parameter_handler.py @@ -32,6 +32,22 @@ def s3_parameter_handler( ) parameter_task_reference = f"{constants.S3_PARAMETERS}-{key}-{jmespath}" + s3_object_task_reference = f"{constants.S3_GET_OBJECT}-{key}" + + if not all_tasks.get(s3_object_task_reference): + new_tasks[s3_object_task_reference] = { + "task_reference": s3_object_task_reference, + "account_id": puppet_account_id, + "region": home_region, + "key": key, + task_reference_constants.MANIFEST_SECTION_NAMES: dict(), + task_reference_constants.MANIFEST_ITEM_NAMES: dict(), + task_reference_constants.MANIFEST_ACCOUNT_IDS: dict(), + "dependencies": [], + "dependencies_by_reference": [], + "execution": constants.EXECUTION_MODE_HUB, + "section_name": constants.S3_GET_OBJECT, + } if all_tasks.get(parameter_task_reference): s3_task_params = all_tasks.get(parameter_task_reference) @@ -40,6 +56,7 @@ def s3_parameter_handler( "task_reference": parameter_task_reference, "account_id": puppet_account_id, "region": home_region, + "s3_object_task_reference": s3_object_task_reference, "key": key, "jmespath": jmespath, "default": default, @@ -47,7 +64,7 @@ def s3_parameter_handler( task_reference_constants.MANIFEST_ITEM_NAMES: dict(), task_reference_constants.MANIFEST_ACCOUNT_IDS: dict(), "dependencies": [], - "dependencies_by_reference": [], + "dependencies_by_reference": [s3_object_task_reference], "execution": constants.EXECUTION_MODE_HUB, "section_name": constants.S3_PARAMETERS, } diff --git a/servicecatalog_puppet/constants.py b/servicecatalog_puppet/constants.py index 9156b1b9d..4ceaa8dc1 100644 --- a/servicecatalog_puppet/constants.py +++ b/servicecatalog_puppet/constants.py @@ -332,6 +332,7 @@ SSM_OUTPUTS = "ssm_outputs" SSM_PARAMETERS = "ssm_parameters" S3_PARAMETERS = "s3_parameters" +S3_GET_OBJECT = "s3_get_object" BOTO3_PARAMETERS = "boto3_parameters" SSM_PARAMETERS_WITH_A_PATH = "ssm_parameters_with_a_path" PORTFOLIO_ASSOCIATIONS = "portfolio-associations" diff --git a/servicecatalog_puppet/waluigi/task_mixins/io_mixin.py b/servicecatalog_puppet/waluigi/task_mixins/io_mixin.py index 15c24ce91..ddb20e927 100644 --- a/servicecatalog_puppet/waluigi/task_mixins/io_mixin.py +++ b/servicecatalog_puppet/waluigi/task_mixins/io_mixin.py @@ -73,10 +73,16 @@ def write_empty_output(self): with self.output().open("wb") as f: f.write(b"{}") - def write_output(self, content): + def write_output(self, content, skip_json_encode=False): if self.should_use_caching: with self.output().open("w") as f: - f.write(serialisation_utils.json_dumps(content).decode("utf-8")) + if skip_json_encode: + f.write(content.decode("utf-8")) + else: + f.write(serialisation_utils.json_dumps(content).decode("utf-8")) else: with self.output().open("wb") as f: - f.write(serialisation_utils.json_dumps(content)) + if skip_json_encode: + f.write(content) + else: + f.write(serialisation_utils.json_dumps(content)) diff --git a/servicecatalog_puppet/workflow/dependencies/resources_factory.py b/servicecatalog_puppet/workflow/dependencies/resources_factory.py index 880e7f8db..630bf799a 100644 --- a/servicecatalog_puppet/workflow/dependencies/resources_factory.py +++ b/servicecatalog_puppet/workflow/dependencies/resources_factory.py @@ -230,6 +230,9 @@ def create(section_name, parameters_to_use, puppet_account_id): resources = [] elif section_name == constants.S3_PARAMETERS: + resources = [] + + elif section_name == constants.S3_GET_OBJECT: resources = [S3_GET_OBJECT_PER_REGION_OF_ACCOUNT] elif section_name == constants.SSM_PARAMETERS_WITH_A_PATH: diff --git a/servicecatalog_puppet/workflow/dependencies/task_factory.py b/servicecatalog_puppet/workflow/dependencies/task_factory.py index cb761fbc2..75365680a 100644 --- a/servicecatalog_puppet/workflow/dependencies/task_factory.py +++ b/servicecatalog_puppet/workflow/dependencies/task_factory.py @@ -166,11 +166,20 @@ def create( return get_s3_parameter_task.GetS3ParameterTask( **common_parameters, + s3_object_task_reference=parameters_to_use.get("s3_object_task_reference"), key=parameters_to_use.get("key"), jmespath_location=parameters_to_use.get("jmespath"), default=parameters_to_use.get("default"), ) + elif section_name == constants.S3_GET_OBJECT: + from servicecatalog_puppet.workflow.s3 import get_s3_object_task + + return get_s3_object_task.GetS3ObjectTask( + **common_parameters, + key=parameters_to_use.get("key"), + ) + elif section_name == constants.SSM_OUTPUTS: from servicecatalog_puppet.workflow.ssm import ssm_outputs_task diff --git a/servicecatalog_puppet/workflow/s3/get_s3_object_task.py b/servicecatalog_puppet/workflow/s3/get_s3_object_task.py new file mode 100644 index 000000000..822deb763 --- /dev/null +++ b/servicecatalog_puppet/workflow/s3/get_s3_object_task.py @@ -0,0 +1,36 @@ +# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import json +from copy import deepcopy + +import jmespath +import luigi + +from servicecatalog_puppet import constants +from servicecatalog_puppet.workflow.dependencies import tasks + + +class GetS3ObjectTask(tasks.TaskWithReference): + account_id = luigi.Parameter() + key = luigi.Parameter() + region = luigi.Parameter() + cachable_level = constants.CACHE_LEVEL_RUN + + def params_for_results_display(self): + return { + "task_reference": self.task_reference, + "account_id": self.account_id, + "region": self.region, + "key": self.key, + } + + def run(self): + with self.spoke_regional_client("s3") as s3: + object = ( + s3.get_object( + Bucket=f"sc-puppet-parameters-{self.account_id}", Key=self.key + ) + .get("Body") + .read() + ) + self.write_output(object, skip_json_encode=True) diff --git a/servicecatalog_puppet/workflow/s3/get_s3_parameter_task.py b/servicecatalog_puppet/workflow/s3/get_s3_parameter_task.py index e94559962..cd0f17c8b 100644 --- a/servicecatalog_puppet/workflow/s3/get_s3_parameter_task.py +++ b/servicecatalog_puppet/workflow/s3/get_s3_parameter_task.py @@ -1,17 +1,17 @@ # Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 import json -from copy import deepcopy +import jmespath import luigi from servicecatalog_puppet import constants from servicecatalog_puppet.workflow.dependencies import tasks -import jmespath class GetS3ParameterTask(tasks.TaskWithReference): account_id = luigi.Parameter() + s3_object_task_reference = luigi.Parameter() key = luigi.Parameter() jmespath_location = luigi.Parameter() default = luigi.Parameter() @@ -28,21 +28,16 @@ def params_for_results_display(self): } def run(self): - with self.spoke_regional_client("s3") as s3: - object = ( - s3.get_object( - Bucket=f"sc-puppet-parameters-{self.account_id}", Key=self.key + object = self.get_output_from_reference_dependency( + self.s3_object_task_reference + ) + result = jmespath.search(self.jmespath_location, object) + if result is None: + if self.default is None: + raise Exception( + "Could not find value in the s3 JSON object and there is no default value." + "Check your JMESPath is correct." ) - .get("Body") - .read() - ) - result = jmespath.search(self.jmespath_location, json.loads(object)) - if result is None: - print("result was none") - if self.default is None: - raise Exception("Could not find value in the s3 JSON object and there is no default value. Check your JMESPath is correct") - else: - print("defauilt was nmot none") - result = self.default - print("result is", result) - self.write_output(result) + else: + result = self.default + self.write_output(result) diff --git a/servicecatalog_puppet/workflow/ssm/get_ssm_parameter_task.py b/servicecatalog_puppet/workflow/ssm/get_ssm_parameter_task.py index c1b29d033..c5f9d24cd 100644 --- a/servicecatalog_puppet/workflow/ssm/get_ssm_parameter_task.py +++ b/servicecatalog_puppet/workflow/ssm/get_ssm_parameter_task.py @@ -3,11 +3,11 @@ import json from copy import deepcopy +import jmespath import luigi from servicecatalog_puppet import constants from servicecatalog_puppet.workflow.dependencies import tasks -import jmespath class GetSSMParameterTask(tasks.TaskWithReference): diff --git a/setup.py b/setup.py index 2b39e86e6..b2d65928a 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ setup_kwargs = { 'name': 'aws-service-catalog-puppet', - 'version': '0.254.1', + 'version': '0.254.2', 'description': 'Making it easier to deploy ServiceCatalog products', 'long_description': '# aws-service-catalog-puppet\n\n![logo](./docs/logo.png) \n\n## Badges\n\n[![codecov](https://codecov.io/gh/awslabs/aws-service-catalog-puppet/branch/master/graph/badge.svg?token=e8M7mdsmy0)](https://codecov.io/gh/awslabs/aws-service-catalog-puppet)\n\n\n## What is it?\nThis is a python3 framework that makes it easier to share multi region AWS Service Catalog portfolios and makes it \npossible to provision products into accounts declaratively using a metadata based rules engine.\n\nWith this framework you define your accounts in a YAML file. You give each account a set of tags, a default region and \na set of enabled regions.\n\nOnce you have done this you can define portfolios should be shared with each set of accounts using the tags and you \ncan specify which regions the shares occur in.\n\nIn addition to this, you can also define products that should be provisioned into accounts using the same tag based \napproach. The framework will assume role into the target account and provision the product on your behalf.\n\n\n## Getting started\n\nYou can read the [installation how to](https://service-catalog-tools-workshop.com/30-how-tos/10-installation/30-service-catalog-puppet.html)\nor you can read through the [every day use](https://service-catalog-tools-workshop.com/30-how-tos/50-every-day-use.html)\nguides.\n\nYou can read the [documentation](https://aws-service-catalog-puppet.readthedocs.io/en/latest/) to understand the inner \nworkings. \n\n\n## Going further\n\nThe framework is one of a pair. The other is [aws-service-catalog-factory](https://github.com/awslabs/aws-service-catalog-factory).\nWith Service Catalog Factory you can create pipelines that deploy multi region portfolios very easily. \n\n## License\n\nThis library is licensed under the Apache 2.0 License. \n \n', 'author': 'Eamonn Faherty',