From c074e504cd3400b141c90f269d41dc4ef96e0b94 Mon Sep 17 00:00:00 2001 From: Eamonn Faherty Date: Wed, 20 Nov 2024 13:24:16 -0800 Subject: [PATCH] Adding jmespath capability to ssm parameters (#726) --- pyproject.toml | 2 +- .../generators/ssm_parameter_handler.py | 1 + .../generators/ssm_parameter_handler_test.py | 3 +++ .../workflow/dependencies/task_factory.py | 2 ++ .../workflow/ssm/get_ssm_parameter_task.py | 23 +++++++++++++++++-- .../ssm/get_ssm_parameter_task_test.py | 10 ++++++-- setup.py | 2 +- 7 files changed, 37 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 08556a55..4caa4bcd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ [tool.poetry] name = "aws-service-catalog-puppet" -version = "0.250.0" +version = "0.251.0" description = "Making it easier to deploy ServiceCatalog products" classifiers = ["Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Natural Language :: English"] homepage = "https://service-catalog-tools-workshop.com/" diff --git a/servicecatalog_puppet/commands/task_reference_helpers/generators/ssm_parameter_handler.py b/servicecatalog_puppet/commands/task_reference_helpers/generators/ssm_parameter_handler.py index 0079e186..ef268612 100644 --- a/servicecatalog_puppet/commands/task_reference_helpers/generators/ssm_parameter_handler.py +++ b/servicecatalog_puppet/commands/task_reference_helpers/generators/ssm_parameter_handler.py @@ -76,6 +76,7 @@ def ssm_parameter_handler( "task_reference": parameter_task_reference, "account_id": parameter_account_id, "region": parameter_region, + "jmespath": ssm_parameter_details.get("jmespath", None), task_reference_constants.MANIFEST_SECTION_NAMES: dict(), task_reference_constants.MANIFEST_ITEM_NAMES: dict(), task_reference_constants.MANIFEST_ACCOUNT_IDS: dict(), diff --git a/servicecatalog_puppet/commands/task_reference_helpers/generators/ssm_parameter_handler_test.py b/servicecatalog_puppet/commands/task_reference_helpers/generators/ssm_parameter_handler_test.py index 500cca12..34acc9d4 100644 --- a/servicecatalog_puppet/commands/task_reference_helpers/generators/ssm_parameter_handler_test.py +++ b/servicecatalog_puppet/commands/task_reference_helpers/generators/ssm_parameter_handler_test.py @@ -108,6 +108,7 @@ def test_for_ssm_parameters_for_spoke_execution_mode(self): task_reference_constants.MANIFEST_ACCOUNT_IDS ), "execution": task.get("execution"), + "jmespath": None, "param_name": f"/sleeper/SleeperFunctionArn/{region}/stacks", "section_name": constants.SSM_PARAMETERS, "task_reference": expected_ssm_parameter_task_ref, @@ -213,6 +214,7 @@ def test_for_ssm_parameters_for_hub_execution_mode(self): task_reference_constants.MANIFEST_ACCOUNT_IDS ), "execution": task.get("execution"), + "jmespath": None, "param_name": f"/sleeper/SleeperFunctionArn/{region}/stacks", "section_name": constants.SSM_PARAMETERS, "task_reference": expected_ssm_parameter_task_ref, @@ -353,6 +355,7 @@ def test_for_ssm_parameters_for_spoke_execution_mode_where_param_is_set_in_hub( task_reference_constants.MANIFEST_ACCOUNT_IDS ), "execution": dependency_task.get("execution"), + "jmespath": None, "param_name": f"/sleeper/SleeperFunctionArn/{region}/stacks", "section_name": constants.SSM_PARAMETERS, "task_reference": expected_ssm_parameter_task_ref, diff --git a/servicecatalog_puppet/workflow/dependencies/task_factory.py b/servicecatalog_puppet/workflow/dependencies/task_factory.py index c7529eac..57598a6e 100644 --- a/servicecatalog_puppet/workflow/dependencies/task_factory.py +++ b/servicecatalog_puppet/workflow/dependencies/task_factory.py @@ -149,6 +149,7 @@ def create( return get_ssm_parameter_task.GetSSMParameterByPathTask( **common_parameters, path=parameters_to_use.get("path"), + jmespath_location=parameters_to_use.get("jmespath"), ) elif section_name == constants.SSM_PARAMETERS: @@ -157,6 +158,7 @@ def create( return get_ssm_parameter_task.GetSSMParameterTask( **common_parameters, param_name=parameters_to_use.get("param_name"), + jmespath_location=parameters_to_use.get("jmespath"), ) elif section_name == constants.SSM_OUTPUTS: diff --git a/servicecatalog_puppet/workflow/ssm/get_ssm_parameter_task.py b/servicecatalog_puppet/workflow/ssm/get_ssm_parameter_task.py index a47d0d69..c1b29d03 100644 --- a/servicecatalog_puppet/workflow/ssm/get_ssm_parameter_task.py +++ b/servicecatalog_puppet/workflow/ssm/get_ssm_parameter_task.py @@ -1,14 +1,19 @@ # Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +import json +from copy import deepcopy + import luigi from servicecatalog_puppet import constants from servicecatalog_puppet.workflow.dependencies import tasks +import jmespath class GetSSMParameterTask(tasks.TaskWithReference): account_id = luigi.Parameter() param_name = luigi.Parameter() + jmespath_location = luigi.Parameter() region = luigi.Parameter() cachable_level = constants.CACHE_LEVEL_RUN @@ -18,6 +23,7 @@ def params_for_results_display(self): "account_id": self.account_id, "region": self.region, "param_name": self.param_name, + "jmespath_location": self.jmespath_location, } def get_parameter_name_to_use(self): @@ -31,7 +37,13 @@ def run(self): with self.spoke_regional_client("ssm") as ssm: try: parameter = ssm.get_parameter(Name=parameter_name_to_use) - result = {parameter_name_to_use: parameter.get("Parameter")} + p = parameter.get("Parameter") + if self.jmespath_location: + p = deepcopy(parameter.get("Parameter")) + p["Value"] = jmespath.search( + self.jmespath_location, json.loads(p.get("Value")) + ) + result = {parameter_name_to_use: p} except ssm.exceptions.ParameterNotFound: pass @@ -41,6 +53,7 @@ def run(self): class GetSSMParameterByPathTask(tasks.TaskWithReference): account_id = luigi.Parameter() path = luigi.Parameter() + jmespath_location = luigi.Parameter() region = luigi.Parameter() cachable_level = constants.CACHE_LEVEL_RUN @@ -50,6 +63,7 @@ def params_for_results_display(self): "account_id": self.account_id, "region": self.region, "path": self.path, + "jmespath_location": self.jmespath_location, } def run(self): @@ -60,7 +74,12 @@ def run(self): Path=self.get_parameter_path_to_use(), Recursive=True ): for parameter in page.get("Parameters", []): - parameters[parameter.get("Name")] = parameter + p = deepcopy(parameter) + if self.jmespath_location: + p["Value"] = jmespath.search( + self.jmespath_location, json.loads(p.get("Value")) + ) + parameters[parameter.get("Name")] = p self.write_output(parameters) def get_parameter_path_to_use(self): diff --git a/servicecatalog_puppet/workflow/ssm/get_ssm_parameter_task_test.py b/servicecatalog_puppet/workflow/ssm/get_ssm_parameter_task_test.py index a34fcaca..0f7c966c 100644 --- a/servicecatalog_puppet/workflow/ssm/get_ssm_parameter_task_test.py +++ b/servicecatalog_puppet/workflow/ssm/get_ssm_parameter_task_test.py @@ -10,6 +10,7 @@ class GetSSMParameterTaskTest(tasks_unit_tests_helper.PuppetTaskUnitTest): account_id = "account_id" param_name = "param_name" region = "region" + jmespath_location = None def setUp(self) -> None: from servicecatalog_puppet.workflow.ssm import get_ssm_parameter_task @@ -20,7 +21,8 @@ def setUp(self) -> None: **self.get_common_args(), account_id=self.account_id, param_name=self.param_name, - region=self.region + region=self.region, + jmespath_location=self.jmespath_location, ) self.wire_up_mocks() @@ -32,6 +34,7 @@ def test_params_for_results_display(self): "account_id": self.account_id, "region": self.region, "param_name": self.param_name, + "jmespath_location": self.jmespath_location, } # exercise @@ -65,6 +68,7 @@ class GetSSMParameterByPathTaskTest(tasks_unit_tests_helper.PuppetTaskUnitTest): account_id = "account_id" path = "path" region = "region" + jmespath_location = None def setUp(self) -> None: from servicecatalog_puppet.workflow.ssm import get_ssm_parameter_task @@ -75,7 +79,8 @@ def setUp(self) -> None: **self.get_common_args(), account_id=self.account_id, path=self.path, - region=self.region + region=self.region, + jmespath_location=self.jmespath_location, ) self.wire_up_mocks() @@ -87,6 +92,7 @@ def test_params_for_results_display(self): "account_id": self.account_id, "region": self.region, "path": self.path, + "jmespath_location": self.jmespath_location, } # exercise diff --git a/setup.py b/setup.py index 7f3c98be..22ee33e4 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ setup_kwargs = { 'name': 'aws-service-catalog-puppet', - 'version': '0.250.0', + 'version': '0.251.0', 'description': 'Making it easier to deploy ServiceCatalog products', 'long_description': '# aws-service-catalog-puppet\n\n![logo](./docs/logo.png) \n\n## Badges\n\n[![codecov](https://codecov.io/gh/awslabs/aws-service-catalog-puppet/branch/master/graph/badge.svg?token=e8M7mdsmy0)](https://codecov.io/gh/awslabs/aws-service-catalog-puppet)\n\n\n## What is it?\nThis is a python3 framework that makes it easier to share multi region AWS Service Catalog portfolios and makes it \npossible to provision products into accounts declaratively using a metadata based rules engine.\n\nWith this framework you define your accounts in a YAML file. You give each account a set of tags, a default region and \na set of enabled regions.\n\nOnce you have done this you can define portfolios should be shared with each set of accounts using the tags and you \ncan specify which regions the shares occur in.\n\nIn addition to this, you can also define products that should be provisioned into accounts using the same tag based \napproach. The framework will assume role into the target account and provision the product on your behalf.\n\n\n## Getting started\n\nYou can read the [installation how to](https://service-catalog-tools-workshop.com/30-how-tos/10-installation/30-service-catalog-puppet.html)\nor you can read through the [every day use](https://service-catalog-tools-workshop.com/30-how-tos/50-every-day-use.html)\nguides.\n\nYou can read the [documentation](https://aws-service-catalog-puppet.readthedocs.io/en/latest/) to understand the inner \nworkings. \n\n\n## Going further\n\nThe framework is one of a pair. The other is [aws-service-catalog-factory](https://github.com/awslabs/aws-service-catalog-factory).\nWith Service Catalog Factory you can create pipelines that deploy multi region portfolios very easily. \n\n## License\n\nThis library is licensed under the Apache 2.0 License. \n \n', 'author': 'Eamonn Faherty',