Optimising S3 Parameter usage to reduce calls to S3

docs/requirements.txt

@@ -1,4 +1,4 @@
 sphinx==2.0.0
 recommonmark==0.5.0
-setuptools==65.5.1
+setuptools==70.0.0
 aws-service-catalog-puppet

pyproject.toml

@@ -3,7 +3,7 @@
 
 [tool.poetry]
 name = "aws-service-catalog-puppet"
-version = "0.254.1"
+version = "0.254.2"
 description = "Making it easier to deploy ServiceCatalog products"
 classifiers = ["Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Natural Language :: English"]
 homepage = "https://service-catalog-tools-workshop.com/"

servicecatalog_puppet/commands/task_reference_helpers/complete_generator.py

@@ -15,15 +15,15 @@
 from servicecatalog_puppet.commands.task_reference_helpers.generators.boto3_parameter_handler import (
     boto3_parameter_handler,
 )
+from servicecatalog_puppet.commands.task_reference_helpers.generators.s3_parameter_handler import (
+    s3_parameter_handler,
+)
 from servicecatalog_puppet.commands.task_reference_helpers.generators.ssm_outputs_handler import (
     ssm_outputs_handler,
 )
 from servicecatalog_puppet.commands.task_reference_helpers.generators.ssm_parameter_handler import (
     ssm_parameter_handler,
 )
-from servicecatalog_puppet.commands.task_reference_helpers.generators.s3_parameter_handler import (
-    s3_parameter_handler,
-)
 from servicecatalog_puppet.waluigi.shared_tasks.task_topological_generations_without_scheduler_unit_test import (
     dependency_task_reference,
 )

servicecatalog_puppet/commands/task_reference_helpers/generators/s3_parameter_handler.py

@@ -32,6 +32,22 @@ def s3_parameter_handler(
         )
 
         parameter_task_reference = f"{constants.S3_PARAMETERS}-{key}-{jmespath}"
+        s3_object_task_reference = f"{constants.S3_GET_OBJECT}-{key}"
+
+        if not all_tasks.get(s3_object_task_reference):
+            new_tasks[s3_object_task_reference] = {
+                "task_reference": s3_object_task_reference,
+                "account_id": puppet_account_id,
+                "region": home_region,
+                "key": key,
+                task_reference_constants.MANIFEST_SECTION_NAMES: dict(),
+                task_reference_constants.MANIFEST_ITEM_NAMES: dict(),
+                task_reference_constants.MANIFEST_ACCOUNT_IDS: dict(),
+                "dependencies": [],
+                "dependencies_by_reference": [],
+                "execution": constants.EXECUTION_MODE_HUB,
+                "section_name": constants.S3_GET_OBJECT,
+            }
 
         if all_tasks.get(parameter_task_reference):
             s3_task_params = all_tasks.get(parameter_task_reference)
@@ -40,14 +56,15 @@ def s3_parameter_handler(
                 "task_reference": parameter_task_reference,
                 "account_id": puppet_account_id,
                 "region": home_region,
+                "s3_object_task_reference": s3_object_task_reference,
                 "key": key,
                 "jmespath": jmespath,
                 "default": default,
                 task_reference_constants.MANIFEST_SECTION_NAMES: dict(),
                 task_reference_constants.MANIFEST_ITEM_NAMES: dict(),
                 task_reference_constants.MANIFEST_ACCOUNT_IDS: dict(),
                 "dependencies": [],
-                "dependencies_by_reference": [],
+                "dependencies_by_reference": [s3_object_task_reference],
                 "execution": constants.EXECUTION_MODE_HUB,
                 "section_name": constants.S3_PARAMETERS,
             }

servicecatalog_puppet/constants.py

@@ -332,6 +332,7 @@
 SSM_OUTPUTS = "ssm_outputs"
 SSM_PARAMETERS = "ssm_parameters"
 S3_PARAMETERS = "s3_parameters"
+S3_GET_OBJECT = "s3_get_object"
 BOTO3_PARAMETERS = "boto3_parameters"
 SSM_PARAMETERS_WITH_A_PATH = "ssm_parameters_with_a_path"
 PORTFOLIO_ASSOCIATIONS = "portfolio-associations"

servicecatalog_puppet/waluigi/task_mixins/io_mixin.py

@@ -73,10 +73,16 @@ def write_empty_output(self):
             with self.output().open("wb") as f:
                 f.write(b"{}")
 
-    def write_output(self, content):
+    def write_output(self, content, skip_json_encode=False):
         if self.should_use_caching:
             with self.output().open("w") as f:
-                f.write(serialisation_utils.json_dumps(content).decode("utf-8"))
+                if skip_json_encode:
+                    f.write(content.decode("utf-8"))
+                else:
+                    f.write(serialisation_utils.json_dumps(content).decode("utf-8"))
         else:
             with self.output().open("wb") as f:
-                f.write(serialisation_utils.json_dumps(content))
+                if skip_json_encode:
+                    f.write(content)
+                else:
+                    f.write(serialisation_utils.json_dumps(content))

servicecatalog_puppet/workflow/dependencies/resources_factory.py

@@ -230,6 +230,9 @@ def create(section_name, parameters_to_use, puppet_account_id):
         resources = []
 
     elif section_name == constants.S3_PARAMETERS:
+        resources = []
+
+    elif section_name == constants.S3_GET_OBJECT:
         resources = [S3_GET_OBJECT_PER_REGION_OF_ACCOUNT]
 
     elif section_name == constants.SSM_PARAMETERS_WITH_A_PATH:

servicecatalog_puppet/workflow/dependencies/task_factory.py

@@ -166,11 +166,20 @@ def create(
 
         return get_s3_parameter_task.GetS3ParameterTask(
             **common_parameters,
+            s3_object_task_reference=parameters_to_use.get("s3_object_task_reference"),
             key=parameters_to_use.get("key"),
             jmespath_location=parameters_to_use.get("jmespath"),
             default=parameters_to_use.get("default"),
         )
 
+    elif section_name == constants.S3_GET_OBJECT:
+        from servicecatalog_puppet.workflow.s3 import get_s3_object_task
+
+        return get_s3_object_task.GetS3ObjectTask(
+            **common_parameters,
+            key=parameters_to_use.get("key"),
+        )
+
     elif section_name == constants.SSM_OUTPUTS:
         from servicecatalog_puppet.workflow.ssm import ssm_outputs_task
 

servicecatalog_puppet/workflow/s3/get_s3_object_task.py

@@ -0,0 +1,36 @@
+#  Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#  SPDX-License-Identifier: Apache-2.0
+import json
+from copy import deepcopy
+
+import jmespath
+import luigi
+
+from servicecatalog_puppet import constants
+from servicecatalog_puppet.workflow.dependencies import tasks
+
+
+class GetS3ObjectTask(tasks.TaskWithReference):
+    account_id = luigi.Parameter()
+    key = luigi.Parameter()
+    region = luigi.Parameter()
+    cachable_level = constants.CACHE_LEVEL_RUN
+
+    def params_for_results_display(self):
+        return {
+            "task_reference": self.task_reference,
+            "account_id": self.account_id,
+            "region": self.region,
+            "key": self.key,
+        }
+
+    def run(self):
+        with self.spoke_regional_client("s3") as s3:
+            object = (
+                s3.get_object(
+                    Bucket=f"sc-puppet-parameters-{self.account_id}", Key=self.key
+                )
+                .get("Body")
+                .read()
+            )
+            self.write_output(object, skip_json_encode=True)

servicecatalog_puppet/workflow/s3/get_s3_parameter_task.py

@@ -1,17 +1,17 @@
 #  Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #  SPDX-License-Identifier: Apache-2.0
 import json
-from copy import deepcopy
 
+import jmespath
 import luigi
 
 from servicecatalog_puppet import constants
 from servicecatalog_puppet.workflow.dependencies import tasks
-import jmespath
 
 
 class GetS3ParameterTask(tasks.TaskWithReference):
     account_id = luigi.Parameter()
+    s3_object_task_reference = luigi.Parameter()
     key = luigi.Parameter()
     jmespath_location = luigi.Parameter()
     default = luigi.Parameter()
@@ -28,21 +28,16 @@ def params_for_results_display(self):
         }
 
     def run(self):
-        with self.spoke_regional_client("s3") as s3:
-            object = (
-                s3.get_object(
-                    Bucket=f"sc-puppet-parameters-{self.account_id}", Key=self.key
+        object = self.get_output_from_reference_dependency(
+            self.s3_object_task_reference
+        )
+        result = jmespath.search(self.jmespath_location, object)
+        if result is None:
+            if self.default is None:
+                raise Exception(
+                    "Could not find value in the s3 JSON object and there is no default value."
+                    "Check your JMESPath is correct."
                 )
-                .get("Body")
-                .read()
-            )
-            result = jmespath.search(self.jmespath_location, json.loads(object))
-            if result is None:
-                print("result was none")
-                if self.default is None:
-                    raise Exception("Could not find value in the s3 JSON object and there is no default value. Check your JMESPath is correct")
-                else:
-                    print("defauilt was nmot none")
-                    result = self.default
-            print("result is", result)
-            self.write_output(result)
+            else:
+                result = self.default
+        self.write_output(result)

servicecatalog_puppet/workflow/ssm/get_ssm_parameter_task.py

@@ -3,11 +3,11 @@
 import json
 from copy import deepcopy
 
+import jmespath
 import luigi
 
 from servicecatalog_puppet import constants
 from servicecatalog_puppet.workflow.dependencies import tasks
-import jmespath
 
 
 class GetSSMParameterTask(tasks.TaskWithReference):

setup.py

@@ -74,7 +74,7 @@
 
 setup_kwargs = {
     'name': 'aws-service-catalog-puppet',
-    'version': '0.254.1',
+    'version': '0.254.2',
     'description': 'Making it easier to deploy ServiceCatalog products',
     'long_description': '# aws-service-catalog-puppet\n\n![logo](./docs/logo.png) \n\n## Badges\n\n[![codecov](https://codecov.io/gh/awslabs/aws-service-catalog-puppet/branch/master/graph/badge.svg?token=e8M7mdsmy0)](https://codecov.io/gh/awslabs/aws-service-catalog-puppet)\n\n\n## What is it?\nThis is a python3 framework that makes it easier to share multi region AWS Service Catalog portfolios and makes it \npossible to provision products into accounts declaratively using a metadata based rules engine.\n\nWith this framework you define your accounts in a YAML file.  You give each account a set of tags, a default region and \na set of enabled regions.\n\nOnce you have done this you can define portfolios should be shared with each set of accounts using the tags and you \ncan specify which regions the shares occur in.\n\nIn addition to this, you can also define products that should be provisioned into accounts using the same tag based \napproach.  The framework will assume role into the target account and provision the product on your behalf.\n\n\n## Getting started\n\nYou can read the [installation how to](https://service-catalog-tools-workshop.com/30-how-tos/10-installation/30-service-catalog-puppet.html)\nor you can read through the [every day use](https://service-catalog-tools-workshop.com/30-how-tos/50-every-day-use.html)\nguides.\n\nYou can read the [documentation](https://aws-service-catalog-puppet.readthedocs.io/en/latest/) to understand the inner \nworkings. \n\n\n## Going further\n\nThe framework is one of a pair.  The other is [aws-service-catalog-factory](https://github.com/awslabs/aws-service-catalog-factory).\nWith Service Catalog Factory you can create pipelines that deploy multi region portfolios very easily. \n\n## License\n\nThis library is licensed under the Apache 2.0 License. \n \n',
     'author': 'Eamonn Faherty',