From 25ec77c066f874866fe3c9e0233554bbd6f19115 Mon Sep 17 00:00:00 2001 From: Alina Buzachis Date: Wed, 18 Jan 2023 17:16:50 +0100 Subject: [PATCH 1/4] Temporary comment task that lists all the buckets Signed-off-by: Alina Buzachis --- tests/integration/targets/s3/tasks/main.yml | 22 +++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/integration/targets/s3/tasks/main.yml b/tests/integration/targets/s3/tasks/main.yml index ade2e86b..4ef2c4a7 100644 --- a/tests/integration/targets/s3/tasks/main.yml +++ b/tests/integration/targets/s3/tasks/main.yml @@ -57,16 +57,18 @@ that: - _result is success - - name: List S3 buckets - amazon.cloud.s3_bucket: - state: list - register: _result - tags: - - docs - - - assert: - that: - - _result is success + # Temporary commented + # Failed to list resource requests: An error occurred (ThrottlingException) when calling the ListResourceRequests operation (reached max retries: 4): Rate exceeded + # - name: List S3 buckets + # amazon.cloud.s3_bucket: + # state: list + # register: _result + # tags: + # - docs + + # - assert: + # that: + # - _result is success - name: Create S3 bucket - idempotence amazon.cloud.s3_bucket: From a491f2061dc5a61e08dd88b195a686be773cefac Mon Sep 17 00:00:00 2001 From: Alina Buzachis Date: Thu, 19 Jan 2023 12:06:49 +0100 Subject: [PATCH 2/4] Add aws_retry=True to avoid ThrottlingException Signed-off-by: Alina Buzachis --- plugins/module_utils/core.py | 20 ++++++++++++----- tests/integration/targets/s3/tasks/main.yml | 24 ++++++++++----------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/plugins/module_utils/core.py b/plugins/module_utils/core.py index 25321a8f..e5b43a43 100644 --- a/plugins/module_utils/core.py +++ b/plugins/module_utils/core.py @@ -113,6 +113,9 @@ def wait_until_resource_request_success(self, request_token: str): ) @to_sync + @AWSRetry.jittered_backoff( + catch_extra_error_codes=["ThrottlingException"], retries=10 + ) async def list_resources( self, type_name: str, identifiers: Optional[List] = None ) -> List: @@ -144,7 +147,7 @@ async def list_resources( if "NextToken" in response: params["NextToken"] = response["NextToken"] try: - response = self.client.list_resources(**params) + response = self.client.list_resources(**params, aws_retry=True) except ( botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError, @@ -173,6 +176,9 @@ async def list_resources( return results + @AWSRetry.jittered_backoff( + catch_extra_error_codes=["ThrottlingException"], retries=10 + ) def list_resource_requests(self, params: Iterable) -> List: """ Returns existing resource operation requests using specific filters. @@ -186,7 +192,9 @@ def list_resource_requests(self, params: Iterable) -> List: if "NextToken" in response: params["NextToken"] = response["NextToken"] try: - response = self.client.list_resource_requests(**params) + response = self.client.list_resource_requests( + **params, aws_retry=True + ) except ( botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError, @@ -387,7 +395,7 @@ def delete_resource(self, type_name: str, identifier: str) -> bool: self.wait_for_in_progress_requests(in_progress_requests, identifier) try: response = self.client.delete_resource( - TypeName=type_name, Identifier=identifier + TypeName=type_name, Identifier=identifier, aws_retry=True ) except self.client.exceptions.ResourceNotFoundException: # If the resource has been deleted by an IN PROGRESS delete operation @@ -414,7 +422,8 @@ def ensure_request_status(self, response: Dict) -> bool: if response and response["ProgressEvent"]["OperationStatus"] == "PENDING": try: response = self.client.get_resource_request_status( - RequestToken=response["ProgressEvent"]["RequestToken"] + RequestToken=response["ProgressEvent"]["RequestToken"], + aws_retry=True, ) except ( botocore.exceptions.BotoCoreError, @@ -461,7 +470,7 @@ def update_resource( try: for e in in_progress_requests: self.client.cancel_resource_request( - RequestToken=e["RequestToken"] + RequestToken=e["RequestToken"], aws_retry=True ) except ( botocore.exceptions.BotoCoreError, @@ -488,6 +497,7 @@ def update_resource( TypeName=type_name, Identifier=identifier, PatchDocument=str(patch), + aws_retry=True, ) except ( botocore.exceptions.BotoCoreError, diff --git a/tests/integration/targets/s3/tasks/main.yml b/tests/integration/targets/s3/tasks/main.yml index 4ef2c4a7..8d31badb 100644 --- a/tests/integration/targets/s3/tasks/main.yml +++ b/tests/integration/targets/s3/tasks/main.yml @@ -56,19 +56,17 @@ - assert: that: - _result is success - - # Temporary commented - # Failed to list resource requests: An error occurred (ThrottlingException) when calling the ListResourceRequests operation (reached max retries: 4): Rate exceeded - # - name: List S3 buckets - # amazon.cloud.s3_bucket: - # state: list - # register: _result - # tags: - # - docs - - # - assert: - # that: - # - _result is success + + - name: List S3 buckets + amazon.cloud.s3_bucket: + state: list + register: _result + tags: + - docs + + - assert: + that: + - _result is success - name: Create S3 bucket - idempotence amazon.cloud.s3_bucket: From 36a2d7060f5a53efd35776e92ce941c5a33473df Mon Sep 17 00:00:00 2001 From: Alina Buzachis Date: Thu, 26 Jan 2023 11:03:37 +0100 Subject: [PATCH 3/4] Consistent use of wait=true Signed-off-by: Alina Buzachis --- tests/integration/targets/s3/tasks/tagging.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/integration/targets/s3/tasks/tagging.yml b/tests/integration/targets/s3/tasks/tagging.yml index e26d8531..614de394 100644 --- a/tests/integration/targets/s3/tasks/tagging.yml +++ b/tests/integration/targets/s3/tasks/tagging.yml @@ -56,6 +56,7 @@ amazon.cloud.s3_bucket: tags: '{{ first_tags }}' purge_tags: true + wait: true register: _result - name: assert that update succeeded assert: @@ -90,8 +91,6 @@ - _result is not changed - _result.result.properties.tags == first_tags - ### - - name: test updating tags with purge on amazon.cloud.s3_bucket (check mode) amazon.cloud.s3_bucket: tags: '{{ second_tags }}' @@ -107,6 +106,7 @@ amazon.cloud.s3_bucket: tags: '{{ second_tags }}' purge_tags: true + wait: true register: _result - name: assert that update succeeded assert: @@ -136,12 +136,11 @@ - _result is not changed - _result.result.properties.tags == second_tags - # - - name: test updating tags without purge on amazon.cloud.s3_bucket (check mode) amazon.cloud.s3_bucket: tags: '{{ third_tags }}' purge_tags: false + wait: true register: _result check_mode: yes - name: assert that update succeeded @@ -195,7 +194,6 @@ - _result is not changed - _result.result.properties.tags == final_tags - - name: test no tags param amazon.cloud.s3_bucket amazon.cloud.s3_bucket: {} register: _result @@ -205,8 +203,6 @@ - _result is not changed - _result.result.properties.tags == final_tags - ### - - name: test removing tags from amazon.cloud.s3_bucket (check mode) amazon.cloud.s3_bucket: tags: {} @@ -222,6 +218,7 @@ amazon.cloud.s3_bucket: tags: {} purge_tags: true + wait: true register: _result - name: assert that update succeeded assert: From 056451d9c9d95e9597717f90c2efee8ca7c0d960 Mon Sep 17 00:00:00 2001 From: Alina Buzachis Date: Thu, 26 Jan 2023 13:51:23 +0100 Subject: [PATCH 4/4] Attempt to disbale s3 integration tests Signed-off-by: Alina Buzachis --- tests/integration/targets/s3/aliases | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/targets/s3/aliases b/tests/integration/targets/s3/aliases index f1e0d7e3..b547e8f2 100644 --- a/tests/integration/targets/s3/aliases +++ b/tests/integration/targets/s3/aliases @@ -1,2 +1,4 @@ cloud/aws zuul/aws/cloud_control + +disabled # Temporary disabled because they throw several ThrottlingExceptions and it needs further investigation