From 15fdb8dd6941612dffee91ed6710e18db3e6e43a Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Tue, 7 Nov 2023 15:05:46 -0800 Subject: [PATCH] Change run_sample_ci.py to be more generic (#502) * Change run_sample_ci.py to be more generic * Run fleet provisioning sample via script --- .github/workflows/ci.yml | 58 ++++++------- .../workflows/ci_run_basic_connect_cfg.json | 6 +- .../workflows/ci_run_cognito_connect_cfg.json | 6 +- .../ci_run_custom_authorizer_connect_cfg.json | 6 +- .../workflows/ci_run_custom_key_ops_cfg.json | 6 +- .../ci_run_fleet_provisioning_cfg.json | 6 +- .../ci_run_fleet_provisioning_mqtt5_cfg.json | 6 +- .../ci_run_greengrass_discovery_cfg.json | 6 +- .github/workflows/ci_run_jobs_cfg.json | 6 +- .github/workflows/ci_run_jobs_mqtt5_cfg.json | 6 +- .../ci_run_keystore_connect_cfg.json | 6 +- .../workflows/ci_run_mqtt5_pubsub_cfg.json | 6 +- .../ci_run_mqtt5_shared_subscription_cfg.json | 6 +- .../workflows/ci_run_pkcs11_connect_cfg.json | 6 +- .../workflows/ci_run_pkcs12_connect_cfg.json | 6 +- .github/workflows/ci_run_pubsub_cfg.json | 6 +- .github/workflows/ci_run_shadow_cfg.json | 6 +- .../workflows/ci_run_shadow_mqtt5_cfg.json | 6 +- .../ci_run_websocket_connect_cfg.json | 6 +- .../ci_run_windows_cert_connect_cfg.json | 6 +- .../workflows/ci_run_x509_connect_cfg.json | 6 +- utils/ci_iot_thing.py | 34 ++++++++ utils/run_fleet_provisioning_sample.py | 41 ++++++++++ utils/{run_sample_ci.py => run_in_ci.py} | 81 ++++++++++--------- 24 files changed, 205 insertions(+), 129 deletions(-) create mode 100644 utils/ci_iot_thing.py create mode 100644 utils/run_fleet_provisioning_sample.py rename utils/{run_sample_ci.py => run_in_ci.py} (85%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8693820b..a42916d13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -187,10 +187,10 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: run PubSub sample run: | - python ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_pubsub_cfg.json + python ./utils/run_in_ci.py --file ./.github/workflows/ci_run_pubsub_cfg.json - name: run Windows Certificate Connect sample run: | - python ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_windows_cert_connect_cfg.json + python ./utils/run_in_ci.py --file ./.github/workflows/ci_run_windows_cert_connect_cfg.json - name: configure AWS credentials (MQTT5) uses: aws-actions/configure-aws-credentials@v2 with: @@ -198,7 +198,7 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: run MQTT5 PubSub sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_mqtt5_pubsub_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_mqtt5_pubsub_cfg.json - name: configure AWS credentials (Device Advisor) uses: aws-actions/configure-aws-credentials@v2 with: @@ -259,14 +259,14 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: run PubSub sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_pubsub_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_pubsub_cfg.json - name: run PKCS12 Connect sample run: | cert=$(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/PubSub/cert" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo -e "$cert" > /tmp/certificate.pem key=$(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/PubSub/key" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo -e "$key" > /tmp/privatekey.pem pkcs12_password=$(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/PubSub/key_pkcs12_password" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") openssl pkcs12 -export -in /tmp/certificate.pem -inkey /tmp/privatekey.pem -out ./pkcs12-key.p12 -name PubSub_Thing_Alias -password pass:$pkcs12_password - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_pkcs12_connect_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_pkcs12_connect_cfg.json - name: configure AWS credentials (MQTT5) uses: aws-actions/configure-aws-credentials@v2 with: @@ -274,7 +274,7 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: run MQTT5 PubSub sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_mqtt5_pubsub_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_mqtt5_pubsub_cfg.json - name: configure AWS credentials (Device Advisor) uses: aws-actions/configure-aws-credentials@v2 with: @@ -334,7 +334,7 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: run PubSub sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_pubsub_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_pubsub_cfg.json - name: configure AWS credentials (MQTT5) uses: aws-actions/configure-aws-credentials@v2 with: @@ -342,7 +342,7 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: run MQTT5 PubSub sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_mqtt5_pubsub_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_mqtt5_pubsub_cfg.json - name: configure AWS credentials (Device Advisor) uses: aws-actions/configure-aws-credentials@v2 with: @@ -478,22 +478,22 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: run Basic Connect sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_basic_connect_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_basic_connect_cfg.json - name: run Websocket Connect sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_websocket_connect_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_websocket_connect_cfg.json - name: run PubSub sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_pubsub_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_pubsub_cfg.json - name: run CustomKeyOperations sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_custom_key_ops_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_custom_key_ops_cfg.json - name: run PKCS11 Connect sample run: | mkdir -p /tmp/tokens export SOFTHSM2_CONF=/tmp/softhsm2.conf echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_pkcs11_connect_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_pkcs11_connect_cfg.json - name: run Java keystore Connect sample run: | cert=$(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/PubSub/cert" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo -e "$cert" > /tmp/certificate.pem @@ -501,7 +501,7 @@ jobs: pkcs12_password=$(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/PubSub/key_pkcs12_password" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") openssl pkcs12 -export -in /tmp/certificate.pem -inkey /tmp/privatekey.pem -out /tmp/pkcs12-key.p12 -name PubSub_Thing_Alias -password pass:$pkcs12_password keytool -importkeystore -srckeystore /tmp/pkcs12-key.p12 -destkeystore ./java_keystore.keys -srcstoretype PKCS12 -alias PubSub_Thing_Alias -srcstorepass $pkcs12_password -deststorepass $pkcs12_password - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_keystore_connect_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_keystore_connect_cfg.json - name: configure AWS credentials (Cognito) uses: aws-actions/configure-aws-credentials@v2 with: @@ -509,7 +509,7 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: run CognitoConnect sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_cognito_connect_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_cognito_connect_cfg.json - name: configure AWS credentials (Custom Authorizer) uses: aws-actions/configure-aws-credentials@v2 with: @@ -517,7 +517,7 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: run CustomAuthorizerConnect sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_custom_authorizer_connect_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_custom_authorizer_connect_cfg.json - name: configure AWS credentials (Shadow) uses: aws-actions/configure-aws-credentials@v2 with: @@ -525,10 +525,10 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: run Shadow sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_shadow_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_shadow_cfg.json - name: run Mqtt5 Shadow sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_shadow_mqtt5_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_shadow_mqtt5_cfg.json - name: configure AWS credentials (Jobs) uses: aws-actions/configure-aws-credentials@v2 with: @@ -536,10 +536,10 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: run Jobs sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_jobs_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_jobs_cfg.json - name: run Mqtt5 Jobs sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_jobs_mqtt5_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_jobs_mqtt5_cfg.json - name: configure AWS credentials (Fleet provisioning) uses: aws-actions/configure-aws-credentials@v2 with: @@ -547,16 +547,10 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: run Fleet Provisioning sample run: | - echo "Generating UUID for IoT thing" - Sample_UUID=$(python3 -c "import uuid; print (uuid.uuid4())") - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_fleet_provisioning_cfg.json --input_uuid ${Sample_UUID} - python3 utils/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1" + python3 ./utils/run_fleet_provisioning_sample.py --file ./.github/workflows/ci_run_fleet_provisioning_cfg.json --thing-name-prefix "Fleet_Thing_" - name: run Mqtt5 Fleet Provisioning sample run: | - echo "Generating UUID for IoT thing" - Sample_UUID=$(python3 -c "import uuid; print (uuid.uuid4())") - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_fleet_provisioning_mqtt5_cfg.json --input_uuid ${Sample_UUID} - python3 utils/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1" + python3 ./utils/run_fleet_provisioning_sample.py --file ./.github/workflows/ci_run_fleet_provisioning_mqtt5_cfg.json --thing-name-prefix "Fleet_Thing_" - name: configure AWS credentials (X509) uses: aws-actions/configure-aws-credentials@v2 with: @@ -564,7 +558,7 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: run X509 sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_x509_connect_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_x509_connect_cfg.json - name: configure AWS credentials (MQTT5) uses: aws-actions/configure-aws-credentials@v2 with: @@ -572,10 +566,10 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: run MQTT5 PubSub sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_mqtt5_pubsub_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_mqtt5_pubsub_cfg.json - name: run MQTT5 Shared Subscription sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_mqtt5_shared_subscription_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_mqtt5_shared_subscription_cfg.json - name: configure AWS credentials (Greengrass) uses: aws-actions/configure-aws-credentials@v2 with: @@ -583,4 +577,4 @@ jobs: aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: run Greengrass Discovery sample run: | - python3 ./utils/run_sample_ci.py --file ./.github/workflows/ci_run_greengrass_discovery_cfg.json + python3 ./utils/run_in_ci.py --file ./.github/workflows/ci_run_greengrass_discovery_cfg.json diff --git a/.github/workflows/ci_run_basic_connect_cfg.json b/.github/workflows/ci_run_basic_connect_cfg.json index 8aff3b374..08650bf0a 100644 --- a/.github/workflows/ci_run_basic_connect_cfg.json +++ b/.github/workflows/ci_run_basic_connect_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/BasicConnect", - "sample_region": "us-east-1", - "sample_main_class": "basicconnect.BasicConnect", + "runnable_file": "samples/BasicConnect", + "runnable_region": "us-east-1", + "runnable_main_class": "basicconnect.BasicConnect", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_cognito_connect_cfg.json b/.github/workflows/ci_run_cognito_connect_cfg.json index d5ababbc7..2a28053cd 100644 --- a/.github/workflows/ci_run_cognito_connect_cfg.json +++ b/.github/workflows/ci_run_cognito_connect_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/CognitoConnect", - "sample_region": "us-east-1", - "sample_main_class": "cognitoconnect.CognitoConnect", + "runnable_file": "samples/CognitoConnect", + "runnable_region": "us-east-1", + "runnable_main_class": "cognitoconnect.CognitoConnect", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_custom_authorizer_connect_cfg.json b/.github/workflows/ci_run_custom_authorizer_connect_cfg.json index dc192136d..d22901d31 100644 --- a/.github/workflows/ci_run_custom_authorizer_connect_cfg.json +++ b/.github/workflows/ci_run_custom_authorizer_connect_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/CustomAuthorizerConnect", - "sample_region": "us-east-1", - "sample_main_class": "customauthorizerconnect.CustomAuthorizerConnect", + "runnable_file": "samples/CustomAuthorizerConnect", + "runnable_region": "us-east-1", + "runnable_main_class": "customauthorizerconnect.CustomAuthorizerConnect", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_custom_key_ops_cfg.json b/.github/workflows/ci_run_custom_key_ops_cfg.json index 2ed5b991b..407f5ba20 100644 --- a/.github/workflows/ci_run_custom_key_ops_cfg.json +++ b/.github/workflows/ci_run_custom_key_ops_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/CustomKeyOpsConnect", - "sample_region": "us-east-1", - "sample_main_class": "customkeyopsconnect.CustomKeyOpsConnect", + "runnable_file": "samples/CustomKeyOpsConnect", + "runnable_region": "us-east-1", + "runnable_main_class": "customkeyopsconnect.CustomKeyOpsConnect", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_fleet_provisioning_cfg.json b/.github/workflows/ci_run_fleet_provisioning_cfg.json index 77f3a575f..06c32e7a1 100644 --- a/.github/workflows/ci_run_fleet_provisioning_cfg.json +++ b/.github/workflows/ci_run_fleet_provisioning_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/FleetProvisioning", - "sample_region": "us-east-1", - "sample_main_class": "fleetprovisioning.FleetProvisioningSample", + "runnable_file": "samples/FleetProvisioning", + "runnable_region": "us-east-1", + "runnable_main_class": "fleetprovisioning.FleetProvisioningSample", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_fleet_provisioning_mqtt5_cfg.json b/.github/workflows/ci_run_fleet_provisioning_mqtt5_cfg.json index fd23a5697..b9a4407af 100644 --- a/.github/workflows/ci_run_fleet_provisioning_mqtt5_cfg.json +++ b/.github/workflows/ci_run_fleet_provisioning_mqtt5_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/FleetProvisioning", - "sample_region": "us-east-1", - "sample_main_class": "fleetprovisioning.Mqtt5FleetProvisioningSample", + "runnable_file": "samples/FleetProvisioning", + "runnable_region": "us-east-1", + "runnable_main_class": "fleetprovisioning.Mqtt5FleetProvisioningSample", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_greengrass_discovery_cfg.json b/.github/workflows/ci_run_greengrass_discovery_cfg.json index ddc6cf47c..ef528915c 100644 --- a/.github/workflows/ci_run_greengrass_discovery_cfg.json +++ b/.github/workflows/ci_run_greengrass_discovery_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/Greengrass", - "sample_region": "us-east-1", - "sample_main_class": "greengrass.BasicDiscovery", + "runnable_file": "samples/Greengrass", + "runnable_region": "us-east-1", + "runnable_main_class": "greengrass.BasicDiscovery", "arguments": [ { "name": "--cert", diff --git a/.github/workflows/ci_run_jobs_cfg.json b/.github/workflows/ci_run_jobs_cfg.json index e0abf9d62..45cb4704c 100644 --- a/.github/workflows/ci_run_jobs_cfg.json +++ b/.github/workflows/ci_run_jobs_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/Jobs", - "sample_region": "us-east-1", - "sample_main_class": "jobs.JobsSample", + "runnable_file": "samples/Jobs", + "runnable_region": "us-east-1", + "runnable_main_class": "jobs.JobsSample", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_jobs_mqtt5_cfg.json b/.github/workflows/ci_run_jobs_mqtt5_cfg.json index 854f88b4c..3666d66cb 100644 --- a/.github/workflows/ci_run_jobs_mqtt5_cfg.json +++ b/.github/workflows/ci_run_jobs_mqtt5_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/Jobs", - "sample_region": "us-east-1", - "sample_main_class": "jobs.Mqtt5JobsSample", + "runnable_file": "samples/Jobs", + "runnable_region": "us-east-1", + "runnable_main_class": "jobs.Mqtt5JobsSample", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_keystore_connect_cfg.json b/.github/workflows/ci_run_keystore_connect_cfg.json index 158b9dbd1..bae3d7437 100644 --- a/.github/workflows/ci_run_keystore_connect_cfg.json +++ b/.github/workflows/ci_run_keystore_connect_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/JavaKeystoreConnect", - "sample_region": "us-east-1", - "sample_main_class": "javakeystoreconnect.JavaKeystoreConnect", + "runnable_file": "samples/JavaKeystoreConnect", + "runnable_region": "us-east-1", + "runnable_main_class": "javakeystoreconnect.JavaKeystoreConnect", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_mqtt5_pubsub_cfg.json b/.github/workflows/ci_run_mqtt5_pubsub_cfg.json index 1b085f9b9..e11348cc5 100644 --- a/.github/workflows/ci_run_mqtt5_pubsub_cfg.json +++ b/.github/workflows/ci_run_mqtt5_pubsub_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/Mqtt5/PubSub", - "sample_region": "us-east-1", - "sample_main_class": "mqtt5.pubsub.PubSub", + "runnable_file": "samples/Mqtt5/PubSub", + "runnable_region": "us-east-1", + "runnable_main_class": "mqtt5.pubsub.PubSub", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_mqtt5_shared_subscription_cfg.json b/.github/workflows/ci_run_mqtt5_shared_subscription_cfg.json index 3968a4b4d..243b0ec81 100644 --- a/.github/workflows/ci_run_mqtt5_shared_subscription_cfg.json +++ b/.github/workflows/ci_run_mqtt5_shared_subscription_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/Mqtt5/SharedSubscription", - "sample_region": "us-east-1", - "sample_main_class": "mqtt5.sharedsubscription.SharedSubscription", + "runnable_file": "samples/Mqtt5/SharedSubscription", + "runnable_region": "us-east-1", + "runnable_main_class": "mqtt5.sharedsubscription.SharedSubscription", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_pkcs11_connect_cfg.json b/.github/workflows/ci_run_pkcs11_connect_cfg.json index 85e21b787..e89b59eba 100644 --- a/.github/workflows/ci_run_pkcs11_connect_cfg.json +++ b/.github/workflows/ci_run_pkcs11_connect_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/Pkcs11Connect", - "sample_region": "us-east-1", - "sample_main_class": "pkcs11connect.Pkcs11Connect", + "runnable_file": "samples/Pkcs11Connect", + "runnable_region": "us-east-1", + "runnable_main_class": "pkcs11connect.Pkcs11Connect", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_pkcs12_connect_cfg.json b/.github/workflows/ci_run_pkcs12_connect_cfg.json index 2bfe2d264..aa7a49bc0 100644 --- a/.github/workflows/ci_run_pkcs12_connect_cfg.json +++ b/.github/workflows/ci_run_pkcs12_connect_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/Pkcs12Connect", - "sample_region": "us-east-1", - "sample_main_class": "pkcs12connect.Pkcs12Connect", + "runnable_file": "samples/Pkcs12Connect", + "runnable_region": "us-east-1", + "runnable_main_class": "pkcs12connect.Pkcs12Connect", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_pubsub_cfg.json b/.github/workflows/ci_run_pubsub_cfg.json index 6d2765ac8..5ff732ca3 100644 --- a/.github/workflows/ci_run_pubsub_cfg.json +++ b/.github/workflows/ci_run_pubsub_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/BasicPubSub", - "sample_region": "us-east-1", - "sample_main_class": "pubsub.PubSub", + "runnable_file": "samples/BasicPubSub", + "runnable_region": "us-east-1", + "runnable_main_class": "pubsub.PubSub", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_shadow_cfg.json b/.github/workflows/ci_run_shadow_cfg.json index 4f6c6124d..f527accb5 100644 --- a/.github/workflows/ci_run_shadow_cfg.json +++ b/.github/workflows/ci_run_shadow_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/Shadow", - "sample_region": "us-east-1", - "sample_main_class": "shadow.ShadowSample", + "runnable_file": "samples/Shadow", + "runnable_region": "us-east-1", + "runnable_main_class": "shadow.ShadowSample", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_shadow_mqtt5_cfg.json b/.github/workflows/ci_run_shadow_mqtt5_cfg.json index d4c464020..eea1d2770 100644 --- a/.github/workflows/ci_run_shadow_mqtt5_cfg.json +++ b/.github/workflows/ci_run_shadow_mqtt5_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/Shadow", - "sample_region": "us-east-1", - "sample_main_class": "shadow.Mqtt5ShadowSample", + "runnable_file": "samples/Shadow", + "runnable_region": "us-east-1", + "runnable_main_class": "shadow.Mqtt5ShadowSample", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_websocket_connect_cfg.json b/.github/workflows/ci_run_websocket_connect_cfg.json index 186d81dd9..ee9e1a9db 100644 --- a/.github/workflows/ci_run_websocket_connect_cfg.json +++ b/.github/workflows/ci_run_websocket_connect_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/WebsocketConnect", - "sample_region": "us-east-1", - "sample_main_class": "websocketconnect.WebsocketConnect", + "runnable_file": "samples/WebsocketConnect", + "runnable_region": "us-east-1", + "runnable_main_class": "websocketconnect.WebsocketConnect", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_windows_cert_connect_cfg.json b/.github/workflows/ci_run_windows_cert_connect_cfg.json index 36e8de72b..f075a9ca0 100644 --- a/.github/workflows/ci_run_windows_cert_connect_cfg.json +++ b/.github/workflows/ci_run_windows_cert_connect_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/WindowsCertConnect", - "sample_region": "us-east-1", - "sample_main_class": "windowscertconnect.WindowsCertConnect", + "runnable_file": "samples/WindowsCertConnect", + "runnable_region": "us-east-1", + "runnable_main_class": "windowscertconnect.WindowsCertConnect", "arguments": [ { "name": "--endpoint", diff --git a/.github/workflows/ci_run_x509_connect_cfg.json b/.github/workflows/ci_run_x509_connect_cfg.json index b91925559..0642d8e86 100644 --- a/.github/workflows/ci_run_x509_connect_cfg.json +++ b/.github/workflows/ci_run_x509_connect_cfg.json @@ -1,8 +1,8 @@ { "language": "Java", - "sample_file": "samples/X509CredentialsProviderConnect", - "sample_region": "us-east-1", - "sample_main_class": "x509credentialsproviderconnect.X509CredentialsProviderConnect", + "runnable_file": "samples/X509CredentialsProviderConnect", + "runnable_region": "us-east-1", + "runnable_main_class": "x509credentialsproviderconnect.X509CredentialsProviderConnect", "arguments": [ { "name": "--endpoint", diff --git a/utils/ci_iot_thing.py b/utils/ci_iot_thing.py new file mode 100644 index 000000000..62f7810cc --- /dev/null +++ b/utils/ci_iot_thing.py @@ -0,0 +1,34 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +import boto3 + + +def delete_iot_thing(thing_name, region): + try: + iot_client = boto3.client('iot', region_name=region) + except Exception as e: + print(f"ERROR: Could not make Boto3 client. Credentials likely could not be sourced. Exception: {e}") + return -1 + + try: + thing_principals = iot_client.list_thing_principals(thingName=thing_name) + print(f"principals: {thing_principals}") + for principal in thing_principals["principals"]: + certificate_id = principal.split("/")[1] + iot_client.detach_thing_principal(thingName=thing_name, principal=principal) + iot_client.update_certificate(certificateId=certificate_id, newStatus='INACTIVE') + iot_client.delete_certificate(certificateId=certificate_id, forceDelete=True) + except Exception as e: + print("ERROR: Could not delete certificate for IoT thing " + f"{thing_name}, probably thing does not exist. Exception: {e}") + return -1 + + try: + iot_client.delete_thing(thingName=thing_name) + except Exception as e: + print(f"ERROR: Could not delete IoT thing {thing_name}. Exception: {e}") + return -1 + + print("IoT thing deleted successfully") + return 0 diff --git a/utils/run_fleet_provisioning_sample.py b/utils/run_fleet_provisioning_sample.py new file mode 100644 index 000000000..c8ab61470 --- /dev/null +++ b/utils/run_fleet_provisioning_sample.py @@ -0,0 +1,41 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + +import argparse +import uuid +import os +import sys +import run_in_ci +import ci_iot_thing + + +def main(): + argument_parser = argparse.ArgumentParser( + description="Run Fleet Provisioning sample in CI") + argument_parser.add_argument("--file", required=True, help="Configuration file to pull CI data from") + argument_parser.add_argument( + "--input-uuid", required=False, + help="Use specified UUID for thing name instead of a random one") + argument_parser.add_argument( + "--thing-name-prefix", required=False, default="", + help="Prefix for a thing name, should be the same that Fleet Provisioning template uses") + argument_parser.add_argument( + "--region", required=False, default="us-east-1", help="The name of the region to use") + parsed_commands = argument_parser.parse_args() + + cfg_file = parsed_commands.file + thing_uuid = parsed_commands.input_uuid if parsed_commands.input_uuid else str(uuid.uuid4()) + + # Perform fleet provisioning. If it's successful, a newly created thing should appear. + test_result = run_in_ci.setup_and_launch(cfg_file, thing_uuid) + + # Delete a thing created by fleet provisioning. + # NOTE We want to try to delete thing even if test was unsuccessful. + thing_name = parsed_commands.thing_name_prefix + thing_uuid + delete_result = ci_iot_thing.delete_iot_thing(thing_name, parsed_commands.region) + + if test_result != 0 or delete_result != 0: + sys.exit(-1) + +if __name__ == "__main__": + main() diff --git a/utils/run_sample_ci.py b/utils/run_in_ci.py similarity index 85% rename from utils/run_sample_ci.py rename to utils/run_in_ci.py index 512d6fe27..ecd86b5ad 100644 --- a/utils/run_sample_ci.py +++ b/utils/run_in_ci.py @@ -9,7 +9,7 @@ import sys import json # Needs to be installed via pip -import boto3 # - for launching sample +import boto3 current_folder = os.path.dirname(pathlib.Path(__file__).resolve()) if sys.platform == "win32" or sys.platform == "cygwin": @@ -23,13 +23,13 @@ pfx_certificate_store_location = "CurrentUser\\My" pfx_password = "" # Setting a password causes issues, but an empty string is valid so we use that -def setup_json_arguments_list(parsed_commands): +def setup_json_arguments_list(file, input_uuid=None): global config_json global config_json_arguments_list print("Attempting to get credentials from secrets using Boto3...") - secrets_client = boto3.client("secretsmanager", region_name=config_json['sample_region']) - print ("Processing arguments...") + secrets_client = boto3.client("secretsmanager", region_name=config_json['runnable_region']) + print("Processing arguments...") for argument in config_json['arguments']: # Add the name of the argument @@ -80,18 +80,18 @@ def setup_json_arguments_list(parsed_commands): # Raw data? just add it directly! elif 'data' in argument: tmp_value = argument['data'] - if isinstance(tmp_value, str) and 'input_uuid' in parsed_commands: + if isinstance(tmp_value, str) and input_uuid is not None: if ("$INPUT_UUID" in tmp_value): - tmp_value = tmp_value.replace("$INPUT_UUID", parsed_commands.input_uuid) + tmp_value = tmp_value.replace("$INPUT_UUID", input_uuid) if (tmp_value != None and tmp_value != ""): config_json_arguments_list.append(tmp_value) # None of the above? Just print an error else: - print ("ERROR - unknown or missing argument value!") + print("ERROR - unknown or missing argument value!") except Exception as e: - print (f"Something went wrong processing {argument['name']}!") + print(f"Something went wrong processing {argument['name']}: {e}!") return -1 return 0 @@ -186,10 +186,10 @@ def make_windows_pfx_file(certificate_file_path, private_key_path, pfx_file_path print("ERROR - Windows PFX file can only be created on a Windows platform!") return 1 -def setup_sample(parsed_commands): +def setup_runnable(file, input_uuid=None): global config_json - file_absolute = pathlib.Path(parsed_commands.file).resolve() + file_absolute = pathlib.Path(file).resolve() json_file_data = "" with open(file_absolute, "r") as json_file: json_file_data = json_file.read() @@ -198,20 +198,20 @@ def setup_sample(parsed_commands): config_json = json.loads(json_file_data) # Make sure required parameters are all there - if not 'language' in config_json or not 'sample_file' in config_json \ - or not 'sample_region' in config_json or not 'sample_main_class' in config_json: + if not 'language' in config_json or not 'runnable_file' in config_json \ + or not 'runnable_region' in config_json or not 'runnable_main_class' in config_json: return -1 - # Preprocess sample arguments (get secret data, etc) - setup_result = setup_json_arguments_list(parsed_commands) + # Preprocess runnable arguments (get secret data, etc) + setup_result = setup_json_arguments_list(file, input_uuid) if setup_result != 0: return setup_result - print ("JSON config file loaded!") + print("JSON config file loaded!") return 0 -def cleanup_sample(): +def cleanup_runnable(): global config_json global config_json_arguments_list @@ -227,8 +227,8 @@ def cleanup_sample(): # Windows 10 certificate store data? if 'windows_cert_certificate' in argument and 'windows_cert_certificate_path' in argument \ - and 'windows_cert_key' in argument and 'windows_cert_key_path' in argument \ - and 'windows_cert_pfx_key_path' in argument: + and 'windows_cert_key' in argument and 'windows_cert_key_path' in argument \ + and 'windows_cert_pfx_key_path' in argument: if (os.path.isfile(str(current_folder) + argument['windows_cert_certificate_path'])): os.remove(str(current_folder) + argument['windows_cert_certificate_path']) @@ -238,26 +238,26 @@ def cleanup_sample(): os.remove(str(current_folder) + argument['windows_cert_pfx_key_path']) except Exception as e: - print (f"Something went wrong cleaning {argument['name']}!") + print(f"Something went wrong cleaning {argument['name']}!") return -1 -def launch_sample(): +def launch_runnable(): global config_json global config_json_arguments_list if (config_json == None): - print ("No configuration JSON file data found!") + print("No configuration JSON file data found!") return -1 exit_code = 0 - print("Launching sample...") + print("Launching runnable...") # Java if (config_json['language'] == "Java"): - # Flatten arguments down into a asingle string + # Flatten arguments down into a single string arguments_as_string = "" for i in range(0, len(config_json_arguments_list)): arguments_as_string += str(config_json_arguments_list[i]) @@ -266,14 +266,15 @@ def launch_sample(): arguments = ["mvn", "compile", "exec:java"] arguments.append("-pl") - arguments.append(config_json['sample_file']) - arguments.append("-Dexec.mainClass=" + config_json['sample_main_class']) + arguments.append(config_json['runnable_file']) + arguments.append("-Dexec.mainClass=" + config_json['runnable_main_class']) arguments.append("-Daws.crt.ci=True") # We have to do this as a string, unfortunately, due to how -Dexec.args= works... argument_string = subprocess.list2cmdline(arguments) + " -Dexec.args=\"" + arguments_as_string + "\"" - sample_return = subprocess.run(argument_string, shell=True) - exit_code = sample_return.returncode + print(f"Running cmd: {argument_string}") + runnable_return = subprocess.run(argument_string, shell=True) + exit_code = runnable_return.returncode # C++ elif (config_json['language'] == "CPP"): @@ -323,27 +324,33 @@ def launch_sample(): else: exit_code = 1 - cleanup_sample() + cleanup_runnable() return exit_code -def setup_sample_and_launch(parsed_commands): - setup_result = setup_sample(parsed_commands) + +def setup_and_launch(file, input_uuid=None): + setup_result = setup_runnable(file, input_uuid) if setup_result != 0: return setup_result - print ("About to launch sample...") - return launch_sample() + print("About to launch runnable...") + return launch_runnable() + def main(): argument_parser = argparse.ArgumentParser( - description="Run Sample in CI") + description="Run runnable in CI") argument_parser.add_argument("--file", required=True, help="Configuration file to pull CI data from") - argument_parser.add_argument("--input_uuid", required=False, help="UUID data to replace '$INPUT_UUID' with. Only works in Data field") + argument_parser.add_argument("--input_uuid", required=False, + help="UUID data to replace '$INPUT_UUID' with. Only works in Data field") parsed_commands = argument_parser.parse_args() - print("Starting to launch sample...") - sample_result = setup_sample_and_launch(parsed_commands) - sys.exit(sample_result) + file = parsed_commands.file + input_uuid = parsed_commands.input_uuid + + print(f"Starting to launch runnable: config {file}; input UUID: {input_uuid}") + test_result = setup_and_launch(file, input_uuid) + sys.exit(test_result) if __name__ == "__main__":