feat: Change super agent recipe to support System Identity registrati…

…on (NR-273901) (#1095) * first iteration for the super agent recipe for debian * add backwards compatibility * add all distributions * Add forgotten NEW_RELIC_LICENSE_KEY * give recipes final touches * add token renewal endpoint * fix identity creation * do not require org id for config * trigger pipeline * debug journalctl * only enable auth if fleet enabled and org_id present --------- Co-authored-by: Ruben Ruiz de Gauna <[email protected]>
newrelic · Jul 31, 2024 · 1bf4c0a · 1bf4c0a
1 parent 7221c08
commit 1bf4c0a
Show file tree

Hide file tree

Showing 3 changed files with 371 additions and 24 deletions.
diff --git a/recipes/newrelic/infrastructure/super-agent/debian.yml b/recipes/newrelic/infrastructure/super-agent/debian.yml
@@ -81,6 +81,7 @@ install:
         - task: config_supervisors
         - task: config_fleet_id
         - task: config_opamp
+        - task: config_super_agent_auth
         - task: config_host_monitoring
         - task: update_otel_mem_limit
         - task: update_otel_end_point
@@ -138,6 +139,24 @@ install:
             echo "touch is required to run the newrelic install. Please install touch and re-run the installation." >&2
             exit 15
           fi
+        - |
+          IS_CURL_INSTALLED=$(which curl | wc -l)
+          if [ $IS_CURL_INSTALLED -eq 0 ] ; then
+            echo "curl is required to run the newrelic install. Please install curl and re-run the installation." >&2
+            exit 16
+          fi
+        - |
+          IS_OPENSSL_INSTALLED=$(which openssl | wc -l)
+          if [ $IS_OPENSSL_INSTALLED -eq 0 ] ; then
+            echo "openssl is required to run the newrelic install. Please install openssl and re-run the installation." >&2
+            exit 17
+          fi
+        - |
+          IS_MKTEMP_INSTALLED=$(which mktemp | wc -l)
+          if [ $IS_MKTEMP_INSTALLED -eq 0 ] ; then
+            echo "mktemp is required to run the newrelic install. Please install coreutils and re-run the installation." >&2
+            exit 18
+          fi
         - |
           if [ -n "{{.DEBIAN_CODENAME}}" ]; then
             IS_AGENT_AVAILABLE=$(curl -Is {{.NEW_RELIC_DOWNLOAD_URL}}preview/linux/apt/dists/{{.DEBIAN_CODENAME}}/InRelease | grep " 2[0-9][0-9] " | wc -l)
@@ -226,10 +245,9 @@ install:
     log_ssl_ciphers:
       cmds:
         - |
-          IS_OPENSSL_INSTALLED=$(which openssl | wc -l)
           IS_SORT_INSTALLED=$(which sort | wc -l)
           IS_UNIQ_INSTALLED=$(which uniq | wc -l)
-          if [ $IS_OPENSSL_INSTALLED -gt 0 ] && [ $IS_SORT_INSTALLED -gt 0 ] && [ $IS_UNIQ_INSTALLED -gt 0 ]; then
+          if [ $IS_SORT_INSTALLED -gt 0 ] && [ $IS_UNIQ_INSTALLED -gt 0 ]; then
             echo "Detecting available SSL ciphers..."
             openssl ciphers -v | awk '{print " - " $2}' | sort | uniq
           fi
@@ -300,7 +318,7 @@ install:
             OPTIONS="$OPTIONS -o Acquire::Http::Proxy={{.HTTPS_PROXY}}"
           fi
           apt-get $OPTIONS update -yq
-      # apt will return an error if fails to update any of its sources. Ignore these errors and let the "install_infra" task fail.
+      # apt will return an error if fails to update any of its sources. Ignore these errors and let the "install_infra" task fail. Just to trigger the pipeline
       ignore_error: true
 
     install_super_agent:
@@ -403,16 +421,24 @@ install:
     config_opamp:
       cmds:
         - |
-          if [ "{{.NEW_RELIC_SUPER_AGENT_FLEET_ENABLED}}" = "false" ] ; then
+          if [ "{{.NEW_RELIC_SUPER_AGENT_FLEET_ENABLED}}" = "false" ]; then
             sed -i '/^\s*opamp:/s/^/#/' /etc/newrelic-super-agent/config.yaml
             sed -i '/^\s*endpoint: https:\/\/opamp/s/^/#/' /etc/newrelic-super-agent/config.yaml
+          else
+            sed -i 's/s*#\s*opamp:/opamp:/g' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*#\s*endpoint: https:\/\/opamp/s/# //' /etc/newrelic-super-agent/config.yaml
+          fi
+        - |
+          if [ "{{.NEW_RELIC_SUPER_AGENT_FLEET_ENABLED}}" = "false" ]; then
             sed -i '/^\s*api-key:/s/^/#/' /etc/newrelic-super-agent/config.yaml
             sed -i '/^\s*headers:/s/^/#/' /etc/newrelic-super-agent/config.yaml
           else
-            sed -i 's/s*#\s*opamp:/opamp:/g' /etc/newrelic-super-agent/config.yaml
-            sed -i '/^\s*#\s*endpoint: https:\/\/opamp/s/#//' /etc/newrelic-super-agent/config.yaml
-            sed -i '/^\s*#\s*api-key:/s/#//' /etc/newrelic-super-agent/config.yaml
-            sed -i '/^\s*#\s*headers:/s/#//' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*#\s*api-key:/s/# //' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*#\s*headers:/s/# //' /etc/newrelic-super-agent/config.yaml
+          fi
+        - |
+          if [ "{{.NEW_RELIC_SUPER_AGENT_FLEET_ENABLED}}" != "false" ] ; then
+            sed -i 's/api-key: API_KEY_HERE/api-key: {{ .NEW_RELIC_LICENSE_KEY }}/g' /etc/newrelic-super-agent/config.yaml
           fi
         - |
           if [ "{{.NEW_RELIC_REGION}}" = "STAGING" ]; then
@@ -422,9 +448,98 @@ install:
           else
             sed -i 's/\(endpoint: https:\/\/opamp.\).*/\1'"service.newrelic.com\/v1\/opamp"'/' /etc/newrelic-super-agent/config.yaml
           fi
+
+    config_super_agent_auth:
+      cmds:
         - |
-          if [ "{{.NEW_RELIC_SUPER_AGENT_FLEET_ENABLED}}" != "false" ] ; then
-            sed -i 's/api-key: API_KEY_HERE/api-key: {{.NEW_RELIC_LICENSE_KEY}}/g' /etc/newrelic-super-agent/config.yaml
+          if [ "{{.NEW_RELIC_SUPER_AGENT_FLEET_ENABLED}}" != "false" ] && [ "{{ .NEW_RELIC_ORGANIZATION }}" != "" ]; then
+            sed -i '/^\s*#\s*auth_config:\s*$/s/# //' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*#\s*token_url: PLACEHOLDER/s/# //' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*#\s*client_id: PLACEHOLDER/s/# //' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*#\s*provider: PLACEHOLDER/s/# //' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*#\s*private_key_path: PLACEHOLDER/s/# //' /etc/newrelic-super-agent/config.yaml
+          else
+            sed -i '/^\s*auth_config:/s/^/#/' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*token_url: PLACEHOLDER/s/^/#/' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*client_id: PLACEHOLDER/s/^/#/' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*provider: PLACEHOLDER/s/^/#/' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*private_key_path: PLACEHOLDER/s/^/#/' /etc/newrelic-super-agent/config.yaml
+          fi
+        - |
+          if [ "{{.NEW_RELIC_SUPER_AGENT_FLEET_ENABLED}}" != "false" ] && [ "{{ .NEW_RELIC_ORGANIZATION }}" != "" ]; then
+            set -uo pipefail
+
+            mkdir -p /etc/newrelic-super-agent/keys
+            chown root:root /etc/newrelic-super-agent/keys
+            chmod 700 /etc/newrelic-super-agent/keys
+
+            TEMPORAL_FOLDER=$(mktemp -d newrelic-super-agent.XXXXXXXXXX)
+            chown root:root "${TEMPORAL_FOLDER}"
+            chmod 700 "${TEMPORAL_FOLDER}"
+            trap "rm -rf $TEMPORAL_FOLDER" EXIT
+            openssl genrsa -out "$TEMPORAL_FOLDER/key" 4096
+            openssl rsa -in "$TEMPORAL_FOLDER/key" -pubout -out "$TEMPORAL_FOLDER/pub"
+
+            if [ "{{.NEW_RELIC_REGION}}" = "STAGING" ]; then
+              REGISTRATION_ENDPOINT=https://staging-api.newrelic.com/graphql
+              TOKEN_RENEWAL_ENDPOINT=https://system-identity-oauth.staging-service.newrelic.com/oauth2/token
+            elif [ "{{.NEW_RELIC_REGION}}" = "EU" ]; then
+              REGISTRATION_ENDPOINT=https://api.eu.newrelic.com/graphql
+              TOKEN_RENEWAL_ENDPOINT=https://system-identity-oauth.service.eu.newrelic.com/oauth2/token
+            else
+              REGISTRATION_ENDPOINT=https://api.newrelic.com/graphql
+              TOKEN_RENEWAL_ENDPOINT=https://system-identity-oauth.service.newrelic.com/oauth2/token
+            fi
+
+            DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+            NAME="System Identity for $(hostname) - $DATE"
+
+            for RETRY in 1 2 3; do                                                                                                          
+              HTTP_CODE=$(echo '{ "query":
+                  "mutation {
+                    systemIdentityCreate(
+                      name: \"'$NAME'\",
+                      organizationId: \"{{ .NEW_RELIC_ORGANIZATION }}\",
+                      publicKey: \"'$(openssl enc -base64 -A -in "$TEMPORAL_FOLDER/pub")'\"
+                    ) {
+                      clientId,
+                      name
+                    }
+                  }"
+                }' | tr -d $'\n' | curl \
+                  -s -w "%{http_code}" \
+                  -H "Content-Type: application/json" \
+                  -H "API-Key: {{ .NEW_RELIC_API_KEY }}" \
+                  -o "$TEMPORAL_FOLDER/response.json" \
+                  --data-binary @- \
+                  "$REGISTRATION_ENDPOINT"
+              )
+
+              if [ $HTTP_CODE -eq 200 ]; then
+                break
+              fi
+
+              echo "Error creating the new system identity. The API endpoint returned $HTTP_CODE. Retrying ($RETRY/3)..."
+              sleep 2
+            done
+
+            if [ $HTTP_CODE -ne 200 ]; then
+              exit 99
+            fi
+
+            ERROR_MESSAGE=$(/usr/local/bin/newrelic utils jq '.errors[0].message // "NOERROR"' < "$TEMPORAL_FOLDER/response.json" | tr -d '"')
+            if [ "$ERROR_MESSAGE" != "NOERROR" ]; then
+              echo "Error creating an identity: $ERROR_MESSAGE"
+              exit 100
+            fi
+
+            CLIENT_ID=$(/usr/local/bin/newrelic utils jq  '.data.systemIdentityCreate.clientId' < "$TEMPORAL_FOLDER/response.json" | tr -d '"' )
+
+            mv "$TEMPORAL_FOLDER/key" "/etc/newrelic-super-agent/keys/$CLIENT_ID.key"
+            sed -i 's~token_url: PLACEHOLDER~token_url: '"$TOKEN_RENEWAL_ENDPOINT"'~g' /etc/newrelic-super-agent/config.yaml
+            sed -i 's/client_id: PLACEHOLDER/client_id: '"$CLIENT_ID"'/g' /etc/newrelic-super-agent/config.yaml
+            sed -i 's/provider: PLACEHOLDER/provider: local/g' /etc/newrelic-super-agent/config.yaml
+            sed -i 's~private_key_path: PLACEHOLDER~private_key_path: '"/etc/newrelic-super-agent/keys/$CLIENT_ID.key"'~g' /etc/newrelic-super-agent/config.yaml
           fi
 
     config_host_monitoring:

diff --git a/recipes/newrelic/infrastructure/super-agent/rhel.yml b/recipes/newrelic/infrastructure/super-agent/rhel.yml
@@ -101,6 +101,7 @@ install:
         - task: config_supervisors
         - task: config_fleet_id
         - task: config_opamp
+        - task: config_super_agent_auth
         - task: config_host_monitoring
         - task: update_otel_mem_limit
         - task: update_otel_end_point
@@ -158,6 +159,24 @@ install:
             echo "touch is required to run the newrelic install. Please install touch and re-run the installation." >&2
             exit 15
           fi
+        - |
+          IS_CURL_INSTALLED=$(which curl | wc -l)
+          if [ $IS_CURL_INSTALLED -eq 0 ] ; then
+            echo "curl is required to run the newrelic install. Please install curl and re-run the installation." >&2
+            exit 16
+          fi
+        - |
+          IS_OPENSSL_INSTALLED=$(which openssl | wc -l)
+          if [ $IS_OPENSSL_INSTALLED -eq 0 ] ; then
+            echo "openssl is required to run the newrelic install. Please install openssl and re-run the installation." >&2
+            exit 17
+          fi
+        - |
+          IS_MKTEMP_INSTALLED=$(which mktemp | wc -l)
+          if [ $IS_MKTEMP_INSTALLED -eq 0 ] ; then
+            echo "mktemp is required to run the newrelic install. Please install coreutils and re-run the installation." >&2
+            exit 18
+          fi
         - |
           if [ "{{.AMAZON_LINUX_VERSION}}" != "2" ] && [ "{{.AMAZON_LINUX_VERSION}}" != "2023" ] ; then
             REPO_URL=$(echo -n "{{.NEW_RELIC_DOWNLOAD_URL}}preview/linux/yum/{{.REPO_DIR}}/{{.DISTRO_VERSION}}/{{.ARCH}}/newrelic-infra.repo")
@@ -342,16 +361,24 @@ install:
     config_opamp:
       cmds:
         - |
-          if [ "{{.NEW_RELIC_SUPER_AGENT_FLEET_ENABLED}}" = "false" ] ; then
+          if [ "{{.NEW_RELIC_SUPER_AGENT_FLEET_ENABLED}}" = "false" ]; then
             sed -i '/^\s*opamp:/s/^/#/' /etc/newrelic-super-agent/config.yaml
             sed -i '/^\s*endpoint: https:\/\/opamp/s/^/#/' /etc/newrelic-super-agent/config.yaml
+          else
+            sed -i 's/s*#\s*opamp:/opamp:/g' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*#\s*endpoint: https:\/\/opamp/s/# //' /etc/newrelic-super-agent/config.yaml
+          fi
+        - |
+          if [ "{{.NEW_RELIC_SUPER_AGENT_FLEET_ENABLED}}" = "false" ]; then
             sed -i '/^\s*api-key:/s/^/#/' /etc/newrelic-super-agent/config.yaml
             sed -i '/^\s*headers:/s/^/#/' /etc/newrelic-super-agent/config.yaml
           else
-            sed -i 's/s*#\s*opamp:/opamp:/g' /etc/newrelic-super-agent/config.yaml
-            sed -i '/^\s*#\s*endpoint: https:\/\/opamp/s/#//' /etc/newrelic-super-agent/config.yaml
-            sed -i '/^\s*#\s*api-key:/s/#//' /etc/newrelic-super-agent/config.yaml
-            sed -i '/^\s*#\s*headers:/s/#//' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*#\s*api-key:/s/# //' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*#\s*headers:/s/# //' /etc/newrelic-super-agent/config.yaml
+          fi
+        - |
+          if [ "{{.NEW_RELIC_SUPER_AGENT_FLEET_ENABLED}}" != "false" ] ; then
+            sed -i 's/api-key: API_KEY_HERE/api-key: {{ .NEW_RELIC_LICENSE_KEY }}/g' /etc/newrelic-super-agent/config.yaml
           fi
         - |
           if [ "{{.NEW_RELIC_REGION}}" = "STAGING" ]; then
@@ -361,9 +388,98 @@ install:
           else
             sed -i 's/\(endpoint: https:\/\/opamp.\).*/\1'"service.newrelic.com\/v1\/opamp"'/' /etc/newrelic-super-agent/config.yaml
           fi
+
+    config_super_agent_auth:
+      cmds:
         - |
-          if [ "{{.NEW_RELIC_SUPER_AGENT_FLEET_ENABLED}}" != "false" ] ; then
-            sed -i 's/api-key: API_KEY_HERE/api-key: {{.NEW_RELIC_LICENSE_KEY}}/g' /etc/newrelic-super-agent/config.yaml
+          if [ "{{.NEW_RELIC_SUPER_AGENT_FLEET_ENABLED}}" != "false" ] && [ "{{ .NEW_RELIC_ORGANIZATION }}" != "" ]; then
+            sed -i '/^\s*#\s*auth_config:\s*$/s/# //' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*#\s*token_url: PLACEHOLDER/s/# //' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*#\s*client_id: PLACEHOLDER/s/# //' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*#\s*provider: PLACEHOLDER/s/# //' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*#\s*private_key_path: PLACEHOLDER/s/# //' /etc/newrelic-super-agent/config.yaml
+          else
+            sed -i '/^\s*auth_config:/s/^/#/' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*token_url: PLACEHOLDER/s/^/#/' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*client_id: PLACEHOLDER/s/^/#/' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*provider: PLACEHOLDER/s/^/#/' /etc/newrelic-super-agent/config.yaml
+            sed -i '/^\s*private_key_path: PLACEHOLDER/s/^/#/' /etc/newrelic-super-agent/config.yaml
+          fi
+        - |
+          if [ "{{.NEW_RELIC_SUPER_AGENT_FLEET_ENABLED}}" != "false" ] && [ "{{ .NEW_RELIC_ORGANIZATION }}" != "" ]; then
+            set -uo pipefail
+
+            mkdir -p /etc/newrelic-super-agent/keys
+            chown root:root /etc/newrelic-super-agent/keys
+            chmod 700 /etc/newrelic-super-agent/keys
+
+            TEMPORAL_FOLDER=$(mktemp -d newrelic-super-agent.XXXXXXXXXX)
+            chown root:root "${TEMPORAL_FOLDER}"
+            chmod 700 "${TEMPORAL_FOLDER}"
+            trap "rm -rf $TEMPORAL_FOLDER" EXIT
+            openssl genrsa -out "$TEMPORAL_FOLDER/key" 4096
+            openssl rsa -in "$TEMPORAL_FOLDER/key" -pubout -out "$TEMPORAL_FOLDER/pub"
+
+            if [ "{{.NEW_RELIC_REGION}}" = "STAGING" ]; then
+              REGISTRATION_ENDPOINT=https://staging-api.newrelic.com/graphql
+              TOKEN_RENEWAL_ENDPOINT=https://system-identity-oauth.staging-service.newrelic.com/oauth2/token
+            elif [ "{{.NEW_RELIC_REGION}}" = "EU" ]; then
+              REGISTRATION_ENDPOINT=https://api.eu.newrelic.com/graphql
+              TOKEN_RENEWAL_ENDPOINT=https://system-identity-oauth.service.eu.newrelic.com/oauth2/token
+            else
+              REGISTRATION_ENDPOINT=https://api.newrelic.com/graphql
+              TOKEN_RENEWAL_ENDPOINT=https://system-identity-oauth.service.newrelic.com/oauth2/token
+            fi
+
+            DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+            NAME="System Identity for $(hostname) - $DATE"
+
+            for RETRY in 1 2 3; do
+              HTTP_CODE=$(echo '{ "query":
+                  "mutation {
+                    systemIdentityCreate(
+                      name: \"'$NAME'\",
+                      organizationId: \"{{ .NEW_RELIC_ORGANIZATION }}\",
+                      publicKey: \"'$(openssl enc -base64 -A -in "$TEMPORAL_FOLDER/pub")'\"
+                    ) {
+                      clientId,
+                      name
+                    }
+                  }"
+                }' | tr -d $'\n' | curl \
+                  -s -w "%{http_code}" \
+                  -H "Content-Type: application/json" \
+                  -H "API-Key: {{ .NEW_RELIC_API_KEY }}" \
+                  -o "$TEMPORAL_FOLDER/response.json" \
+                  --data-binary @- \
+                  "$REGISTRATION_ENDPOINT"
+              )
+
+              if [ $HTTP_CODE -eq 200 ]; then
+                break
+              fi
+
+              echo "Error creating the new system identity. The API endpoint returned $HTTP_CODE. Retrying ($RETRY/3)..."
+              sleep 2
+            done
+
+            if [ $HTTP_CODE -ne 200 ]; then
+              exit 99
+            fi
+
+            ERROR_MESSAGE=$(/usr/local/bin/newrelic utils jq '.errors[0].message // "NOERROR"' < "$TEMPORAL_FOLDER/response.json" | tr -d '"')
+            if [ "$ERROR_MESSAGE" != "NOERROR" ]; then
+              echo "Error creating an identity: $ERROR_MESSAGE"
+              exit 100
+            fi
+
+            CLIENT_ID=$(/usr/local/bin/newrelic utils jq  '.data.systemIdentityCreate.clientId' < "$TEMPORAL_FOLDER/response.json" | tr -d '"' )
+
+            mv "$TEMPORAL_FOLDER/key" "/etc/newrelic-super-agent/keys/$CLIENT_ID.key"
+            sed -i 's~token_url: PLACEHOLDER~token_url: '"$TOKEN_RENEWAL_ENDPOINT"'~g' /etc/newrelic-super-agent/config.yaml
+            sed -i 's/client_id: PLACEHOLDER/client_id: '"$CLIENT_ID"'/g' /etc/newrelic-super-agent/config.yaml
+            sed -i 's/provider: PLACEHOLDER/provider: local/g' /etc/newrelic-super-agent/config.yaml
+            sed -i 's~private_key_path: PLACEHOLDER~private_key_path: '"/etc/newrelic-super-agent/keys/$CLIENT_ID.key"'~g' /etc/newrelic-super-agent/config.yaml
           fi
 
     config_host_monitoring: