Merge pull request #22 from CanDIG/daisieh/s3-token

DIG-1653: aws credentials stored as vault secret in ingest's store

README.md

@@ -31,8 +31,6 @@ Opa also confirms if a user is a site admin: `is_site_admin` checks for whether
 
 Vault acts as the secret store for CanDIGv2.
 
-Services that require S3 access should have an environment variable `VAULT_S3_TOKEN` that is exchanged with Vault as a header `X-Vault-Token` for authorization to get the credentials. These exchanges are handled by the `get_aws_credential` and `store_aws_credential` methods.
-
 Every service can be set up to have its own secret store in Vault. Diff your module's setup against the lib/templates folder to see what you need to add to create a service store:
 
 - your-module_setup.sh needs to call `bash $PWD/create_service_store.sh "your-module"`
@@ -48,6 +46,8 @@ Every service can be set up to have its own secret store in Vault. Diff your mod
 
 Once those changes have been made, your service can read and write to its service store using the get_service_store_secret and set_service_store_secret methods.
 
+Services that require S3 access need to be authorized in `vault_setup.sh` to access candig-ingest's `aws` secret store. Once that authorization is set up, the service can use the get_aws_credential,
+
 
 ## Access to S3 objects: Minio
 Minio acts as the CanDIGv2 client for S3 access. `get_minio_client` returns a Minio object that can be used with the [Python API](https://min.io/docs/minio/linux/developers/python/API.html). This method, by default, returns an object corresponding to the Minio sandbox instance.

pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools >= 61.0"]
 build-backend = "setuptools.build_meta"
 
 [project]
-version = "v2.3.0"
+version = "v2.4.0"
 name = "candigv2_authx"
 dependencies = [
     "requests>=2.25.1",

src/authx/auth.py

@@ -232,38 +232,9 @@ def get_user_email(request, opa_url=OPA_URL, admin_secret=OPA_SECRET):
     return None
 
 
-def get_vault_token(token=None, vault_s3_token=None, vault_url=VAULT_URL):
+def get_aws_credential(endpoint=None, bucket=None, vault_url=VAULT_URL):
     """
-    Given a known vault_s3_token, exchange for a valid X-Vault-Token.
-    Returns token, status_code
-    """
-    if vault_url is None:
-        return {"error": f"Vault error: service did not provide VAULT_URL"}, 500
-    if vault_s3_token is None:
-        if token is None:
-            return {"error": f"Vault error: service did not provide VAULT_S3_TOKEN"}, 500
-        headers = {
-            "Authorization": f"Bearer {token}",
-            "Content-Type": "application/json",
-            "charset": "utf-8"
-        }
-        body = {
-            "jwt": token,
-            "role": "site_admin"
-        }
-        url = f"{vault_url}/v1/auth/jwt/login"
-        response = requests.post(url, json=body, headers=headers)
-        if response.status_code == 200:
-            client_token = response.json()["auth"]["client_token"]
-            return client_token, 200
-        else:
-            return response.json(), response.status_code
-    return vault_s3_token, 200
-
-
-def get_aws_credential(token=None, vault_url=VAULT_URL, endpoint=None, bucket=None, vault_s3_token=VAULT_S3_TOKEN):
-    """
-    Look up S3 credentials in Vault.
+    Look up S3 credentials in Vault. Executing service must be authorized to access candig-ingest's `/aws` Vault secret path.
     Returns credential object, status code
     """
     if endpoint is None or bucket is None:
@@ -279,26 +250,17 @@ def get_aws_credential(token=None, vault_url=VAULT_URL, endpoint=None, bucket=No
     # clean up endpoint name:
     endpoint = re.sub(r"\W", "_", endpoint)
 
-    vault_token, status_code = get_vault_token(token=token, vault_s3_token=vault_s3_token, vault_url=vault_url)
-    if status_code != 200:
-        return f"get_vault_token failed: {vault_token}", status_code
-    response = requests.get(
-        f"{vault_url}/v1/aws/{endpoint}-{bucket}",
-        headers={
-            "X-Vault-Token": vault_token
-            }
-    )
-    if response.status_code == 200:
-        result = response.json()['data']
-        result['endpoint'] = endpoint
-        result['bucket'] = bucket
-        return result, response.status_code
-    return {"error": f"Vault error: could not get credential for endpoint {endpoint} and bucket {bucket}"}, response.status_code
+    response, status_code = get_service_store_secret("candig-ingest", key=f"aws/{endpoint}/{bucket}")
+    if status_code == 200:
+        response['endpoint'] = endpoint
+        response['bucket'] = bucket
+        return response, status_code
+    return {"error": f"Vault error: could not get credential for endpoint {endpoint} and bucket {bucket}"}, status_code
 
 
-def store_aws_credential(token=None, endpoint=None, s3_url=None, bucket=None, access=None, secret=None, vault_s3_token=VAULT_S3_TOKEN, vault_url=VAULT_URL):
+def store_aws_credential(endpoint=None, s3_url=None, bucket=None, access=None, secret=None, vault_url=VAULT_URL):
     """
-    Store aws credentials in Vault.
+    Store aws credentials in Vault. Executing service must be authorized to write to candig-ingest's `/aws` Vault secret path.
     Returns credential object, status code
     """
     if endpoint is None or bucket is None or access is None or secret is None:
@@ -318,28 +280,49 @@ def store_aws_credential(token=None, endpoint=None, s3_url=None, bucket=None, ac
 
     # clean up endpoint name:
     endpoint = re.sub(r"\W", "_", endpoint)
-    vault_token, status_code = get_vault_token(token=token, vault_s3_token=vault_s3_token, vault_url=vault_url)
-    if status_code != 200:
-        return f"get_vault_token failed: {vault_token}", status_code
-
-    headers={
-        "Authorization": f"Bearer {token}",
-        "X-Vault-Token": vault_token
-        }
-    url = f"{vault_url}/v1/aws/{endpoint}-{bucket}"
     body = {
-        "url": s3_url,
-        "access": access,
-        "secret": secret,
-        "secure": secure
-    }
-    response = requests.post(url, headers=headers, json=body)
-    if response.status_code >= 200 and response.status_code < 300:
-        response = requests.get(url, headers=headers)
-        result = response.json()["data"]
-        result["endpoint"] = endpoint
-        return result, 200
-    return response.json(), response.status_code
+            "url": s3_url,
+            "access_key": access,
+            "secret_key": secret,
+            "secure": secure
+        }
+    response, status_code = set_service_store_secret("candig-ingest", key=f"aws/{endpoint}/{bucket}", value=body)
+    if status_code >= 200 and status_code < 300:
+        response, status_code = get_service_store_secret("candig-ingest", key=f"aws/{endpoint}/{bucket}")
+        if status_code == 200:
+            response["endpoint"] = endpoint
+            response["bucket"] = bucket
+            return response, 200
+    return response, status_code
+
+
+def remove_aws_credential(endpoint=None, bucket=None, vault_url=VAULT_URL):
+    """
+    Delete S3 credentials in Vault. Executing service must be authorized to delete from candig-ingest's `/aws` Vault secret path.
+    Returns credential object, status code
+    """
+    if endpoint is None or bucket is None:
+        return {"error": "Error getting S3 credentials: missing either endpoint or bucket"}, 400
+
+    # eat any http stuff from endpoint:
+    endpoint_parse = re.match(r"https*:\/\/(.+)?", endpoint)
+    if endpoint_parse is not None:
+        endpoint = endpoint_parse.group(1)
+    # if it's any sort of amazon endpoint, it can just be s3.amazonaws.com
+    if "amazonaws.com" in endpoint:
+        endpoint = "s3.amazonaws.com"
+    # clean up endpoint name:
+    endpoint = re.sub(r"\W", "_", endpoint)
+
+    status_code = delete_service_store_secret("candig-ingest", key=f"aws/{endpoint}-{bucket}")
+    if status_code == 200:
+        result = {}
+        result['endpoint'] = endpoint
+        result['bucket'] = bucket
+        return result, status_code
+    if status_code >= 400:
+        return {"error": "No such credential exists"}, status_code
+    return {"error": f"Vault error: could not get credential for endpoint {endpoint} and bucket {bucket}"}, status_code
 
 
 def get_minio_client(token=None, s3_endpoint=None, bucket=None, access_key=None, secret_key=None, region=None, secure=True, public=False):
@@ -360,8 +343,8 @@ def get_minio_client(token=None, s3_endpoint=None, bucket=None, access_key=None,
             response, status_code = get_aws_credential(token=token, endpoint=s3_endpoint, bucket=bucket)
             if "error" in response:
                 raise CandigAuthError(response)
-            access_key = response["access"]
-            secret_key = response["secret"]
+            access_key = response["access_key"]
+            secret_key = response["secret_key"]
             url = response["url"]
             secure = response["secure"]
         else:

test_auth.py

@@ -13,7 +13,6 @@
 OPA_URL = os.getenv('OPA_URL', None)
 OPA_SECRET = os.getenv('OPA_SECRET', None)
 VAULT_URL = os.getenv('VAULT_URL', None)
-VAULT_S3_TOKEN = os.getenv('VAULT_S3_TOKEN', None)
 SITE_ADMIN_USER = os.getenv("CANDIG_SITE_ADMIN_USER", None)
 SITE_ADMIN_PASSWORD = os.getenv("CANDIG_SITE_ADMIN_PASSWORD", None)
 NOT_ADMIN_USER = os.getenv("CANDIG_NOT_ADMIN_USER", None)
@@ -162,19 +161,22 @@ def test_put_aws_credential():
     Test adding credentials to Vault
     """
     if VAULT_URL is not None:
+        if os.getenv("SERVICE_NAME") != "candig-ingest":
+            warnings.warn(UserWarning("aws credential tests can only be run within the candig-ingest container"))
+            return
         endpoint = "http://test.endpoint"
-        # store credential using vault_s3_token and not-site-admin token
-        result, status_code = authx.auth.store_aws_credential(token=authx.auth.get_auth_token(FakeRequest()),endpoint=endpoint, bucket="test_bucket", access="test", secret="secret", vault_url=VAULT_URL, vault_s3_token=VAULT_S3_TOKEN)
+        # store credential using not-site-admin token
+        result, status_code = authx.auth.store_aws_credential(token=authx.auth.get_auth_token(FakeRequest()), endpoint=endpoint, bucket="test_bucket", access="test", secret="secret", vault_url=VAULT_URL)
         print(result, status_code)
         assert status_code == 200
 
         # try getting it with a non-site_admin token
-        result, status_code = authx.auth.get_aws_credential(token=authx.auth.get_auth_token(FakeRequest()), vault_url=VAULT_URL, endpoint=endpoint, bucket="test_bucket", vault_s3_token=None)
+        result, status_code = authx.auth.get_aws_credential(token=authx.auth.get_auth_token(FakeRequest()), vault_url=VAULT_URL, endpoint=endpoint, bucket="test_bucket")
         print(result)
         assert "errors" in result
 
         # try getting it with a site_admin token
-        result, status_code = authx.auth.get_aws_credential(token=authx.auth.get_auth_token(FakeRequest(site_admin=True)), vault_url=VAULT_URL, endpoint=endpoint, bucket="test_bucket", vault_s3_token=None)
+        result, status_code = authx.auth.get_aws_credential(token=authx.auth.get_auth_token(FakeRequest(site_admin=True)), vault_url=VAULT_URL, endpoint=endpoint, bucket="test_bucket")
         assert result['secret'] == 'secret'
         assert result['url'] == 'test.endpoint'
     else:
@@ -192,7 +194,10 @@ def test_get_s3_url():
     fp.seek(0)
     if MINIO_URL is not None:
         if VAULT_URL is not None:
-            result, status_code = authx.auth.store_aws_credential(token=authx.auth.get_auth_token(FakeRequest()),endpoint=MINIO_URL, bucket="test", access=MINIO_ACCESS_KEY, secret=MINIO_SECRET_KEY, vault_url=VAULT_URL, vault_s3_token=VAULT_S3_TOKEN)
+            if os.getenv("SERVICE_NAME") != "candig-ingest":
+                warnings.warn(UserWarning("aws credential tests can only be run within the candig-ingest container"))
+                return
+            result, status_code = authx.auth.store_aws_credential(token=authx.auth.get_auth_token(FakeRequest()),endpoint=MINIO_URL, bucket="test", access=MINIO_ACCESS_KEY, secret=MINIO_SECRET_KEY, vault_url=VAULT_URL)
             assert result['url'] in MINIO_URL
             minio = authx.auth.get_minio_client(token=authx.auth.get_auth_token(FakeRequest()), s3_endpoint=MINIO_URL, bucket="test")
             assert minio['endpoint'] == MINIO_URL