Merge pull request #60 from CanDIG/daisieh/federation-vault

DIG-979: Move federation config to Vault service store

.travis.yml

@@ -3,6 +3,16 @@ dist: focal   # required for Python >= 3.7
 python:
   - "3.12"
 
+before_install:
+  - sudo mkdir -p /run/secrets
+  - sudo touch /run/secrets/vault-approle-token
+  - sudo chmod 777 /run/secrets/vault-approle-token
+  - echo "test" > /run/secrets/vault-approle-token
+  - sudo mkdir -p /home/candig
+  - sudo touch /home/candig/roleid
+  - sudo chmod 777 /home/candig/roleid
+  - echo "test" > /home/candig/roleid
+
 install:
   - pip install -r requirements.txt
 
@@ -11,4 +21,4 @@ script:
   - pytest --cov=candig_federation tests/ -vv
 
 env:
-  - CONFIG_DIR=config TRAVIS=true # Used to skip local integration tests in test_local_federation.py
+  - SERVICE_NAME=federation VAULT_URL=http://localhost CONFIG_DIR=config TRAVIS=true # Used to skip local integration tests in test_local_federation.py

candig_federation/apilog.py

@@ -57,5 +57,8 @@ def apilog(func, *args, **kwargs):
 
     logentry = json.dumps(entrydict)
 
-    current_app.logger.info(logentry)
+    try:
+        current_app.logger.info(logentry)
+    except:
+        pass
     return func(*args, **kwargs)

candig_federation/authz.py

@@ -5,7 +5,6 @@
 
 app = Flask(__name__)
 TEST_KEY = os.getenv("TEST_KEY")
-TYK_FEDERATION_API_ID = os.getenv("TYK_FEDERATION_API_ID")
 
 
 def is_testing(request):
@@ -31,19 +30,3 @@ def is_site_admin(request):
             app.logger.warning(f"Couldn't authorize site_admin: {type(e)} {str(e)}")
             return False
     return False
-
-
-def add_provider_to_tyk(token, issuer):
-    return authx.auth.add_provider_to_tyk_api(TYK_FEDERATION_API_ID, token, issuer)
-
-
-def remove_provider_to_tyk(issuer):
-    return authx.auth.remove_provider_to_tyk_api(TYK_FEDERATION_API_ID, issuer)
-
-
-def add_provider_to_opa(token, issuer):
-    return authx.auth.add_provider_to_opa(token, issuer)
-
-
-def remove_provider_to_opa(issuer):
-    return authx.auth.remove_provider_to_opa(issuer)

candig_federation/federation.py

@@ -270,14 +270,14 @@ def async_requests(self, request, endpoint_path, endpoint_payload, endpoint_serv
             try:
                 # self.announce_fed_out(request_type, url, endpoint_path, endpoint_payload)
                 response = {}
-                url = f"{server['url']}/v1/fanout"
+                url = f"{server['server']['url']}/v1/fanout"
                 response["response"] = async_session.post(url, json=args, headers=header, timeout=self.timeout)
-                response["location"] = server["location"]
+                response["location"] = server['server']["location"]
 
-                responses[server['id']] = response
+                responses[server['server']['id']] = response
 
             except Exception as e:
-                responses[server['id']] = f"async_requests {server['id']}: {type(e)} {str(e)}"
+                responses[server['server']['id']] = f"async_requests {server['server']['id']}: {type(e)} {str(e)}"
 
         return responses
 
@@ -373,7 +373,7 @@ def get_response_object(self):
             # add locations:
             response['location'] = {}
             for server in self.servers:
-                response['location'][server] = self.servers[server]['location']
+                response['location'][server] = self.servers[server]['server']['location']
 
             # now deconvolute the result to an array:
             response_array = []

candig_federation/federation.yaml

@@ -28,10 +28,13 @@ paths:
     post:
       description: Add a server to the federation.
       operationId: operations.add_server
+      parameters:
+        - $ref: '#/components/parameters/register'
       requestBody:
         content:
           'application/json':
             schema:
+              nullable: true
               type: object
               required:
                 - server
@@ -107,12 +110,15 @@ paths:
         '500':
           description: Internal Error
     post:
+      parameters:
+        - $ref: '#/components/parameters/register'
       description: Add a service to the federation.
       operationId: operations.add_service
       requestBody:
         content:
           'application/json':
             schema:
+              nullable: true
               $ref: "#/components/schemas/Service"
       responses:
         '201':
@@ -194,6 +200,13 @@ components:
       schema:
         type: boolean
         default: true
+    register:
+      name: register
+      in: query
+      description: re-register all known servers/services
+      required: false
+      schema:
+          type: boolean
   schemas:
     FederatedRequestBody:
       type: object

candig_federation/network.py

@@ -4,17 +4,22 @@
 
 import json
 from flask import current_app
-import authz
+import authx.auth
+import os
+
+TYK_FEDERATION_API_ID = os.getenv("TYK_FEDERATION_API_ID")
 
 
 def get_registered_servers():
-    try:
-        with open(current_app.config['server_file']) as f:
-            return json.load(f)
-    except Exception as e:
-        print(f"Error in get_registered_servers: {type(e)} {str(e)}")
+    stored_servers_dict, status_code = authx.auth.get_service_store_secret("federation", key="servers")
+    if status_code == 404:
+        # no value was found, so this must need to be initialized
+        stored_servers_dict, status_code = authx.auth.set_service_store_secret("federation", key="servers", value=json.dumps({"servers": {}}))
+        return {}
+    if status_code != 200:
+        print(f"Error in get_registered_servers: {stored_servers_dict}")
         return None
-
+    return stored_servers_dict["servers"]
 
 
 def register_server(obj):
@@ -28,7 +33,7 @@ def register_server(obj):
                 found = True
         if found:
             return None
-        servers[new_server['id']] = new_server
+        servers[new_server['id']] = obj
 
     if 'testing' in obj['authentication']:
         new_server['testing'] = True
@@ -37,20 +42,17 @@ def register_server(obj):
             token = obj['authentication']['token']
             issuer = obj['authentication']['issuer']
 
-            authz.add_provider_to_tyk(token, issuer)
+            authx.auth.add_provider_to_tyk_api(TYK_FEDERATION_API_ID, token, issuer)
         except Exception as e:
             raise Exception(f"Failed to register server with tyk: {type(e)} {str(e)}")
         try:
-            authz.add_provider_to_opa(token, issuer)
+            authx.auth.add_provider_to_opa(token, issuer)
         except Exception as e:
             raise Exception(f"Failed to register server with opa: {type(e)} {str(e)}")
 
-    try:
-        with open(current_app.config['server_file'], 'w') as f:
-            f.write(json.dumps(servers))
-    except Exception as e:
-        print(f"Error in register_server: {type(e)} {str(e)}")
-        return None
+    stored_servers_dict, status_code = authx.auth.set_service_store_secret("federation", key="servers", value=json.dumps({"servers": servers}))
+    if status_code != 200:
+        print(f"Error in register_server: {stored_servers_dict}")
     return obj['server']
 
 
@@ -59,30 +61,32 @@ def unregister_server(server_id):
     result = None
     if servers is not None and server_id in servers:
         result = servers.pop(server_id)
-        with open(current_app.config['server_file'], 'w') as f:
-            f.write(json.dumps(servers))
+    stored_servers_dict, status_code = authx.auth.set_service_store_secret("federation", key="servers", value=json.dumps({"servers": servers}))
+    if status_code != 200:
+        print(f"Error in register_server: {stored_servers_dict}")
     return result
 
 
 def get_registered_services():
-    try:
-        with open(current_app.config['service_file']) as f:
-            return json.load(f)
-    except Exception as e:
-        print(f"Error in get_registered_services: {type(e)} {str(e)}")
+    stored_services_dict, status_code = authx.auth.get_service_store_secret("federation", key="services")
+    if status_code == 404:
+        # no value was found, so this must need to be initialized
+        stored_services_dict, status_code = authx.auth.set_service_store_secret("federation", key="services", value=json.dumps({"services": {}}))
+        return {}
+    if status_code != 200:
+        print(f"Error in get_registered_services: {stored_services_dict}")
         return None
+    return stored_services_dict["services"]
 
 
 def register_service(obj):
     services = get_registered_services()
     if services is not None:
         services[obj['id']] = obj
-    try:
-        with open(current_app.config['service_file'], 'w') as f:
-            f.write(json.dumps(services))
-    except Exception as e:
-        print(f"Error in register_service: {type(e)} {str(e)}")
-        return None
+        stored_services_dict, status_code = authx.auth.set_service_store_secret("federation", key="services", value=json.dumps({"services": services}))
+        if status_code != 200:
+            print(f"Error in register_service: {stored_services_dict}")
+            return None
     return obj
 
 
@@ -91,6 +95,5 @@ def unregister_service(service_id):
     result = None
     if services is not None and service_id in services:
         result = services.pop(service_id)
-        with open(current_app.config['service_file'], 'w') as f:
-            f.write(json.dumps(services))
+        stored_services_dict, status_code = authx.auth.set_service_store_secret("federation", key="services",   value=json.dumps({"services": services}))
     return result

candig_federation/operations.py

@@ -4,6 +4,7 @@
 
 from authz import is_site_admin
 import connexion
+from werkzeug.exceptions import UnsupportedMediaType
 from flask import request, Flask
 from apilog import apilog
 from federation import FederationResponse
@@ -39,24 +40,37 @@ def list_servers():
     """
     servers = get_registered_servers()
     if servers is not None:
-        return list(servers.values()), 200
+        result = map(lambda x: x["server"], servers.values())
+        return list(result), 200
     return {"message": "Couldn't list servers"}, 500
 
 
 @apilog
-def add_server():
+def add_server(register=False):
     """
     :return: Server added.
     """
     if not is_site_admin(request):
         return {"message": "User is not authorized to POST"}, 403
     try:
-        new_server = connexion.request.json
-        if register_server(new_server) is None:
-            return {"message": f"Server {new_server['server']['id']} already present"}, 204
-        return get_registered_servers()[new_server['server']['id']], 201
+        # if register=True, list known servers in Vault and register them all
+        if register:
+            existing_servers = get_registered_servers()
+            for server in existing_servers:
+                register_server(existing_servers[server])
     except Exception as e:
-        return {"message": f"Couldn't add server: {type(e)} {str(e)}"}, 500
+        return {"message": f"Couldn't register servers: {type(e)} {str(e)} {connexion.request}"}, 500
+    try:
+        if connexion.request.json is not None and 'server' in connexion.request.json:
+            new_server = connexion.request.json
+            if register_server(new_server) is None:
+                return {"message": f"Server {new_server['server']['id']} already present"}, 204
+            return get_registered_servers()[new_server['server']['id']]['server'], 201
+    except UnsupportedMediaType as e:
+        # this is the exception that gets thrown if the requestbody is null
+        return get_registered_servers(), 200
+    except Exception as e:
+        return {"message": f"Couldn't add server: {type(e)} {str(e)} {connexion.request}"}, 500
 
 
 @apilog
@@ -108,14 +122,25 @@ def get_service(service_id):
 
 
 @apilog
-def add_service():
+def add_service(register=False):
     """
     :return: Service added.
     """
     if not is_site_admin(request):
         return {"message": "User is not authorized to POST"}, 403
-    new_service = connexion.request.json
-    register_service(new_service)
+    try:
+        # if register=True, list known services in Vault and register them all
+        if register:
+            existing_services = get_registered_services()
+            for service in existing_services:
+                register_service(existing_services[service])
+        new_service = connexion.request.json
+        register_service(new_service)
+    except UnsupportedMediaType as e:
+        # this is the exception that gets thrown if the requestbody is null
+        return get_registered_services(), 200
+    except Exception as e:
+        return {"message": f"Couldn't add service: {type(e)} {str(e)} {connexion.request}"}, 500
     return get_registered_services()[new_service['id']], 200
 
 

candig_federation/server.py

@@ -30,16 +30,6 @@ def main():
     APP.app.logger.addHandler(log_handler)
     APP.app.logger.setLevel(numeric_loglevel)
 
-    APP.app.config["service_file"] = os.path.abspath(f"{CONFIG_DIR}/services.json")
-    if not os.path.exists(APP.app.config["service_file"]):
-        with open(APP.app.config["service_file"], "w") as f:
-            f.write("{}")
-
-    APP.app.config["server_file"] = os.path.abspath(f"{CONFIG_DIR}/servers.json")
-    if not os.path.exists(APP.app.config["server_file"]):
-        with open(APP.app.config["server_file"], "w") as f:
-            f.write("{}")
-
     return APP
 
 def configure_app():

requirements.txt

@@ -1,5 +1,5 @@
 attrs~=23.1.0
-candigv2-authx@git+https://github.com/CanDIG/[email protected].2
+candigv2-authx@git+https://github.com/CanDIG/[email protected].3
 connexion==2.14.1
 decorator==4.4.0
 flask==2.2.5
@@ -10,4 +10,5 @@ requests-futures==1.0.0
 swagger-ui-bundle==0.0.5
 uwsgi==2.0.23
 prometheus-flask-exporter==0.13.0
-Flask-Cors==3.0.10
+Flask-Cors==4.0.1
+requests-mock>=1.11.0