Give each DID Finder its own queue to read tasks from

ssl-hep · Jul 8, 2024 · 548f08b · 548f08b
1 parent 6deb330
commit 548f08b
Show file tree

Hide file tree

Showing 9 changed files with 143 additions and 132 deletions.
diff --git a/did_finder_cernopendata/poetry.lock b/did_finder_cernopendata/poetry.lock
diff --git a/did_finder_cernopendata/pyproject.toml b/did_finder_cernopendata/pyproject.toml
@@ -8,7 +8,7 @@ packages = [{include = "did_finder_cernopendata", from = "src"}]
 
 [tool.poetry.dependencies]
 python = "~3.10"
-servicex-did-finder-lib = "^3.0.0-alpha.4"
+servicex-did-finder-lib = "^3.0.0-alpha.5"
 
 [tool.poetry.group.test]
 optional = true

diff --git a/did_finder_rucio/poetry.lock b/did_finder_rucio/poetry.lock
diff --git a/did_finder_rucio/pyproject.toml b/did_finder_rucio/pyproject.toml
@@ -10,7 +10,7 @@ packages = [{include = "rucio_did_finder", from = "src"}]
 python = "^3.9"
 rucio-clients = "^34.2.0"
 xmltodict = "^0.13.0"
-servicex-did-finder-lib = "^3.0.0-alpha.4"
+servicex-did-finder-lib = "^3.0.0-alpha.5"
 requests = ">=2.25.0,<3.0.0"
 
 [tool.poetry.group.test]

diff --git a/did_finder_xrootd/poetry.lock b/did_finder_xrootd/poetry.lock
diff --git a/did_finder_xrootd/pyproject.toml b/did_finder_xrootd/pyproject.toml
@@ -8,7 +8,7 @@ packages = [{include = "src/servicex_did_finder_xrootd"}]
 
 [tool.poetry.dependencies]
 python = ">=3.10,<4.0"
-servicex-did-finder-lib = "^3.0.0-alpha.4"
+servicex-did-finder-lib = "^3.0.0-alpha.5"
 xrootd = ">=5.6.9"
 
 [tool.poetry.group.test]

diff --git a/helm/servicex/templates/app/celery_config.yaml b/helm/servicex/templates/app/celery_config.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-celery-config
+  labels:
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    app: {{ .Release.Name }}
+
+data:
+  celery_config.py: |
+    broker_url = 'amqp://user:{{ .Values.rabbitmq.auth.password }}@{{ .Release.Name }}-rabbitmq:5672/%2F'
+    task_routes = {
+        'did_finder_cernopendata.tasks.lookup_dataset': {'queue': 'did_finder_cernopendata'},
+        'did_finder_rucio.tasks.lookup_dataset': {'queue': 'did_finder_rucio'},
+        'did_finder_xrootd.tasks.lookup_dataset': {'queue': 'did_finder_xrootd'},
+    }
diff --git a/helm/servicex/templates/app/deployment.yaml b/helm/servicex/templates/app/deployment.yaml
@@ -132,6 +132,8 @@ spec:
         volumeMounts:
           - name: app-cfg
             mountPath: /opt/servicex
+          - name: celery-cfg
+            mountPath: /opt/servicex/celery
           - name: sqlite
             mountPath: /sqlite
           {{- if .Values.app.defaultUsers }}
@@ -147,6 +149,9 @@ spec:
         - name: app-cfg
           configMap:
             name:  {{ .Release.Name }}-flask-config
+        - name: celery-cfg
+          configMap:
+            name: {{ .Release.Name }}-celery-config
         - name: sqlite
           emptyDir: {}
         {{- if .Values.app.defaultUsers }}

diff --git a/servicex_app/servicex_app/__init__.py b/servicex_app/servicex_app/__init__.py
@@ -250,7 +250,11 @@ def b64decode(text):  # needed for python selection decoding to web page.
             transformer_manager = provided_transformer_manager
 
         if not provided_celery_app:
-            celery_app = Celery('ServiceX-App', broker=app.config['RABBIT_MQ_URL'])
+            sys.path.append('/opt/servicex/celery')
+
+            celery_app = Celery('ServiceX-App')
+            celery_app.config_from_object('celery_config')
+
         else:
             celery_app = provided_celery_app