Make DIDFinderApp a subclass of Celery App

did_finder_cernopendata/Dockerfile

@@ -9,27 +9,22 @@ WORKDIR /usr/src/app
 USER root
 
 ENV  POETRY_VERSION=1.5.1
-#ENV POETRY_HOME=/opt/poetry
-#RUN python3 -m venv $POETRY_HOME && \
-#    $POETRY_HOME/bin/pip install poetry==$POETRY_VERSION
-
 
 # There is a bug in poetry with no virtual env where it can delete this package
 # during install. If we delete it before installing poetry it will
 # recognize it as its own and handle it correctly
 # See: https://github.com/python-poetry/poetry/issues/5977
-#RUN pip uninstall -y certifi
+RUN pip uninstall -y certifi
 
 RUN pip install poetry==$POETRY_VERSION
 
 COPY pyproject.toml pyproject.toml
 COPY poetry.lock poetry.lock
 
-#RUN poetry config virtualenvs.create false
-RUN poetry install --no-root --no-interaction --no-ansi
+RUN  poetry install --no-root --no-interaction --no-ansi
 
 # Bring over the main scripts
-COPY src/did_finder_cernopendata .
+COPY . .
 
 # build stamp
 RUN echo "Timestamp:" `date --utc` | tee /image-build-info.txt
@@ -40,4 +35,7 @@ ENV PYTHONUNBUFFERED=1
 # Needed for running as non-root
 ENV POETRY_CACHE_DIR=/root/.cache/pypoetry
 
-ENTRYPOINT [ "poetry", "run", "python", "/usr/src/app/did_finder.py" ]
+ENV PYTHONPATH /usr/src/app/src
+
+ENV BROKER_URL="amqp://guest:guest@localhost:5672//"
+ENTRYPOINT [ "scripts/start_celery_worker.sh"]

did_finder_cernopendata/pyproject.toml

@@ -8,7 +8,7 @@ packages = [{include = "did_finder_cernopendata", from = "src"}]
 
 [tool.poetry.dependencies]
 python = "~3.10"
-servicex-did-finder-lib = "^3.0.0-alpha.5"
+servicex-did-finder-lib = "^3.0.0-alpha.6"
 
 [tool.poetry.group.test]
 optional = true

did_finder_cernopendata/scripts/start_celery_worker.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+poetry run celery --broker="$BROKER_URL" -A did_finder_cernopendata worker \
+                  --loglevel=info -Q did_finder_cernopendata \
+                  --concurrency=1 --hostname=did_finder_cernopendata@%h

did_finder_cernopendata/src/did_finder_cernopendata/did_finder.py → did_finder_cernopendata/src/did_finder_cernopendata/celery.py

@@ -9,6 +9,14 @@
 
 cache_prefix = os.environ.get('CACHE_PREFIX', '')
 
+app = DIDFinderApp('cernopendata')
+
+
+@app.did_lookup_task(name="did_finder_cernopendata.lookup_dataset")
+def lookup_dataset(self, did: str, dataset_id: int, endpoint: str) -> None:
+    self.do_lookup(did=did, dataset_id=dataset_id,
+                   endpoint=endpoint, user_did_finder=find_files)
+
 
 def find_files(did_name: str,
                info: Dict[str, Any],
@@ -64,19 +72,3 @@ def find_files(did_name: str,
         if non_root_uri:
             raise Exception('CMSOpenData: Opendata record returned a strange url'
                             '\n\t' + '\n\t'.join(all_lines))
-
-
-def run_open_data():
-    __log.info('Starting CERNOpenData DID finder')
-    app = DIDFinderApp('cernopendata')
-
-    @app.did_lookup_task(name="did_finder_cernopendata.lookup_dataset")
-    def lookup_dataset(self, did: str, dataset_id: int, endpoint: str) -> None:
-        self.do_lookup(did=did, dataset_id=dataset_id,
-                       endpoint=endpoint, user_did_finder=find_files)
-
-    app.start()
-
-
-if __name__ == "__main__":
-    run_open_data()

did_finder_cernopendata/tests/did_finder_cernopendata_tests/test_servicex_did_finder_cernopendata.py

@@ -4,7 +4,7 @@
 
 import pytest
 
-from did_finder_cernopendata.did_finder import find_files
+from did_finder_cernopendata.celery import find_files
 
 
 @pytest.fixture

did_finder_rucio/Dockerfile

@@ -33,3 +33,8 @@ RUN echo "Timestamp:" `date --utc` | tee /image-build-info.txt
 
 ENV X509_USER_PROXY /tmp/grid-security/x509up
 ENV X509_CERT_DIR /etc/grid-security/certificates
+ENV PYTHONPATH /usr/src/app/src
+
+ENV BROKER_URL="amqp://guest:guest@localhost:5672//"
+ENTRYPOINT [ "scripts/start_celery_worker.sh"]
+

did_finder_rucio/pyproject.toml

@@ -10,7 +10,7 @@ packages = [{include = "rucio_did_finder", from = "src"}]
 python = "^3.9"
 rucio-clients = "^34.2.0"
 xmltodict = "^0.13.0"
-servicex-did-finder-lib = "^3.0.0-alpha.5"
+servicex-did-finder-lib = "^3.0.0-alpha.6"
 requests = ">=2.25.0,<3.0.0"
 
 [tool.poetry.group.test]

did_finder_rucio/runme.sh → did_finder_rucio/scripts/start_celery_worker.sh

@@ -1,8 +1,7 @@
 #!/usr/bin/env bash
-
 /usr/src/app/proxy-exporter.sh &
 
-while true; do 
+while true; do
     date
     ls ${X509_USER_PROXY}
     RESULT=$?
@@ -13,10 +12,6 @@ while true; do
     sleep 5
 done
 
-export PYTHONPATH=src
-
-# Assume $REPORT_LOGICAL_FILES is set to --report-logical-files to activate
-echo "----------->$PYTHONPATH"
-ls -lht $PYTHONPATH
-python3 scripts/did_finder.py --rabbit-uri $RMQ_URI $REPORT_LOGICAL_FILES
-
+poetry run celery --broker="$BROKER_URL" -A rucio_did_finder worker \
+                  --loglevel=info -Q did_finder_rucio \
+                  --concurrency=1 --hostname=did_finder_rucio@%h

did_finder_rucio/scripts/did_finder.py → did_finder_rucio/src/rucio_did_finder/celery.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, IRIS-HEP
+# Copyright (c) 2024, IRIS-HEP
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -25,15 +25,25 @@
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-import argparse
 import logging
+import os
 
 from rucio.client.didclient import DIDClient
 from rucio.client.replicaclient import ReplicaClient
+
+from rucio_did_finder.lookup_request import LookupRequest
 from rucio_did_finder.rucio_adapter import RucioAdapter
 from servicex_did_finder_lib import DIDFinderApp
-from rucio_did_finder.lookup_request import LookupRequest
+
+__log = logging.getLogger(__name__)
+
+cache_prefix = os.environ.get('CACHE_PREFIX', '')
+# Initialize the finder
+did_client = DIDClient()
+replica_client = ReplicaClient()
+rucio_adapter = RucioAdapter(did_client, replica_client, False)
+
+app = DIDFinderApp('rucio', did_finder_args={"rucio_adapter": rucio_adapter})
 
 
 def find_files(did_name, info, did_finder_args):
@@ -46,38 +56,7 @@ def find_files(did_name, info, did_finder_args):
         yield file
 
 
-def run_rucio_finder():
-    logger = logging.getLogger()
-
-    # Parse the command line arguments
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--report-logical-files", action="store_true")
-    DIDFinderApp.add_did_finder_cnd_arguments(parser)
-
-    args = parser.parse_args()
-
-    logger.info("ServiceX DID Finder starting up. ")
-
-    if args.report_logical_files:
-        logger.info("---- DID Finder Only Returning Logical Names, not replicas -----")
-
-    # Initialize the finder
-    did_client = DIDClient()
-    replica_client = ReplicaClient()
-    rucio_adapter = RucioAdapter(did_client, replica_client, args.report_logical_files)
-
-    # Sneak the run_query method into the args. These values will be made available to the task
-    args.rucio_adapter = rucio_adapter
-
-    # Run the DID Finder
-    app = DIDFinderApp('rucio', parsed_args=args)
-
-    @app.did_lookup_task(name="did_finder_rucio.lookup_dataset")
-    def lookup_dataset(self, did: str, dataset_id: int, endpoint: str) -> None:
-        self.do_lookup(did=did, dataset_id=dataset_id,
-                       endpoint=endpoint, user_did_finder=find_files)
-    app.start()
-
-
-if __name__ == "__main__":
-    run_rucio_finder()
+@app.did_lookup_task(name="did_finder_rucio.lookup_dataset")
+def lookup_dataset(self, did: str, dataset_id: int, endpoint: str) -> None:
+    self.do_lookup(did=did, dataset_id=dataset_id,
+                   endpoint=endpoint, user_did_finder=find_files)

did_finder_xrootd/Dockerfile

@@ -9,10 +9,6 @@ WORKDIR /usr/src/app
 USER root
 
 ENV  POETRY_VERSION=1.5.1
-#ENV POETRY_HOME=/opt/poetry
-#RUN python3 -m venv $POETRY_HOME && \
-#    $POETRY_HOME/bin/pip install poetry==$POETRY_VERSION
-
 
 # There is a bug in poetry with no virtual env where it can delete this package
 # during install. If we delete it before installing poetry it will
@@ -28,11 +24,11 @@ RUN pip install poetry==$POETRY_VERSION
 COPY pyproject.toml pyproject.toml
 COPY poetry.lock poetry.lock
 
-#RUN poetry config virtualenvs.create false
-RUN poetry install --no-root --no-interaction --no-ansi
+RUN poetry config virtualenvs.create false && \
+    poetry install --no-root --no-interaction --no-ansi
 
 # Bring over the main scripts
-COPY src/servicex_did_finder_xrootd .
+COPY . .
 
 # build stamp
 RUN echo "Timestamp:" `date --utc` | tee /image-build-info.txt
@@ -42,5 +38,7 @@ ENV PYTHONUNBUFFERED=1
 
 # Needed for running as non-root
 ENV POETRY_CACHE_DIR=/root/.cache/pypoetry
+ENV PYTHONPATH /usr/src/app/src
 
-ENTRYPOINT [ "poetry", "run", "python", "/usr/src/app/did_finder.py" ]
+ENV BROKER_URL="amqp://guest:guest@localhost:5672//"
+ENTRYPOINT [ "scripts/start_celery_worker.sh"]

did_finder_xrootd/pyproject.toml

@@ -8,7 +8,7 @@ packages = [{include = "src/servicex_did_finder_xrootd"}]
 
 [tool.poetry.dependencies]
 python = ">=3.10,<4.0"
-servicex-did-finder-lib = "^3.0.0-alpha.5"
+servicex-did-finder-lib = "^3.0.0-alpha.6"
 xrootd = ">=5.6.9"
 
 [tool.poetry.group.test]

did_finder_xrootd/scripts/start_celery_worker.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+poetry run celery --broker="$BROKER_URL" -A servicex_did_finder_xrootd worker \
+                  --loglevel=info -Q did_finder_xrootd \
+                  --concurrency=1 --hostname=did_finder_xrootd@%h