Merge pull request #36 from bento-platform/chore/logging-and-updates

chore: update dependencies, add workflow fetch time logging

Dockerfile

@@ -1,8 +1,8 @@
-FROM ghcr.io/bento-platform/bento_base_image:python-debian-2024.02.01
+FROM ghcr.io/bento-platform/bento_base_image:python-debian-2024.03.01
 
 # Use uvicorn (instead of hypercorn) in production since I've found
 # multiple benchmarks showing it to be faster - David L
-RUN pip install --no-cache-dir "uvicorn[standard]==0.27.1"
+RUN pip install --no-cache-dir "uvicorn[standard]==0.28.0"
 
 # Backwards-compatible with old BentoV2 container layout
 WORKDIR /service-registry

bento_service_registry/services.py

@@ -70,7 +70,7 @@ async def get_service(
 
                 try:
                     service_resp = {**(await r.json()), "url": s_url}
-                    self._logger.info(f"{service_info_url}: Took {(datetime.now() - dt).total_seconds():.1f}s")
+                    self._logger.debug(f"{service_info_url}: Took {(datetime.now() - dt).total_seconds():.1f}s")
                 except (JSONDecodeError, aiohttp.ContentTypeError, TypeError) as e:
                     # JSONDecodeError can happen if the JSON is invalid
                     # ContentTypeError can happen if the Content-Type is not application/json

bento_service_registry/workflows.py

@@ -2,6 +2,7 @@
 import asyncio
 import logging
 
+from datetime import datetime
 from fastapi import Depends, status
 from typing import Annotated
 from urllib.parse import urljoin
@@ -26,6 +27,7 @@ async def get_workflows_from_service(
     http_session: aiohttp.ClientSession,
     logger: logging.Logger,
     service: dict,
+    start_dt: datetime,
 ) -> WorkflowsByPurpose:
     service_url: str | None = service.get("url")
 
@@ -34,16 +36,23 @@ async def get_workflows_from_service(
         return {}
 
     service_url_norm: str = service_url.rstrip("/") + "/"
+    workflows_url: str = urljoin(service_url_norm, "workflows")
+
+    async with http_session.get(workflows_url, headers=authz_header) as res:
+        data = await res.json()
+        time_taken = (datetime.now() - start_dt).total_seconds()
 
-    async with http_session.get(urljoin(service_url_norm, "workflows"), headers=authz_header) as res:
         if res.status != status.HTTP_200_OK:
             logger.error(
-                f"Got non-200 response from data type service ({service_url=}): {res.status=}; body={await res.json()}")
+                f"Got non-200 response from data type service ({service_url=}): {res.status=}; body={data}; "
+                f"took {time_taken:.1f}s")
             return {}
 
+        logger.debug(f"{workflows_url}: took {time_taken:.1f}s")
+
         wfs: dict[str, dict[str, dict]] = {}
 
-        for purpose, purpose_wfs in (await res.json()).items():
+        for purpose, purpose_wfs in data.items():
             if purpose not in wfs:
                 wfs[purpose] = {}
             # TODO: pydantic model + validation
@@ -67,9 +76,11 @@ async def get_workflows(
 
     logger.debug(f"Found {len(workflow_services)} workflow-providing services")
 
+    start_dt = datetime.now()
     service_wfs = await asyncio.gather(
-        *(get_workflows_from_service(authz_header, http_session, logger, s) for s in workflow_services)
+        *(get_workflows_from_service(authz_header, http_session, logger, s, start_dt) for s in workflow_services)
     )
+    logger.debug(f"Fetching all workflows took {(datetime.now() - start_dt).total_seconds():.1f}s")
 
     workflows_from_services: WorkflowsByPurpose = {}
     workflows_found: int = 0

dev.Dockerfile

@@ -1,6 +1,6 @@
-FROM ghcr.io/bento-platform/bento_base_image:python-debian-2024.02.01
+FROM ghcr.io/bento-platform/bento_base_image:python-debian-2024.03.01
 
-RUN pip install --no-cache-dir "uvicorn[standard]==0.27.1"
+RUN pip install --no-cache-dir "uvicorn[standard]==0.28.0"
 
 # Backwards-compatible with old BentoV2 container layout
 WORKDIR /service-registry