Merge pull request #28 from bento-platform/chore/fastapi-rewrite

refact!: FastAPI rewrite

.github/workflows/test.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.8", "3.10" ]
+        python-version: [ "3.10", "3.11" ]
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
@@ -21,6 +21,6 @@ jobs:
       - name: Install dependencies
         run: poetry install
       - name: Test
-        run: pytest -svv --cov=bento_drop_box_service --cov-branch
+        run: poetry run pytest -svv --cov=bento_drop_box_service --cov-branch
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3

.gitignore

@@ -8,7 +8,6 @@ __pycache__
 .tox
 
 # IDE stuff
-.idea/workspace.xml
-.idea/dataSources*
+.idea/*
 *.swp
 *.swo

Dockerfile

@@ -2,19 +2,18 @@ FROM ghcr.io/bento-platform/bento_base_image:python-debian-2023.05.12
 
 # Use uvicorn (instead of hypercorn) in production since I've found
 # multiple benchmarks showing it to be faster - David L
-RUN pip install --no-cache-dir "uvicorn[standard]==0.22.0"
+RUN pip install --no-cache-dir poetry==1.5.1 "uvicorn[standard]==0.22.0"
 
 # Backwards-compatible with old BentoV2 container layout
 WORKDIR /drop-box
 
 COPY pyproject.toml .
-COPY poetry.toml .
 COPY poetry.lock .
 
 # Install production dependencies
 # Without --no-root, we get errors related to the code not being copied in yet.
 # But we don't want the code here, otherwise Docker cache doesn't work well.
-RUN poetry install --without dev --no-root
+RUN poetry config virtualenvs.create false && poetry install --without dev --no-root
 
 # Manually copy only what's relevant
 # (Don't use .dockerignore, which allows us to have development containers too)

README.md

@@ -8,6 +8,8 @@ This is a small Quart application providing files for ingestion (through `bento_
 for `bento_wes`). By default, the file served are read on the existing filesystem, but
 these can also be read from a minIO instance (or AWS S3 for that matter).
 
+**Requires:** Python 3.10+ and Poetry 1.5+
+
 
 
 ## Environment Variables
@@ -18,10 +20,6 @@ This is used for file URI generation.
 If using the current filesystem to serve file, you can use the `SERVICE_DATA`
 environment variable to point to some location (./data by default).
 
-If the `MINIO_URL` variable is set, the application will try to connect to
-a minIO instance. To do so, you will also need to set `MINIO_USERNAME`,
-`MINIO_PASSWORD` and `MINIO_BUCKET`.
-
 
 
 ## Running in Development
@@ -30,16 +28,11 @@ Poetry is used to manage dependencies.
 
 ### Getting set up
 
-1. Create a virtual environment for the project:
-   ```bash
-   virtualenv -p python3 ./env
-   source env/bin/activate
-   ```
-2. Install `poetry`:
+1. Install `poetry`:
    ```bash
    pip install poetry
    ```
-3. Install project dependencies:
+2. Install project dependencies inside a Poetry-managed virtual environment:
    ```bash
    poetry install
    ```
@@ -49,27 +42,29 @@ Poetry is used to manage dependencies.
 To run the service in development mode, use the following command:
 
 ```bash
-QUART_ENV=development QUART_APP=bento_service_registry.app quart run
+poetry run python -m debugpy --listen "0.0.0.0:5678" -m uvicorn \
+  "bento_drop_box_service.app:application" \
+  --host 0.0.0.0 \
+  --port 5000 \
+  --reload
 ```
 
 ### Running tests
 
 To run tests and linting, run Tox:
 
 ```bash
-tox
+poetry run tox
 ```
 
 
+
 ## Deploying
 
 
 The `bento_drop_box_service` service can be deployed with an ASGI server like 
-Hypercorn, specifying `bento_drop_box_service.app:application` as the 
+Uvicorn, specifying `bento_drop_box_service.app:application` as the 
 ASGI application.
 
 It is best to then put an HTTP server software such as NGINX in front of 
 Hypercorn. 
-
-**Quart applications should NEVER be deployed in production via the Quart 
-development server, i.e. `quart run`!**

bento_drop_box_service/app.py

@@ -1,46 +1,36 @@
-import os
-
-from bento_lib.responses.quart_errors import (
-    quart_error_wrap,
-    quart_error_wrap_with_traceback,
-    quart_bad_request_error,
-    quart_not_found_error,
-    quart_internal_server_error
+from bento_lib.responses.fastapi_errors import http_exception_handler_factory, validation_exception_handler_factory
+from fastapi import FastAPI
+from fastapi.exceptions import RequestValidationError
+from fastapi.middleware.cors import CORSMiddleware
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+from .authz import authz_middleware
+from .config import get_config
+from .logger import get_logger
+from .routes import drop_box_router
+
+__all__ = [
+    "application",
+]
+
+
+application = FastAPI()
+application.include_router(drop_box_router)
+
+# TODO: Find a way to DI this
+config_for_setup = get_config()
+
+application.add_middleware(
+    CORSMiddleware,
+    allow_origins=config_for_setup.cors_origins,
+    allow_headers=["Authorization"],
+    allow_credentials=True,
+    allow_methods=["*"],
 )
-from quart import Quart
-from werkzeug.exceptions import BadRequest, NotFound
-
-from bento_drop_box_service.backend import close_backend
-from bento_drop_box_service.constants import SERVICE_NAME, SERVICE_TYPE
-from bento_drop_box_service.routes import drop_box_service
-
-
-SERVICE_DATA = os.environ.get("SERVICE_DATA", "data/")
-MINIO_URL = os.environ.get("MINIO_URL", None)
-
-application = Quart(__name__)
-application.config.from_mapping(
-    BENTO_DEBUG=os.environ.get(
-        "CHORD_DEBUG", os.environ.get("BENTO_DEBUG", os.environ.get("QUART_ENV", "production"))
-    ).strip().lower() in ("true", "1", "development"),
-    SERVICE_ID=os.environ.get("SERVICE_ID", str(":".join(list(SERVICE_TYPE.values())[:2]))),
-    SERVICE_DATA_SOURCE="minio" if MINIO_URL else "local",
-    SERVICE_DATA=None if MINIO_URL else SERVICE_DATA,
-    SERVICE_URL=os.environ.get("SERVICE_URL", "http://127.0.0.1:5000"),  # base URL to construct object URIs from
-    MINIO_URL=MINIO_URL,
-    MINIO_USERNAME=os.environ.get("MINIO_USERNAME") if MINIO_URL else None,
-    MINIO_PASSWORD=os.environ.get("MINIO_PASSWORD") if MINIO_URL else None,
-    MINIO_BUCKET=os.environ.get("MINIO_BUCKET") if MINIO_URL else None,
-    MINIO_RESOURCE=None,  # manual application-wide override for MinIO boto3 resource
-    TRAVERSAL_LIMIT=16,
-)
-
-application.register_blueprint(drop_box_service)
 
-# Generic catch-all
-application.register_error_handler(Exception, quart_error_wrap_with_traceback(quart_internal_server_error,
-                                                                              service_name=SERVICE_NAME))
-application.register_error_handler(BadRequest, quart_error_wrap(quart_bad_request_error))
-application.register_error_handler(NotFound, quart_error_wrap(quart_not_found_error))
+# Non-standard middleware setup so that we can import the instance and use it for dependencies too
+authz_middleware.attach(application)
 
-application.teardown_appcontext(close_backend)
+application.exception_handler(StarletteHTTPException)(
+    http_exception_handler_factory(get_logger(config_for_setup), authz_middleware))
+application.exception_handler(RequestValidationError)(validation_exception_handler_factory(authz_middleware))

bento_drop_box_service/authz.py

@@ -0,0 +1,17 @@
+from bento_lib.auth.middleware.fastapi import FastApiAuthMiddleware
+from .config import get_config
+
+__all__ = [
+    "authz_middleware",
+]
+
+# TODO: Find a way to DI this
+config = get_config()
+
+
+# Non-standard middleware setup so that we can import the instance and use it for dependencies too
+authz_middleware = FastApiAuthMiddleware(
+    config.bento_authz_service_url,
+    debug_mode=config.bento_debug,
+    enabled=config.authz_enabled,
+)

bento_drop_box_service/backends/__init__.py

@@ -1,10 +1,8 @@
 from .base import DropBoxBackend
 from .local import LocalBackend
-from .minio import MinioBackend
 
 
 __all__ = [
     "DropBoxBackend",
     "LocalBackend",
-    "MinioBackend",
 ]

bento_drop_box_service/backends/base.py

@@ -1,27 +1,51 @@
 import logging
 from abc import ABC, abstractmethod
-from typing import Tuple
-from werkzeug import Request, Response
+from fastapi import Request, Response
+from typing import TypedDict
 
+from ..config import Config
 
-__all__ = ["DropBoxBackend"]
+
+__all__ = ["DropBoxEntry", "DropBoxBackend"]
+
+
+# TODO: py3.11: individual optional fields
+class DropBoxEntry(TypedDict, total=False):
+    name: str
+    filePath: str
+    relativePath: str
+    uri: str
+    size: int
+    lastModified: float
+    lastMetadataChange: float
+    contents: tuple["DropBoxEntry", ...]
 
 
 class DropBoxBackend(ABC):
-    def __init__(self, logger: logging.Logger):
-        self.logger = logger
+    def __init__(self, config: Config, logger: logging.Logger):
+        self._config = config
+        self._logger = logger
+
+    @property
+    def config(self) -> Config:
+        return self._config
+
+    @property
+    def logger(self) -> logging.Logger:
+        return self._logger
 
     @abstractmethod
-    async def get_directory_tree(self) -> Tuple[dict]:
+    async def get_directory_tree(self) -> tuple[DropBoxEntry, ...]:  # pragma: no cover
         pass
 
     @abstractmethod
-    async def upload_to_path(self, request: Request, path: str, content_length: int) -> Response:
+    async def upload_to_path(self, request: Request, path: str, content_length: int) -> Response:  # pragma: no cover
         pass
 
     @abstractmethod
-    async def retrieve_from_path(self, path: str) -> Response:
+    async def retrieve_from_path(self, path: str) -> Response:  # pragma: no cover
         pass
 
-    async def close(self):
+    @abstractmethod
+    async def delete_at_path(self, path: str) -> None:  # pragma: no cover
         pass