diff --git a/.idea/chord_lib.iml b/.idea/bento_lib.iml similarity index 84% rename from .idea/chord_lib.iml rename to .idea/bento_lib.iml index 0fe9ca9..15c4bca 100644 --- a/.idea/chord_lib.iml +++ b/.idea/bento_lib.iml @@ -2,7 +2,7 @@ - + @@ -13,4 +13,4 @@ - \ No newline at end of file + diff --git a/MANIFEST.in b/MANIFEST.in index 34a5dc8..576cb3c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -include chord_lib/schemas/*.json -include chord_lib/package.cfg +include bento_lib/schemas/*.json +include bento_lib/package.cfg diff --git a/README.md b/README.md index eee9d54..8da9058 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# CHORD Library (for Python CHORD microservices) +# Bento Library (for Python Bento microservices) -![Build Status](https://api.travis-ci.org/c3g/chord_lib.svg?branch=master) -[![codecov](https://codecov.io/gh/c3g/chord_lib/branch/master/graph/badge.svg)](https://codecov.io/gh/c3g/chord_lib) -[![PyPI version](https://badge.fury.io/py/chord-lib.svg)](https://badge.fury.io/py/chord-lib) +![Build Status](https://api.travis-ci.org/bento-platform/bento_lib.svg?branch=master) +[![codecov](https://codecov.io/gh/bento-platform/bento_lib/branch/master/graph/badge.svg)](https://codecov.io/gh/bento-platform/bento_lib) +[![PyPI version](https://badge.fury.io/py/bento-lib.svg)](https://badge.fury.io/py/bento-lib) -Common utilities and helpers for CHORD services. +Common utilities and helpers for Bento platform services. ## Running Tests @@ -22,7 +22,7 @@ python3 -m tox * [ ] All tests pass and test coverage has not been reduced * [ ] Package version has been updated (following semver) in - `chord_lib/package.cfg` + `bento_lib/package.cfg` * [ ] The latest changes have been merged from the `develop` branch into the `master` branch @@ -46,7 +46,7 @@ git pull source env/bin/activate # Remove existing build files -rm -rf build/ dist/ chord_lib.egg-info/ +rm -rf build/ dist/ bento_lib.egg-info/ # Build the new package python3 setup.py sdist bdist_wheel @@ -64,58 +64,58 @@ twine upload dist/* ### `auth` `auth` provides Python service decorators and Django / DRF backends for dealing -with the CHORD container authentication headers (derived from +with the Bento container authentication headers (derived from `lua-resty-openidc`, set by the internal container NGINX instance.) ### `events` -`events` facilitates JSON-serialized message-passing between CHORD +`events` facilitates JSON-serialized message-passing between Bento microservices. Serialized objects can be at most 512 MB. Events should have a lower-case type which is type-insensitively unique and adequately describes the associated data. -All CHORD channels are prefixed with `chord.`. +All Bento channels are prefixed with `bento.`. ### `ingestion` `ingestion` contains common code used for handling ingestion routines in -different CHORD data services. +different Bento data services. ### `schemas` `schemas` contains common JSON schemas which may be useful to a variety of -different CHORD services. +different Bento services. -`schemas.chord` contains CHORD-specific schemas, and `schemas.ga4gh` contains +`schemas.bento` contains Bento-specific schemas, and `schemas.ga4gh` contains GA4GH-standardized schemas (possibly not exactly to spec.) ### `search` `search` contains definitions, validators, and transformations for the query -syntax for CHORD, as well as a transpiler to the `psycopg2` PostgreSQL IR. +syntax for Bento, as well as a transpiler to the `psycopg2` PostgreSQL IR. -The query syntax for CHORD takes advantage of JSON schemas augmented with +The query syntax for Bento takes advantage of JSON schemas augmented with additional properties about the field's accessibility and, in the case of Postgres, how the field maps to a table column (or JSON column sub-field.) -`search.data_structure` contains code for evaluating a CHORD query against a +`search.data_structure` contains code for evaluating a Bento query against a Python data structure. `search.operations` contains constants representing valid search operations one can allow against particular fields from within an augmented JSON schema. -`search.postgres` contains a "transpiler" from the CHORD query syntax to the +`search.postgres` contains a "transpiler" from the Bento query syntax to the `psycopg2`-provided [intermediate representation (IR)](https://www.psycopg.org/docs/sql.html) for PostgreSQL, allowing safe queries against a Postgres database. -`search.queries` provides definitions for the CHORD query AST and some helper +`search.queries` provides definitions for the Bento query AST and some helper methods for creating and processing ASTs. ### `utils` -`utils` contains miscellaneous utilities commonly required by CHORD services. +`utils` contains miscellaneous utilities commonly required by Bento services. ### `workflows` diff --git a/chord_lib/__init__.py b/bento_lib/__init__.py similarity index 100% rename from chord_lib/__init__.py rename to bento_lib/__init__.py diff --git a/chord_lib/auth/__init__.py b/bento_lib/auth/__init__.py similarity index 100% rename from chord_lib/auth/__init__.py rename to bento_lib/auth/__init__.py diff --git a/chord_lib/auth/django_remote_user.py b/bento_lib/auth/django_remote_user.py similarity index 59% rename from chord_lib/auth/django_remote_user.py rename to bento_lib/auth/django_remote_user.py index 0bc5c3a..4ae5165 100644 --- a/chord_lib/auth/django_remote_user.py +++ b/bento_lib/auth/django_remote_user.py @@ -2,26 +2,26 @@ from django.contrib.auth.middleware import RemoteUserMiddleware from rest_framework.authentication import RemoteUserAuthentication -from chord_lib.auth.headers import DJANGO_USER_HEADER, DJANGO_USER_ROLE_HEADER -from chord_lib.auth.roles import ROLE_OWNER, ROLE_USER +from bento_lib.auth.headers import DJANGO_USER_HEADER, DJANGO_USER_ROLE_HEADER +from bento_lib.auth.roles import ROLE_OWNER, ROLE_USER __all__ = [ - "CHORDRemoteUserAuthentication", - "CHORDRemoteUserBackend", - "CHORDRemoteUserMiddleware", + "BentoRemoteUserAuthentication", + "BentoRemoteUserBackend", + "BentoRemoteUserMiddleware", ] -class CHORDRemoteUserAuthentication(RemoteUserAuthentication): +class BentoRemoteUserAuthentication(RemoteUserAuthentication): header = DJANGO_USER_HEADER -class CHORDRemoteUserMiddleware(RemoteUserMiddleware): +class BentoRemoteUserMiddleware(RemoteUserMiddleware): header = DJANGO_USER_HEADER -class CHORDRemoteUserBackend(RemoteUserBackend): +class BentoRemoteUserBackend(RemoteUserBackend): # noinspection PyMethodMayBeStatic def configure_user(self, request, user): is_owner = request.META.get(DJANGO_USER_ROLE_HEADER, ROLE_USER) == ROLE_OWNER diff --git a/chord_lib/auth/flask_decorators.py b/bento_lib/auth/flask_decorators.py similarity index 67% rename from chord_lib/auth/flask_decorators.py rename to bento_lib/auth/flask_decorators.py index b87b71f..31ab363 100644 --- a/chord_lib/auth/flask_decorators.py +++ b/bento_lib/auth/flask_decorators.py @@ -4,9 +4,9 @@ from functools import wraps from typing import Union -from chord_lib.auth.headers import CHORD_USER_HEADER, CHORD_USER_ROLE_HEADER -from chord_lib.auth.roles import ROLE_OWNER, ROLE_USER -from chord_lib.responses.flask_errors import flask_forbidden_error +from bento_lib.auth.headers import BENTO_USER_HEADER, BENTO_USER_ROLE_HEADER +from bento_lib.auth.roles import ROLE_OWNER, ROLE_USER +from bento_lib.responses.flask_errors import flask_forbidden_error __all__ = [ @@ -17,16 +17,16 @@ # TODO: Centralize this -CHORD_DEBUG = os.environ.get("CHORD_DEBUG", "true").lower() == "true" -CHORD_PERMISSIONS = os.environ.get("CHORD_PERMISSIONS", str(not CHORD_DEBUG)).lower() == "true" +BENTO_DEBUG = os.environ.get("CHORD_DEBUG", "true").lower() == "true" +BENTO_PERMISSIONS = os.environ.get("CHORD_PERMISSIONS", str(not BENTO_DEBUG)).lower() == "true" def _check_roles(headers, roles: Union[set, dict]): method_roles = roles if not isinstance(roles, dict) else roles.get(request.method, set()) return ( - not CHORD_PERMISSIONS or + not BENTO_PERMISSIONS or len(method_roles) == 0 or - (CHORD_USER_HEADER in headers and headers.get(CHORD_USER_ROLE_HEADER, "") in method_roles) + (BENTO_USER_HEADER in headers and headers.get(BENTO_USER_ROLE_HEADER, "") in method_roles) ) diff --git a/bento_lib/auth/headers.py b/bento_lib/auth/headers.py new file mode 100644 index 0000000..e253b6b --- /dev/null +++ b/bento_lib/auth/headers.py @@ -0,0 +1,18 @@ +__all__ = [ + "BENTO_USER_HEADER", + "BENTO_USER_ROLE_HEADER", + + "DJANGO_USER_HEADER", + "DJANGO_USER_ROLE_HEADER", +] + + +def _to_django_header(header: str): + return f"HTTP_{header.replace('-', '_').upper()}" + + +BENTO_USER_HEADER = "X-User" +BENTO_USER_ROLE_HEADER = "X-User-Role" + +DJANGO_USER_HEADER = _to_django_header(BENTO_USER_HEADER) +DJANGO_USER_ROLE_HEADER = _to_django_header(BENTO_USER_ROLE_HEADER) diff --git a/chord_lib/auth/roles.py b/bento_lib/auth/roles.py similarity index 100% rename from chord_lib/auth/roles.py rename to bento_lib/auth/roles.py diff --git a/chord_lib/events/__init__.py b/bento_lib/events/__init__.py similarity index 97% rename from chord_lib/events/__init__.py rename to bento_lib/events/__init__.py index 49046fc..1bcf87d 100644 --- a/chord_lib/events/__init__.py +++ b/bento_lib/events/__init__.py @@ -18,11 +18,11 @@ ] -ALL_SERVICE_EVENTS = "chord.service.*" -ALL_DATA_TYPE_EVENTS = "chord.data_type.*" +ALL_SERVICE_EVENTS = "bento.service.*" +ALL_DATA_TYPE_EVENTS = "bento.data_type.*" -_SERVICE_CHANNEL_TPL = "chord.service.{}" -_DATA_TYPE_CHANNEL_TPL = "chord.data_type.{}" +_SERVICE_CHANNEL_TPL = "bento.service.{}" +_DATA_TYPE_CHANNEL_TPL = "bento.data_type.{}" # Types diff --git a/chord_lib/events/notifications.py b/bento_lib/events/notifications.py similarity index 100% rename from chord_lib/events/notifications.py rename to bento_lib/events/notifications.py diff --git a/chord_lib/events/types.py b/bento_lib/events/types.py similarity index 100% rename from chord_lib/events/types.py rename to bento_lib/events/types.py diff --git a/chord_lib/ingestion.py b/bento_lib/ingestion.py similarity index 100% rename from chord_lib/ingestion.py rename to bento_lib/ingestion.py diff --git a/chord_lib/package.cfg b/bento_lib/package.cfg similarity index 70% rename from chord_lib/package.cfg rename to bento_lib/package.cfg index a5b7f5e..a3515d0 100644 --- a/chord_lib/package.cfg +++ b/bento_lib/package.cfg @@ -1,5 +1,5 @@ [package] -name = chord_lib -version = 0.9.0 +name = bento_lib +version = 0.10.0 authors = David Lougheed author_emails = david.lougheed@mail.mcgill.ca diff --git a/chord_lib/responses/__init__.py b/bento_lib/responses/__init__.py similarity index 100% rename from chord_lib/responses/__init__.py rename to bento_lib/responses/__init__.py diff --git a/chord_lib/responses/errors.py b/bento_lib/responses/errors.py similarity index 90% rename from chord_lib/responses/errors.py rename to bento_lib/responses/errors.py index 55d4cc1..242e376 100644 --- a/chord_lib/responses/errors.py +++ b/bento_lib/responses/errors.py @@ -22,12 +22,12 @@ def _error_message(message): def http_error(code: int, *errors): if code not in HTTP_STATUS_CODES: - print(f"[CHORD Lib] Error: Could not find code {code} in valid HTTP status codes.") + print(f"[Bento Lib] Error: Could not find code {code} in valid HTTP status codes.") code = 500 errors = (*errors, f"An invalid status code of {code} was specified by the service.") if code < 400: - print(f"[CHORD Lib] Error: Code {code} is not an HTTP error code.") + print(f"[Bento Lib] Error: Code {code} is not an HTTP error code.") code = 500 errors = (*errors, f"A non-error status code of {code} was specified by the service.") diff --git a/chord_lib/responses/flask_errors.py b/bento_lib/responses/flask_errors.py similarity index 96% rename from chord_lib/responses/flask_errors.py rename to bento_lib/responses/flask_errors.py index 7942189..51c4d8a 100644 --- a/chord_lib/responses/flask_errors.py +++ b/bento_lib/responses/flask_errors.py @@ -5,7 +5,7 @@ from functools import partial from typing import Callable -from chord_lib.responses import errors +from bento_lib.responses import errors __all__ = [ @@ -24,7 +24,7 @@ ] -def flask_error_wrap_with_traceback(fn: Callable, service_name="CHORD Service") -> Callable: +def flask_error_wrap_with_traceback(fn: Callable, service_name="Bento Service") -> Callable: """ Function to wrap flask_* error creators with something that supports the application.register_error_handler method, while also printing a traceback. diff --git a/bento_lib/schemas/__init__.py b/bento_lib/schemas/__init__.py new file mode 100644 index 0000000..0ed2160 --- /dev/null +++ b/bento_lib/schemas/__init__.py @@ -0,0 +1,4 @@ +from . import bento +from . import ga4gh + +__all__ = ["bento", "ga4gh"] diff --git a/chord_lib/schemas/_utils.py b/bento_lib/schemas/_utils.py similarity index 100% rename from chord_lib/schemas/_utils.py rename to bento_lib/schemas/_utils.py diff --git a/bento_lib/schemas/bento.py b/bento_lib/schemas/bento.py new file mode 100644 index 0000000..fb19155 --- /dev/null +++ b/bento_lib/schemas/bento.py @@ -0,0 +1,13 @@ +from ._utils import load_json_schema + + +__all__ = [ + "BENTO_INGEST_SCHEMA", + "BENTO_DATA_USE_SCHEMA", +] + + +# TODO: Refactor this schema and semi-combine with workflow schema +BENTO_INGEST_SCHEMA = load_json_schema("bento_ingest.schema.json") + +BENTO_DATA_USE_SCHEMA = load_json_schema("bento_data_use.schema.json") diff --git a/chord_lib/schemas/chord_data_use.schema.json b/bento_lib/schemas/bento_data_use.schema.json similarity index 92% rename from chord_lib/schemas/chord_data_use.schema.json rename to bento_lib/schemas/bento_data_use.schema.json index ba90b20..8e6b9bb 100644 --- a/chord_lib/schemas/chord_data_use.schema.json +++ b/bento_lib/schemas/bento_data_use.schema.json @@ -1,7 +1,7 @@ { - "$id": "https://raw.githubusercontent.com/c3g/chord_lib/master/chord_lib/schemas/chord_data_use.schema.json", + "$id": "https://raw.githubusercontent.com/bento-platform/bento_lib/master/bento_lib/schemas/bento_data_use.schema.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "CHORD Data Use File", + "title": "Bento Data Use File", "description": "Schema defining data usage conditions from the GA4GH DUO ontology.", "type": "object", "properties": { diff --git a/chord_lib/schemas/chord_ingest.schema.json b/bento_lib/schemas/bento_ingest.schema.json similarity index 100% rename from chord_lib/schemas/chord_ingest.schema.json rename to bento_lib/schemas/bento_ingest.schema.json diff --git a/chord_lib/schemas/ga4gh.py b/bento_lib/schemas/ga4gh.py similarity index 100% rename from chord_lib/schemas/ga4gh.py rename to bento_lib/schemas/ga4gh.py diff --git a/chord_lib/schemas/ga4gh_service_info.schema.json b/bento_lib/schemas/ga4gh_service_info.schema.json similarity index 85% rename from chord_lib/schemas/ga4gh_service_info.schema.json rename to bento_lib/schemas/ga4gh_service_info.schema.json index 2db632f..78b2b73 100644 --- a/chord_lib/schemas/ga4gh_service_info.schema.json +++ b/bento_lib/schemas/ga4gh_service_info.schema.json @@ -1,5 +1,5 @@ { - "$id": "https://raw.githubusercontent.com/c3g/chord_lib/master/chord_lib/schemas/ga4gh_service_info.schema.json", + "$id": "https://raw.githubusercontent.com/bento-platform/bento_lib/master/bento_lib/schemas/ga4gh_service_info.schema.json", "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { diff --git a/chord_lib/search/__init__.py b/bento_lib/search/__init__.py similarity index 100% rename from chord_lib/search/__init__.py rename to bento_lib/search/__init__.py diff --git a/bento_lib/search/_types.py b/bento_lib/search/_types.py new file mode 100644 index 0000000..57ee7d0 --- /dev/null +++ b/bento_lib/search/_types.py @@ -0,0 +1,5 @@ +from typing import Dict + +__all__ = ["JSONSchema"] + +JSONSchema = Dict diff --git a/chord_lib/search/data_structure.py b/bento_lib/search/data_structure.py similarity index 95% rename from chord_lib/search/data_structure.py rename to bento_lib/search/data_structure.py index ae4e7d4..728ecf6 100644 --- a/chord_lib/search/data_structure.py +++ b/bento_lib/search/data_structure.py @@ -4,7 +4,8 @@ from operator import and_, or_, not_, lt, le, eq, gt, ge, contains, is_not from typing import Callable, Dict, List, Iterable, Optional, Tuple, Union -from chord_lib.search import queries as q +from . import queries as q +from ._types import JSONSchema __all__ = ["check_ast_against_data_structure"] @@ -19,7 +20,7 @@ ArrayLengthData = Tuple[str, int, Tuple["ArrayLengthData", ...]] -def _validate_data_structure_against_schema(data_structure: QueryableStructure, schema: dict): +def _validate_data_structure_against_schema(data_structure: QueryableStructure, schema: JSONSchema): """ Validates a queryable data structure of some type against a JSON schema. This is an important validation step, because (assuming the schema is correct) it allows methods to make more assumptions about the integrity of the @@ -46,7 +47,7 @@ def _validate_not_wc(e: q.Expression): def evaluate_no_validate( ast: q.AST, data_structure: QueryableStructure, - schema: dict, + schema: JSONSchema, index_combination: Optional[IndexCombination], internal: bool = False, resolve_checks: bool = True, @@ -92,7 +93,7 @@ def evaluate_no_validate( def evaluate( ast: q.AST, data_structure: QueryableStructure, - schema: dict, + schema: JSONSchema, index_combination: Optional[IndexCombination], internal: bool = False, resolve_checks: bool = True, @@ -104,7 +105,7 @@ def evaluate( check_permissions) -def _collect_array_lengths(ast: q.AST, data_structure: QueryableStructure, schema: dict, +def _collect_array_lengths(ast: q.AST, data_structure: QueryableStructure, schema: JSONSchema, resolve_checks: bool) -> Iterable[ArrayLengthData]: """ To evaluate a query in a manner consistent with the Postgres evaluator (and facilitate richer queries), each array @@ -203,7 +204,7 @@ def _create_all_index_combinations(arrays_data: Iterable[ArrayLengthData], paren def check_ast_against_data_structure( ast: q.AST, data_structure: QueryableStructure, - schema: dict, + schema: JSONSchema, internal: bool = False, return_all_index_combinations: bool = False, ) -> Union[bool, Iterable[IndexCombination]]: @@ -242,7 +243,7 @@ def _evaluate(i: int, ic: IndexCombination) -> bool: def _binary_op(op: BBOperator)\ - -> Callable[[FunctionArgs, QueryableStructure, dict, Optional[IndexCombination], bool, bool], bool]: + -> Callable[[FunctionArgs, QueryableStructure, JSONSchema, Optional[IndexCombination], bool, bool, bool], bool]: """ Returns a boolean-returning binary operator on a pair of arguments against a data structure/object of some type and return a Boolean result. @@ -253,8 +254,9 @@ def _binary_op(op: BBOperator)\ is_and = op == and_ is_or = op == or_ - def uncurried_binary_op(args: FunctionArgs, ds: QueryableStructure, schema: dict, ic: Optional[IndexCombination], - internal: bool, resolve_checks: bool, check_permissions: bool) -> bool: + def uncurried_binary_op(args: FunctionArgs, ds: QueryableStructure, schema: JSONSchema, + ic: Optional[IndexCombination], internal: bool, resolve_checks: bool, + check_permissions: bool) -> bool: # TODO: Standardize type safety / behaviour!!! # Evaluate both sides of the binary expression. If there's a type error while trying to use a Python built-in, @@ -282,7 +284,7 @@ def uncurried_binary_op(args: FunctionArgs, ds: QueryableStructure, schema: dict return uncurried_binary_op -def _resolve_checks(resolve_value: str, schema: dict): +def _resolve_checks(resolve_value: str, schema: JSONSchema): """ Performs standard checks while going through any type of "resolve"-based function (where a #resolve call is being processed) to prevent access errors. @@ -305,7 +307,7 @@ def _resolve_checks(resolve_value: str, schema: dict): def _get_child_resolve_array_lengths( new_resolve: List[q.Literal], resolving_ds: List, - item_schema: dict, + item_schema: JSONSchema, new_path: str, resolve_checks: bool, ) -> Iterable[ArrayLengthData]: @@ -327,7 +329,7 @@ def _get_child_resolve_array_lengths( def _resolve_array_lengths( resolve: List[q.Literal], resolving_ds: QueryableStructure, - schema: dict, + schema: JSONSchema, path: str = "_root", resolve_checks: bool = True, ) -> Optional[ArrayLengthData]: @@ -370,7 +372,7 @@ def _resolve_array_lengths( def _resolve_properties_and_check( resolve: List[q.Literal], - schema: dict, + schema: JSONSchema, index_combination: Optional[IndexCombination], ) -> dict: """ @@ -400,7 +402,7 @@ def _resolve_properties_and_check( return r_schema.get("search", {}) -def _resolve(resolve: List[q.Literal], resolving_ds: QueryableStructure, _schema: dict, +def _resolve(resolve: List[q.Literal], resolving_ds: QueryableStructure, _schema: JSONSchema, index_combination: Optional[IndexCombination], _internal, _resolve_checks, _check_permissions): """ Resolves / evaluates a path (either object or array) into a value. Assumes the data structure has already been @@ -424,12 +426,13 @@ def _resolve(resolve: List[q.Literal], resolving_ds: QueryableStructure, _schema QUERY_CHECK_SWITCH: Dict[ q.FunctionName, - Callable[[FunctionArgs, QueryableStructure, dict, Optional[IndexCombination], bool, bool], QueryableStructure] + Callable[[FunctionArgs, QueryableStructure, JSONSchema, Optional[IndexCombination], bool, bool, bool], + QueryableStructure] ] = { q.FUNCTION_AND: _binary_op(and_), q.FUNCTION_OR: _binary_op(or_), - q.FUNCTION_NOT: lambda args, ds, schema, internal, ic, r_chk, p_chk: - not_(evaluate_no_validate(args[0], ds, schema, internal, ic, r_chk, p_chk)), + q.FUNCTION_NOT: lambda args, ds, schema, ic, internal, r_chk, p_chk: + not_(evaluate_no_validate(args[0], ds, schema, ic, internal, r_chk, p_chk)), q.FUNCTION_LT: _binary_op(lt), q.FUNCTION_LE: _binary_op(le), diff --git a/chord_lib/search/operations.py b/bento_lib/search/operations.py similarity index 100% rename from chord_lib/search/operations.py rename to bento_lib/search/operations.py diff --git a/chord_lib/search/postgres.py b/bento_lib/search/postgres.py similarity index 90% rename from chord_lib/search/postgres.py rename to bento_lib/search/postgres.py index 339bfcb..a3caa25 100644 --- a/chord_lib/search/postgres.py +++ b/bento_lib/search/postgres.py @@ -3,7 +3,8 @@ from psycopg2 import sql from typing import Callable, Dict, List, Optional, Tuple -from chord_lib.search import queries as q +from . import queries as q +from ._types import JSONSchema # Search Rules: @@ -54,7 +55,7 @@ def __repr__(self): # pragma: no cover return f"" -def json_schema_to_postgres_type(schema: dict) -> str: +def json_schema_to_postgres_type(schema: JSONSchema) -> str: """ Maps a JSON schema to a Postgres type for on the fly mapping. :param schema: JSON schema to map. @@ -76,7 +77,7 @@ def json_schema_to_postgres_type(schema: dict) -> str: return "TEXT" # TODO -def json_schema_to_postgres_schema(name: str, schema: dict) -> Tuple[Optional[sql.Composable], Optional[str]]: +def json_schema_to_postgres_schema(name: str, schema: JSONSchema) -> Tuple[Optional[sql.Composable], Optional[str]]: """ Maps a JSON object schema to a Postgres schema for on-the-fly mapping. :param name: the name to give the fake table. @@ -96,14 +97,14 @@ def json_schema_to_postgres_schema(name: str, schema: dict) -> Tuple[Optional[sq ) -def _get_search_and_database_properties(schema: dict) -> Tuple[dict, dict]: +def _get_search_and_database_properties(schema: JSONSchema) -> Tuple[dict, dict]: search_properties = schema.get("search", {}) return search_properties, search_properties.get("database", {}) def collect_resolve_join_tables( resolve: Tuple[q.Literal, ...], - schema: dict, + schema: JSONSchema, parent_relation: Optional[Tuple[Optional[sql.Composable], Optional[sql.Composable]]] = None, resolve_path: Optional[str] = None ) -> Tuple[JoinAndSelectData, ...]: @@ -216,7 +217,7 @@ def collect_resolve_join_tables( resolve_path=new_resolve_path if current_relation is not None else None) -def collect_join_tables(ast: q.AST, terms: tuple, schema: dict) -> Tuple[JoinAndSelectData, ...]: +def collect_join_tables(ast: q.AST, terms: tuple, schema: JSONSchema) -> Tuple[JoinAndSelectData, ...]: if isinstance(ast, q.Literal): return terms @@ -239,7 +240,7 @@ def collect_join_tables(ast: q.AST, terms: tuple, schema: dict) -> Tuple[JoinAnd return new_terms -def join_fragment(ast: q.AST, schema: dict) -> sql.Composable: +def join_fragment(ast: q.AST, schema: JSONSchema) -> sql.Composable: terms = collect_join_tables(ast, (), schema) if not terms: # Query was probably just a literal # TODO: Don't hard-code _root? @@ -263,7 +264,7 @@ def join_fragment(ast: q.AST, schema: dict) -> sql.Composable: )) -def search_ast_to_psycopg2_expr(ast: q.AST, params: tuple, schema: dict, internal: bool = False) \ +def search_ast_to_psycopg2_expr(ast: q.AST, params: tuple, schema: JSONSchema, internal: bool = False) \ -> SQLComposableWithParams: if isinstance(ast, q.Literal): return sql.Placeholder(), (*params, ast.value) @@ -273,7 +274,7 @@ def search_ast_to_psycopg2_expr(ast: q.AST, params: tuple, schema: dict, interna return POSTGRES_SEARCH_LANGUAGE_FUNCTIONS[ast.fn](ast.args, params, schema, internal) -def search_query_to_psycopg2_sql(query, schema: dict, internal: bool = False) -> SQLComposableWithParams: +def search_query_to_psycopg2_sql(query, schema: JSONSchema, internal: bool = False) -> SQLComposableWithParams: # TODO: Shift recursion to not have to add in the extra SELECT for the root? ast = q.convert_query_to_ast_and_preprocess(query) sql_obj, params = search_ast_to_psycopg2_expr(ast, (), schema, internal) @@ -281,7 +282,7 @@ def search_query_to_psycopg2_sql(query, schema: dict, internal: bool = False) -> return sql.SQL("SELECT {}.* FROM {} WHERE {}").format(SQL_ROOT, join_fragment(ast, schema), sql_obj), params -def uncurried_binary_op(op: str, args: List[q.AST], params: tuple, schema: dict, internal: bool = False) \ +def uncurried_binary_op(op: str, args: List[q.AST], params: tuple, schema: JSONSchema, internal: bool = False) \ -> SQLComposableWithParams: # TODO: Need to fix params!! Use named params lhs_sql, lhs_params = search_ast_to_psycopg2_expr(args[0], params, schema, internal) @@ -298,7 +299,8 @@ def _not(args: list, params: tuple, schema: dict, internal: bool = False) -> SQL return sql.SQL("NOT ({})").format(child_sql), params + child_params -def _wildcard(args: List[q.AST], params: tuple, _schema: dict, _internal: bool = False) -> SQLComposableWithParams: +def _wildcard(args: List[q.AST], params: tuple, _schema: JSONSchema, _internal: bool = False) \ + -> SQLComposableWithParams: if isinstance(args[0], q.Expression): raise NotImplementedError("Cannot currently use #co on an expression") # TODO @@ -308,33 +310,36 @@ def _wildcard(args: List[q.AST], params: tuple, _schema: dict, _internal: bool = raise TypeError("Type-invalid use of binary function #co") -def get_relation(resolve: List[q.Literal], schema: dict): +def get_relation(resolve: List[q.Literal], schema: JSONSchema): aliases = collect_resolve_join_tables((QUERY_ROOT, *resolve), schema)[-1].aliases return aliases.current if aliases.current is not None else aliases.parent -def get_field(resolve: List[q.Literal], schema: dict) -> Optional[str]: +def get_field(resolve: List[q.Literal], schema: JSONSchema) -> Optional[str]: return collect_resolve_join_tables((QUERY_ROOT, *resolve), schema)[-1].field_alias -def get_search_properties(resolve: List[q.Literal], schema: dict) -> dict: +def get_search_properties(resolve: List[q.Literal], schema: JSONSchema) -> dict: return collect_resolve_join_tables((QUERY_ROOT, *resolve), schema)[-1].search_properties -def _resolve(args: List[q.AST], params: tuple, schema: dict, _internal: bool = False) -> SQLComposableWithParams: +def _resolve(args: List[q.AST], params: tuple, schema: JSONSchema, _internal: bool = False) -> SQLComposableWithParams: f_id = get_field(args, schema) return sql.SQL("{}.{}").format(get_relation(args, schema), sql.Identifier(f_id) if f_id is not None else sql.SQL("*")), params -def _contains(args: List[q.AST], params: tuple, schema: dict, internal: bool = False) -> SQLComposableWithParams: +def _contains(args: List[q.AST], params: tuple, schema: JSONSchema, internal: bool = False) -> SQLComposableWithParams: lhs_sql, lhs_params = search_ast_to_psycopg2_expr(args[0], params, schema, internal) rhs_sql, rhs_params = search_ast_to_psycopg2_expr(q.Expression(fn=q.FUNCTION_HELPER_WC, args=[args[1]]), params, schema, internal) return sql.SQL("({}) LIKE ({})").format(lhs_sql, rhs_sql), params + lhs_params + rhs_params -POSTGRES_SEARCH_LANGUAGE_FUNCTIONS: Dict[str, Callable[[List[q.AST], tuple, dict, bool], SQLComposableWithParams]] = { +POSTGRES_SEARCH_LANGUAGE_FUNCTIONS: Dict[ + str, + Callable[[List[q.AST], tuple, JSONSchema, bool], SQLComposableWithParams] +] = { q.FUNCTION_AND: _binary_op("AND"), q.FUNCTION_OR: _binary_op("OR"), q.FUNCTION_NOT: _not, diff --git a/chord_lib/search/queries.py b/bento_lib/search/queries.py similarity index 97% rename from chord_lib/search/queries.py rename to bento_lib/search/queries.py index c68328d..4da3409 100644 --- a/chord_lib/search/queries.py +++ b/bento_lib/search/queries.py @@ -1,5 +1,6 @@ from typing import Callable, List, Optional, Tuple, Union +from ._types import JSONSchema from .operations import ( SEARCH_OP_LT, SEARCH_OP_LE, @@ -214,7 +215,7 @@ def and_asts_to_ast(asts: Tuple[AST, ...]) -> Optional[AST]: return Expression(FUNCTION_AND, [asts[0], and_asts_to_ast(asts[1:])]) -def check_operation_permissions(ast: AST, schema: dict, search_getter: Callable[[List[Literal], dict], dict], +def check_operation_permissions(ast: AST, schema: JSONSchema, search_getter: Callable[[List[Literal], dict], dict], internal: bool = False): if ast.type == "l": return diff --git a/chord_lib/workflows.py b/bento_lib/workflows.py similarity index 100% rename from chord_lib/workflows.py rename to bento_lib/workflows.py diff --git a/chord_lib/auth/headers.py b/chord_lib/auth/headers.py deleted file mode 100644 index 2809989..0000000 --- a/chord_lib/auth/headers.py +++ /dev/null @@ -1,18 +0,0 @@ -__all__ = [ - "CHORD_USER_HEADER", - "CHORD_USER_ROLE_HEADER", - - "DJANGO_USER_HEADER", - "DJANGO_USER_ROLE_HEADER", -] - - -def _to_django_header(header: str): - return f"HTTP_{header.replace('-', '_').upper()}" - - -CHORD_USER_HEADER = "X-User" -CHORD_USER_ROLE_HEADER = "X-User-Role" - -DJANGO_USER_HEADER = _to_django_header(CHORD_USER_HEADER) -DJANGO_USER_ROLE_HEADER = _to_django_header(CHORD_USER_ROLE_HEADER) diff --git a/chord_lib/schemas/__init__.py b/chord_lib/schemas/__init__.py deleted file mode 100644 index a33efb2..0000000 --- a/chord_lib/schemas/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from . import chord -from . import ga4gh - -__all__ = ["chord", "ga4gh"] diff --git a/chord_lib/schemas/chord.py b/chord_lib/schemas/chord.py deleted file mode 100644 index daaa7f1..0000000 --- a/chord_lib/schemas/chord.py +++ /dev/null @@ -1,13 +0,0 @@ -from ._utils import load_json_schema - - -__all__ = [ - "CHORD_INGEST_SCHEMA", - "CHORD_DATA_USE_SCHEMA", -] - - -# TODO: Refactor this schema and semi-combine with workflow schema -CHORD_INGEST_SCHEMA = load_json_schema("chord_ingest.schema.json") - -CHORD_DATA_USE_SCHEMA = load_json_schema("chord_data_use.schema.json") diff --git a/requirements.txt b/requirements.txt index 82a6945..78ca800 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,26 +1,26 @@ appdirs==1.4.4 attrs==19.3.0 -certifi==2020.4.5.1 +certifi==2020.4.5.2 chardet==3.0.4 click==7.1.2 -codecov==2.0.22 +codecov==2.1.4 coverage==5.1 distlib==0.3.0 -Django==2.2.12 +Django==2.2.13 djangorestframework==3.11.0 entrypoints==0.3 filelock==3.0.12 -flake8==3.8.1 +flake8==3.8.3 Flask==1.1.2 idna==2.9 -importlib-metadata==1.6.0 +importlib-metadata==1.6.1 itsdangerous==1.1.0 Jinja2==2.11.2 jsonschema==3.2.0 MarkupSafe==1.1.1 mccabe==0.6.1 -more-itertools==8.2.0 -packaging==20.3 +more-itertools==8.3.0 +packaging==20.4 pluggy==0.13.1 psycopg2-binary==2.8.5 py==1.8.1 @@ -28,19 +28,19 @@ pycodestyle==2.6.0 pyflakes==2.2.0 pyparsing==2.4.7 pyrsistent==0.16.0 -pytest==5.4.2 -pytest-cov==2.8.1 +pytest==5.4.3 +pytest-cov==2.9.0 pytest-django==3.9.0 python-dateutil==2.8.1 pytz==2020.1 -redis==3.5.1 +redis==3.5.3 requests==2.23.0 -six==1.14.0 +six==1.15.0 sqlparse==0.3.1 toml==0.10.1 -tox==3.15.0 +tox==3.15.2 urllib3==1.25.9 -virtualenv==20.0.20 -wcwidth==0.1.9 +virtualenv==20.0.21 +wcwidth==0.2.4 Werkzeug==1.0.1 zipp==3.1.0 diff --git a/setup.py b/setup.py index 23e8552..69bc24a 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ long_description = rf.read() config = configparser.ConfigParser() -config.read(os.path.join(os.path.dirname(os.path.realpath(__file__)), "chord_lib", "package.cfg")) +config.read(os.path.join(os.path.dirname(os.path.realpath(__file__)), "bento_lib", "package.cfg")) setuptools.setup( name=config["package"]["name"], @@ -18,25 +18,25 @@ install_requires=[ "jsonschema>=3.2.0,<4", "psycopg2-binary>=2.8.5,<3.0", - "redis>=3.5.1,<4.0", + "redis>=3.5.3,<4.0", "Werkzeug>=1.0.1,<2.0", ], extras_require={ "flask": ["Flask>=1.1.2,<2.0"], - "django": ["Django>=2.2.12,<3.0", "djangorestframework>=3.11,<3.12"] + "django": ["Django>=2.2.13,<3.0", "djangorestframework>=3.11,<3.12"] }, author=config["package"]["authors"], author_email=config["package"]["author_emails"], - description="A set of common utilities and helpers for CHORD.", + description="A set of common utilities and helpers for Bento platform services.", long_description=long_description, long_description_content_type="text/markdown", packages=setuptools.find_packages(), include_package_data=True, - url="https://github.com/c3g/chord_lib", + url="https://github.com/bento-platform/bento_lib", license="LGPLv3", classifiers=[ "Programming Language :: Python :: 3", diff --git a/tests/test_events.py b/tests/test_events.py index 400d6a9..750853d 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -1,4 +1,4 @@ -import chord_lib.events +import bento_lib.events import pytest import redis import time @@ -16,7 +16,7 @@ TEST_EVENT_BODY = "test" -event_bus = chord_lib.events.EventBus() +event_bus = bento_lib.events.EventBus() def test_registration(): @@ -51,7 +51,7 @@ def handle_service_event(message): assert event["type"] == TEST_SERVICE_EVENT assert event["data"] == TEST_EVENT_BODY - event_bus.add_handler(chord_lib.events.ALL_SERVICE_EVENTS, handle_service_event) + event_bus.add_handler(bento_lib.events.ALL_SERVICE_EVENTS, handle_service_event) event_bus.start_event_loop() r = event_bus.publish_service_event(TEST_SERVICE, TEST_SERVICE_EVENT, TEST_EVENT_BODY) @@ -88,8 +88,8 @@ def handle_data_type_event(message): assert event["type"] == TEST_DATA_TYPE_EVENT assert event["data"] == TEST_EVENT_BODY - event_bus.add_handler(chord_lib.events.ALL_DATA_TYPE_EVENTS, handle_data_type_event) - r = event_bus.add_handler(chord_lib.events.ALL_DATA_TYPE_EVENTS, handle_data_type_event) + event_bus.add_handler(bento_lib.events.ALL_DATA_TYPE_EVENTS, handle_data_type_event) + r = event_bus.add_handler(bento_lib.events.ALL_DATA_TYPE_EVENTS, handle_data_type_event) assert not r event_bus.start_event_loop() @@ -109,7 +109,7 @@ def test_premature_stop(): def test_late_handler(): try: event_bus.start_event_loop() - r = event_bus.add_handler(chord_lib.events.ALL_SERVICE_EVENTS, lambda _: None) + r = event_bus.add_handler(bento_lib.events.ALL_SERVICE_EVENTS, lambda _: None) assert not r finally: event_bus.stop_event_loop() @@ -117,12 +117,12 @@ def test_late_handler(): def test_fake_event_bus(): global event_bus - chord_lib.events._connection_info = {"unix_socket_path": "/road/to/nowhere.sock"} + bento_lib.events._connection_info = {"unix_socket_path": "/road/to/nowhere.sock"} with pytest.raises(redis.exceptions.ConnectionError): - chord_lib.events.EventBus() + bento_lib.events.EventBus() - event_bus = chord_lib.events.EventBus(allow_fake=True) + event_bus = bento_lib.events.EventBus(allow_fake=True) test_registration() @@ -130,7 +130,7 @@ def test_fake_event_bus(): def handle_service_event(_message): pass - event_bus.add_handler(chord_lib.events.ALL_SERVICE_EVENTS, handle_service_event) + event_bus.add_handler(bento_lib.events.ALL_SERVICE_EVENTS, handle_service_event) event_bus.start_event_loop() r = event_bus.publish_service_event(TEST_SERVICE, TEST_SERVICE_EVENT, TEST_EVENT_BODY) @@ -143,11 +143,11 @@ def handle_service_event(_message): def test_notification_format(): - n = chord_lib.events.notifications.format_notification("test", "test2", "go_somewhere", "https://google.ca") + n = bento_lib.events.notifications.format_notification("test", "test2", "go_somewhere", "https://google.ca") assert isinstance(n, dict) assert len(list(n.keys())) == 4 assert n["title"] == "test" assert n["description"] == "test2" assert n["notification_type"] == "go_somewhere" assert n["action_target"] == "https://google.ca" - validate(n, chord_lib.events.types.EVENT_CREATE_NOTIFICATION_SCHEMA) + validate(n, bento_lib.events.types.EVENT_CREATE_NOTIFICATION_SCHEMA) diff --git a/tests/test_ingestion.py b/tests/test_ingestion.py index a22e9c2..f76d837 100644 --- a/tests/test_ingestion.py +++ b/tests/test_ingestion.py @@ -2,7 +2,7 @@ import os import pytest -from chord_lib.ingestion import ( +from bento_lib.ingestion import ( file_with_prefix, find_common_prefix, formatted_output, diff --git a/tests/test_platform_django.py b/tests/test_platform_django.py index 3754d2a..76c7b2e 100644 --- a/tests/test_platform_django.py +++ b/tests/test_platform_django.py @@ -8,12 +8,12 @@ @pytest.mark.django_db def test_remote_auth_backend(): - import chord_lib.auth.django_remote_user - from chord_lib.auth.headers import DJANGO_USER_HEADER, DJANGO_USER_ROLE_HEADER + import bento_lib.auth.django_remote_user + from bento_lib.auth.headers import DJANGO_USER_HEADER, DJANGO_USER_ROLE_HEADER from django.contrib.auth.models import User from django.http.request import HttpRequest - b = chord_lib.auth.django_remote_user.CHORDRemoteUserBackend() + b = bento_lib.auth.django_remote_user.BentoRemoteUserBackend() r = HttpRequest() r.META = { DJANGO_USER_HEADER: "test", diff --git a/tests/test_platform_flask.py b/tests/test_platform_flask.py index bdb58d0..d3995bb 100644 --- a/tests/test_platform_flask.py +++ b/tests/test_platform_flask.py @@ -1,5 +1,5 @@ -import chord_lib.auth.flask_decorators as fd -import chord_lib.responses.flask_errors as fe +import bento_lib.auth.flask_decorators as fd +import bento_lib.responses.flask_errors as fe import pytest from flask import Flask @@ -39,7 +39,7 @@ def test3(): def test_flask_forbidden_error(flask_client): # Turn CHORD permissions mode on to make sure we're getting real permissions checks - fd.CHORD_PERMISSIONS = True + fd.BENTO_PERMISSIONS = True # non-existent endpoint diff --git a/tests/test_responses.py b/tests/test_responses.py index 3ece94d..3f78d59 100644 --- a/tests/test_responses.py +++ b/tests/test_responses.py @@ -1,4 +1,4 @@ -import chord_lib.responses as responses +import bento_lib.responses as responses import json from dateutil.parser import isoparse diff --git a/tests/test_search.py b/tests/test_search.py index 202b1ca..70fd1ca 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -1,4 +1,4 @@ -from chord_lib.search import build_search_response, data_structure, operations, postgres, queries +from bento_lib.search import build_search_response, data_structure, operations, postgres, queries from datetime import datetime from pytest import raises diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 2f15464..8a4e7a3 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -1,6 +1,6 @@ import werkzeug.utils -from chord_lib import workflows +from bento_lib import workflows TEST_WORKFLOWS = { "ingestion": { diff --git a/tox.ini b/tox.ini index 2daab7b..8755714 100644 --- a/tox.ini +++ b/tox.ini @@ -6,5 +6,5 @@ exclude = .git,.tox,__pycache__ skip_install = true commands = pip install -r requirements.txt - pytest -svv --cov=chord_lib --cov-branch {posargs} - flake8 ./chord_lib ./tests + pytest -svv --cov=bento_lib --cov-branch {posargs} + flake8 ./bento_lib ./tests