From 64288b14f91f49718d3c68f26a5abd71df6a51d2 Mon Sep 17 00:00:00 2001 From: Clemens Rudert Date: Mon, 28 Oct 2024 13:53:43 +0100 Subject: [PATCH 1/8] initialize processor on server boot --- pyramid_oereb/__init__.py | 19 +++++++++++++++++++ pyramid_oereb/core/views/webservice.py | 9 ++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/pyramid_oereb/__init__.py b/pyramid_oereb/__init__.py index fa9af4ff4d..4ffa677f01 100644 --- a/pyramid_oereb/__init__.py +++ b/pyramid_oereb/__init__.py @@ -2,11 +2,14 @@ import logging +from pyramid.request import Request + from pyramid_oereb.core.adapter import DatabaseAdapter from pyramid_oereb.core.config import Config from pyramid.config import Configurator + log = logging.getLogger(__name__) route_prefix = None # initially instantiate database adapter for global session handling @@ -34,6 +37,7 @@ def includeme(config): Args: config (Configurator): The pyramid apps config object """ + from pyramid_oereb.core.processor import create_processor, Processor global route_prefix @@ -57,6 +61,21 @@ def includeme(config): settings.update({ 'pyramid_oereb': Config.get_config() }) + processor = None + try: + processor = create_processor() + except Exception as e: + log.error(f'Initialisation of processor failed with an error: {e}') + exit(1) + + def get_processor(request: Request) -> Processor: + return processor + + config.add_request_method( + get_processor, + 'pyramid_oereb_processor', + reify=True + ) config.add_renderer('pyramid_oereb_extract_json', 'pyramid_oereb.core.renderer.extract.json_.Renderer') config.add_renderer('pyramid_oereb_extract_xml', 'pyramid_oereb.core.renderer.extract.xml_.Renderer') diff --git a/pyramid_oereb/core/views/webservice.py b/pyramid_oereb/core/views/webservice.py index f60e3381a0..d12b19f793 100644 --- a/pyramid_oereb/core/views/webservice.py +++ b/pyramid_oereb/core/views/webservice.py @@ -16,7 +16,6 @@ from pyramid_oereb import Config from pyreproj import Reprojector -from pyramid_oereb.core.processor import create_processor from pyramid_oereb.core.readers.address import AddressReader from pyramid_oereb.core.renderer import Base as Renderer from timeit import default_timer as timer @@ -195,7 +194,7 @@ def _get_egrid_coord(self, params): Config.get('srid'), self.__parse_gnss__(gnss).wkt ) - processor = create_processor(real_estate_only=True) + processor = self._request.pyramid_oereb_processor return processor.real_estate_reader.read(params, **{'geometry': geom_wkt}) else: raise HTTPBadRequest('EN or GNSS must be defined.') @@ -214,7 +213,7 @@ def _get_egrid_ident(self, params): identdn = self._params.get('IDENTDN') number = self._params.get('NUMBER') if identdn and number: - processor = create_processor(real_estate_only=True) + processor = self._request.pyramid_oereb_processor return processor.real_estate_reader.read( params, **{ @@ -251,7 +250,7 @@ def _get_egrid_address(self, params): srid=Config.get('srid'), wkt=addresses[0].geom.wkt ) - processor = create_processor(real_estate_only=True) + processor = self._request.pyramid_oereb_processor return processor.real_estate_reader.read(params, **{'geometry': geometry}) else: raise HTTPBadRequest('POSTALCODE, LOCALISATION and NUMBER must be defined.') @@ -267,7 +266,7 @@ def get_extract_by_id(self): log.debug("get_extract_by_id() start") try: params = self.__validate_extract_params__() - processor = create_processor() + processor = self._request.pyramid_oereb_processor # read the real estate from configured source by the passed parameters real_estate_reader = processor.real_estate_reader if params.egrid: From e749ab26ed4b8c89cdba1438e0bdfeaff676d4e4 Mon Sep 17 00:00:00 2001 From: Clemens Rudert Date: Mon, 28 Oct 2024 13:53:43 +0100 Subject: [PATCH 2/8] initialize processor on server boot --- pyramid_oereb/__init__.py | 19 +++++++++++++++++++ pyramid_oereb/core/views/webservice.py | 9 ++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/pyramid_oereb/__init__.py b/pyramid_oereb/__init__.py index fa9af4ff4d..4ffa677f01 100644 --- a/pyramid_oereb/__init__.py +++ b/pyramid_oereb/__init__.py @@ -2,11 +2,14 @@ import logging +from pyramid.request import Request + from pyramid_oereb.core.adapter import DatabaseAdapter from pyramid_oereb.core.config import Config from pyramid.config import Configurator + log = logging.getLogger(__name__) route_prefix = None # initially instantiate database adapter for global session handling @@ -34,6 +37,7 @@ def includeme(config): Args: config (Configurator): The pyramid apps config object """ + from pyramid_oereb.core.processor import create_processor, Processor global route_prefix @@ -57,6 +61,21 @@ def includeme(config): settings.update({ 'pyramid_oereb': Config.get_config() }) + processor = None + try: + processor = create_processor() + except Exception as e: + log.error(f'Initialisation of processor failed with an error: {e}') + exit(1) + + def get_processor(request: Request) -> Processor: + return processor + + config.add_request_method( + get_processor, + 'pyramid_oereb_processor', + reify=True + ) config.add_renderer('pyramid_oereb_extract_json', 'pyramid_oereb.core.renderer.extract.json_.Renderer') config.add_renderer('pyramid_oereb_extract_xml', 'pyramid_oereb.core.renderer.extract.xml_.Renderer') diff --git a/pyramid_oereb/core/views/webservice.py b/pyramid_oereb/core/views/webservice.py index f60e3381a0..d12b19f793 100644 --- a/pyramid_oereb/core/views/webservice.py +++ b/pyramid_oereb/core/views/webservice.py @@ -16,7 +16,6 @@ from pyramid_oereb import Config from pyreproj import Reprojector -from pyramid_oereb.core.processor import create_processor from pyramid_oereb.core.readers.address import AddressReader from pyramid_oereb.core.renderer import Base as Renderer from timeit import default_timer as timer @@ -195,7 +194,7 @@ def _get_egrid_coord(self, params): Config.get('srid'), self.__parse_gnss__(gnss).wkt ) - processor = create_processor(real_estate_only=True) + processor = self._request.pyramid_oereb_processor return processor.real_estate_reader.read(params, **{'geometry': geom_wkt}) else: raise HTTPBadRequest('EN or GNSS must be defined.') @@ -214,7 +213,7 @@ def _get_egrid_ident(self, params): identdn = self._params.get('IDENTDN') number = self._params.get('NUMBER') if identdn and number: - processor = create_processor(real_estate_only=True) + processor = self._request.pyramid_oereb_processor return processor.real_estate_reader.read( params, **{ @@ -251,7 +250,7 @@ def _get_egrid_address(self, params): srid=Config.get('srid'), wkt=addresses[0].geom.wkt ) - processor = create_processor(real_estate_only=True) + processor = self._request.pyramid_oereb_processor return processor.real_estate_reader.read(params, **{'geometry': geometry}) else: raise HTTPBadRequest('POSTALCODE, LOCALISATION and NUMBER must be defined.') @@ -267,7 +266,7 @@ def get_extract_by_id(self): log.debug("get_extract_by_id() start") try: params = self.__validate_extract_params__() - processor = create_processor() + processor = self._request.pyramid_oereb_processor # read the real estate from configured source by the passed parameters real_estate_reader = processor.real_estate_reader if params.egrid: From c82e2058b8a92b804ae5462e7031918b8a0e62ed Mon Sep 17 00:00:00 2001 From: Clemens Rudert Date: Wed, 11 Dec 2024 08:57:16 +0100 Subject: [PATCH 3/8] fix oereblex caching issue --- .../contrib/data_sources/oereblex/sources/plr_oereblex.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyramid_oereb/contrib/data_sources/oereblex/sources/plr_oereblex.py b/pyramid_oereb/contrib/data_sources/oereblex/sources/plr_oereblex.py index 61bbf7cf35..007e12ada8 100644 --- a/pyramid_oereb/contrib/data_sources/oereblex/sources/plr_oereblex.py +++ b/pyramid_oereb/contrib/data_sources/oereblex/sources/plr_oereblex.py @@ -131,3 +131,8 @@ def collect_related_geometries_by_real_estate(self, session, real_estate): selectinload(self.models.Geometry.public_law_restriction) .selectinload(self.models.PublicLawRestriction.responsible_office), ).all() + + def read(self, params, real_estate, bbox): + # resetting the local per request cache of queried oereblex links + self._queried_geolinks = {} + return super(DatabaseOEREBlexSource, self).read(params, real_estate, bbox) From 3be27fd587670a85baec0051c3399702227d5df9 Mon Sep 17 00:00:00 2001 From: Clemens Rudert Date: Wed, 11 Dec 2024 09:06:59 +0100 Subject: [PATCH 4/8] fix lint issues --- pyproject.toml | 2 +- pyramid_oereb/__init__.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6002099fb5..2ffe8cdc93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,7 +109,7 @@ create_legend_entries = "pyramid_oereb.contrib.data_sources.standard.load_legend create_stats_tables = "pyramid_oereb.contrib.stats.scripts.create_stats_tables:create_stats_tables" [tool.flake8] -exclude = [".venv", "tests/init_db.py"] +exclude = [".venv", "tests/init_db.py", ".build"] max-line-length = 110 [tool.pytest.ini_options] diff --git a/pyramid_oereb/__init__.py b/pyramid_oereb/__init__.py index 4ffa677f01..2bfff03a6b 100644 --- a/pyramid_oereb/__init__.py +++ b/pyramid_oereb/__init__.py @@ -9,7 +9,6 @@ from pyramid.config import Configurator - log = logging.getLogger(__name__) route_prefix = None # initially instantiate database adapter for global session handling From d4bd2459936afc2095fe5b17e967188c0c9dc2da Mon Sep 17 00:00:00 2001 From: Clemens Rudert Date: Mon, 16 Dec 2024 14:16:43 +0100 Subject: [PATCH 5/8] make oereblex cache thred safe --- .../oereblex/sources/plr_oereblex.py | 16 ++++++++++------ pyramid_oereb/core/views/webservice.py | 4 ++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pyramid_oereb/contrib/data_sources/oereblex/sources/plr_oereblex.py b/pyramid_oereb/contrib/data_sources/oereblex/sources/plr_oereblex.py index 007e12ada8..709885d8a6 100644 --- a/pyramid_oereb/contrib/data_sources/oereblex/sources/plr_oereblex.py +++ b/pyramid_oereb/contrib/data_sources/oereblex/sources/plr_oereblex.py @@ -97,15 +97,15 @@ def document_records_from_oereblex(self, params, geolink, law_status, oereblex_p log.debug("document_records_from_oereblex() start, GEO-Link {}, law status {}, oereblex_params {}" .format(geolink, law_status.code, oereblex_params)) identifier = '{}{}{}'.format(geolink, law_status.code, params.language) - if identifier in self._queried_geolinks: + if identifier in self._queried_geolinks[params.identifier]: log.debug('skip querying this geolink "{}" because it was fetched already.'.format(identifier)) log.debug('use already queried instead') else: self._oereblex_source.read(params, geolink, law_status, oereblex_params) log.debug("document_records_from_oereblex() returning {} records" .format(len(self._oereblex_source.records))) - self._queried_geolinks[identifier] = self._oereblex_source.records - return self._queried_geolinks[identifier] + self._queried_geolinks[params.identifier][identifier] = self._oereblex_source.records + return self._queried_geolinks[params.identifier][identifier] def collect_related_geometries_by_real_estate(self, session, real_estate): """ @@ -133,6 +133,10 @@ def collect_related_geometries_by_real_estate(self, session, real_estate): ).all() def read(self, params, real_estate, bbox): - # resetting the local per request cache of queried oereblex links - self._queried_geolinks = {} - return super(DatabaseOEREBlexSource, self).read(params, real_estate, bbox) + # adding a local cache depending on the request identifier + self._queried_geolinks[params.identifier] = {} + # calling the original logic + result = super(DatabaseOEREBlexSource, self).read(params, real_estate, bbox) + # removing the cache after work is done + del self._queried_geolinks[params.identifier] + return result diff --git a/pyramid_oereb/core/views/webservice.py b/pyramid_oereb/core/views/webservice.py index d12b19f793..f1dc896a3d 100644 --- a/pyramid_oereb/core/views/webservice.py +++ b/pyramid_oereb/core/views/webservice.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import logging +from uuid import uuid4 + # import yappi import qrcode import io @@ -637,6 +639,8 @@ def __init__(self, response_format, with_geometry=False, images=False, signed=Fa self.__topics__ = topics self.__extract_url__ = extract_url self.__qr_code_ref__ = qr_code_ref + #uniquely identifier to reference the original request in the pyramid_oereb system + self.identifier = str(uuid4()) def set_identdn(self, identdn): """ From a7ecfc1e1071ce8b33a4c44cab0ed88954b9abbf Mon Sep 17 00:00:00 2001 From: Clemens Rudert Date: Mon, 16 Dec 2024 14:24:23 +0100 Subject: [PATCH 6/8] fix lint issue --- pyramid_oereb/core/views/webservice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyramid_oereb/core/views/webservice.py b/pyramid_oereb/core/views/webservice.py index f1dc896a3d..0d42e861b6 100644 --- a/pyramid_oereb/core/views/webservice.py +++ b/pyramid_oereb/core/views/webservice.py @@ -639,7 +639,7 @@ def __init__(self, response_format, with_geometry=False, images=False, signed=Fa self.__topics__ = topics self.__extract_url__ = extract_url self.__qr_code_ref__ = qr_code_ref - #uniquely identifier to reference the original request in the pyramid_oereb system + # uniquely identifier to reference the original request in the pyramid_oereb system self.identifier = str(uuid4()) def set_identdn(self, identdn): From e196190fed2aa0c9bad908db2131754c2130b0a7 Mon Sep 17 00:00:00 2001 From: Clemens Rudert Date: Tue, 14 Jan 2025 12:47:07 +0100 Subject: [PATCH 7/8] fix lint --- tests/mockrequest.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/mockrequest.py b/tests/mockrequest.py index fbc939d04b..e94800bf37 100644 --- a/tests/mockrequest.py +++ b/tests/mockrequest.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from pyramid.testing import DummyRequest +from pyramid_oereb.core.processor import create_processor from pyramid_oereb.core.views.webservice import Parameter @@ -9,11 +10,19 @@ def __init__(self): super(MockParameter, self).__init__('JSON', with_geometry=False, images=False) +class MockProcessor(): + + @property + def real_estate_reader(self): + return + + class MockRequest(DummyRequest): def __init__(self, current_route_url=None): super(MockRequest, self).__init__() self._current_route_url = current_route_url + self.pyramid_oereb_processor = create_processor() def current_route_url(self, *elements, **kw): if self._current_route_url: From f05ed18742ef6c5c5a9f7d87a0c40e6adad04150 Mon Sep 17 00:00:00 2001 From: Clemens Rudert Date: Tue, 14 Jan 2025 12:55:52 +0100 Subject: [PATCH 8/8] fix tests --- .../contrib.data_sources.oereblex/sources/test_plr_oereblex.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/contrib.data_sources.oereblex/sources/test_plr_oereblex.py b/tests/contrib.data_sources.oereblex/sources/test_plr_oereblex.py index c84330e723..2c0e9b442a 100644 --- a/tests/contrib.data_sources.oereblex/sources/test_plr_oereblex.py +++ b/tests/contrib.data_sources.oereblex/sources/test_plr_oereblex.py @@ -710,6 +710,9 @@ def test_document_records_from_oereblex(plr_source_params, document_records, par from pyramid_oereb.contrib.data_sources.oereblex.sources.plr_oereblex import DatabaseOEREBlexSource source = DatabaseOEREBlexSource(**plr_source_params) + # since we patch the original read method out of the way, we need to mimikri the logic encapsulated + # there + source._queried_geolinks[params.identifier] = {} assert source.document_records_from_oereblex( params, 1,