From ebb2ceb75d18f071f87d5b62421233fd80ba3813 Mon Sep 17 00:00:00 2001 From: Margriet Palm Date: Thu, 19 Dec 2024 12:41:53 +0100 Subject: [PATCH 1/6] Change CRS in all tests to 28892 --- pyproject.toml | 2 +- threedi_modelchecker/checks/geo_query.py | 2 +- threedi_modelchecker/checks/other.py | 2 +- threedi_modelchecker/checks/raster.py | 12 +- threedi_modelchecker/config.py | 13 -- threedi_modelchecker/tests/conftest.py | 3 +- threedi_modelchecker/tests/factories.py | 48 +++--- .../tests/test_checks_base.py | 59 ++------ .../tests/test_checks_location.py | 134 +++++++---------- .../tests/test_checks_other.py | 140 ++++++++---------- 10 files changed, 164 insertions(+), 251 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 00ae03db..d49f8ebd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ "Click", "GeoAlchemy2>=0.9,!=0.11.*", "SQLAlchemy>=1.4", - "threedi-schema @ git+https://github.com/nens/threedi-schema.git@margriet_schema_300_leftovers" + "threedi-schema @ git+https://github.com/nens/threedi-schema.git@margriet_111_reproject_schematisation" ] [project.optional-dependencies] diff --git a/threedi_modelchecker/checks/geo_query.py b/threedi_modelchecker/checks/geo_query.py index 50750c10..324ea1d5 100644 --- a/threedi_modelchecker/checks/geo_query.py +++ b/threedi_modelchecker/checks/geo_query.py @@ -13,7 +13,7 @@ def epsg_code_query(): def transform(col): - return geo_func.ST_Transform(col, epsg_code_query()) + return geo_func.ST_Transform(col, DEFAULT_EPSG) def distance(col_1, col_2): diff --git a/threedi_modelchecker/checks/other.py b/threedi_modelchecker/checks/other.py index bd99e9b8..82e2fc20 100644 --- a/threedi_modelchecker/checks/other.py +++ b/threedi_modelchecker/checks/other.py @@ -324,7 +324,7 @@ def get_invalid(self, session: Session) -> List[NamedTuple]: f"""SELECT * FROM connection_node AS cn1, connection_node AS cn2 WHERE - distance(cn1.geom, cn2.geom, 1) < :min_distance + distance(cn1.geom, cn2.geom) < :min_distance AND cn1.ROWID != cn2.ROWID AND cn2.ROWID IN ( SELECT ROWID diff --git a/threedi_modelchecker/checks/raster.py b/threedi_modelchecker/checks/raster.py index 0fcc1f3a..fea4fc8f 100644 --- a/threedi_modelchecker/checks/raster.py +++ b/threedi_modelchecker/checks/raster.py @@ -171,11 +171,13 @@ class RasterHasMatchingEPSGCheck(BaseRasterCheck): """Check whether a raster's EPSG code matches the EPSG code in the global settings for the SQLite.""" def get_invalid(self, session): - epsg_code_query = session.query(models.ModelSettings.epsg_code).first() - if epsg_code_query is not None: - self.epsg_code = epsg_code_query[0] - else: - self.epsg_code = None + # TODO: fix this + # epsg_code_query = session.query(models.ModelSettings.epsg_code).first() + # if epsg_code_query is not None: + # self.epsg_code = epsg_code_query[0] + # else: + # self.epsg_code = None + self.epsg_code = 28992 return super().get_invalid(session) def is_valid(self, path: str, interface_cls: Type[RasterInterface]): diff --git a/threedi_modelchecker/config.py b/threedi_modelchecker/config.py index 58887a4c..67021ff1 100644 --- a/threedi_modelchecker/config.py +++ b/threedi_modelchecker/config.py @@ -1579,19 +1579,6 @@ def is_none_or_empty(col): column=models.ModelSettings.manhole_aboveground_storage_area, min_value=0, ), - QueryCheck( - error_code=317, - column=models.ModelSettings.epsg_code, - invalid=CONDITIONS["has_no_dem"].filter(models.ModelSettings.epsg_code == None), - message="model_settings.epsg_code may not be NULL if no dem file is provided", - ), - QueryCheck( - error_code=318, - level=CheckLevel.WARNING, - column=models.ModelSettings.epsg_code, - invalid=CONDITIONS["has_dem"].filter(models.ModelSettings.epsg_code == None), - message="if model_settings.epsg_code is NULL, it will be extracted from the DEM later, however, the modelchecker will use ESPG:28992 for its spatial checks", - ), QueryCheck( error_code=319, column=models.ModelSettings.use_2d_flow, diff --git a/threedi_modelchecker/tests/conftest.py b/threedi_modelchecker/tests/conftest.py index b5dbee94..d194a04d 100644 --- a/threedi_modelchecker/tests/conftest.py +++ b/threedi_modelchecker/tests/conftest.py @@ -24,7 +24,7 @@ def threedi_db(tmpdir_factory): shutil.copyfile(data_dir / "empty.sqlite", tmp_sqlite) db = ThreediDatabase(tmp_sqlite) schema = ModelSchema(db) - schema.upgrade(backup=False, upgrade_spatialite_version=False) + schema.upgrade(backup=False, upgrade_spatialite_version=False, custom_epsg_code=28992) schema.set_spatial_indexes() return db @@ -40,7 +40,6 @@ def session(threedi_db): :return: sqlalchemy.orm.session.Session """ s = threedi_db.get_session(future=True) - factories.inject_session(s) s.model_checker_context = LocalContext(base_path=data_dir) diff --git a/threedi_modelchecker/tests/factories.py b/threedi_modelchecker/tests/factories.py index 5086fbc3..1abe8a1e 100644 --- a/threedi_modelchecker/tests/factories.py +++ b/threedi_modelchecker/tests/factories.py @@ -1,9 +1,17 @@ + from inspect import isclass import factory from factory import Faker from threedi_schema import constants, models +# Default geometry strings + +DEFAULT_POINT = "SRID=28992;POINT (142742 473443)" +# line from DEFAULT_POINT to another point +DEFAULT_LINE = "SRID=28992;LINESTRING (142742 473443, 142747 473448)" +# polygon containing DEFAULT_POINT +DEFAULT_POLYGON = "SRID=28992;POLYGON ((142742 473443, 142743 473443, 142744 473444, 142742 473444, 142742 473443))" def inject_session(session): """Inject the session into all factories""" @@ -47,7 +55,7 @@ class Meta: sqlalchemy_session = None code = Faker("name") - geom = "SRID=4326;POINT(-71.064544 42.28787)" + geom = DEFAULT_POINT class ChannelFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -58,7 +66,7 @@ class Meta: display_name = Faker("name") code = Faker("name") exchange_type = constants.CalculationType.CONNECTED - geom = "SRID=4326;LINESTRING(-71.064544 42.28787, -71.0645 42.287)" + geom = DEFAULT_LINE class WeirFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -75,7 +83,7 @@ class Meta: sewerage = False connection_node_id_start = 1 connection_node_id_end = 1 - geom = "SRID=4326;LINESTRING(4.885534714757985 52.38513158257129,4.88552805617346 52.38573773758626)" + geom = DEFAULT_LINE class BoundaryConditions2DFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -86,7 +94,7 @@ class Meta: type = constants.BoundaryType.WATERLEVEL.value timeseries = "0,-0.5" display_name = Faker("name") - geom = "SRID=4326;LINESTRING(4.885534714757985 52.38513158257129,4.88552805617346 52.38573773758626)" + geom = DEFAULT_LINE class BoundaryConditions1DFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -97,7 +105,7 @@ class Meta: type = constants.BoundaryType.WATERLEVEL timeseries = "0,-0.5" connection_node_id = 1 - geom = "SRID=4326;POINT(-71.064544 42.28787)" + geom = DEFAULT_POINT class PumpMapFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -105,7 +113,7 @@ class Meta: model = models.PumpMap sqlalchemy_session = None - geom = "SRID=4326;POINT(-71.064544 42.28787)" + geom = DEFAULT_POINT class PumpFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -120,7 +128,7 @@ class Meta: start_level = 1.0 lower_stop_level = 0.0 capacity = 5.0 - geom = "SRID=4326;POINT(-71.064544 42.28787)" + geom = DEFAULT_POINT class CrossSectionLocationFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -132,7 +140,7 @@ class Meta: reference_level = 0.0 friction_type = constants.FrictionType.CHEZY friction_value = 0.0 - geom = "SRID=4326;POINT(-71.064544 42.28787)" + geom = DEFAULT_POINT class AggregationSettingsFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -163,7 +171,7 @@ class Meta: timeseries = "0,-0.1" connection_node_id = 1 - geom = "SRID=4326;POINT(-71.064544 42.28787)" + geom = DEFAULT_POINT class Lateral2DFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -172,7 +180,7 @@ class Meta: sqlalchemy_session = None timeseries = "0,-0.2" - geom = "SRID=4326;POINT(-71.064544 42.28787)" + geom = DEFAULT_POINT type = constants.Later2dType.SURFACE @@ -195,7 +203,7 @@ class Meta: model = models.DryWeatherFlowMap sqlalchemy_session = None - geom = "SRID=4326;POINT(-71.064544 42.28787)" + geom = DEFAULT_POINT class DryWeatherFlowFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -203,7 +211,7 @@ class Meta: model = models.DryWeatherFlow sqlalchemy_session = None - geom = "SRID=4326;POLYGON((30 10, 40 40, 20 40, 10 20, 30 10))" + geom = DEFAULT_POLYGON class DryWeatherFlowDistributionFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -223,7 +231,7 @@ class Meta: sqlalchemy_session = None area = 0.0 - geom = "SRID=4326;POLYGON((30 10, 40 40, 20 40, 10 20, 30 10))" + geom = DEFAULT_POLYGON # surface_parameters = factory.SubFactory(SurfaceParameterFactory) @@ -234,7 +242,7 @@ class Meta: sqlalchemy_session = None percentage = 100.0 - geom = "SRID=4326;LINESTRING(30 10, 10 30, 40 40)" + geom = DEFAULT_LINE class TagsFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -253,7 +261,7 @@ class Meta: measure_operator = constants.MeasureOperators.greater_than target_type = constants.StructureControlTypes.channel target_id = 10 - geom = "SRID=4326;POINT(-71.064544 42.28787)" + geom = DEFAULT_POINT class ControlMemoryFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -270,7 +278,7 @@ class Meta: is_active = True upper_threshold = 1.0 lower_threshold = -1.0 - geom = "SRID=4326;POINT(-71.064544 42.28787)" + geom = DEFAULT_POINT class ControlMeasureMapFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -279,7 +287,7 @@ class Meta: sqlalchemy_session = None control_type = constants.MeasureVariables.waterlevel - geom = "SRID=4326;LINESTRING(30 10, 10 30, 40 40)" + geom = DEFAULT_LINE class ControlMeasureLocationFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -287,7 +295,7 @@ class Meta: model = models.ControlMeasureLocation sqlalchemy_session = None - geom = "SRID=4326;POINT(-71.064544 42.28787)" + geom = DEFAULT_POINT class CulvertFactory(factory.alchemy.SQLAlchemyModelFactory): @@ -298,7 +306,7 @@ class Meta: code = "code" display_name = Faker("name") exchange_type = constants.CalculationTypeCulvert.ISOLATED_NODE - geom = "SRID=4326;LINESTRING(-71.064544 42.28787, -71.0645 42.287)" + geom = DEFAULT_LINE friction_value = 0.03 friction_type = 2 invert_level_start = 0.1 @@ -314,7 +322,7 @@ class Meta: display_name = Faker("name") code = "code" - geom = "SRID=4326;LINESTRING(-71.06452 42.2874, -71.06452 42.286)" + geom = DEFAULT_LINE class VegetationDragFactory(factory.alchemy.SQLAlchemyModelFactory): diff --git a/threedi_modelchecker/tests/test_checks_base.py b/threedi_modelchecker/tests/test_checks_base.py index 18a48cef..a5b596c5 100644 --- a/threedi_modelchecker/tests/test_checks_base.py +++ b/threedi_modelchecker/tests/test_checks_base.py @@ -158,7 +158,6 @@ def test_unique_check_multiple_description(): def test_all_equal_check(session): factories.ModelSettingsFactory(minimum_table_step_size=0.5) factories.ModelSettingsFactory(minimum_table_step_size=0.5) - check = AllEqualCheck(models.ModelSettings.minimum_table_step_size) invalid_rows = check.get_invalid(session) assert len(invalid_rows) == 0 @@ -310,7 +309,7 @@ def test_type_check_boolean(session): def test_geometry_check(session): - factories.ConnectionNodeFactory(geom="SRID=4326;POINT(-371.064544 42.28787)") + factories.ConnectionNodeFactory(geom=factories.DEFAULT_POINT) geometry_check = GeometryCheck(models.ConnectionNode.geom) invalid_rows = geometry_check.get_invalid(session) @@ -319,9 +318,7 @@ def test_geometry_check(session): def test_geometry_type_check(session): - factories.ConnectionNodeFactory.create_batch( - 2, geom="SRID=4326;POINT(-71.064544 42.28787)" - ) + factories.ConnectionNodeFactory.create_batch(2, geom=factories.DEFAULT_POINT) geometry_type_check = GeometryTypeCheck(models.ConnectionNode.geom) invalid_rows = geometry_type_check.get_invalid(session) @@ -532,58 +529,28 @@ def test_global_settings_use_1d_flow_and_no_1d_elements(session): assert len(errors) == 0 -def test_length_geom_linestring_in_4326(session): - factories.ModelSettingsFactory(epsg_code=28992) - channel_too_short = factories.ChannelFactory( - geom="SRID=4326;LINESTRING(" - "-0.38222938832999598 -0.13872236685816669, " - "-0.38222930900909202 -0.13872236685816669)", - ) - factories.ChannelFactory( - geom="SRID=4326;LINESTRING(" - "-0.38222938468305784 -0.13872235682908687, " - "-0.38222931083256106 -0.13872235591735235, " - "-0.38222930992082654 -0.13872207236791409, " - "-0.38222940929989008 -0.13872235591735235)", - ) - +def test_length_geom_linestring(session): + # TODO: reconsider this check because it tests geo_query.length, + # not query_check so this should not be here! + factories.ModelSettingsFactory() + x, y = [ + float(coord) + for coord in factories.DEFAULT_LINE.split("(")[1].split(",")[0].split() + ] + short_line = f"SRID=28992;LINESTRING({x} {y}, {x + 0.03} {y + 0.03})" + channel_too_short = factories.ChannelFactory(geom=short_line) + factories.ChannelFactory(geom=factories.DEFAULT_LINE) q = Query(models.Channel).filter(geo_query.length(models.Channel.geom) < 0.05) check_length_linestring = QueryCheck( column=models.Channel.geom, invalid=q, message="Length of the channel is too short, should be at least 0.05m", ) - errors = check_length_linestring.get_invalid(session) assert len(errors) == 1 assert errors[0].id == channel_too_short.id -def test_length_geom_linestring_missing_epsg_from_global_settings(session): - factories.ChannelFactory( - geom="SRID=4326;LINESTRING(" - "-0.38222938832999598 -0.13872236685816669, " - "-0.38222930900909202 -0.13872236685816669)", - ) - factories.ChannelFactory( - geom="SRID=4326;LINESTRING(" - "-0.38222938468305784 -0.13872235682908687, " - "-0.38222931083256106 -0.13872235591735235, " - "-0.38222930992082654 -0.13872207236791409, " - "-0.38222940929989008 -0.13872235591735235)", - ) - - q = Query(models.Channel).filter(geo_query.length(models.Channel.geom) < 0.05) - check_length_linestring = QueryCheck( - column=models.Channel.geom, - invalid=q, - message="Length of the channel is too short, should be at least 0.05m", - ) - - errors = check_length_linestring.get_invalid(session) - assert len(errors) == 1 - - @pytest.mark.parametrize( "min_value,max_value,left_inclusive,right_inclusive", [ diff --git a/threedi_modelchecker/tests/test_checks_location.py b/threedi_modelchecker/tests/test_checks_location.py index af3c3572..668f0460 100644 --- a/threedi_modelchecker/tests/test_checks_location.py +++ b/threedi_modelchecker/tests/test_checks_location.py @@ -12,51 +12,47 @@ ) from threedi_modelchecker.tests import factories +SRID = 28992 +POINT1 = "142742 473443" +POINT2 = "142743 473443" # POINT1 + 1 meter +POINT3 = "142747 473443" # POINT1 + 5 meter -def test_point_location_check(session): - factories.ChannelFactory( - id=1, - geom="SRID=4326;LINESTRING(5.387204 52.155172, 5.387204 52.155262)", - ) - factories.CrossSectionLocationFactory( - channel_id=1, geom="SRID=4326;POINT(5.387204 52.155200)" - ) - factories.CrossSectionLocationFactory( - channel_id=1, geom="SRID=4326;POINT(5.387218 52.155244)" + +@pytest.mark.parametrize( + "bc_geom, nof_invalid", [(POINT1, 0), (POINT2, 0), (POINT3, 1)] +) +def test_point_location_check(session, bc_geom, nof_invalid): + factories.ConnectionNodeFactory(id=0, geom=f"SRID={SRID};POINT({POINT1})") + factories.BoundaryConditions1DFactory( + id=0, connection_node_id=0, geom=f"SRID={SRID};POINT({bc_geom})" ) errors = PointLocationCheck( - column=models.CrossSectionLocation.geom, - ref_column=models.CrossSectionLocation.channel_id, - ref_table=models.Channel, - max_distance=0.1, + column=models.BoundaryCondition1D.geom, + ref_column=models.BoundaryCondition1D.connection_node_id, + ref_table=models.ConnectionNode, + max_distance=1, ).get_invalid(session) - assert len(errors) == 1 + assert len(errors) == nof_invalid @pytest.mark.parametrize( "channel_geom, nof_invalid", [ - ("LINESTRING(5.387204 52.155172, 5.387204 52.155262)", 0), - ("LINESTRING(5.387218 52.155172, 5.387218 52.155262)", 0), # within tolerance - ("LINESTRING(5.387204 52.155262, 5.387204 52.155172)", 0), # reversed - ( - "LINESTRING(5.387218 52.155262, 5.387218 52.155172)", - 0, - ), # reversed, within tolerance - ( - "LINESTRING(5.387204 52.164151, 5.387204 52.155262)", - 1, - ), # startpoint is wrong - ("LINESTRING(5.387204 52.155172, 5.387204 52.164151)", 1), # endpoint is wrong + (f"LINESTRING({POINT1}, {POINT3})", 0), + (f"LINESTRING({POINT2}, {POINT3})", 0), # within tolerance + (f"LINESTRING({POINT3}, {POINT1})", 0), # reversed + (f"LINESTRING({POINT3}, {POINT2})", 0), # within tolerance, within tolerance + (f"LINESTRING(142732 473443, {POINT3})", 1), # startpoint is wrong + (f"LINESTRING({POINT1}, 142752 473443)", 1), # endpoint is wrong ], ) def test_linestring_location_check(session, channel_geom, nof_invalid): - factories.ConnectionNodeFactory(id=1, geom="SRID=4326;POINT(5.387204 52.155172)") - factories.ConnectionNodeFactory(id=2, geom="SRID=4326;POINT(5.387204 52.155262)") + factories.ConnectionNodeFactory(id=1, geom=f"SRID={SRID};POINT({POINT1})") + factories.ConnectionNodeFactory(id=2, geom=f"SRID={SRID};POINT({POINT3})") factories.ChannelFactory( connection_node_id_start=1, connection_node_id_end=2, - geom=f"SRID=4326;{channel_geom}", + geom=f"SRID={SRID};{channel_geom}", ) errors = LinestringLocationCheck( column=models.Channel.geom, @@ -72,20 +68,17 @@ def test_linestring_location_check(session, channel_geom, nof_invalid): @pytest.mark.parametrize( "channel_geom, nof_invalid", [ - ("LINESTRING(5.387204 52.155172, 5.387204 52.155262)", 0), - ( - "LINESTRING(5.387204 52.164151, 5.387204 52.155262)", - 1, - ), # startpoint is wrong + (f"LINESTRING({POINT1}, {POINT3})", 0), + (f"LINESTRING(142732 473443, {POINT3})", 1), # startpoint is wrong ], ) def test_connection_node_linestring_location_check(session, channel_geom, nof_invalid): - factories.ConnectionNodeFactory(id=1, geom="SRID=4326;POINT(5.387204 52.155172)") - factories.ConnectionNodeFactory(id=2, geom="SRID=4326;POINT(5.387204 52.155262)") + factories.ConnectionNodeFactory(id=1, geom=f"SRID={SRID};POINT({POINT1})") + factories.ConnectionNodeFactory(id=2, geom=f"SRID={SRID};POINT({POINT3})") factories.ChannelFactory( connection_node_id_start=1, connection_node_id_end=2, - geom=f"SRID=4326;{channel_geom}", + geom=f"SRID={SRID};{channel_geom}", ) errors = ConnectionNodeLinestringLocationCheck( column=models.Channel.geom, max_distance=1.01 @@ -96,11 +89,8 @@ def test_connection_node_linestring_location_check(session, channel_geom, nof_in @pytest.mark.parametrize( "channel_geom, nof_invalid", [ - ("LINESTRING(5.387204 52.155172, 5.387204 52.155262)", 0), - ( - "LINESTRING(5.387204 52.164151, 5.387204 52.155262)", - 1, - ), # startpoint is wrong + (f"LINESTRING({POINT1}, {POINT3})", 0), + (f"LINESTRING(142732 473443, {POINT3})", 1), # startpoint is wrong ], ) @pytest.mark.parametrize( @@ -110,22 +100,20 @@ def test_connection_node_linestring_location_check(session, channel_geom, nof_in def test_control_measure_map_linestring_map_location_check( session, control_table, control_type, channel_geom, nof_invalid ): - factories.ControlMeasureLocationFactory( - id=1, geom="SRID=4326;POINT(5.387204 52.155172)" - ) - factories.ControlMemoryFactory(id=1, geom="SRID=4326;POINT(5.387204 52.155262)") - factories.ControlTableFactory(id=1, geom="SRID=4326;POINT(5.387204 52.155262)") + factories.ControlMeasureLocationFactory(id=1, geom=f"SRID={SRID};POINT({POINT1})") + factories.ControlMemoryFactory(id=1, geom=f"SRID={SRID};POINT({POINT3})") + factories.ControlTableFactory(id=1, geom=f"SRID={SRID};POINT({POINT3})") factories.ControlMeasureMapFactory( measure_location_id=1, control_id=1, control_type="memory", - geom=f"SRID=4326;{channel_geom}", + geom=f"SRID={SRID};{channel_geom}", ) factories.ControlMeasureMapFactory( measure_location_id=1, control_id=1, control_type="table", - geom=f"SRID=4326;{channel_geom}", + geom=f"SRID={SRID};{channel_geom}", ) errors = ControlMeasureMapLinestringMapLocationCheck( control_table=control_table, @@ -138,23 +126,19 @@ def test_control_measure_map_linestring_map_location_check( @pytest.mark.parametrize( "channel_geom, nof_invalid", [ - ("LINESTRING(5.387204 52.155172, 5.387204 52.155262)", 0), - ( - "LINESTRING(5.387204 52.164151, 5.387204 52.155262)", - 1, - ), # startpoint is wrong + (f"LINESTRING({POINT1}, {POINT3})", 0), + (f"LINESTRING(142732 473443, {POINT3})", 1), # startpoint is wrong ], ) def test_dwf_map_linestring_map_location_check(session, channel_geom, nof_invalid): - factories.ConnectionNodeFactory(id=1, geom="SRID=4326;POINT(5.387204 52.155172)") + factories.ConnectionNodeFactory(id=1, geom=f"SRID={SRID};POINT({POINT1})") + factories.DryWeatherFlowFactory( id=1, - geom="SRID=4326;POLYGON((5.387204 52.155262, 5.387204 52.155172, 5.387304 52.155172, 5.387304 52.155262, 5.387204 52.155262))", + geom=f"SRID={SRID};POLYGON(({POINT1}, {POINT3}, 142742 473445, {POINT1}))", ) factories.DryWheatherFlowMapFactory( - connection_node_id=1, - dry_weather_flow_id=1, - geom=f"SRID=4326;{channel_geom}", + connection_node_id=1, dry_weather_flow_id=1, geom=f"SRID={SRID};{channel_geom}" ) errors = DWFMapLinestringLocationCheck(max_distance=1.01).get_invalid(session) assert len(errors) == nof_invalid @@ -163,20 +147,15 @@ def test_dwf_map_linestring_map_location_check(session, channel_geom, nof_invali @pytest.mark.parametrize( "channel_geom, nof_invalid", [ - ("LINESTRING(5.387204 52.155172, 5.387204 52.155262)", 0), - ( - "LINESTRING(5.387204 52.164151, 5.387204 52.155262)", - 1, - ), # startpoint is wrong + (f"LINESTRING({POINT1}, {POINT3})", 0), + (f"LINESTRING(142732 473443, {POINT3})", 1), # startpoint is wrong ], ) def test_pump_map_linestring_map_location_check(session, channel_geom, nof_invalid): - factories.ConnectionNodeFactory(id=1, geom="SRID=4326;POINT(5.387204 52.155172)") - factories.PumpFactory(id=1, geom="SRID=4326;POINT(5.387204 52.155262)") + factories.ConnectionNodeFactory(id=1, geom=f"SRID={SRID};POINT({POINT1})") + factories.PumpFactory(id=1, geom=f"SRID={SRID};POINT({POINT3})") factories.PumpMapFactory( - connection_node_id_end=1, - pump_id=1, - geom=f"SRID=4326;{channel_geom}", + connection_node_id_end=1, pump_id=1, geom=f"SRID={SRID};{channel_geom}" ) errors = PumpMapLinestringLocationCheck(max_distance=1.01).get_invalid(session) assert len(errors) == nof_invalid @@ -185,23 +164,18 @@ def test_pump_map_linestring_map_location_check(session, channel_geom, nof_inval @pytest.mark.parametrize( "channel_geom, nof_invalid", [ - ("LINESTRING(5.387204 52.155172, 5.387204 52.155262)", 0), - ( - "LINESTRING(5.387204 52.164151, 5.387204 52.155262)", - 1, - ), # startpoint is wrong + (f"LINESTRING({POINT1}, {POINT3})", 0), + (f"LINESTRING(142732 473443, {POINT3})", 1), # startpoint is wrong ], ) def test_surface_map_linestring_map_location_check(session, channel_geom, nof_invalid): - factories.ConnectionNodeFactory(id=1, geom="SRID=4326;POINT(5.387204 52.155172)") + factories.ConnectionNodeFactory(id=1, geom=f"SRID={SRID};POINT({POINT1})") factories.SurfaceFactory( id=1, - geom="SRID=4326;POLYGON((5.387204 52.155262, 5.387204 52.155172, 5.387304 52.155172, 5.387304 52.155262, 5.387204 52.155262))", + geom=f"SRID={SRID};POLYGON(({POINT1}, {POINT3}, 142742 473445, {POINT1}))", ) factories.SurfaceMapFactory( - connection_node_id=1, - surface_id=1, - geom=f"SRID=4326;{channel_geom}", + connection_node_id=1, surface_id=1, geom=f"SRID={SRID};{channel_geom}" ) errors = SurfaceMapLinestringLocationCheck(max_distance=1.01).get_invalid(session) assert len(errors) == nof_invalid diff --git a/threedi_modelchecker/tests/test_checks_other.py b/threedi_modelchecker/tests/test_checks_other.py index 73aec95a..37b222be 100644 --- a/threedi_modelchecker/tests/test_checks_other.py +++ b/threedi_modelchecker/tests/test_checks_other.py @@ -41,6 +41,11 @@ from . import factories +SRID = 28992 +POINT1 = "142742 473443" +POINT2 = "142743 473443" # POINT1 + 1 meter +POINT3 = "142747 473443" # POINT1 + 5 meter + @pytest.mark.parametrize( "aggregation_method,flow_variable,expected_result", @@ -73,29 +78,23 @@ def test_aggregation_settings( def test_connection_nodes_length(session): - factories.ModelSettingsFactory(epsg_code=28992) - factories.ConnectionNodeFactory(id=1, geom="SRID=4326;POINT(-71.064544 42.28787)") - factories.ConnectionNodeFactory(id=2, geom="SRID=4326;POINT(-71.0645 42.287)") - factories.ConnectionNodeFactory( - id=3, geom="SRID=4326;POINT(-0.38222938832999598 -0.13872236685816669)" - ), - factories.ConnectionNodeFactory( - id=4, geom="SRID=4326;POINT(-0.38222930900909202 -0.13872236685816669)" - ), + factories.ModelSettingsFactory() + factories.ConnectionNodeFactory(id=1, geom=f"SRID={SRID};POINT({POINT1})") + factories.ConnectionNodeFactory(id=2, geom=f"SRID={SRID};POINT({POINT2})") factories.WeirFactory( connection_node_id_start=1, connection_node_id_end=2, ) weir_too_short = factories.WeirFactory( - connection_node_id_start=3, - connection_node_id_end=4, + connection_node_id_start=1, + connection_node_id_end=1, ) check_length = ConnectionNodesLength( column=models.Weir.id, start_node=models.Weir.connection_node_id_start, end_node=models.Weir.connection_node_id_end, - min_distance=0.05, + min_distance=0.01, ) errors = check_length.get_invalid(session) @@ -104,11 +103,9 @@ def test_connection_nodes_length(session): def test_connection_nodes_length_missing_start_node(session): - factories.ModelSettingsFactory(epsg_code=28992) + factories.ModelSettingsFactory() factories.WeirFactory(connection_node_id_start=1, connection_node_id_end=2) - factories.ConnectionNodeFactory( - id=2, geom="SRID=4326;POINT(-0.38222930900909202 -0.13872236685816669)" - ) + factories.ConnectionNodeFactory(id=2) check_length = ConnectionNodesLength( column=models.Weir.id, start_node=models.Weir.connection_node_id_start, @@ -123,12 +120,9 @@ def test_connection_nodes_length_missing_start_node(session): def test_connection_nodes_length_missing_end_node(session): if session.bind.name == "postgresql": pytest.skip("Postgres only accepts coords in epsg 4326") - factories.ModelSettingsFactory(epsg_code=28992) + factories.ModelSettingsFactory() factories.WeirFactory(connection_node_id_start=1, connection_node_id_end=2) - factories.ConnectionNodeFactory( - id=1, geom="SRID=4326;POINT(-0.38222930900909202 -0.13872236685816669)" - ) - + factories.ConnectionNodeFactory(id=1) check_length = ConnectionNodesLength( column=models.Weir.id, start_node=models.Weir.connection_node_id_start, @@ -142,29 +136,29 @@ def test_connection_nodes_length_missing_end_node(session): def test_open_channels_with_nested_newton(session): factories.NumericalSettingsFactory(use_nested_newton=0) - factories.ConnectionNodeFactory(id=1, geom="SRID=4326;POINT(-71.064544 42.28787)") - factories.ConnectionNodeFactory(id=2, geom="SRID=4326;POINT(-71.0645 42.287)") + factories.ConnectionNodeFactory(id=1, geom=f"SRID={SRID};POINT({POINT1})") + factories.ConnectionNodeFactory(id=2, geom=f"SRID={SRID};POINT({POINT2})") factories.ChannelFactory( id=1, connection_node_id_start=1, connection_node_id_end=2, - geom="SRID=4326;LINESTRING(-71.064544 42.28787, -71.0645 42.287)", + geom=f"SRID={SRID};LINESTRING({POINT1}, {POINT2})", ) factories.CrossSectionLocationFactory( channel_id=1, cross_section_shape=constants.CrossSectionShape.TABULATED_TRAPEZIUM, cross_section_table="0,1\n1,0", - geom="SRID=4326;POINT(-71.0645 42.287)", + geom=f"SRID={SRID};POINT({POINT2})", ) factories.ChannelFactory( id=2, connection_node_id_start=1, connection_node_id_end=2, - geom="SRID=4326;LINESTRING(-71.064544 42.28787, -71.0645 42.287)", + geom=f"SRID={SRID};LINESTRING({POINT1}, {POINT2})", ) factories.CrossSectionLocationFactory( channel_id=2, - geom="SRID=4326;POINT(-71.0645 42.287)", + geom=f"SRID={SRID};POINT({POINT2})", cross_section_shape=constants.CrossSectionShape.EGG, ) @@ -197,33 +191,31 @@ def test_channel_manhole_level_check( # using factories, create one minimal test case which passes, and one which fails # once that works, parametrise. # use nested factories for channel and connectionNode - starting_coordinates = "4.718300 52.696686" - ending_coordinates = "4.718255 52.696709" factories.ConnectionNodeFactory( id=1, - geom=f"SRID=4326;POINT({starting_coordinates})", + geom=f"SRID={SRID};POINT({POINT1})", bottom_level=manhole_level, ) factories.ConnectionNodeFactory( id=2, - geom=f"SRID=4326;POINT({ending_coordinates})", + geom=f"SRID={SRID};POINT({POINT2})", bottom_level=manhole_level, ) factories.ChannelFactory( id=1, - geom=f"SRID=4326;LINESTRING({starting_coordinates}, {ending_coordinates})", + geom=f"SRID={SRID};LINESTRING({POINT1}, {POINT2})", connection_node_id_start=1, connection_node_id_end=2, ) # starting cross-section location factories.CrossSectionLocationFactory( - geom="SRID=4326;POINT(4.718278 52.696697)", + geom=f"SRID={SRID};POINT(142742.25 473443)", reference_level=starting_reference_level, channel_id=1, ) # ending cross-section location factories.CrossSectionLocationFactory( - geom="SRID=4326;POINT(4.718264 52.696704)", + geom=f"SRID={SRID};POINT(142743.25 473443)", reference_level=ending_reference_level, channel_id=1, ) @@ -234,25 +226,23 @@ def test_channel_manhole_level_check( def test_node_distance(session): con1_too_close = factories.ConnectionNodeFactory( - geom="SRID=4326;POINT(4.728282 52.64579283592512)" + geom=f"SRID={SRID};POINT(142740 473443)", ) con2_too_close = factories.ConnectionNodeFactory( - geom="SRID=4326;POINT(4.72828 52.64579283592512)" + geom=f"SRID={SRID};POINT(142743 473443)", ) # Good distance factories.ConnectionNodeFactory( - geom="SRID=4326;POINT(4.726838755789598 52.64514133594995)" + geom=f"SRID={SRID};POINT(142755 473443)", ) - # sanity check to see the distances between the nodes node_a = aliased(models.ConnectionNode) node_b = aliased(models.ConnectionNode) - distances_query = Query(func.ST_Distance(node_a.geom, node_b.geom, 1)).filter( + distances_query = Query(func.ST_Distance(node_a.geom, node_b.geom)).filter( node_a.id != node_b.id ) # Shows the distances between all 3 nodes: node 1 and 2 are too close distances_query.with_session(session).all() - check = ConnectionNodesDistance(minimum_distance=10) invalid = check.get_invalid(session) assert len(invalid) == 2 @@ -390,16 +380,13 @@ def test_cross_section_same_configuration( """ factories.ChannelFactory( id=1, - geom="SRID=4326;LINESTRING(4.718301 52.696686, 4.718255 52.696709)", ) factories.ChannelFactory( id=2, - geom="SRID=4326;LINESTRING(4.718301 52.696686, 4.718255 52.696709)", ) # shape 1 is always open factories.CrossSectionLocationFactory( channel_id=1, - geom="SRID=4326;POINT(4.718278 52.696697)", cross_section_shape=1, cross_section_width=3, cross_section_height=4, @@ -407,7 +394,6 @@ def test_cross_section_same_configuration( # the second one is parametrised factories.CrossSectionLocationFactory( channel_id=1 if same_channels else 2, - geom="SRID=4326;POINT(4.718265 52.696704)", cross_section_shape=shape, cross_section_width=width, cross_section_height=height, @@ -434,20 +420,20 @@ def test_spatial_index_disabled(empty_sqlite_v4): @pytest.mark.parametrize( - "x,y,ok", + "start,ok", [ - (-71.064544, 42.28787, True), # at start - (-71.0645, 42.287, True), # at end - (-71.06452, 42.2874, True), # middle - (-71.064544, 42.287869, False), # close to start - (-71.064499, 42.287001, False), # close to end + ("142742 473443", True), # at start + ("142747 473448", True), # at end + ("142744.5 473445.5", True), # middle + ("142742 473443.01", False), # close to start + ("142747 473447.99", False), # close to end ], ) -def test_potential_breach_start_end(session, x, y, ok): - # channel geom: LINESTRING (-71.064544 42.28787, -71.0645 42.287) +def test_potential_breach_start_end(session, start, ok): + # channel geom: "SRID={SRID}};LINESTRING (142742 473443, 142747 473448)" factories.ChannelFactory(id=1) factories.PotentialBreachFactory( - geom=f"SRID=4326;LINESTRING({x} {y}, -71.064544 42.286)", channel_id=1 + geom=f"SRID={SRID};LINESTRING({start}, 142750 473450)", channel_id=1 ) check = PotentialBreachStartEndCheck(models.PotentialBreach.geom, min_distance=1.0) invalid = check.get_invalid(session) @@ -458,21 +444,23 @@ def test_potential_breach_start_end(session, x, y, ok): @pytest.mark.parametrize( - "x,y,ok", + "start,channel_id,ok", [ - (-71.06452, 42.2874, True), # exactly on other - (-71.06452, 42.287401, False), # too close to other - (-71.0645, 42.287, True), # far enough from other + (POINT1, 1, True), # exactly on other + ("142742.5 473443.5", 1, False), # too close to other + ("142742.5 473443.5", 2, True), # too close to other but other channel + ("142740 473440", 1, True), # far enough from other ], ) -def test_potential_breach_interdistance(session, x, y, ok): - # channel geom: LINESTRING (-71.064544 42.28787, -71.0645 42.287) +def test_potential_breach_interdistance(session, start, channel_id, ok): + # channel geom: "SRID=28992;LINESTRING (142742 473443, 142747 473448)" factories.ChannelFactory(id=1) + factories.ChannelFactory(id=2) factories.PotentialBreachFactory( - geom="SRID=4326;LINESTRING(-71.06452 42.2874, -71.0646 42.286)", channel_id=1 + geom=f"SRID={SRID};LINESTRING({POINT1}, {POINT3})", channel_id=1 ) factories.PotentialBreachFactory( - geom=f"SRID=4326;LINESTRING({x} {y}, -71.064544 42.286)", channel_id=1 + geom=f"SRID={SRID};LINESTRING({start}, 142737 473438)", channel_id=channel_id ) check = PotentialBreachInterdistanceCheck( models.PotentialBreach.geom, min_distance=1.0 @@ -484,23 +472,6 @@ def test_potential_breach_interdistance(session, x, y, ok): assert len(invalid) == 1 -def test_potential_breach_interdistance_other_channel(session): - factories.ChannelFactory(id=1) - factories.ChannelFactory(id=2) - factories.PotentialBreachFactory( - geom="SRID=4326;LINESTRING(-71.06452 42.2874, -71.0646 42.286)", channel_id=1 - ) - factories.PotentialBreachFactory( - geom="SRID=4326;LINESTRING(-71.06452 42.287401, -71.064544 42.286)", - channel_id=2, - ) - check = PotentialBreachInterdistanceCheck( - models.PotentialBreach.geom, min_distance=1.0 - ) - invalid = check.get_invalid(session) - assert len(invalid) == 0 - - @pytest.mark.parametrize( "storage_area,time_step,expected_result,capacity", [ @@ -605,13 +576,18 @@ def test_feature_closed_cross_section(session, shape, expected_result): @pytest.mark.parametrize( "defined_area, max_difference, expected_result", [ - (1.5, 1.2, 0), - (2, 1, 1), - (1, 0.5, 1), + (1.2, 0.5, 0), + (1, 1, 0), + (2.1, 1, 1), + (1.6, 0.5, 1), ], ) def test_defined_area(session, defined_area, max_difference, expected_result): - geom = "SRID=4326;POLYGON((4.7 52.5, 4.7 52.50001, 4.70001 52.50001, 4.70001 52.50001))" + # create square polygon with area 1 + x0 = int(POINT1.split()[0]) + y0 = int(POINT1.split()[1]) + geom = f"SRID={SRID};POLYGON(({x0} {y0}, {x0 + 1} {y0}, {x0 + 1} {y0 + 1}, {x0} {y0 + 1}, {x0} {y0}))" + factories.SurfaceFactory(area=defined_area, geom=geom) check = DefinedAreaCheck(models.Surface.area, max_difference=max_difference) invalid = check.get_invalid(session) From 1bd91295e8862a19b43f6dea36afe74ff561f9a9 Mon Sep 17 00:00:00 2001 From: Margriet Palm Date: Mon, 23 Dec 2024 15:19:40 +0100 Subject: [PATCH 2/6] Disable RasterHasMatchingEPSGCheck (should be fixed in ticket 414) --- threedi_modelchecker/checks/raster.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/threedi_modelchecker/checks/raster.py b/threedi_modelchecker/checks/raster.py index fea4fc8f..ed0e761b 100644 --- a/threedi_modelchecker/checks/raster.py +++ b/threedi_modelchecker/checks/raster.py @@ -171,14 +171,10 @@ class RasterHasMatchingEPSGCheck(BaseRasterCheck): """Check whether a raster's EPSG code matches the EPSG code in the global settings for the SQLite.""" def get_invalid(self, session): - # TODO: fix this - # epsg_code_query = session.query(models.ModelSettings.epsg_code).first() - # if epsg_code_query is not None: - # self.epsg_code = epsg_code_query[0] - # else: - # self.epsg_code = None - self.epsg_code = 28992 - return super().get_invalid(session) + # TODO: replace this check with check that ensures that all EPSG + # in a schematisation match (ticket 414) + return [] + # return super().get_invalid(session) def is_valid(self, path: str, interface_cls: Type[RasterInterface]): with interface_cls(path) as raster: From 5857b601b9c3907ee7d46e619f0bcafbfaeddb71 Mon Sep 17 00:00:00 2001 From: Margriet Palm Date: Mon, 23 Dec 2024 15:20:32 +0100 Subject: [PATCH 3/6] Fix linting issues --- threedi_modelchecker/tests/conftest.py | 4 +++- threedi_modelchecker/tests/factories.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/threedi_modelchecker/tests/conftest.py b/threedi_modelchecker/tests/conftest.py index d194a04d..a3cfb890 100644 --- a/threedi_modelchecker/tests/conftest.py +++ b/threedi_modelchecker/tests/conftest.py @@ -24,7 +24,9 @@ def threedi_db(tmpdir_factory): shutil.copyfile(data_dir / "empty.sqlite", tmp_sqlite) db = ThreediDatabase(tmp_sqlite) schema = ModelSchema(db) - schema.upgrade(backup=False, upgrade_spatialite_version=False, custom_epsg_code=28992) + schema.upgrade( + backup=False, upgrade_spatialite_version=False, custom_epsg_code=28992 + ) schema.set_spatial_indexes() return db diff --git a/threedi_modelchecker/tests/factories.py b/threedi_modelchecker/tests/factories.py index 1abe8a1e..b496cae0 100644 --- a/threedi_modelchecker/tests/factories.py +++ b/threedi_modelchecker/tests/factories.py @@ -1,4 +1,3 @@ - from inspect import isclass import factory @@ -13,6 +12,7 @@ # polygon containing DEFAULT_POINT DEFAULT_POLYGON = "SRID=28992;POLYGON ((142742 473443, 142743 473443, 142744 473444, 142742 473444, 142742 473443))" + def inject_session(session): """Inject the session into all factories""" for _, cls in globals().items(): From d2d3b5be59cf8f328b4366b3836657a54be2bd36 Mon Sep 17 00:00:00 2001 From: Margriet Palm Date: Mon, 23 Dec 2024 15:43:29 +0100 Subject: [PATCH 4/6] Remove geo_query and use explicit queries instead now transformations are no longer needed --- threedi_modelchecker/checks/geo_query.py | 24 ------------------- threedi_modelchecker/checks/location.py | 12 +++++----- threedi_modelchecker/checks/other.py | 18 +++++++------- threedi_modelchecker/config.py | 12 +++++----- .../tests/test_checks_base.py | 23 ------------------ 5 files changed, 20 insertions(+), 69 deletions(-) delete mode 100644 threedi_modelchecker/checks/geo_query.py diff --git a/threedi_modelchecker/checks/geo_query.py b/threedi_modelchecker/checks/geo_query.py deleted file mode 100644 index 324ea1d5..00000000 --- a/threedi_modelchecker/checks/geo_query.py +++ /dev/null @@ -1,24 +0,0 @@ -from geoalchemy2 import functions as geo_func -from sqlalchemy import func -from sqlalchemy.orm import Query -from sqlalchemy.sql import literal -from threedi_schema import models - -DEFAULT_EPSG = 28992 - - -def epsg_code_query(): - epsg_code = Query(models.ModelSettings.epsg_code).limit(1).scalar_subquery() - return func.coalesce(epsg_code, literal(DEFAULT_EPSG)).label("epsg_code") - - -def transform(col): - return geo_func.ST_Transform(col, DEFAULT_EPSG) - - -def distance(col_1, col_2): - return geo_func.ST_Distance(transform(col_1), transform(col_2)) - - -def length(col): - return geo_func.ST_Length(transform(col)) diff --git a/threedi_modelchecker/checks/location.py b/threedi_modelchecker/checks/location.py index 65709c6d..d7784e49 100644 --- a/threedi_modelchecker/checks/location.py +++ b/threedi_modelchecker/checks/location.py @@ -1,11 +1,11 @@ from typing import List, NamedTuple +from geoalchemy2.functions import ST_Distance from sqlalchemy import func from sqlalchemy.orm import aliased, Session from threedi_schema.domain import models from threedi_modelchecker.checks.base import BaseCheck -from threedi_modelchecker.checks.geo_query import distance class PointLocationCheck(BaseCheck): @@ -32,7 +32,7 @@ def get_invalid(self, session): self.ref_table, self.ref_table.id == self.ref_column, ) - .filter(distance(self.column, self.ref_table.geom) > self.max_distance) + .filter(ST_Distance(self.column, self.ref_table.geom) > self.max_distance) .all() ) @@ -75,10 +75,10 @@ def get_invalid(self, session: Session) -> List[NamedTuple]: start_point = func.ST_PointN(self.column, 1) end_point = func.ST_PointN(self.column, func.ST_NPoints(self.column)) - start_ok = distance(start_point, start_node.geom) <= tol - end_ok = distance(end_point, end_node.geom) <= tol - start_ok_if_reversed = distance(end_point, start_node.geom) <= tol - end_ok_if_reversed = distance(start_point, end_node.geom) <= tol + start_ok = ST_Distance(start_point, start_node.geom) <= tol + end_ok = ST_Distance(end_point, end_node.geom) <= tol + start_ok_if_reversed = ST_Distance(end_point, start_node.geom) <= tol + end_ok_if_reversed = ST_Distance(start_point, end_node.geom) <= tol return ( self.to_check(session) .join(start_node, start_node.id == self.ref_column_start) diff --git a/threedi_modelchecker/checks/other.py b/threedi_modelchecker/checks/other.py index 82e2fc20..7f8669bd 100644 --- a/threedi_modelchecker/checks/other.py +++ b/threedi_modelchecker/checks/other.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from typing import List, Literal, NamedTuple +from geoalchemy2.functions import ST_Distance, ST_Length from sqlalchemy import ( and_, case, @@ -19,7 +20,6 @@ from .base import BaseCheck, CheckLevel from .cross_section_definitions import cross_section_configuration_for_record -from .geo_query import distance, length, transform class CorrectAggregationSettingsExist(BaseCheck): @@ -290,7 +290,7 @@ def get_invalid(self, session): self.to_check(session) .join(start_node, start_node.id == self.start_node) .join(end_node, end_node.id == self.end_node) - .filter(distance(start_node.geom, end_node.geom) < self.min_distance) + .filter(ST_Distance(start_node.geom, end_node.geom) < self.min_distance) ) return list(q.with_session(session).all()) @@ -548,10 +548,10 @@ def get_invalid(self, session: Session) -> List[NamedTuple]: linestring = models.Channel.geom tol = self.min_distance breach_point = func.Line_Locate_Point( - transform(linestring), transform(func.ST_PointN(self.column, 1)) + linestring, func.ST_PointN(self.column, 1) ) - dist_1 = breach_point * length(linestring) - dist_2 = (1 - breach_point) * length(linestring) + dist_1 = breach_point * ST_Length(linestring) + dist_2 = (1 - breach_point) * ST_Length(linestring) return ( self.to_check(session) .join(models.Channel, self.table.c.channel_id == models.Channel.id) @@ -577,10 +577,8 @@ def get_invalid(self, session: Session) -> List[NamedTuple]: # First fetch the position of each potential breach per channel def get_position(point, linestring): - breach_point = func.Line_Locate_Point( - transform(linestring), transform(func.ST_PointN(point, 1)) - ) - return (breach_point * length(linestring)).label("position") + breach_point = func.Line_Locate_Point(linestring, func.ST_PointN(point, 1)) + return (breach_point * ST_Length(linestring)).label("position") potential_breaches = sorted( session.query(self.table, get_position(self.column, models.Channel.geom)) @@ -776,7 +774,7 @@ def get_invalid(self, session: Session) -> List[NamedTuple]: self.table.c.id, self.table.c.area, self.table.c.geom, - func.ST_Area(transform(self.table.c.geom)).label("calculated_area"), + func.ST_Area(self.table.c.geom).label("calculated_area"), ).subquery() return ( session.query(all_results) diff --git a/threedi_modelchecker/config.py b/threedi_modelchecker/config.py index 67021ff1..3053ef21 100644 --- a/threedi_modelchecker/config.py +++ b/threedi_modelchecker/config.py @@ -1,11 +1,11 @@ from typing import List +from geoalchemy2 import functions as geo_func from sqlalchemy import and_, func, or_, true from sqlalchemy.orm import Query from threedi_schema import constants, models from threedi_schema.beta_features import BETA_COLUMNS, BETA_VALUES -from .checks import geo_query from .checks.base import ( AllEqualCheck, BaseCheck, @@ -1068,7 +1068,7 @@ def is_none_or_empty(col): error_code=202, level=CheckLevel.WARNING, column=table.id, - invalid=Query(table).filter(geo_query.length(table.geom) < 5), + invalid=Query(table).filter(geo_func.ST_Length(table.geom) < 5), message=f"The length of {table.__tablename__} is very short (< 5 m). A length of at least 5.0 m is recommended to avoid timestep reduction.", ) for table in [models.Channel, models.Culvert] @@ -1369,8 +1369,8 @@ def is_none_or_empty(col): invalid=Query(models.ExchangeLine) .join(models.Channel, models.Channel.id == models.ExchangeLine.channel_id) .filter( - geo_query.length(models.ExchangeLine.geom) - < (0.8 * geo_query.length(models.Channel.geom)) + geo_func.ST_Length(models.ExchangeLine.geom) + < (0.8 * geo_func.ST_Length(models.Channel.geom)) ), message=( "exchange_line.geom should not be significantly shorter than its " @@ -1384,7 +1384,7 @@ def is_none_or_empty(col): invalid=Query(models.ExchangeLine) .join(models.Channel, models.Channel.id == models.ExchangeLine.channel_id) .filter( - geo_query.distance(models.ExchangeLine.geom, models.Channel.geom) > 500.0 + geo_func.ST_Distance(models.ExchangeLine.geom, models.Channel.geom) > 500.0 ), message=("exchange_line.geom is far (> 500 m) from its corresponding channel"), ), @@ -1455,7 +1455,7 @@ def is_none_or_empty(col): invalid=Query(models.PotentialBreach) .join(models.Channel, models.Channel.id == models.PotentialBreach.channel_id) .filter( - geo_query.distance( + geo_func.ST_Distance( func.PointN(models.PotentialBreach.geom, 1), models.Channel.geom ) > TOLERANCE_M diff --git a/threedi_modelchecker/tests/test_checks_base.py b/threedi_modelchecker/tests/test_checks_base.py index a5b596c5..3eae2c33 100644 --- a/threedi_modelchecker/tests/test_checks_base.py +++ b/threedi_modelchecker/tests/test_checks_base.py @@ -4,7 +4,6 @@ from sqlalchemy.orm import Query from threedi_schema import constants, custom_types, models -from threedi_modelchecker.checks import geo_query from threedi_modelchecker.checks.base import ( _sqlalchemy_to_sqlite_types, AllEqualCheck, @@ -529,28 +528,6 @@ def test_global_settings_use_1d_flow_and_no_1d_elements(session): assert len(errors) == 0 -def test_length_geom_linestring(session): - # TODO: reconsider this check because it tests geo_query.length, - # not query_check so this should not be here! - factories.ModelSettingsFactory() - x, y = [ - float(coord) - for coord in factories.DEFAULT_LINE.split("(")[1].split(",")[0].split() - ] - short_line = f"SRID=28992;LINESTRING({x} {y}, {x + 0.03} {y + 0.03})" - channel_too_short = factories.ChannelFactory(geom=short_line) - factories.ChannelFactory(geom=factories.DEFAULT_LINE) - q = Query(models.Channel).filter(geo_query.length(models.Channel.geom) < 0.05) - check_length_linestring = QueryCheck( - column=models.Channel.geom, - invalid=q, - message="Length of the channel is too short, should be at least 0.05m", - ) - errors = check_length_linestring.get_invalid(session) - assert len(errors) == 1 - assert errors[0].id == channel_too_short.id - - @pytest.mark.parametrize( "min_value,max_value,left_inclusive,right_inclusive", [ From 0c4d6ff3b3bba4881897fedf7767c1f0c2e3a096 Mon Sep 17 00:00:00 2001 From: Margriet Palm Date: Tue, 24 Dec 2024 10:24:43 +0100 Subject: [PATCH 5/6] Remove mention on removed transformation from docstring of ConnectionNodesLength --- threedi_modelchecker/checks/other.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/threedi_modelchecker/checks/other.py b/threedi_modelchecker/checks/other.py index 7f8669bd..0e04bbfa 100644 --- a/threedi_modelchecker/checks/other.py +++ b/threedi_modelchecker/checks/other.py @@ -258,8 +258,7 @@ def get_invalid(self, session): class ConnectionNodesLength(BaseCheck): """Check that the distance between `start_node` and `end_node` is at least - `min_distance`. The coords will be transformed into (the first entry) of - ModelSettings.epsg_code. + `min_distance`. """ def __init__( From 3c3e8f859039595a038bb9ccbdf80ccf3f7e4be8 Mon Sep 17 00:00:00 2001 From: Margriet Palm Date: Tue, 24 Dec 2024 11:22:53 +0100 Subject: [PATCH 6/6] Update changes --- CHANGES.rst | 11 ++++++++++- threedi_modelchecker/__init__.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b0a20cbb..54c9f18e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,7 +3,16 @@ Changelog of threedi-modelchecker -2.14.2 (unreleased) +2.16.0 (unreleased) +------------------- + +- Adapt to schema 230 where all geometries use the model CRS and model_settings.epsg_code is no longer available +- Remove checks for model_settings.epsg_code (317 and 318) +- Remove usage of epsg 4326 in the tests because this CRS is no longer valid +- Remove no longer needed transformations + + +2.15.0 (unreleased) ------------------- - Change minimum python version to 3.9 in pyproject.toml, update test matrix. diff --git a/threedi_modelchecker/__init__.py b/threedi_modelchecker/__init__.py index 13d93cd4..e5742e9a 100644 --- a/threedi_modelchecker/__init__.py +++ b/threedi_modelchecker/__init__.py @@ -1,5 +1,5 @@ from .model_checks import * # NOQA # fmt: off -__version__ = '2.14.2.dev0' +__version__ = '2.16.0.dev0' # fmt: on