Skip to content

Commit

Permalink
Added intersects filter (#916)
Browse files Browse the repository at this point in the history
* added intersects filter
  • Loading branch information
barrydaniels-nl authored Jan 8, 2025
1 parent 5422970 commit 030adce
Show file tree
Hide file tree
Showing 8 changed files with 322 additions and 23 deletions.
12 changes: 10 additions & 2 deletions src/dso_api/dynamic_api/filters/openapi.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
"gte": "Greater than or equal to; ",
"not": "Exclude matches; ",
"contains": "Should contain; ",
"intersects": "Use WKT (POLYGON((x1 y1, x2 y2, ...))) or GeoJSON",
}

OPENAPI_LOOKUP_EXAMPLES = {
Expand All @@ -60,9 +61,16 @@
},
"https://geojson.org/schema/Point.json": {
"": "Use x,y or POINT(x y)", # only for no lookup
"intersects": "Use WKT (POLYGON((x1 y1, x2 y2, ...))) or GeoJSON",
},
"https://geojson.org/schema/Polygon.json": {
"contains": "Use x,y or POINT(x y)",
"intersects": "Use WKT (POLYGON((x1 y1, x2 y2, ...))) or GeoJSON",
},
"https://geojson.org/schema/MultiPolygon.json": {
"contains": "Use x,y or POINT(x y)",
"intersects": "Use WKT (POLYGON((x1 y1, x2 y2, ...))) or GeoJSON",
},
"https://geojson.org/schema/Polygon.json": {"contains": "Use x,y or POINT(x y)"},
"https://geojson.org/schema/MultiPolygon.json": {"contains": "Use x,y or POINT(x y)"},
}


Expand Down
4 changes: 2 additions & 2 deletions src/dso_api/dynamic_api/filters/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
# The empty value is there to indicate the field also supports no lookup operator.
# This is mostly used by the OpenAPI generator.
_comparison_lookups = {"", "gte", "gt", "lt", "lte", "in", "not", "isnull"}
_polygon_lookups = {"", "contains", "isnull", "not"}
_polygon_lookups = {"", "contains", "isnull", "not", "intersects"}
_string_lookups = {"", "in", "isnull", "not", "isempty", "like"}

ALLOWED_IDENTIFIER_LOOKUPS = {"", "in", "not", "isnull"}
Expand All @@ -69,7 +69,7 @@
"array": {"", "contains"},
"object": set(),
"https://geojson.org/schema/Geometry.json": _polygon_lookups, # Assume it works.
"https://geojson.org/schema/Point.json": {"", "isnull", "not"},
"https://geojson.org/schema/Point.json": {"", "isnull", "not", "intersects"},
"https://geojson.org/schema/Polygon.json": _polygon_lookups,
"https://geojson.org/schema/MultiPolygon.json": _polygon_lookups,
# Format variants for type string:
Expand Down
155 changes: 149 additions & 6 deletions src/dso_api/dynamic_api/filters/values.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,22 @@

from __future__ import annotations

import logging
import math
import re
from datetime import date, datetime, time
from decimal import Decimal

from django.contrib.gis.gdal.error import GDALException
from django.contrib.gis.geos import GEOSException, GEOSGeometry, Point
from django.contrib.gis.geos.prototypes import geom # noqa: F401
from django.utils.dateparse import parse_datetime
from django.utils.translation import gettext_lazy as _
from gisserver.geometries import CRS
from rest_framework.exceptions import ValidationError

logger = logging.getLogger(__name__)

# Don't want Decimal("NaN"), Decimal("-inf") or '0.321000e+2' to be accepted.
RE_DECIMAL = re.compile(r"^[0-9]+(\.[0-9]+)?$")

Expand Down Expand Up @@ -72,17 +77,122 @@ def str2time(value: str) -> time:

def str2geo(value: str, crs: CRS | None = None) -> GEOSGeometry:
"""Convert a string to a geometry object.
Currently only parses point objects.
Supports Point, Polygon and MultiPolygon objects in WKT or GeoJSON format.
Args:
value: String representation of geometry (WKT, GeoJSON, or x,y format)
crs: Optional coordinate reference system
Returns:
GEOSGeometry object
"""
srid = crs.srid if crs else 4326
stripped_value = value.lstrip()
# Try parsing as GeoJSON first
if stripped_value.startswith(("{", "[")):
return _parse_geojson(stripped_value, srid)

# Try x,y format if it looks like two numbers separated by a comma
if stripped_value.startswith(("POINT", "POLYGON", "MULTIPOLYGON")):
return _parse_wkt_geometry(stripped_value, srid)
else:
return _parse_point_geometry(stripped_value, srid)


def _parse_geojson(value: str, srid: int | None) -> GEOSGeometry:
"""Parse GeoJSON string and validate basic structure.
Args:
value: GeoJSON string
Returns:
GEOSGeometry object
Raises:
ValidationError: If GeoJSON is invalid
"""
try:
return GEOSGeometry(value, srid)
except (GEOSException, ValueError, GDALException) as e:
raise ValidationError(f"Invalid GeoJSON: {e}") from e


def _parse_wkt_geometry(value: str, srid: int | None) -> GEOSGeometry:
"""Parse and validate a WKT geometry string.
Args:
value: WKT geometry string
srid: Optional spatial reference identifier
Returns:
Validated GEOSGeometry object
Raises:
ValidationError: If geometry is invalid or unsupported type
"""
srid = crs.srid if crs else None
x, y = _parse_point(value)
try:
geom = GEOSGeometry(value, srid)
except (GEOSException, ValueError) as e:
raise ValidationError(f"Invalid WKT format in {value}. Error: {e}") from e

if geom.geom_type not in ("Point", "Polygon", "MultiPolygon"):
raise ValidationError(
f"Unsupported geometry type: {geom.geom_type}. "
"Only Point, Polygon and MultiPolygon are supported."
)

# check if the geometry is within the Netherlands, only warn if not
if geom.geom_type in ("Polygon", "MultiPolygon") and not _validate_bounds(geom, srid):
logger.warning("Geometry bounds outside Netherlands")

# Try parsing as point
if geom.geom_type == "Point":
try:
return _validate_point_geometry(geom, srid)
except ValidationError as e:
raise ValidationError(f"Invalid point format in {value}. Error: {e}") from e

return geom


def _parse_point_geometry(value: str, srid: int | None) -> GEOSGeometry:
"""Parse and validate a point in x,y format.
Args:
value: String in "x,y" format
srid: Optional spatial reference identifier
Returns:
Validated Point geometry
Raises:
ValidationError: If point format or coordinates are invalid
"""
try:
x, y = _parse_point(value)
return _validate_correct_x_y(x, y, srid)
except ValueError as e:
raise ValidationError(f"{e} in {value!r}") from None
except GEOSException as e:
raise ValidationError(f"Invalid x,y values {x},{y} with SRID {srid}") from e
raise ValidationError(f"Invalid point format in {value!r}. Error: {e}") from e


def _validate_point_geometry(point: GEOSGeometry, srid: int | None) -> GEOSGeometry:
"""Validate a point geometry's coordinates.
Args:
point: Point geometry to validate
srid: Optional spatial reference identifier
Returns:
Validated point geometry
Raises:
ValidationError: If coordinates are invalid
"""
try:
x, y = point.coords
return _validate_correct_x_y(x, y, srid)
except (ValueError, GEOSException) as e:
raise ValidationError(f"Invalid point coordinates {point.coords} with SRID {srid}") from e


def _parse_point(value: str) -> tuple[float, float]:
Expand All @@ -106,6 +216,39 @@ def _parse_point(value: str) -> tuple[float, float]:
return x, y


def _validate_bounds(geom: GEOSGeometry, srid: int | None) -> bool: # noqa: F811
"""Validate if geometry bounds are within Netherlands extent.
Returns True if geometry is within bounds, False otherwise.
Args:
geom: GEOSGeometry object to validate
srid: Spatial reference system identifier
Returns:
bool: True if geometry is within Netherlands bounds, False otherwise
"""
bounds = geom.extent # (xmin, ymin, xmax, ymax)
corners = [(bounds[0], bounds[1]), (bounds[2], bounds[3])] # SW, NE corners

if srid == 4326:
return _validate_wgs84_bounds(corners)
elif srid == 28992:
return _validate_rd_bounds(corners)
return True # If srid is not 4326 or 28992, assume valid


def _validate_wgs84_bounds(corners: list[tuple[float, float]]) -> bool:
"""Check if WGS84 coordinates are within Netherlands bounds.
Note: Expects (x,y) format, will swap to (lat,lon) internally.
"""
return all(_valid_nl_wgs84(y, x) for x, y in corners)


def _validate_rd_bounds(corners: list[tuple[float, float]]) -> bool:
"""Check if RD coordinates are within Netherlands bounds."""
return all(_valid_rd(x, y) for x, y in corners)


def _validate_correct_x_y(x: float, y: float, srid: int | None) -> Point:
"""Auto-correct various input variations."""

Expand Down
9 changes: 8 additions & 1 deletion src/dso_api/dynamic_api/views/doc.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,11 @@ def lookup_context(op, example, descr):
lookup_context(
"contains", "Kommagescheiden lijst", "Test of er een intersectie is met de waarde."
),
lookup_context(
"intersects",
"GeoJSON of <code>POLYGON(x y ...)</code>",
"Test of er een intersectie is met de waarde.",
),
lookup_context(
"isnull",
"<code>true</code> | <code>false</code>",
Expand Down Expand Up @@ -425,7 +430,9 @@ def _field_data(field: DatasetFieldSchema):
# Catch-all for other geometry types
type = type[len("https://geojson.org/schema/") : -5]
value_example = f"GeoJSON of <code>{type.upper()}(x y ...)<code>"
lookups = []
# Keep the lookups from get_allowed_lookups for geometry fields
if not lookups:
lookups = QueryFilterEngine.get_allowed_lookups(field) - {""}
elif field.relation or "://" in type:
lookups = _identifier_lookups
if field.type == "string":
Expand Down
12 changes: 8 additions & 4 deletions src/templates/dso_api/dynamic_api/docs/rest/filtering.md
Original file line number Diff line number Diff line change
Expand Up @@ -136,10 +136,14 @@ als `exact`. Er is geen *escaping* van de jokertekens mogelijk.

### Bij waarden met een geometrie

| Operator | Werking | SQL Equivalent |
| --------------------------------- | -------------------------------------------------- | ------------------------------------------ |
| `?{geoveld}[contains]={x},{y}` | Geometrie moet voorkomen op een punt (intersectie) | `ST_Intersects({geoveld}, POINT({x} {y}))` |
| `?{geoveld}[contains]=POINT(x y)` | Idem, nu in de WKT (well-known text) notatie. | `ST_Intersects({geoveld}, POINT({x} {y}))` |
| Operator | Werking | SQL Equivalent |
| --------------------------------------------------- | ------------------------------------------------------ | -----------------------------------------------------|
| `?{geoveld}[contains]={x},{y}` | Geometrie moet voorkomen op een punt (intersectie) | `ST_Intersects({geoveld}, POINT({x} {y}))` |
| `?{geoveld}[contains]=POINT(x y)` | Idem, nu in de WKT (well-known text) notatie. | `ST_Intersects({geoveld}, POINT({x} {y}))` |
| `?{geoveld}[intersects]={x},{y}` | Geometrie moet voorkomen op een punt (intersectie) | `ST_Intersects({geoveld}, POINT({x} {y}))` |
| `?{geoveld}[intersects]=POINT(x y)` | Idem, nu in de WKT (well-known text) notatie. | `ST_Intersects({geoveld}, POINT({x} {y}))` |
| `?{geoveld}[intersects]=POLYGON ((4.89...))` | Geometry moet overlappen met een polygon (intersectie).| `ST_Intersects({geoveld}, POLYGON ((4.89...)))` |
| `?{geoveld}[intersects]=MULTIPOLYGON (((4.89...)))` | Geometry moet overlappen met een MULTIPOLYGON | `ST_Intersects({geoveld}, MULTIPOLYGON ((4.89...)))` |

Bij het doorzoeken van geometrievelden wordt gebruik gemaakt van de
projectie opgegeven in de header `Accept-CRS`. Afhankelijk van de
Expand Down
37 changes: 34 additions & 3 deletions src/tests/test_dynamic_api/test_filters.py
Original file line number Diff line number Diff line change
Expand Up @@ -407,32 +407,63 @@ def test_parse_point_invalid(value):
@pytest.mark.parametrize(
"value",
[
# Basic invalid formats
"",
"a",
"foo",
"inf,nan",
"0, 0",
"0," + 314 * "1",
# Invalid WKT formats
"POINT",
"POINT ",
"POINT(x y)",
"POINT(1.0 2.0",
"POINT(1.0,2.0)",
"POINT 1.0 2.0",
"POINT(1. .1)",
# Outside range of Netherlands:
# Out of bounds points
"POINT(0 0)",
"POINT(0.0 0.0)",
"POINT(-1.1 -3.3)",
"POINT(-1 2)",
"POINT(100.0 42.0)",
# Invalid GeoJSON
'{"type": "Point"}',
'{"coordinates": [1,2]}',
'{"type": "Invalid", "coordinates": [1,2]}',
],
)
def test_str2geo_invalid(value):
with pytest.raises(ValidationError) as exc_info:
"""Test str2geo with invalid input formats."""
with pytest.raises(ValidationError):
str2geo(value)

assert repr(value) in str(exc_info.value)

@pytest.mark.parametrize(
"value,expected_type",
[
# WKT formats with valid Netherlands coordinates (Amsterdam area)
("POINT(4.9 52.4)", "Point"), # WGS84
("POLYGON((4.9 52.4, 4.9 52.5, 5.0 52.5, 5.0 52.4, 4.9 52.4))", "Polygon"),
("MULTIPOLYGON(((4.9 52.4, 4.9 52.5, 5.0 52.5, 5.0 52.4, 4.9 52.4)))", "MultiPolygon"),
# GeoJSON formats with valid Netherlands coordinates (Amsterdam area)
('{"type": "Point", "coordinates": [4.9, 52.4]}', "Point"), # WGS84
(
'{"type": "Polygon", "coordinates": [[[4.9, 52.4], [4.9, 52.5], [5.0, 52.5], [5.0, 52.4], [4.9, 52.4]]]}', # noqa: E501
"Polygon",
),
(
'{"type": "MultiPolygon", "coordinates": [[[[4.9, 52.4], [4.9, 52.5], [5.0, 52.5], [5.0, 52.4], [4.9, 52.4]]]]}', # noqa: E501
"MultiPolygon",
),
],
)
def test_str2geo_valid_formats(value, expected_type):
"""Test str2geo with valid WKT and GeoJSON formats for Point, Polygon and MultiPolygon."""
result = str2geo(value)
assert result.geom_type == expected_type
assert result.srid == 4326 # Default SRID


@pytest.mark.parametrize(
Expand Down
3 changes: 2 additions & 1 deletion src/tests/test_dynamic_api/test_openapi.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,8 +153,9 @@ def test_openapi_json(api_client, afval_dataset, fietspaaltjes_dataset, filled_r
"eigenaarNaam[like]",
"eigenaarNaam[not]",
"geometry",
"geometry[isnull]",
"geometry[not]",
"geometry[isnull]",
"geometry[intersects]",
"id",
"id[gt]",
"id[gte]",
Expand Down
Loading

0 comments on commit 030adce

Please sign in to comment.