diff --git a/src/bag/bag_loader.py b/src/bag/bag_loader.py index e173888..5bdacf1 100644 --- a/src/bag/bag_loader.py +++ b/src/bag/bag_loader.py @@ -14,7 +14,7 @@ class BagLoader: tmp_folder = "/tmp" - + tables = { "bag_ligplaats": "bag_ligplaatsen.csv.zip", "bag_openbareruimte": "bag_openbareruimtes.csv.zip", @@ -77,19 +77,21 @@ def unpack_zip(self, *, table_name: str, endpoint: str): def preprocess_csv_rows(self, reader: csv.DictReader, writer: csv.DictWriter): seen = {} for row in reader: - key = row['Identificatie'] - if (key not in seen): + key = row["Identificatie"] + if key not in seen: seen[key] = row - elif row['Volgnummer'] > seen[key]['Volgnummer']: + elif row["Volgnummer"] > seen[key]["Volgnummer"]: seen[key] = row - + filtered_rows = list(seen.values()) writer.writeheader() for row in filtered_rows: writer.writerow(row) def preprocess_csv_files(self, csv_path_in, csv_path_out): - with open(csv_path_in, 'r') as infile, open(csv_path_out, 'w', newline='') as outfile: + with open(csv_path_in, "r") as infile, open( + csv_path_out, "w", newline="" + ) as outfile: reader = csv.DictReader(infile) writer = csv.DictWriter(outfile, fieldnames=reader.fieldnames) self.preprocess_csv_rows(reader, writer) @@ -98,9 +100,7 @@ def preprocess_csv_files(self, csv_path_in, csv_path_out): def download_zip(self, *, table_name: str, endpoint: str, path_base: str): base_url = settings.BAG_CSV_BASE_URL url = f"{base_url}/{endpoint}" - logger.info( - f"Downloading {table_name} from {url}" - ) + logger.info(f"Downloading {table_name} from {url}") response = requests.get( url, timeout=300, @@ -125,9 +125,13 @@ def import_tables_from_endpoint(self): for table, endpoint in self.tables.items(): logger.info(f"Loading table {table}") path_base = f"{self.tmp_folder}" - path_zip = self.download_zip(table_name=table, endpoint=endpoint, path_base=path_base) + path_zip = self.download_zip( + table_name=table, endpoint=endpoint, path_base=path_base + ) path_csv = self.unpack_zip(table_name=table, endpoint=path_zip) path_csv_processed = f"{path_csv}-processed.csv" - self.preprocess_csv_files(csv_path_in=path_csv, csv_path_out=path_csv_processed) + self.preprocess_csv_files( + csv_path_in=path_csv, csv_path_out=path_csv_processed + ) csv_files[table] = path_csv_processed - self.load_tables_from_csv(csv_files) \ No newline at end of file + self.load_tables_from_csv(csv_files) diff --git a/src/bag/migrations/0001_initial.py b/src/bag/migrations/0001_initial.py index 089d787..d9e4426 100644 --- a/src/bag/migrations/0001_initial.py +++ b/src/bag/migrations/0001_initial.py @@ -12,220 +12,712 @@ class Migration(migrations.Migration): initial = True - dependencies = [ - ] + dependencies = [] operations = [ migrations.CreateModel( - name='Ligplaats', + name="Ligplaats", fields=[ - ('id', models.CharField(db_column='identificatie', max_length=16, primary_key=True, serialize=False)), - ('document_mutatie', models.DateField(db_column='documentdatum')), - ('document_nummer', models.CharField(db_column='documentnummer', max_length=20)), - ('begin_geldigheid', models.DateField(db_column='begingeldigheid')), - ('einde_geldigheid', models.DateField(db_column='eindgeldigheid', null=True)), - ('indicatie_geconstateerd', models.BooleanField(db_column='geconstateerd')), - ('geometrie', django.contrib.gis.db.models.fields.PolygonField(srid=28992)), - ('status', models.CharField(db_column='statusomschrijving', max_length=80)), - ('buurt', models.CharField(db_column='ligtinbuurtid', max_length=16, null=True)), - ('volgnummer', models.IntegerField()), - ('registratiedatum', models.DateField()), - ('statuscode', models.CharField(max_length=1)), - ('heeft_hoofdadres', models.CharField(db_column='heefthoofdadresid', max_length=16, null=True)), - ('dossier', models.CharField(db_column='heeftdossierid', max_length=16, null=True)), - ('proces_code', models.CharField(db_column='bagprocescode', max_length=4, null=True)), - ('proces_omschrijving', models.CharField(db_column='bagprocesomschrijving', max_length=80, null=True)), + ( + "id", + models.CharField( + db_column="identificatie", + max_length=16, + primary_key=True, + serialize=False, + ), + ), + ("document_mutatie", models.DateField(db_column="documentdatum")), + ( + "document_nummer", + models.CharField(db_column="documentnummer", max_length=20), + ), + ("begin_geldigheid", models.DateField(db_column="begingeldigheid")), + ( + "einde_geldigheid", + models.DateField(db_column="eindgeldigheid", null=True), + ), + ( + "indicatie_geconstateerd", + models.BooleanField(db_column="geconstateerd"), + ), + ( + "geometrie", + django.contrib.gis.db.models.fields.PolygonField(srid=28992), + ), + ( + "status", + models.CharField(db_column="statusomschrijving", max_length=80), + ), + ( + "buurt", + models.CharField( + db_column="ligtinbuurtid", max_length=16, null=True + ), + ), + ("volgnummer", models.IntegerField()), + ("registratiedatum", models.DateField()), + ("statuscode", models.CharField(max_length=1)), + ( + "heeft_hoofdadres", + models.CharField( + db_column="heefthoofdadresid", max_length=16, null=True + ), + ), + ( + "dossier", + models.CharField( + db_column="heeftdossierid", max_length=16, null=True + ), + ), + ( + "proces_code", + models.CharField( + db_column="bagprocescode", max_length=4, null=True + ), + ), + ( + "proces_omschrijving", + models.CharField( + db_column="bagprocesomschrijving", max_length=80, null=True + ), + ), ], options={ - 'db_table': 'bag_ligplaats', + "db_table": "bag_ligplaats", }, bases=(models.Model, bag.models.BagObject), ), migrations.CreateModel( - name='Openbareruimte', + name="Openbareruimte", fields=[ - ('id', models.CharField(db_column='identificatie', max_length=16, primary_key=True, serialize=False)), - ('document_mutatie', models.DateField(db_column='documentdatum')), - ('document_nummer', models.CharField(db_column='documentnummer', max_length=20)), - ('begin_geldigheid', models.DateField(db_column='begingeldigheid')), - ('einde_geldigheid', models.DateField(db_column='eindgeldigheid', null=True)), - ('type', models.CharField(db_column='typeomschrijving', max_length=80)), - ('naam', models.CharField(max_length=80)), - ('naam_nen', models.CharField(db_column='naamnen', max_length=80)), - ('geometrie', django.contrib.gis.db.models.fields.GeometryField(null=True, srid=28992)), - ('omschrijving', models.TextField(db_column='beschrijvingnaam', null=True)), - ('woonplaats_id', models.CharField(db_column='ligtinwoonplaatsid', max_length=16, null=True)), - ('status', models.CharField(db_column='statusomschrijving', max_length=80)), - ('volgnummer', models.IntegerField()), - ('registratiedatum', models.DateField()), - ('straatcode', models.CharField(max_length=16, null=True)), - ('straatnaam', models.CharField(db_column='straatnaamptt', max_length=80, null=True)), - ('statuscode', models.CharField(max_length=1)), - ('geconstateerd', models.BooleanField()), - ('typecode', models.CharField(max_length=1)), - ('dossier', models.CharField(db_column='heeftdossierid', max_length=16, null=True)), - ('proces_code', models.CharField(db_column='bagprocescode', max_length=4, null=True)), - ('proces_omschrijving', models.CharField(db_column='bagprocesomschrijving', max_length=80, null=True)), + ( + "id", + models.CharField( + db_column="identificatie", + max_length=16, + primary_key=True, + serialize=False, + ), + ), + ("document_mutatie", models.DateField(db_column="documentdatum")), + ( + "document_nummer", + models.CharField(db_column="documentnummer", max_length=20), + ), + ("begin_geldigheid", models.DateField(db_column="begingeldigheid")), + ( + "einde_geldigheid", + models.DateField(db_column="eindgeldigheid", null=True), + ), + ("type", models.CharField(db_column="typeomschrijving", max_length=80)), + ("naam", models.CharField(max_length=80)), + ("naam_nen", models.CharField(db_column="naamnen", max_length=80)), + ( + "geometrie", + django.contrib.gis.db.models.fields.GeometryField( + null=True, srid=28992 + ), + ), + ( + "omschrijving", + models.TextField(db_column="beschrijvingnaam", null=True), + ), + ( + "woonplaats_id", + models.CharField( + db_column="ligtinwoonplaatsid", max_length=16, null=True + ), + ), + ( + "status", + models.CharField(db_column="statusomschrijving", max_length=80), + ), + ("volgnummer", models.IntegerField()), + ("registratiedatum", models.DateField()), + ("straatcode", models.CharField(max_length=16, null=True)), + ( + "straatnaam", + models.CharField( + db_column="straatnaamptt", max_length=80, null=True + ), + ), + ("statuscode", models.CharField(max_length=1)), + ("geconstateerd", models.BooleanField()), + ("typecode", models.CharField(max_length=1)), + ( + "dossier", + models.CharField( + db_column="heeftdossierid", max_length=16, null=True + ), + ), + ( + "proces_code", + models.CharField( + db_column="bagprocescode", max_length=4, null=True + ), + ), + ( + "proces_omschrijving", + models.CharField( + db_column="bagprocesomschrijving", max_length=80, null=True + ), + ), ], options={ - 'db_table': 'bag_openbareruimte', + "db_table": "bag_openbareruimte", }, managers=[ - ('transformed_geo_objects', django.db.models.manager.Manager()), + ("transformed_geo_objects", django.db.models.manager.Manager()), ], ), migrations.CreateModel( - name='Pand', + name="Pand", fields=[ - ('id', models.CharField(db_column='identificatie', max_length=16, primary_key=True, serialize=False)), - ('document_mutatie', models.DateField(db_column='documentdatum')), - ('document_nummer', models.CharField(db_column='documentnummer', max_length=80)), - ('begin_geldigheid', models.DateField(db_column='begingeldigheid')), - ('einde_geldigheid', models.DateField(db_column='eindgeldigheid', null=True)), - ('bouwjaar', models.IntegerField(db_column='oorspronkelijkbouwjaar', null=True)), - ('laagste_bouwlaag', models.IntegerField(db_column='laagstebouwlaag', null=True)), - ('hoogste_bouwlaag', models.IntegerField(db_column='hoogstebouwlaag', null=True)), - ('geometrie', django.contrib.gis.db.models.fields.PolygonField(srid=28992)), - ('pandnaam', models.CharField(db_column='naam', max_length=80, null=True)), - ('bouwblok', models.CharField(db_column='ligtinbouwblokid', max_length=16, null=True)), - ('bouwlagen', models.IntegerField(db_column='aantalbouwlagen', null=True)), - ('ligging', models.CharField(db_column='liggingomschrijving', max_length=80, null=True)), - ('type_woonobject', models.CharField(db_column='typewoonobject', max_length=80, null=True)), - ('status', models.CharField(db_column='statusomschrijving', max_length=80)), - ('volgnummer', models.IntegerField()), - ('registratiedatum', models.DateField()), - ('geconstateerd', models.BooleanField()), - ('statuscode', models.CharField(max_length=3)), - ('ligging_code', models.CharField(db_column='liggingcode', max_length=2, null=True)), - ('buurt', models.CharField(db_column='ligtinbuurtid', max_length=16, null=True)), - ('dossier', models.CharField(db_column='heeftdossierid', max_length=16, null=True)), - ('proces_code', models.CharField(db_column='bagprocescode', max_length=4, null=True)), - ('proces_omschrijving', models.CharField(db_column='bagprocesomschrijving', max_length=80, null=True)), + ( + "id", + models.CharField( + db_column="identificatie", + max_length=16, + primary_key=True, + serialize=False, + ), + ), + ("document_mutatie", models.DateField(db_column="documentdatum")), + ( + "document_nummer", + models.CharField(db_column="documentnummer", max_length=80), + ), + ("begin_geldigheid", models.DateField(db_column="begingeldigheid")), + ( + "einde_geldigheid", + models.DateField(db_column="eindgeldigheid", null=True), + ), + ( + "bouwjaar", + models.IntegerField(db_column="oorspronkelijkbouwjaar", null=True), + ), + ( + "laagste_bouwlaag", + models.IntegerField(db_column="laagstebouwlaag", null=True), + ), + ( + "hoogste_bouwlaag", + models.IntegerField(db_column="hoogstebouwlaag", null=True), + ), + ( + "geometrie", + django.contrib.gis.db.models.fields.PolygonField(srid=28992), + ), + ( + "pandnaam", + models.CharField(db_column="naam", max_length=80, null=True), + ), + ( + "bouwblok", + models.CharField( + db_column="ligtinbouwblokid", max_length=16, null=True + ), + ), + ( + "bouwlagen", + models.IntegerField(db_column="aantalbouwlagen", null=True), + ), + ( + "ligging", + models.CharField( + db_column="liggingomschrijving", max_length=80, null=True + ), + ), + ( + "type_woonobject", + models.CharField( + db_column="typewoonobject", max_length=80, null=True + ), + ), + ( + "status", + models.CharField(db_column="statusomschrijving", max_length=80), + ), + ("volgnummer", models.IntegerField()), + ("registratiedatum", models.DateField()), + ("geconstateerd", models.BooleanField()), + ("statuscode", models.CharField(max_length=3)), + ( + "ligging_code", + models.CharField(db_column="liggingcode", max_length=2, null=True), + ), + ( + "buurt", + models.CharField( + db_column="ligtinbuurtid", max_length=16, null=True + ), + ), + ( + "dossier", + models.CharField( + db_column="heeftdossierid", max_length=16, null=True + ), + ), + ( + "proces_code", + models.CharField( + db_column="bagprocescode", max_length=4, null=True + ), + ), + ( + "proces_omschrijving", + models.CharField( + db_column="bagprocesomschrijving", max_length=80, null=True + ), + ), ], options={ - 'db_table': 'bag_pand', + "db_table": "bag_pand", }, ), migrations.CreateModel( - name='Standplaats', + name="Standplaats", fields=[ - ('id', models.CharField(db_column='identificatie', max_length=16, primary_key=True, serialize=False)), - ('document_mutatie', models.DateField(db_column='documentdatum')), - ('document_nummer', models.CharField(db_column='documentnummer', max_length=20)), - ('begin_geldigheid', models.DateField(db_column='begingeldigheid')), - ('einde_geldigheid', models.DateField(db_column='eindgeldigheid', null=True)), - ('indicatie_geconstateerd', models.BooleanField(db_column='geconstateerd')), - ('geometrie', django.contrib.gis.db.models.fields.PolygonField(srid=28992)), - ('buurt', models.CharField(db_column='ligtinbuurtid', max_length=16, null=True)), - ('status', models.CharField(db_column='statusomschrijving', max_length=80)), - ('volgnummer', models.IntegerField()), - ('registratiedatum', models.DateField()), - ('statuscode', models.CharField(max_length=1)), - ('hoofdadres', models.CharField(db_column='heefthoofdadresid', max_length=16, null=True)), - ('dossier', models.CharField(db_column='heeftdossierid', max_length=16, null=True)), - ('proces_code', models.CharField(db_column='bagprocescode', max_length=4, null=True)), - ('proces_omschrijving', models.CharField(db_column='bagprocesomschrijving', max_length=80, null=True)), + ( + "id", + models.CharField( + db_column="identificatie", + max_length=16, + primary_key=True, + serialize=False, + ), + ), + ("document_mutatie", models.DateField(db_column="documentdatum")), + ( + "document_nummer", + models.CharField(db_column="documentnummer", max_length=20), + ), + ("begin_geldigheid", models.DateField(db_column="begingeldigheid")), + ( + "einde_geldigheid", + models.DateField(db_column="eindgeldigheid", null=True), + ), + ( + "indicatie_geconstateerd", + models.BooleanField(db_column="geconstateerd"), + ), + ( + "geometrie", + django.contrib.gis.db.models.fields.PolygonField(srid=28992), + ), + ( + "buurt", + models.CharField( + db_column="ligtinbuurtid", max_length=16, null=True + ), + ), + ( + "status", + models.CharField(db_column="statusomschrijving", max_length=80), + ), + ("volgnummer", models.IntegerField()), + ("registratiedatum", models.DateField()), + ("statuscode", models.CharField(max_length=1)), + ( + "hoofdadres", + models.CharField( + db_column="heefthoofdadresid", max_length=16, null=True + ), + ), + ( + "dossier", + models.CharField( + db_column="heeftdossierid", max_length=16, null=True + ), + ), + ( + "proces_code", + models.CharField( + db_column="bagprocescode", max_length=4, null=True + ), + ), + ( + "proces_omschrijving", + models.CharField( + db_column="bagprocesomschrijving", max_length=80, null=True + ), + ), ], options={ - 'db_table': 'bag_standplaats', + "db_table": "bag_standplaats", }, bases=(models.Model, bag.models.BagObject), ), migrations.CreateModel( - name='Verblijfsobject', + name="Verblijfsobject", fields=[ - ('id', models.CharField(db_column='identificatie', max_length=16, primary_key=True, serialize=False)), - ('document_mutatie', models.DateField(db_column='documentdatum')), - ('document_nummer', models.CharField(db_column='documentnummer', max_length=80)), - ('begin_geldigheid', models.DateField(db_column='begingeldigheid')), - ('einde_geldigheid', models.DateField(db_column='eindgeldigheid', null=True)), - ('oppervlakte', models.IntegerField(null=True)), - ('verdieping_toegang', models.CharField(db_column='verdiepingtoegang', max_length=80, null=True)), - ('aantal_eenheden_complex', models.IntegerField(db_column='aantaleenhedencomplex', null=True)), - ('bouwlagen', models.IntegerField(db_column='aantalbouwlagen', null=True)), - ('aantal_kamers', models.IntegerField(db_column='aantalkamers', null=True)), - ('indicatie_geconstateerd', models.BooleanField(db_column='geconstateerd')), - ('geometrie', django.contrib.gis.db.models.fields.GeometryField(null=True, srid=28992)), - ('buurt', models.CharField(db_column='ligtinbuurtid', max_length=16, null=True)), - ('gebruiksdoel_gezondheidszorgfunctie', models.CharField(db_column='gebruiksdoelgezondheidszorgfunctieomschrijving', max_length=80, null=True)), - ('gebruiksdoel_woonfunctie', models.CharField(db_column='gebruiksdoelwoonfunctieomschrijving', max_length=80, null=True)), - ('hoogste_bouwlaag', models.IntegerField(db_column='hoogstebouwlaag', null=True)), - ('laagste_bouwlaag', models.IntegerField(db_column='laagstebouwlaag', null=True)), - ('status', models.CharField(db_column='statusomschrijving', max_length=80, null=True)), - ('reden_afvoer', models.CharField(db_column='redenafvoeromschrijving', max_length=80, null=True)), - ('reden_opvoer', models.CharField(db_column='redenopvoeromschrijving', max_length=80, null=True)), - ('eigendomsverhouding', models.CharField(db_column='eigendomsverhoudingomschrijving', max_length=80, null=True)), - ('gebruiksdoel', models.CharField(db_column='feitelijkgebruikomschrijving', max_length=80, null=True)), - ('volgnummer', models.IntegerField()), - ('registratiedatum', models.DateField()), - ('cbsnummer', models.CharField(max_length=16, null=True)), - ('indicatie_woningvoorraad', models.CharField(db_column='indicatiewoningvoorraad', max_length=1, null=True)), - ('financierings_code', models.CharField(db_column='financieringscodecode', max_length=4, null=True)), - ('financierings_omschrijving', models.CharField(db_column='financieringscodeomschrijving', max_length=80, null=True)), - ('hoofdadres', models.CharField(db_column='heefthoofdadresid', max_length=16, null=True)), - ('status_code', models.IntegerField(db_column='statuscode', null=True)), - ('gebruiksdoel_woonfunctie_code', models.CharField(db_column='gebruiksdoelwoonfunctiecode', max_length=4, null=True)), - ('gebruiksdoel_gezondheidszorgfunctie_code', models.CharField(db_column='gebruiksdoelgezondheidszorgfunctiecode', max_length=4, null=True)), - ('eigendomsverhouding_code', models.CharField(db_column='eigendomsverhoudingcode', max_length=4, null=True)), - ('feitelijk_gebruik_code', models.CharField(db_column='feitelijkgebruikcode', max_length=4, null=True)), - ('reden_opvoer_code', models.CharField(db_column='redenopvoercode', max_length=4, null=True)), - ('reden_afvoer_code', models.CharField(db_column='redenafvoercode', max_length=4, null=True)), - ('dossier', models.CharField(db_column='heeftdossierid', max_length=16, null=True)), - ('proces_code', models.CharField(db_column='bagprocescode', max_length=4, null=True)), - ('proces_omschrijving', models.CharField(db_column='bagprocesomschrijving', max_length=80, null=True)), + ( + "id", + models.CharField( + db_column="identificatie", + max_length=16, + primary_key=True, + serialize=False, + ), + ), + ("document_mutatie", models.DateField(db_column="documentdatum")), + ( + "document_nummer", + models.CharField(db_column="documentnummer", max_length=80), + ), + ("begin_geldigheid", models.DateField(db_column="begingeldigheid")), + ( + "einde_geldigheid", + models.DateField(db_column="eindgeldigheid", null=True), + ), + ("oppervlakte", models.IntegerField(null=True)), + ( + "verdieping_toegang", + models.CharField( + db_column="verdiepingtoegang", max_length=80, null=True + ), + ), + ( + "aantal_eenheden_complex", + models.IntegerField(db_column="aantaleenhedencomplex", null=True), + ), + ( + "bouwlagen", + models.IntegerField(db_column="aantalbouwlagen", null=True), + ), + ( + "aantal_kamers", + models.IntegerField(db_column="aantalkamers", null=True), + ), + ( + "indicatie_geconstateerd", + models.BooleanField(db_column="geconstateerd"), + ), + ( + "geometrie", + django.contrib.gis.db.models.fields.GeometryField( + null=True, srid=28992 + ), + ), + ( + "buurt", + models.CharField( + db_column="ligtinbuurtid", max_length=16, null=True + ), + ), + ( + "gebruiksdoel_gezondheidszorgfunctie", + models.CharField( + db_column="gebruiksdoelgezondheidszorgfunctieomschrijving", + max_length=80, + null=True, + ), + ), + ( + "gebruiksdoel_woonfunctie", + models.CharField( + db_column="gebruiksdoelwoonfunctieomschrijving", + max_length=80, + null=True, + ), + ), + ( + "hoogste_bouwlaag", + models.IntegerField(db_column="hoogstebouwlaag", null=True), + ), + ( + "laagste_bouwlaag", + models.IntegerField(db_column="laagstebouwlaag", null=True), + ), + ( + "status", + models.CharField( + db_column="statusomschrijving", max_length=80, null=True + ), + ), + ( + "reden_afvoer", + models.CharField( + db_column="redenafvoeromschrijving", max_length=80, null=True + ), + ), + ( + "reden_opvoer", + models.CharField( + db_column="redenopvoeromschrijving", max_length=80, null=True + ), + ), + ( + "eigendomsverhouding", + models.CharField( + db_column="eigendomsverhoudingomschrijving", + max_length=80, + null=True, + ), + ), + ( + "gebruiksdoel", + models.CharField( + db_column="feitelijkgebruikomschrijving", + max_length=80, + null=True, + ), + ), + ("volgnummer", models.IntegerField()), + ("registratiedatum", models.DateField()), + ("cbsnummer", models.CharField(max_length=16, null=True)), + ( + "indicatie_woningvoorraad", + models.CharField( + db_column="indicatiewoningvoorraad", max_length=1, null=True + ), + ), + ( + "financierings_code", + models.CharField( + db_column="financieringscodecode", max_length=4, null=True + ), + ), + ( + "financierings_omschrijving", + models.CharField( + db_column="financieringscodeomschrijving", + max_length=80, + null=True, + ), + ), + ( + "hoofdadres", + models.CharField( + db_column="heefthoofdadresid", max_length=16, null=True + ), + ), + ("status_code", models.IntegerField(db_column="statuscode", null=True)), + ( + "gebruiksdoel_woonfunctie_code", + models.CharField( + db_column="gebruiksdoelwoonfunctiecode", max_length=4, null=True + ), + ), + ( + "gebruiksdoel_gezondheidszorgfunctie_code", + models.CharField( + db_column="gebruiksdoelgezondheidszorgfunctiecode", + max_length=4, + null=True, + ), + ), + ( + "eigendomsverhouding_code", + models.CharField( + db_column="eigendomsverhoudingcode", max_length=4, null=True + ), + ), + ( + "feitelijk_gebruik_code", + models.CharField( + db_column="feitelijkgebruikcode", max_length=4, null=True + ), + ), + ( + "reden_opvoer_code", + models.CharField( + db_column="redenopvoercode", max_length=4, null=True + ), + ), + ( + "reden_afvoer_code", + models.CharField( + db_column="redenafvoercode", max_length=4, null=True + ), + ), + ( + "dossier", + models.CharField( + db_column="heeftdossierid", max_length=16, null=True + ), + ), + ( + "proces_code", + models.CharField( + db_column="bagprocescode", max_length=4, null=True + ), + ), + ( + "proces_omschrijving", + models.CharField( + db_column="bagprocesomschrijving", max_length=80, null=True + ), + ), ], options={ - 'db_table': 'bag_verblijfsobject', + "db_table": "bag_verblijfsobject", }, bases=(models.Model, bag.models.BagObject), ), migrations.CreateModel( - name='Verblijfsobjectpandrelatie', + name="Verblijfsobjectpandrelatie", fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('pand', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bag.pand')), - ('verblijfsobject', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='bag.verblijfsobject')), + ("id", models.AutoField(primary_key=True, serialize=False)), + ( + "pand", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, to="bag.pand" + ), + ), + ( + "verblijfsobject", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="bag.verblijfsobject", + ), + ), ], options={ - 'db_table': 'bag_verblijfsobjectpandrelatie', + "db_table": "bag_verblijfsobjectpandrelatie", }, ), migrations.AddField( - model_name='verblijfsobject', - name='panden', - field=models.ManyToManyField(related_name='verblijfsobjecten', through='bag.Verblijfsobjectpandrelatie', to='bag.pand'), + model_name="verblijfsobject", + name="panden", + field=models.ManyToManyField( + related_name="verblijfsobjecten", + through="bag.Verblijfsobjectpandrelatie", + to="bag.pand", + ), ), migrations.CreateModel( - name='Nummeraanduiding', + name="Nummeraanduiding", fields=[ - ('id', models.CharField(db_column='identificatie', max_length=16, primary_key=True, serialize=False)), - ('document_mutatie', models.DateField(db_column='documentdatum')), - ('document_nummer', models.CharField(db_column='documentnummer', max_length=40)), - ('begin_geldigheid', models.DateTimeField(db_column='begingeldigheid')), - ('eind_geldigheid', models.DateTimeField(db_column='eindgeldigheid', null=True)), - ('huisnummer', models.IntegerField()), - ('huisletter', models.CharField(max_length=1, null=True)), - ('huisnummer_toevoeging', models.CharField(db_column='huisnummertoevoeging', max_length=4, null=True)), - ('postcode', models.CharField(max_length=6, null=True)), - ('type_adres', models.CharField(db_column='typeadres', max_length=80, null=True)), - ('status', models.CharField(db_column='statusomschrijving', max_length=80, null=True)), - ('volgnummer', models.IntegerField()), - ('registratiedatum', models.DateTimeField()), - ('geconstateerd', models.BooleanField()), - ('woonplaats', models.CharField(db_column='ligtinwoonplaatsid', max_length=80, null=True)), - ('type_adresseerbaar_object_code', models.CharField(db_column='typeadresseerbaarobjectcode', max_length=4, null=True)), - ('type_adresseerbaar_object_omschrijving', models.CharField(db_column='typeadresseerbaarobjectomschrijving', max_length=80, null=True)), - ('status_code', models.CharField(db_column='statuscode', max_length=4, null=True)), - ('dossier', models.CharField(db_column='heeftdossierid', max_length=40, null=True)), - ('proces_code', models.CharField(db_column='bagprocescode', max_length=4, null=True)), - ('proces_omschrijving', models.CharField(db_column='bagprocesomschrijving', max_length=80, null=True)), - ('ligplaats', models.ForeignKey(db_column='adresseertligplaatsid', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='nummeraanduidingen', to='bag.ligplaats')), - ('openbare_ruimte', models.ForeignKey(db_column='ligtaanopenbareruimteid', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='nummeraanduidingen', to='bag.openbareruimte')), - ('standplaats', models.ForeignKey(db_column='adresseertstandplaatsid', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='nummeraanduidingen', to='bag.standplaats')), - ('verblijfsobject', models.ForeignKey(db_column='adresseertverblijfsobjectid', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='nummeraanduidingen', to='bag.verblijfsobject')), + ( + "id", + models.CharField( + db_column="identificatie", + max_length=16, + primary_key=True, + serialize=False, + ), + ), + ("document_mutatie", models.DateField(db_column="documentdatum")), + ( + "document_nummer", + models.CharField(db_column="documentnummer", max_length=40), + ), + ("begin_geldigheid", models.DateTimeField(db_column="begingeldigheid")), + ( + "eind_geldigheid", + models.DateTimeField(db_column="eindgeldigheid", null=True), + ), + ("huisnummer", models.IntegerField()), + ("huisletter", models.CharField(max_length=1, null=True)), + ( + "huisnummer_toevoeging", + models.CharField( + db_column="huisnummertoevoeging", max_length=4, null=True + ), + ), + ("postcode", models.CharField(max_length=6, null=True)), + ( + "type_adres", + models.CharField(db_column="typeadres", max_length=80, null=True), + ), + ( + "status", + models.CharField( + db_column="statusomschrijving", max_length=80, null=True + ), + ), + ("volgnummer", models.IntegerField()), + ("registratiedatum", models.DateTimeField()), + ("geconstateerd", models.BooleanField()), + ( + "woonplaats", + models.CharField( + db_column="ligtinwoonplaatsid", max_length=80, null=True + ), + ), + ( + "type_adresseerbaar_object_code", + models.CharField( + db_column="typeadresseerbaarobjectcode", max_length=4, null=True + ), + ), + ( + "type_adresseerbaar_object_omschrijving", + models.CharField( + db_column="typeadresseerbaarobjectomschrijving", + max_length=80, + null=True, + ), + ), + ( + "status_code", + models.CharField(db_column="statuscode", max_length=4, null=True), + ), + ( + "dossier", + models.CharField( + db_column="heeftdossierid", max_length=40, null=True + ), + ), + ( + "proces_code", + models.CharField( + db_column="bagprocescode", max_length=4, null=True + ), + ), + ( + "proces_omschrijving", + models.CharField( + db_column="bagprocesomschrijving", max_length=80, null=True + ), + ), + ( + "ligplaats", + models.ForeignKey( + db_column="adresseertligplaatsid", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="nummeraanduidingen", + to="bag.ligplaats", + ), + ), + ( + "openbare_ruimte", + models.ForeignKey( + db_column="ligtaanopenbareruimteid", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="nummeraanduidingen", + to="bag.openbareruimte", + ), + ), + ( + "standplaats", + models.ForeignKey( + db_column="adresseertstandplaatsid", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="nummeraanduidingen", + to="bag.standplaats", + ), + ), + ( + "verblijfsobject", + models.ForeignKey( + db_column="adresseertverblijfsobjectid", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="nummeraanduidingen", + to="bag.verblijfsobject", + ), + ), ], options={ - 'db_table': 'bag_nummeraanduiding', + "db_table": "bag_nummeraanduiding", }, ), ] diff --git a/src/bag/migrations/0002_alter_pand_managers_alter_verblijfsobject_managers_and_more.py b/src/bag/migrations/0002_alter_pand_managers_alter_verblijfsobject_managers_and_more.py index a4d6ce2..2c1fbaf 100644 --- a/src/bag/migrations/0002_alter_pand_managers_alter_verblijfsobject_managers_and_more.py +++ b/src/bag/migrations/0002_alter_pand_managers_alter_verblijfsobject_managers_and_more.py @@ -1,56 +1,56 @@ # Generated by Django 4.1.13 on 2024-02-19 12:16 -from django.db import migrations, models import django.db.models.manager +from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('bag', '0001_initial'), + ("bag", "0001_initial"), ] operations = [ migrations.AlterModelManagers( - name='pand', + name="pand", managers=[ - ('transformed_geo_objects', django.db.models.manager.Manager()), + ("transformed_geo_objects", django.db.models.manager.Manager()), ], ), migrations.AlterModelManagers( - name='verblijfsobject', + name="verblijfsobject", managers=[ - ('transformed_geo_objects', django.db.models.manager.Manager()), + ("transformed_geo_objects", django.db.models.manager.Manager()), ], ), migrations.AddField( - model_name='ligplaats', - name='plusvolgnummer', + model_name="ligplaats", + name="plusvolgnummer", field=models.IntegerField(null=True), ), migrations.AddField( - model_name='nummeraanduiding', - name='plusvolgnummer', + model_name="nummeraanduiding", + name="plusvolgnummer", field=models.IntegerField(null=True), ), migrations.AddField( - model_name='openbareruimte', - name='plusvolgnummer', + model_name="openbareruimte", + name="plusvolgnummer", field=models.IntegerField(null=True), ), migrations.AddField( - model_name='pand', - name='plusvolgnummer', + model_name="pand", + name="plusvolgnummer", field=models.IntegerField(null=True), ), migrations.AddField( - model_name='standplaats', - name='plusvolgnummer', + model_name="standplaats", + name="plusvolgnummer", field=models.IntegerField(null=True), ), migrations.AddField( - model_name='verblijfsobject', - name='plusvolgnummer', + model_name="verblijfsobject", + name="plusvolgnummer", field=models.IntegerField(null=True), ), ] diff --git a/src/bag/migrations/0003_alter_pand_bouwblok.py b/src/bag/migrations/0003_alter_pand_bouwblok.py index 902ca7e..9a8ed61 100644 --- a/src/bag/migrations/0003_alter_pand_bouwblok.py +++ b/src/bag/migrations/0003_alter_pand_bouwblok.py @@ -6,13 +6,15 @@ class Migration(migrations.Migration): dependencies = [ - ('bag', '0002_alter_pand_managers_alter_verblijfsobject_managers_and_more'), + ("bag", "0002_alter_pand_managers_alter_verblijfsobject_managers_and_more"), ] operations = [ migrations.AlterField( - model_name='pand', - name='bouwblok', - field=models.CharField(db_column='ligtinbouwblokid', db_index=True, max_length=16, null=True), + model_name="pand", + name="bouwblok", + field=models.CharField( + db_column="ligtinbouwblokid", db_index=True, max_length=16, null=True + ), ), ] diff --git a/src/bag/models.py b/src/bag/models.py index ad9245b..90b3687 100644 --- a/src/bag/models.py +++ b/src/bag/models.py @@ -1,7 +1,6 @@ from django.conf import settings from django.contrib.gis.db import models as gis_models from django.contrib.gis.db.models.functions import Centroid, Transform -from django.contrib.postgres.fields import ArrayField from django.db import models @@ -40,6 +39,7 @@ class GeoModel(models.Model): class Meta: abstract = True + class Ligplaats(GeoModel, BagObject): id = models.CharField(max_length=16, primary_key=True, db_column="identificatie") document_mutatie = models.DateField(db_column="documentdatum") @@ -179,7 +179,9 @@ class Pand(GeoModel): hoogste_bouwlaag = models.IntegerField(null=True, db_column="hoogstebouwlaag") geometrie = gis_models.PolygonField(srid=28992) pandnaam = models.CharField(max_length=80, null=True, db_column="naam") - bouwblok = models.CharField(max_length=16, db_column="ligtinbouwblokid", db_index=True, null=True) + bouwblok = models.CharField( + max_length=16, db_column="ligtinbouwblokid", db_index=True, null=True + ) bouwlagen = models.IntegerField(null=True, db_column="aantalbouwlagen") ligging = models.CharField( @@ -381,4 +383,4 @@ def __str__(self): return f"{self.openbare_ruimte} {self.huisnummer}{huisletter}{huisnr_toevoeging} ({self.type_adres})" class Meta: - db_table = "bag_nummeraanduiding" \ No newline at end of file + db_table = "bag_nummeraanduiding" diff --git a/src/bag/tests/test_bag_loader.py b/src/bag/tests/test_bag_loader.py index 78802c6..b1c791c 100644 --- a/src/bag/tests/test_bag_loader.py +++ b/src/bag/tests/test_bag_loader.py @@ -1,18 +1,18 @@ import logging -from django.conf import settings from django.test import TestCase from bag.bag_loader import BagLoader log = logging.getLogger(__name__) + class MockWriter: content = [] def writeheader(self): pass - + def writerow(self, row): self.content.append(row) @@ -22,7 +22,6 @@ def __str__(self): class APITest(TestCase): @classmethod - def setUp(self): pass @@ -30,7 +29,7 @@ def test_preprocess_csv(self): csv_data = [ {"Identificatie": "0457100000068588", "Volgnummer": "3"}, {"Identificatie": "0457100000068549", "Volgnummer": "3"}, - {"Identificatie": "0457100000068588", "Volgnummer": "1"} + {"Identificatie": "0457100000068588", "Volgnummer": "1"}, ] reader = iter(csv_data) @@ -38,8 +37,12 @@ def test_preprocess_csv(self): bag = BagLoader() bag.preprocess_csv_rows(reader, writer) - self.assertNotIn({'Identificatie': '0457100000068588', 'Volgnummer': '1'}, writer.content) - self.assertIn({'Identificatie': '0457100000068588', 'Volgnummer': '3'}, writer.content) - self.assertIn({'Identificatie': '0457100000068549', 'Volgnummer': '3'}, writer.content) - - + self.assertNotIn( + {"Identificatie": "0457100000068588", "Volgnummer": "1"}, writer.content + ) + self.assertIn( + {"Identificatie": "0457100000068588", "Volgnummer": "3"}, writer.content + ) + self.assertIn( + {"Identificatie": "0457100000068549", "Volgnummer": "3"}, writer.content + ) diff --git a/src/bag/utils.py b/src/bag/utils.py index 13a5fa8..c02a4c1 100644 --- a/src/bag/utils.py +++ b/src/bag/utils.py @@ -4,12 +4,20 @@ from django.conf import settings from django.core.exceptions import ObjectDoesNotExist -from bag.constants import (BAG_TYPE_LIGPLAATS, BAG_TYPE_NUMMERAANDUIDING, - BAG_TYPE_OPENBARE_RUIMTE, BAG_TYPE_PAND, - BAG_TYPE_STANDPLAATS, BAG_TYPE_VERBLIJFSOBJECT) -from bag.exceptions import (BagIdException, IncorrectBagIdLengthException, - IncorrectGemeenteCodeException, - IncorrectObjectTypeException) +from bag.constants import ( + BAG_TYPE_LIGPLAATS, + BAG_TYPE_NUMMERAANDUIDING, + BAG_TYPE_OPENBARE_RUIMTE, + BAG_TYPE_PAND, + BAG_TYPE_STANDPLAATS, + BAG_TYPE_VERBLIJFSOBJECT, +) +from bag.exceptions import ( + BagIdException, + IncorrectBagIdLengthException, + IncorrectGemeenteCodeException, + IncorrectObjectTypeException, +) from bag.models import Ligplaats, Standplaats, Verblijfsobject logger = logging.getLogger(__name__) @@ -83,10 +91,7 @@ def get_object_from_id(id): ) try: - bag_object = ( - bag_model.objects.filter(id=id) - .get() - ) + bag_object = bag_model.objects.filter(id=id).get() except ObjectDoesNotExist: raise bag_model.DoesNotExist( f"{bag_model.__name__} met landelijk id {id} bestaat niet." diff --git a/src/bouwdossiers/apps.py b/src/bouwdossiers/apps.py index 6fa29cf..3c66750 100644 --- a/src/bouwdossiers/apps.py +++ b/src/bouwdossiers/apps.py @@ -2,4 +2,4 @@ class BouwDossiersConfig(AppConfig): - name = 'bouwdossiers' + name = "bouwdossiers" diff --git a/src/bouwdossiers/migrations/0001_initial.py b/src/bouwdossiers/migrations/0001_initial.py index c6cfcb5..e87d164 100644 --- a/src/bouwdossiers/migrations/0001_initial.py +++ b/src/bouwdossiers/migrations/0001_initial.py @@ -10,82 +10,181 @@ class Migration(migrations.Migration): initial = True - dependencies = [ - ] + dependencies = [] operations = [ migrations.CreateModel( - name='BouwDossier', + name="BouwDossier", fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('source', models.CharField(choices=[('EDEPOT', 'edepot'), ('WABO', 'wabo')], default='EDEPOT', max_length=20)), - ('dossiernr', models.IntegerField()), - ('stadsdeel', models.CharField(db_index=True, max_length=3)), - ('titel', models.CharField(db_index=True, max_length=512)), - ('datering', models.DateField(null=True)), - ('dossier_type', models.CharField(max_length=64, null=True)), - ('dossier_status', models.CharField(choices=[('A', 'Aanvraag'), ('B', 'Behandeling')], max_length=1, null=True)), - ('access', models.CharField(choices=[('PUBLIC', 'Public'), ('RESTRICTED', 'Restricted')], max_length=20, null=True)), + ("id", models.AutoField(primary_key=True, serialize=False)), + ( + "source", + models.CharField( + choices=[("EDEPOT", "edepot"), ("WABO", "wabo")], + default="EDEPOT", + max_length=20, + ), + ), + ("dossiernr", models.IntegerField()), + ("stadsdeel", models.CharField(db_index=True, max_length=3)), + ("titel", models.CharField(db_index=True, max_length=512)), + ("datering", models.DateField(null=True)), + ("dossier_type", models.CharField(max_length=64, null=True)), + ( + "dossier_status", + models.CharField( + choices=[("A", "Aanvraag"), ("B", "Behandeling")], + max_length=1, + null=True, + ), + ), + ( + "access", + models.CharField( + choices=[("PUBLIC", "Public"), ("RESTRICTED", "Restricted")], + max_length=20, + null=True, + ), + ), ], options={ - 'ordering': ('stadsdeel', 'dossiernr'), + "ordering": ("stadsdeel", "dossiernr"), }, ), migrations.CreateModel( - name='ImportFile', + name="ImportFile", fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('name', models.CharField(max_length=512, unique=True)), - ('status', models.CharField(choices=[('B', 'Busy'), ('F', 'Finished'), ('E', 'Error')], max_length=1)), - ('last_import', models.DateTimeField(auto_now=True)), + ("id", models.AutoField(primary_key=True, serialize=False)), + ("name", models.CharField(max_length=512, unique=True)), + ( + "status", + models.CharField( + choices=[("B", "Busy"), ("F", "Finished"), ("E", "Error")], + max_length=1, + ), + ), + ("last_import", models.DateTimeField(auto_now=True)), ], options={ - 'ordering': ('name',), + "ordering": ("name",), }, ), migrations.CreateModel( - name='Document', + name="Document", fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('subdossier_titel', models.CharField(max_length=128)), - ('barcode', models.CharField(db_index=True, max_length=250)), - ('bestanden', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, size=None)), - ('access', models.CharField(choices=[('PUBLIC', 'Public'), ('RESTRICTED', 'Restricted')], max_length=20, null=True)), - ('bouwdossier', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='documenten', to='bouwdossiers.BouwDossier')), + ("id", models.AutoField(primary_key=True, serialize=False)), + ("subdossier_titel", models.CharField(max_length=128)), + ("barcode", models.CharField(db_index=True, max_length=250)), + ( + "bestanden", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=128), + blank=True, + size=None, + ), + ), + ( + "access", + models.CharField( + choices=[("PUBLIC", "Public"), ("RESTRICTED", "Restricted")], + max_length=20, + null=True, + ), + ), + ( + "bouwdossier", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="documenten", + to="bouwdossiers.BouwDossier", + ), + ), ], ), migrations.AddField( - model_name='bouwdossier', - name='importfile', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='bouwdossiers', to='bouwdossiers.ImportFile'), + model_name="bouwdossier", + name="importfile", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="bouwdossiers", + to="bouwdossiers.ImportFile", + ), ), migrations.CreateModel( - name='Adres', + name="Adres", fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('straat', models.CharField(max_length=150)), - ('huisnummer_van', models.IntegerField(null=True)), - ('huisnummer_tot', models.IntegerField(null=True)), - ('openbareruimte_id', models.CharField(db_index=True, max_length=16)), - ('stadsdeel', models.CharField(db_index=True, max_length=3)), - ('nummeraanduidingen', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=16), blank=True, size=None)), - ('nummeraanduidingen_label', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=256), blank=True, size=None)), - ('panden', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=16), blank=True, size=None)), - ('verblijfsobjecten', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=16), blank=True, size=None)), - ('verblijfsobjecten_label', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=256), blank=True, size=None)), - ('bouwdossier', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='adressen', to='bouwdossiers.BouwDossier')), + ("id", models.AutoField(primary_key=True, serialize=False)), + ("straat", models.CharField(max_length=150)), + ("huisnummer_van", models.IntegerField(null=True)), + ("huisnummer_tot", models.IntegerField(null=True)), + ("openbareruimte_id", models.CharField(db_index=True, max_length=16)), + ("stadsdeel", models.CharField(db_index=True, max_length=3)), + ( + "nummeraanduidingen", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=16), + blank=True, + size=None, + ), + ), + ( + "nummeraanduidingen_label", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=256), + blank=True, + size=None, + ), + ), + ( + "panden", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=16), + blank=True, + size=None, + ), + ), + ( + "verblijfsobjecten", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=16), + blank=True, + size=None, + ), + ), + ( + "verblijfsobjecten_label", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=256), + blank=True, + size=None, + ), + ), + ( + "bouwdossier", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="adressen", + to="bouwdossiers.BouwDossier", + ), + ), ], ), migrations.AddConstraint( - model_name='bouwdossier', - constraint=models.UniqueConstraint(fields=('stadsdeel', 'dossiernr'), name='unique_bouwdossier'), + model_name="bouwdossier", + constraint=models.UniqueConstraint( + fields=("stadsdeel", "dossiernr"), name="unique_bouwdossier" + ), ), migrations.AddIndex( - model_name='adres', - index=django.contrib.postgres.indexes.GinIndex(fields=['nummeraanduidingen'], name='bouwdossier_nummera_fb3aa1_gin'), + model_name="adres", + index=django.contrib.postgres.indexes.GinIndex( + fields=["nummeraanduidingen"], name="bouwdossier_nummera_fb3aa1_gin" + ), ), migrations.AddIndex( - model_name='adres', - index=django.contrib.postgres.indexes.GinIndex(fields=['panden'], name='bouwdossier_panden_b87c3c_gin'), + model_name="adres", + index=django.contrib.postgres.indexes.GinIndex( + fields=["panden"], name="bouwdossier_panden_b87c3c_gin" + ), ), ] diff --git a/src/bouwdossiers/migrations/0002_auto_20200416_1318.py b/src/bouwdossiers/migrations/0002_auto_20200416_1318.py index 332e680..178414c 100644 --- a/src/bouwdossiers/migrations/0002_auto_20200416_1318.py +++ b/src/bouwdossiers/migrations/0002_auto_20200416_1318.py @@ -7,68 +7,75 @@ class Migration(migrations.Migration): dependencies = [ - ('bouwdossiers', '0001_initial'), + ("bouwdossiers", "0001_initial"), ] operations = [ migrations.AddField( - model_name='adres', - name='huisnummer_letter', + model_name="adres", + name="huisnummer_letter", field=models.CharField(default=None, max_length=10, null=True), ), migrations.AddField( - model_name='adres', - name='huisnummer_toevoeging', + model_name="adres", + name="huisnummer_toevoeging", field=models.CharField(default=None, max_length=10, null=True), ), migrations.AddField( - model_name='adres', - name='locatie_aanduiding', + model_name="adres", + name="locatie_aanduiding", field=models.CharField(max_length=250, null=True), ), migrations.AddField( - model_name='bouwdossier', - name='olo_liaan_nummer', + model_name="bouwdossier", + name="olo_liaan_nummer", field=models.IntegerField(default=None, null=True), ), migrations.AddField( - model_name='bouwdossier', - name='wabo_bron', + model_name="bouwdossier", + name="wabo_bron", field=models.CharField(default=None, max_length=30, null=True), ), migrations.AddField( - model_name='document', - name='document_type', + model_name="document", + name="document_type", field=models.CharField(blank=True, max_length=250, null=True), ), migrations.AddField( - model_name='document', - name='oorspronkelijk_pad', - field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=250), blank=True, default=list, size=None), + model_name="document", + name="oorspronkelijk_pad", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=250), + blank=True, + default=list, + size=None, + ), ), migrations.AlterField( - model_name='adres', - name='openbareruimte_id', + model_name="adres", + name="openbareruimte_id", field=models.CharField(db_index=True, max_length=16, null=True), ), migrations.AlterField( - model_name='adres', - name='straat', + model_name="adres", + name="straat", field=models.CharField(max_length=150, null=True), ), migrations.AlterField( - model_name='document', - name='barcode', + model_name="document", + name="barcode", field=models.CharField(db_index=True, max_length=250, null=True), ), migrations.AlterField( - model_name='document', - name='bestanden', - field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=250), blank=True, size=None), + model_name="document", + name="bestanden", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=250), blank=True, size=None + ), ), migrations.AlterField( - model_name='document', - name='subdossier_titel', + model_name="document", + name="subdossier_titel", field=models.TextField(blank=True, null=True), ), ] diff --git a/src/bouwdossiers/migrations/0003_auto_20200501_1642.py b/src/bouwdossiers/migrations/0003_auto_20200501_1642.py index 2372b72..355ce78 100644 --- a/src/bouwdossiers/migrations/0003_auto_20200501_1642.py +++ b/src/bouwdossiers/migrations/0003_auto_20200501_1642.py @@ -7,28 +7,43 @@ class Migration(migrations.Migration): dependencies = [ - ('bouwdossiers', '0002_auto_20200416_1318'), + ("bouwdossiers", "0002_auto_20200416_1318"), ] operations = [ migrations.RenameField( - model_name='document', - old_name='document_type', - new_name='document_omschrijving', + model_name="document", + old_name="document_type", + new_name="document_omschrijving", ), migrations.AddField( - model_name='bouwdossier', - name='activiteiten', - field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=250), blank=True, default=list, size=None), + model_name="bouwdossier", + name="activiteiten", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=250), + blank=True, + default=list, + size=None, + ), ), migrations.AlterField( - model_name='bouwdossier', - name='source', - field=models.CharField(choices=[('EDEPOT', 'edepot'), ('WABO', 'wabo')], default='EDEPOT', help_text='Field that defines wabo and pre_wabo dossier', max_length=20), + model_name="bouwdossier", + name="source", + field=models.CharField( + choices=[("EDEPOT", "edepot"), ("WABO", "wabo")], + default="EDEPOT", + help_text="Field that defines wabo and pre_wabo dossier", + max_length=20, + ), ), migrations.AlterField( - model_name='bouwdossier', - name='wabo_bron', - field=models.CharField(default=None, help_text='Should contain the origin of the dossier. Can be for example digital or paper dossier.', max_length=30, null=True), + model_name="bouwdossier", + name="wabo_bron", + field=models.CharField( + default=None, + help_text="Should contain the origin of the dossier. Can be for example digital or paper dossier.", + max_length=30, + null=True, + ), ), ] diff --git a/src/bouwdossiers/migrations/0004_auto_20200907_1507.py b/src/bouwdossiers/migrations/0004_auto_20200907_1507.py index 1ed51ef..825cbd2 100644 --- a/src/bouwdossiers/migrations/0004_auto_20200907_1507.py +++ b/src/bouwdossiers/migrations/0004_auto_20200907_1507.py @@ -6,18 +6,18 @@ class Migration(migrations.Migration): dependencies = [ - ('bouwdossiers', '0003_auto_20200501_1642'), + ("bouwdossiers", "0003_auto_20200501_1642"), ] operations = [ migrations.AlterField( - model_name='adres', - name='stadsdeel', + model_name="adres", + name="stadsdeel", field=models.CharField(db_index=True, max_length=10), ), migrations.AlterField( - model_name='bouwdossier', - name='stadsdeel', + model_name="bouwdossier", + name="stadsdeel", field=models.CharField(db_index=True, max_length=10), ), ] diff --git a/src/bouwdossiers/migrations/0005_auto_20200908_1438.py b/src/bouwdossiers/migrations/0005_auto_20200908_1438.py index d0e03ab..c711a77 100644 --- a/src/bouwdossiers/migrations/0005_auto_20200908_1438.py +++ b/src/bouwdossiers/migrations/0005_auto_20200908_1438.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('bouwdossiers', '0004_auto_20200907_1507'), + ("bouwdossiers", "0004_auto_20200907_1507"), ] operations = [ migrations.AlterField( - model_name='bouwdossier', - name='dossier_type', + model_name="bouwdossier", + name="dossier_type", field=models.CharField(max_length=255, null=True), ), ] diff --git a/src/bouwdossiers/migrations/0006_auto_20201228_1449.py b/src/bouwdossiers/migrations/0006_auto_20201228_1449.py index 929fd34..25d238e 100644 --- a/src/bouwdossiers/migrations/0006_auto_20201228_1449.py +++ b/src/bouwdossiers/migrations/0006_auto_20201228_1449.py @@ -6,48 +6,50 @@ class Migration(migrations.Migration): dependencies = [ - ('bouwdossiers', '0005_auto_20200908_1438'), + ("bouwdossiers", "0005_auto_20200908_1438"), ] operations = [ migrations.AddField( - model_name='bouwdossier', - name='access_restricted_until', + model_name="bouwdossier", + name="access_restricted_until", field=models.DateField(null=True), ), migrations.AddField( - model_name='bouwdossier', - name='bwt_nummer', + model_name="bouwdossier", + name="bwt_nummer", field=models.CharField(max_length=127, null=True), ), migrations.AddField( - model_name='bouwdossier', - name='gebruiksdoel', + model_name="bouwdossier", + name="gebruiksdoel", field=models.CharField(max_length=255, null=True), ), migrations.AddField( - model_name='document', - name='access_restricted_until', + model_name="document", + name="access_restricted_until", field=models.DateField(null=True), ), migrations.AddField( - model_name='document', - name='copyright', - field=models.CharField(choices=[('Y', 'Yes'), ('N', 'No')], max_length=1, null=True), + model_name="document", + name="copyright", + field=models.CharField( + choices=[("Y", "Yes"), ("N", "No")], max_length=1, null=True + ), ), migrations.AddField( - model_name='document', - name='copyright_holders', + model_name="document", + name="copyright_holders", field=models.CharField(max_length=512, null=True), ), migrations.AddField( - model_name='document', - name='copyright_manufacturers', + model_name="document", + name="copyright_manufacturers", field=models.CharField(max_length=512, null=True), ), migrations.AddField( - model_name='document', - name='copyright_until', + model_name="document", + name="copyright_until", field=models.DateField(null=True), ), ] diff --git a/src/bouwdossiers/migrations/0007_alter_bouwdossier_options.py b/src/bouwdossiers/migrations/0007_alter_bouwdossier_options.py index 6e8e702..f8ac331 100644 --- a/src/bouwdossiers/migrations/0007_alter_bouwdossier_options.py +++ b/src/bouwdossiers/migrations/0007_alter_bouwdossier_options.py @@ -6,12 +6,12 @@ class Migration(migrations.Migration): dependencies = [ - ('bouwdossiers', '0006_auto_20201228_1449'), + ("bouwdossiers", "0006_auto_20201228_1449"), ] operations = [ migrations.AlterModelOptions( - name='bouwdossier', + name="bouwdossier", options={}, ), ] diff --git a/src/bouwdossiers/models.py b/src/bouwdossiers/models.py index c86b38a..9d8b144 100644 --- a/src/bouwdossiers/models.py +++ b/src/bouwdossiers/models.py @@ -3,46 +3,34 @@ from django.contrib.postgres.indexes import GinIndex from django.db.models import CASCADE -SOURCE_EDEPOT = 'EDEPOT' -SOURCE_WABO = 'WABO' +SOURCE_EDEPOT = "EDEPOT" +SOURCE_WABO = "WABO" -SOURCE_CHOICES = ( - (SOURCE_EDEPOT, 'edepot'), - (SOURCE_WABO, 'wabo') -) +SOURCE_CHOICES = ((SOURCE_EDEPOT, "edepot"), (SOURCE_WABO, "wabo")) -ACCESS_PUBLIC = 'PUBLIC' -ACCESS_RESTRICTED = 'RESTRICTED' +ACCESS_PUBLIC = "PUBLIC" +ACCESS_RESTRICTED = "RESTRICTED" -ACCESS_CHOICES = ( - (ACCESS_PUBLIC, 'Public'), - (ACCESS_RESTRICTED, 'Restricted') -) +ACCESS_CHOICES = ((ACCESS_PUBLIC, "Public"), (ACCESS_RESTRICTED, "Restricted")) -COPYRIGHT_YES = 'Y' -COPYRIGHT_NO = 'N' +COPYRIGHT_YES = "Y" +COPYRIGHT_NO = "N" -COPYRIGHT_CHOICES = ( - (COPYRIGHT_YES, 'Yes'), - (COPYRIGHT_NO, 'No') -) +COPYRIGHT_CHOICES = ((COPYRIGHT_YES, "Yes"), (COPYRIGHT_NO, "No")) -STATUS_AANVRAAG = 'A' -STATUS_BEHANDELING = 'B' +STATUS_AANVRAAG = "A" +STATUS_BEHANDELING = "B" -STATUS_CHOICES = ( - (STATUS_AANVRAAG, 'Aanvraag'), - (STATUS_BEHANDELING, 'Behandeling') -) +STATUS_CHOICES = ((STATUS_AANVRAAG, "Aanvraag"), (STATUS_BEHANDELING, "Behandeling")) -IMPORT_BUSY = 'B' -IMPORT_FINISHED = 'F' -IMPORT_ERROR = 'E' +IMPORT_BUSY = "B" +IMPORT_FINISHED = "F" +IMPORT_ERROR = "E" IMPORT_CHOICES = ( - (IMPORT_BUSY, 'Busy'), - (IMPORT_FINISHED, 'Finished'), - (IMPORT_ERROR, 'Error') + (IMPORT_BUSY, "Busy"), + (IMPORT_FINISHED, "Finished"), + (IMPORT_ERROR, "Error"), ) @@ -52,17 +40,19 @@ class ImportFileBase(models.Model): last_import = models.DateTimeField(auto_now=True) def __str__(self): - return f'{self.name}' + return f"{self.name}" class Meta: - ordering = ('name',) + ordering = ("name",) abstract = True + class ImportFile(ImportFileBase): id = models.AutoField(primary_key=True) - + + class BouwDossierBase(models.Model): - importfile = models.ForeignKey(ImportFile, related_name='+', on_delete=CASCADE) + importfile = models.ForeignKey(ImportFile, related_name="+", on_delete=CASCADE) dossiernr = models.IntegerField(null=False) stadsdeel = models.CharField(max_length=10, db_index=True) titel = models.CharField(max_length=512, null=False, db_index=True) @@ -77,7 +67,8 @@ class BouwDossierBase(models.Model): max_length=20, choices=SOURCE_CHOICES, default=SOURCE_EDEPOT, - help_text='Field that defines wabo and pre_wabo dossier') + help_text="Field that defines wabo and pre_wabo dossier", + ) # WABO Related fields olo_liaan_nummer = models.IntegerField(null=True, default=None) @@ -86,41 +77,61 @@ class BouwDossierBase(models.Model): max_length=30, null=True, default=None, - help_text='Should contain the origin of the dossier. Can be for example digital or paper dossier.') - activiteiten = ArrayField(models.CharField(max_length=250, null=False), blank=True, default=list) + help_text="Should contain the origin of the dossier. Can be for example digital or paper dossier.", + ) + activiteiten = ArrayField( + models.CharField(max_length=250, null=False), blank=True, default=list + ) def __str__(self): - return f'{self.dossiernr} - {self.titel}' + return f"{self.dossiernr} - {self.titel}" class Meta: - ordering = ('stadsdeel', 'dossiernr',) + ordering = ( + "stadsdeel", + "dossiernr", + ) abstract = True + class BouwDossier(BouwDossierBase): id = models.AutoField(primary_key=True) - importfile = models.ForeignKey(ImportFile, related_name='bouwdossiers', on_delete=CASCADE) + importfile = models.ForeignKey( + ImportFile, related_name="bouwdossiers", on_delete=CASCADE + ) class Meta: constraints = [ - models.UniqueConstraint(fields=['stadsdeel', 'dossiernr'], name='unique_bouwdossier'), + models.UniqueConstraint( + fields=["stadsdeel", "dossiernr"], name="unique_bouwdossier" + ), ] + # TODO Do we need multiple adres instances for the same street and huisnummer van/tot # ?? If there are multiple bouwdossiers for the same adres can we use the same adres instance ? class AdresBase(models.Model): - bouwdossier = models.ForeignKey(BouwDossier, - related_name='+', - on_delete=CASCADE) + bouwdossier = models.ForeignKey(BouwDossier, related_name="+", on_delete=CASCADE) straat = models.CharField(max_length=150, null=True) huisnummer_van = models.IntegerField(null=True) huisnummer_tot = models.IntegerField(null=True) - openbareruimte_id = models.CharField(max_length=16, db_index=True, null=True) # landelijk_id + openbareruimte_id = models.CharField( + max_length=16, db_index=True, null=True + ) # landelijk_id stadsdeel = models.CharField(max_length=10, db_index=True) - nummeraanduidingen = ArrayField(models.CharField(max_length=16, null=False), blank=True) - nummeraanduidingen_label = ArrayField(models.CharField(max_length=256, null=False), blank=True) + nummeraanduidingen = ArrayField( + models.CharField(max_length=16, null=False), blank=True + ) + nummeraanduidingen_label = ArrayField( + models.CharField(max_length=256, null=False), blank=True + ) panden = ArrayField(models.CharField(max_length=16, null=False), blank=True) - verblijfsobjecten = ArrayField(models.CharField(max_length=16, null=False), blank=True) - verblijfsobjecten_label = ArrayField(models.CharField(max_length=256, null=False), blank=True) + verblijfsobjecten = ArrayField( + models.CharField(max_length=16, null=False), blank=True + ) + verblijfsobjecten_label = ArrayField( + models.CharField(max_length=256, null=False), blank=True + ) # WABO Related fields locatie_aanduiding = models.CharField(max_length=250, null=True) @@ -128,17 +139,18 @@ class AdresBase(models.Model): huisnummer_letter = models.CharField(max_length=10, null=True, default=None) def __str__(self): - return f'{self.straat} {self.huisnummer_van} - {self.huisnummer_tot}' + return f"{self.straat} {self.huisnummer_van} - {self.huisnummer_tot}" class Meta: - indexes = [GinIndex(fields=['nummeraanduidingen']), GinIndex(fields=['panden'])] + indexes = [GinIndex(fields=["nummeraanduidingen"]), GinIndex(fields=["panden"])] abstract = True + class Adres(AdresBase): id = models.AutoField(primary_key=True) - bouwdossier = models.ForeignKey(BouwDossier, - related_name='adressen', - on_delete=CASCADE) + bouwdossier = models.ForeignKey( + BouwDossier, related_name="adressen", on_delete=CASCADE + ) # SubDossier Model has been replaced by Document Model for the following reasons: @@ -148,7 +160,7 @@ class Adres(AdresBase): # The previous structure did not differentiate between documents in a dossier. # All bestanden (scans) that are public were grouped in the same subdossier and nonpublic ones were ignored class DocumentBase(models.Model): - bouwdossier = models.ForeignKey(BouwDossier, related_name='+', on_delete=CASCADE) + bouwdossier = models.ForeignKey(BouwDossier, related_name="+", on_delete=CASCADE) subdossier_titel = models.TextField(blank=True, null=True) document_omschrijving = models.CharField(max_length=250, blank=True, null=True) barcode = models.CharField(max_length=250, db_index=True, null=True) @@ -161,15 +173,19 @@ class DocumentBase(models.Model): copyright_manufacturers = models.CharField(max_length=512, null=True) # WABO Related fields - oorspronkelijk_pad = ArrayField(models.CharField(max_length=250, null=False), blank=True, default=list) + oorspronkelijk_pad = ArrayField( + models.CharField(max_length=250, null=False), blank=True, default=list + ) def __str__(self): - return f'{self.barcode}' - + return f"{self.barcode}" + class Meta: abstract = True + class Document(DocumentBase): id = models.AutoField(primary_key=True) - bouwdossier = models.ForeignKey(BouwDossier, related_name='documenten', on_delete=CASCADE) - \ No newline at end of file + bouwdossier = models.ForeignKey( + BouwDossier, related_name="documenten", on_delete=CASCADE + ) diff --git a/src/bouwdossiers/serializers.py b/src/bouwdossiers/serializers.py index a13d01d..a4ad789 100644 --- a/src/bouwdossiers/serializers.py +++ b/src/bouwdossiers/serializers.py @@ -6,8 +6,14 @@ from rest_framework.reverse import reverse from rest_framework.serializers import ModelSerializer -from importer.models import (SOURCE_CHOICES, SOURCE_EDEPOT, SOURCE_WABO, Adres, - BouwDossier, Document) +from importer.models import ( + SOURCE_CHOICES, + SOURCE_EDEPOT, + SOURCE_WABO, + Adres, + BouwDossier, + Document, +) log = logging.getLogger(__name__) @@ -15,9 +21,20 @@ class AdresSerializer(ModelSerializer): class Meta: model = Adres - fields = ('straat', 'huisnummer_van', 'huisnummer_tot', 'nummeraanduidingen', 'nummeraanduidingen_label', - 'panden', 'verblijfsobjecten', 'verblijfsobjecten_label', 'openbareruimte_id', 'huisnummer_letter', - 'huisnummer_toevoeging', 'locatie_aanduiding') + fields = ( + "straat", + "huisnummer_van", + "huisnummer_tot", + "nummeraanduidingen", + "nummeraanduidingen_label", + "panden", + "verblijfsobjecten", + "verblijfsobjecten_label", + "openbareruimte_id", + "huisnummer_letter", + "huisnummer_toevoeging", + "locatie_aanduiding", + ) class DocumentSerializer(ModelSerializer): @@ -31,14 +48,18 @@ def to_representation(self, instance): result = super().to_representation(instance) _bestanden = [] - for bestand in result['bestanden']: - filename = bestand.replace(' ', '%20') + for bestand in result["bestanden"]: + filename = bestand.replace(" ", "%20") if instance.bouwdossier.source == SOURCE_EDEPOT: - filename = filename.replace('/', '-') + filename = filename.replace("/", "-") # If stadsdeel en dossiernr are not part of the filename, they will be added to the beginning of the filename in the form of # SD12345. In this way iiif-auth-proxy can determine the access for this bestand in de the dossier. m = re.search(r"^([A-Z]+)-(\d+)-", filename) - if m and m.group(1) == instance.bouwdossier.stadsdeel and int(m.group(2)) == instance.bouwdossier.dossiernr: + if ( + m + and m.group(1) == instance.bouwdossier.stadsdeel + and int(m.group(2)) == instance.bouwdossier.dossiernr + ): url = f"{settings.IIIF_BASE_URL}{dict(SOURCE_CHOICES)[instance.bouwdossier.source]}:{filename}" else: url = f"{settings.IIIF_BASE_URL}{dict(SOURCE_CHOICES)[instance.bouwdossier.source]}:{instance.bouwdossier.stadsdeel}{instance.bouwdossier.dossiernr}-{filename}" @@ -47,38 +68,36 @@ def to_representation(self, instance): file_reference = f"{instance.bouwdossier.stadsdeel}-{instance.bouwdossier.dossiernr}-{instance.bouwdossier.olo_liaan_nummer}_{instance.barcode}" url = f"{settings.IIIF_BASE_URL}{dict(SOURCE_CHOICES)[instance.bouwdossier.source]}:{file_reference}" - _bestanden.append({'filename': filename, 'url': url}) + _bestanden.append({"filename": filename, "url": url}) - result['bestanden'] = _bestanden + result["bestanden"] = _bestanden return result class Meta: model = Document fields = ( - 'subdossier_titel', - 'barcode', - 'bestanden', - 'oorspronkelijk_pad', - 'document_omschrijving', - 'access', - 'access_restricted_until', - 'copyright', - 'copyright_until', - 'copyright_holders', - 'copyright_manufacturers') + "subdossier_titel", + "barcode", + "bestanden", + "oorspronkelijk_pad", + "document_omschrijving", + "access", + "access_restricted_until", + "copyright", + "copyright_until", + "copyright_holders", + "copyright_manufacturers", + ) class CustomLinksField(LinksField): def get_url(self, obj, view_name, request, _format): - url_kwargs = { - 'pk': obj.stadsdeel + str(obj.dossiernr) - } + url_kwargs = {"pk": obj.stadsdeel + str(obj.dossiernr)} - return reverse( - view_name, kwargs=url_kwargs, request=request, format=_format) + return reverse(view_name, kwargs=url_kwargs, request=request, format=_format) class CustomHalSerializer(HALSerializer): @@ -93,21 +112,21 @@ class BouwDossierSerializer(CustomHalSerializer): class Meta: model = BouwDossier fields = ( - '_links', - 'titel', - '_display', - 'dossiernr', - 'stadsdeel', - 'datering', - 'dossier_type', - 'gebruiksdoel', - 'bwt_nummer', - 'dossier_status', - 'olo_liaan_nummer', - 'access', - 'access_restricted_until', - 'activiteiten', - 'documenten', - 'adressen', - 'source', + "_links", + "titel", + "_display", + "dossiernr", + "stadsdeel", + "datering", + "dossier_type", + "gebruiksdoel", + "bwt_nummer", + "dossier_status", + "olo_liaan_nummer", + "access", + "access_restricted_until", + "activiteiten", + "documenten", + "adressen", + "source", ) diff --git a/src/bouwdossiers/tests/factories.py b/src/bouwdossiers/tests/factories.py index 377cf49..5ca55f8 100644 --- a/src/bouwdossiers/tests/factories.py +++ b/src/bouwdossiers/tests/factories.py @@ -5,56 +5,56 @@ class ImportFileFactory(factory.django.DjangoModelFactory): class Meta: - django_get_or_create = ('name',) + django_get_or_create = ("name",) model = models.ImportFile - name = 'test.xml' + name = "test.xml" status = models.IMPORT_FINISHED last_import = "2019-04-12 13:38:23" class BouwDossierFactory(factory.django.DjangoModelFactory): class Meta: - django_get_or_create = ('dossiernr', 'stadsdeel', 'olo_liaan_nummer', 'source') + django_get_or_create = ("dossiernr", "stadsdeel", "olo_liaan_nummer", "source") model = models.BouwDossier source = models.SOURCE_EDEPOT importfile = factory.SubFactory(ImportFileFactory) - dossiernr = '12345' + dossiernr = "12345" olo_liaan_nummer = 123456 - stadsdeel = 'AA' - titel = 'weesperstraat 113 - 117' + stadsdeel = "AA" + titel = "weesperstraat 113 - 117" datering = "1998-01-01" dossier_status = None - dossier_type = 'verbouwing' + dossier_type = "verbouwing" access = models.ACCESS_PUBLIC class AdresFactory(factory.django.DjangoModelFactory): class Meta: - django_get_or_create = ('straat', 'huisnummer_van') + django_get_or_create = ("straat", "huisnummer_van") model = models.Adres bouwdossier = factory.SubFactory(BouwDossierFactory) - straat = 'weesperstraat' + straat = "weesperstraat" huisnummer_van = 113 huisnummer_tot = 117 - openbareruimte_id = '0363300000004835' - nummeraanduidingen = ['0363200000406187'] - nummeraanduidingen_label = ['Weesperstraat 113'] - panden = ['0363100012165490'] - verblijfsobjecten = ['036301000xxxxxxx'] - verblijfsobjecten_label = ['Weesperstraat 113'] - stadsdeel = 'AA' + openbareruimte_id = "0363300000004835" + nummeraanduidingen = ["0363200000406187"] + nummeraanduidingen_label = ["Weesperstraat 113"] + panden = ["0363100012165490"] + verblijfsobjecten = ["036301000xxxxxxx"] + verblijfsobjecten_label = ["Weesperstraat 113"] + stadsdeel = "AA" class DocumentFactory(factory.django.DjangoModelFactory): class Meta: - django_get_or_create = ('subdossier_titel',) + django_get_or_create = ("subdossier_titel",) model = models.Document bouwdossier = factory.SubFactory(BouwDossierFactory) - subdossier_titel = 'Tekeningen (plattegrond)' - bestanden = ['SU10000010_00001.jpg'] + subdossier_titel = "Tekeningen (plattegrond)" + bestanden = ["SU10000010_00001.jpg"] access = models.ACCESS_RESTRICTED - barcode = 'ST100' + barcode = "ST100" diff --git a/src/bouwdossiers/tests/test_api.py b/src/bouwdossiers/tests/test_api.py index 9915982..a48320c 100644 --- a/src/bouwdossiers/tests/test_api.py +++ b/src/bouwdossiers/tests/test_api.py @@ -4,15 +4,21 @@ from django.urls import reverse from rest_framework.test import APITestCase -from bouwdossiers.models import (SOURCE_EDEPOT, SOURCE_WABO, Adres, - BouwDossier, Document) +from bouwdossiers.models import SOURCE_EDEPOT, SOURCE_WABO, Adres, BouwDossier, Document from bouwdossiers.tests import factories from bouwdossiers.tests.tools_for_testing import create_authz_token -def create_bouwdossiers(n, stadsdeel='AA', source=SOURCE_EDEPOT): - return [factories.BouwDossierFactory(dossiernr=randint(10, 10000), stadsdeel=stadsdeel, - olo_liaan_nummer=randint(10, 10000), source=source) for i in range(n)] +def create_bouwdossiers(n, stadsdeel="AA", source=SOURCE_EDEPOT): + return [ + factories.BouwDossierFactory( + dossiernr=randint(10, 10000), + stadsdeel=stadsdeel, + olo_liaan_nummer=randint(10, 10000), + source=source, + ) + for i in range(n) + ] def delete_all_records(): @@ -30,25 +36,29 @@ def setUpClass(cls): def test_api_list(self): create_bouwdossiers(3, source=SOURCE_EDEPOT) create_bouwdossiers(4, source=SOURCE_WABO) - url = reverse('bouwdossier-list') + url = reverse("bouwdossier-list") test_parameters = [ (None, 3), (settings.BOUWDOSSIER_PUBLIC_SCOPE, 3), (settings.BOUWDOSSIER_READ_SCOPE, 7), - (settings.BOUWDOSSIER_EXTENDED_SCOPE, 7) + (settings.BOUWDOSSIER_EXTENDED_SCOPE, 7), ] for scope, num_exptected in test_parameters: - header = {'HTTP_AUTHORIZATION': "Bearer " + create_authz_token(scope)} if scope else {} + header = ( + {"HTTP_AUTHORIZATION": "Bearer " + create_authz_token(scope)} + if scope + else {} + ) response = self.client.get(url, **header) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], num_exptected) + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], num_exptected) delete_all_records() def test_api_malformed_code(self): - url = reverse('bouwdossier-detail', kwargs={'pk': 1}) + url = reverse("bouwdossier-detail", kwargs={"pk": 1}) response = self.client.get(url) self.assertEqual(response.status_code, 400) @@ -56,24 +66,28 @@ def test_api_detail_using_stadsdeel_and_dossier(self): dossiers = create_bouwdossiers(3) pk = dossiers[0].stadsdeel + str(dossiers[0].dossiernr) - url = reverse('bouwdossier-detail', kwargs={'pk': pk}) + url = reverse("bouwdossier-detail", kwargs={"pk": pk}) response = self.client.get(url) - self.assertEqual(response.data['stadsdeel'], dossiers[0].stadsdeel) - self.assertEqual(response.data['dossiernr'], dossiers[0].dossiernr) + self.assertEqual(response.data["stadsdeel"], dossiers[0].stadsdeel) + self.assertEqual(response.data["dossiernr"], dossiers[0].dossiernr) delete_all_records() def test_api_detail_wabo_using_stadsdeel_and_dossier_without_auth(self): dossiers = create_bouwdossiers(4, source=SOURCE_WABO) pk = dossiers[0].stadsdeel + str(dossiers[0].dossiernr) - url = reverse('bouwdossier-detail', kwargs={'pk': pk}) + url = reverse("bouwdossier-detail", kwargs={"pk": pk}) test_parameters = [ None, settings.BOUWDOSSIER_PUBLIC_SCOPE, - 'non-existing', + "non-existing", ] for scope in test_parameters: - header = {'HTTP_AUTHORIZATION': "Bearer " + create_authz_token(scope)} if scope else {} + header = ( + {"HTTP_AUTHORIZATION": "Bearer " + create_authz_token(scope)} + if scope + else {} + ) response = self.client.get(url, **header) self.assertEqual(response.status_code, 404) @@ -82,355 +96,439 @@ def test_api_detail_wabo_using_stadsdeel_and_dossier_without_auth(self): def test_api_detail_wabo_using_stadsdeel_and_dossier_with_auth(self): dossiers = create_bouwdossiers(4, source=SOURCE_WABO) pk = dossiers[0].stadsdeel + str(dossiers[0].dossiernr) - url = reverse('bouwdossier-detail', kwargs={'pk': pk}) + url = reverse("bouwdossier-detail", kwargs={"pk": pk}) test_parameters = [ settings.BOUWDOSSIER_READ_SCOPE, - settings.BOUWDOSSIER_EXTENDED_SCOPE + settings.BOUWDOSSIER_EXTENDED_SCOPE, ] for scope in test_parameters: - header = {'HTTP_AUTHORIZATION': "Bearer " + create_authz_token(scope)} + header = {"HTTP_AUTHORIZATION": "Bearer " + create_authz_token(scope)} response = self.client.get(url, **header) - self.assertEqual(response.data['stadsdeel'], dossiers[0].stadsdeel) - self.assertEqual(response.data['dossiernr'], dossiers[0].dossiernr) + self.assertEqual(response.data["stadsdeel"], dossiers[0].stadsdeel) + self.assertEqual(response.data["dossiernr"], dossiers[0].dossiernr) delete_all_records() def test_api_detail_using_stadsdeel_3_letters(self): create_bouwdossiers(3) dossier = BouwDossier.objects.first() - dossier.stadsdeel = 'AAA' + dossier.stadsdeel = "AAA" dossier.save() pk = dossier.stadsdeel + str(dossier.dossiernr) - url = reverse('bouwdossier-detail', kwargs={'pk': pk}) + url = reverse("bouwdossier-detail", kwargs={"pk": pk}) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['stadsdeel'], dossier.stadsdeel) - self.assertEqual(response.data['dossiernr'], dossier.dossiernr) + self.assertEqual(response.data["stadsdeel"], dossier.stadsdeel) + self.assertEqual(response.data["dossiernr"], dossier.dossiernr) delete_all_records() def test_api_detail_using_stadsdeel_4_letters(self): create_bouwdossiers(3) dossier = BouwDossier.objects.first() - dossier.stadsdeel = 'AAAA' + dossier.stadsdeel = "AAAA" dossier.save() pk = dossier.stadsdeel + str(dossier.dossiernr) - url = reverse('bouwdossier-detail', kwargs={'pk': pk}) + url = reverse("bouwdossier-detail", kwargs={"pk": pk}) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['stadsdeel'], dossier.stadsdeel) - self.assertEqual(response.data['dossiernr'], dossier.dossiernr) + self.assertEqual(response.data["stadsdeel"], dossier.stadsdeel) + self.assertEqual(response.data["dossiernr"], dossier.dossiernr) delete_all_records() def test_api_with_lowercase_stadsdeel(self): create_bouwdossiers(3) dossier = BouwDossier.objects.first() - dossier.stadsdeel = 'AAA' + dossier.stadsdeel = "AAA" dossier.save() pk = dossier.stadsdeel.lower() + str(dossier.dossiernr) - url = reverse('bouwdossier-detail', kwargs={'pk': pk}) + url = reverse("bouwdossier-detail", kwargs={"pk": pk}) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['stadsdeel'], dossier.stadsdeel) - self.assertEqual(response.data['dossiernr'], dossier.dossiernr) + self.assertEqual(response.data["stadsdeel"], dossier.stadsdeel) + self.assertEqual(response.data["dossiernr"], dossier.dossiernr) delete_all_records() def test_api_with_underscore_between_stadsdeel_and_dossiernr(self): create_bouwdossiers(3) dossier = BouwDossier.objects.first() - dossier.stadsdeel = 'AAA' + dossier.stadsdeel = "AAA" dossier.save() - pk = dossier.stadsdeel + '_' + str(dossier.dossiernr) # add an underscore - url = reverse('bouwdossier-detail', kwargs={'pk': pk}) + pk = dossier.stadsdeel + "_" + str(dossier.dossiernr) # add an underscore + url = reverse("bouwdossier-detail", kwargs={"pk": pk}) response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['stadsdeel'], dossier.stadsdeel) - self.assertEqual(response.data['dossiernr'], dossier.dossiernr) + self.assertEqual(response.data["stadsdeel"], dossier.stadsdeel) + self.assertEqual(response.data["dossiernr"], dossier.dossiernr) delete_all_records() def test_api_with_wrongly_formatted_stadsdeel_dossiernr_combination(self): create_bouwdossiers(3) dossier = BouwDossier.objects.first() - dossier.stadsdeel = 'AAA' + dossier.stadsdeel = "AAA" dossier.save() pk = "A" - url = reverse('bouwdossier-detail', kwargs={'pk': pk}) + url = reverse("bouwdossier-detail", kwargs={"pk": pk}) response = self.client.get(url) self.assertEqual(response.status_code, 400) delete_all_records() def test_dossiernr_stadsdeel(self): factories.DocumentFactory(bouwdossier__dossiernr=111) - factories.AdresFactory(bouwdossier__dossiernr=111) # Also add an address to the bouwdossier + factories.AdresFactory( + bouwdossier__dossiernr=111 + ) # Also add an address to the bouwdossier # And add two more dossiers to make sure it's only selecting the one we need factories.DocumentFactory(bouwdossier__dossiernr=222) factories.DocumentFactory(bouwdossier__dossiernr=333) - url = reverse('bouwdossier-list') + '?dossiernr=111&stadsdeel=AA' + url = reverse("bouwdossier-list") + "?dossiernr=111&stadsdeel=AA" response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 1) - self.assertEqual(response.data['results'][0]['titel'], 'weesperstraat 113 - 117') - self.assertEqual(response.data['results'][0]['documenten'][0]['bestanden'][0]['filename'], - 'SU10000010_00001.jpg') - self.assertEqual(response.data['results'][0]['documenten'][0]['bestanden'][0]['url'], - f"{settings.IIIF_BASE_URL}edepot:AA111-SU10000010_00001.jpg") - self.assertEqual(response.data['results'][0]['documenten'][0]['access'], 'RESTRICTED') - self.assertEqual(response.data['results'][0]['documenten'][0]['barcode'], 'ST100') - self.assertEqual(response.data['results'][0]['adressen'][0]['nummeraanduidingen'][0], - '0363200000406187') - self.assertEqual(response.data['results'][0]['adressen'][0]['panden'][0], '0363100012165490') + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 1) + self.assertEqual( + response.data["results"][0]["titel"], "weesperstraat 113 - 117" + ) + self.assertEqual( + response.data["results"][0]["documenten"][0]["bestanden"][0]["filename"], + "SU10000010_00001.jpg", + ) + self.assertEqual( + response.data["results"][0]["documenten"][0]["bestanden"][0]["url"], + f"{settings.IIIF_BASE_URL}edepot:AA111-SU10000010_00001.jpg", + ) + self.assertEqual( + response.data["results"][0]["documenten"][0]["access"], "RESTRICTED" + ) + self.assertEqual( + response.data["results"][0]["documenten"][0]["barcode"], "ST100" + ) + self.assertEqual( + response.data["results"][0]["adressen"][0]["nummeraanduidingen"][0], + "0363200000406187", + ) + self.assertEqual( + response.data["results"][0]["adressen"][0]["panden"][0], "0363100012165490" + ) delete_all_records() def test_stadsdeel_None(self): dossiers = create_bouwdossiers(3) - url = reverse('bouwdossier-list') + f'?dossiernr={dossiers[0].dossiernr}&stadsdeel=CC' + url = ( + reverse("bouwdossier-list") + + f"?dossiernr={dossiers[0].dossiernr}&stadsdeel=CC" + ) response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 0) + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 0) delete_all_records() def test_dossiernr_None(self): create_bouwdossiers(3) - url = reverse('bouwdossier-list') + '?dossiernr=123456&stadsdeel=AA' # existing stadsdeel, but non existing dossiernr + url = ( + reverse("bouwdossier-list") + "?dossiernr=123456&stadsdeel=AA" + ) # existing stadsdeel, but non existing dossiernr response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 0) + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 0) delete_all_records() def test_nummeraanduiding(self): factories.AdresFactory() factories.AdresFactory() adres = factories.AdresFactory() - adres.nummeraanduidingen = ['1111111111111111'] + adres.nummeraanduidingen = ["1111111111111111"] adres.save() - url = reverse('bouwdossier-list') + '?nummeraanduiding=1111111111111111' + url = reverse("bouwdossier-list") + "?nummeraanduiding=1111111111111111" response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 1) - self.assertEqual(response.data['results'][0]['titel'], 'weesperstraat 113 - 117') + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 1) + self.assertEqual( + response.data["results"][0]["titel"], "weesperstraat 113 - 117" + ) delete_all_records() def test_nummeraanduiding_non_existent(self): factories.AdresFactory() - url = reverse('bouwdossier-list') + '?nummeraanduiding=1111111111111111' + url = reverse("bouwdossier-list") + "?nummeraanduiding=1111111111111111" response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 0) + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 0) delete_all_records() def test_pand(self): factories.AdresFactory() factories.AdresFactory() adres = factories.AdresFactory() - adres.panden = ['1111111111111111'] + adres.panden = ["1111111111111111"] adres.save() - url = reverse('bouwdossier-list') + '?pand=1111111111111111' + url = reverse("bouwdossier-list") + "?pand=1111111111111111" response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 1) - self.assertEqual(response.data['results'][0]['titel'], 'weesperstraat 113 - 117') + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 1) + self.assertEqual( + response.data["results"][0]["titel"], "weesperstraat 113 - 117" + ) delete_all_records() def test_verblijfsobject(self): factories.AdresFactory() factories.AdresFactory() adres = factories.AdresFactory() - adres.verblijfsobjecten = ['1111111111111111'] + adres.verblijfsobjecten = ["1111111111111111"] adres.save() - url = reverse('bouwdossier-list') + '?verblijfsobject=1111111111111111' + url = reverse("bouwdossier-list") + "?verblijfsobject=1111111111111111" response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 1) - self.assertEqual(response.data['results'][0]['titel'], 'weesperstraat 113 - 117') + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 1) + self.assertEqual( + response.data["results"][0]["titel"], "weesperstraat 113 - 117" + ) delete_all_records() def test_openbareruimte(self): factories.AdresFactory() factories.AdresFactory() adres = factories.AdresFactory() - adres.openbareruimte_id = '1111111111111111' + adres.openbareruimte_id = "1111111111111111" adres.save() - url = reverse('bouwdossier-list') + '?verblijfsobject=036301000xxxxxxx/?openbareruimte=1111111111111111' + url = ( + reverse("bouwdossier-list") + + "?verblijfsobject=036301000xxxxxxx/?openbareruimte=1111111111111111" + ) response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 1) - self.assertEqual(response.data['results'][0]['titel'], 'weesperstraat 113 - 117') + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 1) + self.assertEqual( + response.data["results"][0]["titel"], "weesperstraat 113 - 117" + ) delete_all_records() def test_dossiernr_stadsdeel_max_datering_none(self): create_bouwdossiers(3) - url = reverse('bouwdossier-list') + '?dossiernr=12345&stadsdeel=AA&max_datering=1997' + url = ( + reverse("bouwdossier-list") + + "?dossiernr=12345&stadsdeel=AA&max_datering=1997" + ) response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 0) + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 0) delete_all_records() def test_dossiernr_stadsdeel_max_datering(self): dossiers = create_bouwdossiers(3) - url = reverse('bouwdossier-list') + f'?dossiernr={dossiers[0].dossiernr}&stadsdeel=AA&max_datering=2000' + url = ( + reverse("bouwdossier-list") + + f"?dossiernr={dossiers[0].dossiernr}&stadsdeel=AA&max_datering=2000" + ) response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 1) - self.assertEqual(response.data['results'][0]['titel'], 'weesperstraat 113 - 117') + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 1) + self.assertEqual( + response.data["results"][0]["titel"], "weesperstraat 113 - 117" + ) delete_all_records() def test_dossiernr_stadsdeel_min_datering_none(self): dossiers = create_bouwdossiers(3) - url = reverse('bouwdossier-list') + f'?dossiernr={dossiers[0].dossiernr}&stadsdeel=A&min_datering=1999' + url = ( + reverse("bouwdossier-list") + + f"?dossiernr={dossiers[0].dossiernr}&stadsdeel=A&min_datering=1999" + ) response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 0) + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 0) delete_all_records() def test_dossiernr_stadsdeel_min_datering(self): dossiers = create_bouwdossiers(3) - url = reverse('bouwdossier-list') + f'?dossiernr={dossiers[0].dossiernr}&stadsdeel=AA&min_datering=1997' + url = ( + reverse("bouwdossier-list") + + f"?dossiernr={dossiers[0].dossiernr}&stadsdeel=AA&min_datering=1997" + ) response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 1) - self.assertEqual(response.data['results'][0]['titel'], 'weesperstraat 113 - 117') + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 1) + self.assertEqual( + response.data["results"][0]["titel"], "weesperstraat 113 - 117" + ) delete_all_records() def test_olo_liaan_nummer(self): bd = factories.BouwDossierFactory(olo_liaan_nummer=7777777) factories.DocumentFactory(bouwdossier__dossiernr=bd.dossiernr) - factories.AdresFactory(bouwdossier__dossiernr=bd.dossiernr) # Also add an address to the bouwdossier + factories.AdresFactory( + bouwdossier__dossiernr=bd.dossiernr + ) # Also add an address to the bouwdossier # And add two more dossiers to make sure it's only selecting the one we need factories.BouwDossierFactory(dossiernr=222) factories.BouwDossierFactory(dossiernr=333) - url = reverse('bouwdossier-list') + '?olo_liaan_nummer=7777777' + url = reverse("bouwdossier-list") + "?olo_liaan_nummer=7777777" response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 1) - self.assertEqual(response.data['results'][0]['olo_liaan_nummer'], 7777777) - self.assertEqual(response.data['results'][0]['titel'], 'weesperstraat 113 - 117') - self.assertEqual(response.data['results'][0]['documenten'][0]['bestanden'][0]['filename'], - 'SU10000010_00001.jpg') - self.assertEqual(response.data['results'][0]['documenten'][0]['bestanden'][0]['url'], - f"{settings.IIIF_BASE_URL}edepot:AA12345-SU10000010_00001.jpg") - self.assertEqual(response.data['results'][0]['documenten'][0]['access'], 'RESTRICTED') - self.assertEqual(response.data['results'][0]['documenten'][0]['barcode'], 'ST100') - self.assertEqual(response.data['results'][0]['adressen'][0]['nummeraanduidingen'][0], - '0363200000406187') - self.assertEqual(response.data['results'][0]['adressen'][0]['panden'][0], '0363100012165490') + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 1) + self.assertEqual(response.data["results"][0]["olo_liaan_nummer"], 7777777) + self.assertEqual( + response.data["results"][0]["titel"], "weesperstraat 113 - 117" + ) + self.assertEqual( + response.data["results"][0]["documenten"][0]["bestanden"][0]["filename"], + "SU10000010_00001.jpg", + ) + self.assertEqual( + response.data["results"][0]["documenten"][0]["bestanden"][0]["url"], + f"{settings.IIIF_BASE_URL}edepot:AA12345-SU10000010_00001.jpg", + ) + self.assertEqual( + response.data["results"][0]["documenten"][0]["access"], "RESTRICTED" + ) + self.assertEqual( + response.data["results"][0]["documenten"][0]["barcode"], "ST100" + ) + self.assertEqual( + response.data["results"][0]["adressen"][0]["nummeraanduidingen"][0], + "0363200000406187", + ) + self.assertEqual( + response.data["results"][0]["adressen"][0]["panden"][0], "0363100012165490" + ) delete_all_records() def test_filter_subdossier(self): """ subdossier should match with case insensitive start of titel of subdossier """ - factories.DocumentFactory(subdossier_titel='Tekeningen one') - factories.DocumentFactory(subdossier_titel='Tekeningen two') - factories.DocumentFactory(subdossier_titel='Tekeningen three') - factories.DocumentFactory(subdossier_titel='No tekingen') + factories.DocumentFactory(subdossier_titel="Tekeningen one") + factories.DocumentFactory(subdossier_titel="Tekeningen two") + factories.DocumentFactory(subdossier_titel="Tekeningen three") + factories.DocumentFactory(subdossier_titel="No tekingen") - url = reverse('bouwdossier-list') + '?subdossier=tekeningen' + url = reverse("bouwdossier-list") + "?subdossier=tekeningen" response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 3) - self.assertEqual(response.data['results'][0]['titel'], 'weesperstraat 113 - 117') + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 3) + self.assertEqual( + response.data["results"][0]["titel"], "weesperstraat 113 - 117" + ) delete_all_records() def test_subdossier_none(self): - factories.DocumentFactory(subdossier_titel='One') - factories.DocumentFactory(subdossier_titel='Two') - factories.DocumentFactory(subdossier_titel='Three') + factories.DocumentFactory(subdossier_titel="One") + factories.DocumentFactory(subdossier_titel="Two") + factories.DocumentFactory(subdossier_titel="Three") - url = reverse('bouwdossier-list') + '?subdossier=dit_match_niet' + url = reverse("bouwdossier-list") + "?subdossier=dit_match_niet" response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 0) + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 0) delete_all_records() def test_dossier_type(self): create_bouwdossiers(3) - factories.BouwDossierFactory(dossier_type='differenttype') + factories.BouwDossierFactory(dossier_type="differenttype") - url = reverse('bouwdossier-list') + '?dossier_type=differenttype' + url = reverse("bouwdossier-list") + "?dossier_type=differenttype" response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 1) - self.assertEqual(response.data['results'][0]['titel'], 'weesperstraat 113 - 117') + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 1) + self.assertEqual( + response.data["results"][0]["titel"], "weesperstraat 113 - 117" + ) delete_all_records() def test_dossier_type_none(self): create_bouwdossiers(3) - url = reverse('bouwdossier-list') + '?dossier_type=non_existing_type' + url = reverse("bouwdossier-list") + "?dossier_type=non_existing_type" response = self.client.get(url) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 0) + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 0) delete_all_records() def test_dossier_with_stadsdeel(self): bd = factories.BouwDossierFactory() factories.DocumentFactory(bouwdossier__dossiernr=bd.dossiernr) - factories.AdresFactory(bouwdossier__dossiernr=bd.dossiernr) # Also add an address to the bouwdossier + factories.AdresFactory( + bouwdossier__dossiernr=bd.dossiernr + ) # Also add an address to the bouwdossier # And add two more dossiers to make sure it's only selecting the one we need factories.BouwDossierFactory(dossiernr=222) factories.BouwDossierFactory(dossiernr=333) - url = reverse('bouwdossier-list') + f'?dossier={bd.stadsdeel}{bd.dossiernr}' + url = reverse("bouwdossier-list") + f"?dossier={bd.stadsdeel}{bd.dossiernr}" response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertIn('results', response.data) - self.assertIn('count', response.data) - self.assertEqual(response.data['count'], 1) - self.assertEqual(response.data['results'][0]['titel'], 'weesperstraat 113 - 117') - self.assertEqual(response.data['results'][0]['documenten'][0]['bestanden'][0]['filename'], - 'SU10000010_00001.jpg') - self.assertEqual(response.data['results'][0]['documenten'][0]['bestanden'][0]['url'], - f"{settings.IIIF_BASE_URL}edepot:AA12345-SU10000010_00001.jpg") - self.assertEqual(response.data['results'][0]['documenten'][0]['access'], 'RESTRICTED') - self.assertEqual(response.data['results'][0]['documenten'][0]['barcode'], 'ST100') - self.assertEqual(response.data['results'][0]['adressen'][0]['nummeraanduidingen'][0], - '0363200000406187') - self.assertEqual(response.data['results'][0]['adressen'][0]['panden'][0], '0363100012165490') + self.assertIn("results", response.data) + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 1) + self.assertEqual( + response.data["results"][0]["titel"], "weesperstraat 113 - 117" + ) + self.assertEqual( + response.data["results"][0]["documenten"][0]["bestanden"][0]["filename"], + "SU10000010_00001.jpg", + ) + self.assertEqual( + response.data["results"][0]["documenten"][0]["bestanden"][0]["url"], + f"{settings.IIIF_BASE_URL}edepot:AA12345-SU10000010_00001.jpg", + ) + self.assertEqual( + response.data["results"][0]["documenten"][0]["access"], "RESTRICTED" + ) + self.assertEqual( + response.data["results"][0]["documenten"][0]["barcode"], "ST100" + ) + self.assertEqual( + response.data["results"][0]["adressen"][0]["nummeraanduidingen"][0], + "0363200000406187", + ) + self.assertEqual( + response.data["results"][0]["adressen"][0]["panden"][0], "0363100012165490" + ) delete_all_records() def test_dossier_incorrectly_without_stadsdeel(self): create_bouwdossiers(3) - url = reverse('bouwdossier-list') + '?dossier=12345' + url = reverse("bouwdossier-list") + "?dossier=12345" response = self.client.get(url) self.assertEqual(response.status_code, 400) delete_all_records() def test_invalid_dossiernr(self): create_bouwdossiers(3) - url = reverse('bouwdossier-list') + '?dossiernr=wrong' + url = reverse("bouwdossier-list") + "?dossiernr=wrong" response = self.client.get(url) self.assertEqual(response.status_code, 400) delete_all_records() @@ -438,36 +536,48 @@ def test_invalid_dossiernr(self): def test_dossier_wabo_fields(self): bd = factories.BouwDossierFactory() factories.DocumentFactory(bouwdossier__dossiernr=bd.dossiernr) - factories.AdresFactory(bouwdossier__dossiernr=bd.dossiernr) # Also add an address to the bouwdossier + factories.AdresFactory( + bouwdossier__dossiernr=bd.dossiernr + ) # Also add an address to the bouwdossier - dossier = BouwDossier.objects.get(stadsdeel='AA', dossiernr='12345') - dossier.olo_liaan_nummer = '67890' - dossier.wabo_bron = 'test' + dossier = BouwDossier.objects.get(stadsdeel="AA", dossiernr="12345") + dossier.olo_liaan_nummer = "67890" + dossier.wabo_bron = "test" dossier.source = SOURCE_WABO dossier.save() document = dossier.documenten.first() - document.bestanden = ['https://conversiestraatwabo.amsterdam.nl/webDAV/SDC/PUA/1234567.PDF'] - document.barcode = document.bestanden[0].split('/')[-1].split('.')[0] - document.oorspronkelijk_pad = ['/path/to/bestand'] + document.bestanden = [ + "https://conversiestraatwabo.amsterdam.nl/webDAV/SDC/PUA/1234567.PDF" + ] + document.barcode = document.bestanden[0].split("/")[-1].split(".")[0] + document.oorspronkelijk_pad = ["/path/to/bestand"] document.save() adres = dossier.adressen.first() - adres.locatie_aanduiding = 'aanduiding' - adres.huisnummer_letter = 'A' - adres.huisnummer_toevoeging = 'B' + adres.locatie_aanduiding = "aanduiding" + adres.huisnummer_letter = "A" + adres.huisnummer_toevoeging = "B" adres.save() - url = reverse('bouwdossier-detail', kwargs={'pk': 'AA12345'}) - header = {'HTTP_AUTHORIZATION': "Bearer " + create_authz_token(settings.BOUWDOSSIER_READ_SCOPE)} + url = reverse("bouwdossier-detail", kwargs={"pk": "AA12345"}) + header = { + "HTTP_AUTHORIZATION": "Bearer " + + create_authz_token(settings.BOUWDOSSIER_READ_SCOPE) + } response = self.client.get(url, **header) - documents = response.data.get('documenten') - adressen = response.data['adressen'] - self.assertEqual(response.data['olo_liaan_nummer'], 67890) - self.assertEqual(response.data.get('wabo_bron'), None) # Bron is not needed in the api Check model - self.assertEqual(documents[0]['oorspronkelijk_pad'], ['/path/to/bestand']) - self.assertEqual(documents[0]['bestanden'][0]['url'], f"{settings.IIIF_BASE_URL}wabo:AA-12345-67890_1234567") - self.assertEqual(adressen[0]['locatie_aanduiding'], 'aanduiding') - self.assertEqual(adressen[0]['huisnummer_letter'], 'A') - self.assertEqual(adressen[0]['huisnummer_toevoeging'], 'B') + documents = response.data.get("documenten") + adressen = response.data["adressen"] + self.assertEqual(response.data["olo_liaan_nummer"], 67890) + self.assertEqual( + response.data.get("wabo_bron"), None + ) # Bron is not needed in the api Check model + self.assertEqual(documents[0]["oorspronkelijk_pad"], ["/path/to/bestand"]) + self.assertEqual( + documents[0]["bestanden"][0]["url"], + f"{settings.IIIF_BASE_URL}wabo:AA-12345-67890_1234567", + ) + self.assertEqual(adressen[0]["locatie_aanduiding"], "aanduiding") + self.assertEqual(adressen[0]["huisnummer_letter"], "A") + self.assertEqual(adressen[0]["huisnummer_toevoeging"], "B") delete_all_records() diff --git a/src/bouwdossiers/tests/test_tools.py b/src/bouwdossiers/tests/test_tools.py index b5f4038..c8a423b 100644 --- a/src/bouwdossiers/tests/test_tools.py +++ b/src/bouwdossiers/tests/test_tools.py @@ -6,21 +6,38 @@ @pytest.mark.parametrize( "test_input, expected_stadsdeel, expected_dossiernr", [ - ('AB1234', 'AB', '1234'), # default formatting - ('AB_1234', 'AB', '1234'), # with underscore formatting - ('ab1234', 'AB', '1234'), # with lowercase letter stadsdeelcode formatting - ('ab_1234', 'AB', '1234'), # with underscore and lowercase letter stadsdeelcode formatting - ('ab_1234', 'AB', '1234'), # with underscore and lowercase letter stadsdeelcode formatting - + ("AB1234", "AB", "1234"), # default formatting + ("AB_1234", "AB", "1234"), # with underscore formatting + ("ab1234", "AB", "1234"), # with lowercase letter stadsdeelcode formatting + ( + "ab_1234", + "AB", + "1234", + ), # with underscore and lowercase letter stadsdeelcode formatting + ( + "ab_1234", + "AB", + "1234", + ), # with underscore and lowercase letter stadsdeelcode formatting # The same but with four letters - ('ABCD1234', 'ABCD', '1234'), # default formatting - ('ABCD_1234', 'ABCD', '1234'), # with underscore formatting - ('abcd1234', 'ABCD', '1234'), # with lowercase letter stadsdeelcode formatting - ('abcd_1234', 'ABCD', '1234'), # with underscore and lowercase letter stadsdeelcode formatting - ('abcd_1234', 'ABCD', '1234'), # with underscore and lowercase letter stadsdeelcode formatting - ] + ("ABCD1234", "ABCD", "1234"), # default formatting + ("ABCD_1234", "ABCD", "1234"), # with underscore formatting + ("abcd1234", "ABCD", "1234"), # with lowercase letter stadsdeelcode formatting + ( + "abcd_1234", + "ABCD", + "1234", + ), # with underscore and lowercase letter stadsdeelcode formatting + ( + "abcd_1234", + "ABCD", + "1234", + ), # with underscore and lowercase letter stadsdeelcode formatting + ], ) -def test_separate_dossier_four_letters(test_input, expected_stadsdeel, expected_dossiernr): +def test_separate_dossier_four_letters( + test_input, expected_stadsdeel, expected_dossiernr +): stadsdeel, dossiernr = tools.separate_dossier(test_input) assert stadsdeel == expected_stadsdeel assert dossiernr == expected_dossiernr @@ -29,11 +46,11 @@ def test_separate_dossier_four_letters(test_input, expected_stadsdeel, expected_ @pytest.mark.parametrize( "test_input", [ - 'ABC', # missing dossiernr - '1234', # missing stadsdeelcode - 'A1234', # only one letter stadsdeelcode should also fail - 'ABCDE1234' # five or more letter stadsdeelcodes should also fail - ] + "ABC", # missing dossiernr + "1234", # missing stadsdeelcode + "A1234", # only one letter stadsdeelcode should also fail + "ABCDE1234", # five or more letter stadsdeelcodes should also fail + ], ) def test_separate_dossier_errors(test_input): # Test whether wrongly formatted inputs fail diff --git a/src/bouwdossiers/tests/tools_for_testing.py b/src/bouwdossiers/tests/tools_for_testing.py index 3aecb33..537f59b 100644 --- a/src/bouwdossiers/tests/tools_for_testing.py +++ b/src/bouwdossiers/tests/tools_for_testing.py @@ -24,21 +24,15 @@ def create_authz_token(scopes=None): scopes = [scopes] load_jwks(settings.JWKS_TEST_KEY) - key = next(iter(_keyset['keys'])) + key = next(iter(_keyset["keys"])) now = int(time.time()) - header = { - 'alg': 'ES256', - 'kid': key.key_id - } + header = {"alg": "ES256", "kid": key.key_id} claims = { - 'iat': now, - 'exp': now + 3600, - 'scopes': scopes, - 'sub': 'authztest@amsterdam.nl', + "iat": now, + "exp": now + 3600, + "scopes": scopes, + "sub": "authztest@amsterdam.nl", } - token = JWT( - header=header, - claims=claims - ) + token = JWT(header=header, claims=claims) token.make_signed_token(key) return token.serialize() diff --git a/src/bouwdossiers/tools.py b/src/bouwdossiers/tools.py index 27ef14f..4a49a6f 100644 --- a/src/bouwdossiers/tools.py +++ b/src/bouwdossiers/tools.py @@ -13,14 +13,14 @@ def separate_dossier(dossier): If format is not correct, an APIException is raised """ - dossier = dossier.replace('_', '') # Also allow for searching by "SDC_1234" + dossier = dossier.replace("_", "") # Also allow for searching by "SDC_1234" dossier = dossier.upper() # Also allow to search by lower case letters try: # Check if dossier has correct format - assert re.match(r'^\D{2,4}\d+$', dossier) + assert re.match(r"^\D{2,4}\d+$", dossier) # split the stadsdeel and dossier by using | - stadsdeel, dossiernr = re.findall(r'\D{2,4}|\d+', dossier) + stadsdeel, dossiernr = re.findall(r"\D{2,4}|\d+", dossier) return stadsdeel, dossiernr except AssertionError: raise InvalidDossier( diff --git a/src/bouwdossiers/urls.py b/src/bouwdossiers/urls.py index d57f9c7..3d8c2e1 100644 --- a/src/bouwdossiers/urls.py +++ b/src/bouwdossiers/urls.py @@ -4,8 +4,7 @@ class BouwdossiersView(routers.APIRootView): - """ - """ + """ """ class BouwdossiersRouter(routers.DefaultRouter): @@ -14,7 +13,8 @@ class BouwdossiersRouter(routers.DefaultRouter): bouwdossiers = BouwdossiersRouter() -bouwdossiers.register(r'bouwdossier', api_views.BouwDossierViewSet, - basename='bouwdossier') +bouwdossiers.register( + r"bouwdossier", api_views.BouwDossierViewSet, basename="bouwdossier" +) urls = bouwdossiers.urls diff --git a/src/bouwdossiers/views.py b/src/bouwdossiers/views.py index 5e109b2..6e37dc3 100644 --- a/src/bouwdossiers/views.py +++ b/src/bouwdossiers/views.py @@ -11,15 +11,23 @@ class BouwDossierFilter(FilterSet): - nummeraanduiding = filters.CharFilter(field_name='adressen__nummeraanduidingen', method='array_contains_filter') - pand = filters.CharFilter(field_name='adressen__panden', method='array_contains_filter') - verblijfsobject = filters.CharFilter(field_name='adressen__verblijfsobjecten', method='array_contains_filter') - openbareruimte = filters.CharFilter(field_name='adressen__openbareruimte_id') - min_datering = filters.CharFilter(field_name='datering__year', lookup_expr='gte') - max_datering = filters.CharFilter(field_name='datering__year', lookup_expr='lte') - subdossier = filters.CharFilter(field_name='documenten__subdossier_titel', lookup_expr='istartswith') + nummeraanduiding = filters.CharFilter( + field_name="adressen__nummeraanduidingen", method="array_contains_filter" + ) + pand = filters.CharFilter( + field_name="adressen__panden", method="array_contains_filter" + ) + verblijfsobject = filters.CharFilter( + field_name="adressen__verblijfsobjecten", method="array_contains_filter" + ) + openbareruimte = filters.CharFilter(field_name="adressen__openbareruimte_id") + min_datering = filters.CharFilter(field_name="datering__year", lookup_expr="gte") + max_datering = filters.CharFilter(field_name="datering__year", lookup_expr="lte") + subdossier = filters.CharFilter( + field_name="documenten__subdossier_titel", lookup_expr="istartswith" + ) dossiernr = filters.NumberFilter() - dossier = filters.CharFilter(method='dossier_with_stadsdeel') + dossier = filters.CharFilter(method="dossier_with_stadsdeel") stadsdeel = filters.CharFilter() dossier_type = filters.CharFilter() @@ -27,16 +35,16 @@ class Meta: model = models.BouwDossier fields = ( - 'dossiernr', - 'stadsdeel', - 'nummeraanduiding', - 'verblijfsobject', - 'pand', - 'openbareruimte', - 'min_datering', - 'max_datering', - 'subdossier', - 'olo_liaan_nummer', + "dossiernr", + "stadsdeel", + "nummeraanduiding", + "verblijfsobject", + "pand", + "openbareruimte", + "min_datering", + "max_datering", + "subdossier", + "olo_liaan_nummer", ) def dossier_with_stadsdeel(self, queryset, _filter_name, value): @@ -46,7 +54,7 @@ def dossier_with_stadsdeel(self, queryset, _filter_name, value): def array_contains_filter(self, queryset, _filter_name, value): if not isinstance(value, list): value = [value] - lookup = '%s__%s' % (_filter_name, 'contains') + lookup = "%s__%s" % (_filter_name, "contains") return queryset.filter(**{lookup: value}).distinct() @@ -54,11 +62,18 @@ class BouwDossierViewSet(DatapuntViewSet): filterset_class = BouwDossierFilter def get_queryset(self): - allowed_scopes = [settings.BOUWDOSSIER_READ_SCOPE, settings.BOUWDOSSIER_EXTENDED_SCOPE] + allowed_scopes = [ + settings.BOUWDOSSIER_READ_SCOPE, + settings.BOUWDOSSIER_EXTENDED_SCOPE, + ] if any(scope in self.request.get_token_scopes for scope in allowed_scopes): - return models.BouwDossier.objects.all().prefetch_related('adressen', 'documenten') + return models.BouwDossier.objects.all().prefetch_related( + "adressen", "documenten" + ) else: - return models.BouwDossier.objects.filter(source='EDEPOT').prefetch_related('adressen', 'documenten') + return models.BouwDossier.objects.filter(source="EDEPOT").prefetch_related( + "adressen", "documenten" + ) def get_serializer_class(self): return serializers.BouwDossierSerializer @@ -66,7 +81,9 @@ def get_serializer_class(self): def get_object(self): # We expect a key of the form AA0000123 in which AA is the code for the # stadsdeel and the numberic part (which can vary in length) is the dossiernumber - stadsdeel, dossiernr = tools.separate_dossier(self.kwargs['pk']) - obj = get_object_or_404(self.get_queryset(), stadsdeel=stadsdeel.upper(), dossiernr=dossiernr) + stadsdeel, dossiernr = tools.separate_dossier(self.kwargs["pk"]) + obj = get_object_or_404( + self.get_queryset(), stadsdeel=stadsdeel.upper(), dossiernr=dossiernr + ) return obj diff --git a/src/health/apps.py b/src/health/apps.py index ea5593f..e860540 100644 --- a/src/health/apps.py +++ b/src/health/apps.py @@ -2,4 +2,4 @@ class HealthConfig(AppConfig): - name = 'health' + name = "health" diff --git a/src/health/urls.py b/src/health/urls.py index 03f54a9..36577ad 100644 --- a/src/health/urls.py +++ b/src/health/urls.py @@ -3,6 +3,6 @@ from . import views urlpatterns = [ - re_path(r'^health$', views.health), - re_path(r'^data$', views.check_data), + re_path(r"^health$", views.health), + re_path(r"^data$", views.check_data), ] diff --git a/src/health/views.py b/src/health/views.py index 912ecd0..4b525b5 100644 --- a/src/health/views.py +++ b/src/health/views.py @@ -8,6 +8,7 @@ try: # noinspection PyUnresolvedReferences from django.apps import apps + get_model = apps.get_model except ImportError: from django.db.models.loading import get_model @@ -17,8 +18,9 @@ model = get_model(settings.HEALTH_MODEL) except AttributeError: raise ImproperlyConfigured( - 'settings.HEALTH_MODEL {} doesn\'t resolve to ' - 'a useable model'.format(settings.HEALTH_MODEL)) + "settings.HEALTH_MODEL {} doesn't resolve to " + "a useable model".format(settings.HEALTH_MODEL) + ) log = logging.getLogger(__name__) @@ -32,21 +34,19 @@ def health(_request): with connection.cursor() as cursor: cursor.execute("select 1") assert cursor.fetchone() - except DatabaseError: # noqa + except DatabaseError: # noqa log.exception("Database connectivity failed") return HttpResponse( - "Database connectivity failed", - content_type="text/plain", status=500) + "Database connectivity failed", content_type="text/plain", status=500 + ) - return HttpResponse( - "Connectivity OK", content_type='text/plain', status=200) + return HttpResponse("Connectivity OK", content_type="text/plain", status=200) def check_data(request): if model.objects.all().count() < 100: return HttpResponse( - "Too few metadata in the database", - content_type="text/plain", status=500) + "Too few metadata in the database", content_type="text/plain", status=500 + ) - return HttpResponse( - "Data OK", content_type='text/plain', status=200) + return HttpResponse("Data OK", content_type="text/plain", status=200) diff --git a/src/importer/apps.py b/src/importer/apps.py index f8e2e0e..ff9040f 100644 --- a/src/importer/apps.py +++ b/src/importer/apps.py @@ -2,5 +2,5 @@ class ImporterConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'importer' + default_auto_field = "django.db.models.BigAutoField" + name = "importer" diff --git a/src/importer/batch.py b/src/importer/batch.py index 9e0a434..62139a9 100644 --- a/src/importer/batch.py +++ b/src/importer/batch.py @@ -77,7 +77,7 @@ def delete_all(): def _normalize_bestand(bestand): - bestand_parts = bestand.split('/')[4:] + bestand_parts = bestand.split("/")[4:] if len(bestand_parts) < 3: log.error(f"Unexpected bestand : {bestand}") return None @@ -85,15 +85,15 @@ def _normalize_bestand(bestand): bestand_parts[1] = f"{int(bestand_parts[1]):05d}" elif bestand_parts[2].isdigit(): bestand_parts[2] = f"{int(bestand_parts[2]):05d}" - return '/'.join(bestand_parts) + return "/".join(bestand_parts) def get_access(el): checks = { - 'openbaarheidsBeperking': ('N', 'j'), - 'openbaar': ('J', 'n'), - 'gevoelig_object': ('N', 'j'), - 'bevat_persoonsgegevens': ('false', 'true') + "openbaarheidsBeperking": ("N", "j"), + "openbaar": ("J", "n"), + "gevoelig_object": ("N", "j"), + "bevat_persoonsgegevens": ("false", "true"), } if not any(el.get(check) for check in checks.keys()): @@ -107,12 +107,14 @@ def get_access(el): def openbaar_to_copyright(copyright): - if copyright == 'J': + if copyright == "J": return models.COPYRIGHT_YES return models.COPYRIGHT_NO -def add_wabo_dossier(x_dossier, file_path, import_file, count, total_count): # noqa C901 +def add_wabo_dossier( + x_dossier, file_path, import_file, count, total_count +): # noqa C901 """ For information about wabo and pre_wabo please check the README Add wabo dossier to the the bouwdossier model. Structure of import is @@ -127,7 +129,7 @@ def add_wabo_dossier(x_dossier, file_path, import_file, count, total_count): # # The intern number can be something like sdz_prewabo_1274 or sdc_33 # If prewabo is present it is a prewabo dossier. - dossier = x_dossier.get('intern_nummer') + dossier = x_dossier.get("intern_nummer") m = re.match(r"([a-z]+)_(?:([a-z]+)_)?(\d+)", dossier) if not m: log.error(f"Invalid intern_nummer {dossier} in {file_path}") @@ -137,39 +139,44 @@ def add_wabo_dossier(x_dossier, file_path, import_file, count, total_count): # wabo_tag = m.group(2) dossiernr = m.group(3) - if wabo_tag and wabo_tag == 'prewabo': + if wabo_tag and wabo_tag == "prewabo": # prewabo key2 dossiers numbers can have the same values, for the same stadsdeel as existing # prewabo dossiers imported from the edepot. Therefore we add the letter p to the stadsdeel # to make the combination unique. - stadsdeel += 'p' + stadsdeel += "p" # There were titels longer than the allowed 512 characters, so to avoid errors we cut them off at 512 - titel = x_dossier.get('dossier_titel')[:509] + '...' if len(x_dossier.get('dossier_titel', '')) > 512 \ - else x_dossier.get('dossier_titel') + titel = ( + x_dossier.get("dossier_titel")[:509] + "..." + if len(x_dossier.get("dossier_titel", "")) > 512 + else x_dossier.get("dossier_titel") + ) if not titel: - titel = '' + titel = "" log.warning(f"Missing titel for Wabo dossier {dossiernr} in {file_path}") - datering = x_dossier.get('begindatum') - dossier_type = x_dossier.get('omschrijving').lower() + datering = x_dossier.get("begindatum") + dossier_type = x_dossier.get("omschrijving").lower() if type(dossier_type) is str and len(dossier_type) > 255: dossier_type = dossier_type[:255] # Cap at 255 characters - olo_liaan_nummer = x_dossier.get('OLO_liaan_nummer') + olo_liaan_nummer = x_dossier.get("OLO_liaan_nummer") if type(olo_liaan_nummer) is str and len(olo_liaan_nummer): # In some cases the string starts with 'OLO'. We need to remove this - olo_liaan_nummer = olo_liaan_nummer.replace('OLO', '') + olo_liaan_nummer = olo_liaan_nummer.replace("OLO", "") # prewabo key2 dossiers do not have a olo number. Because we do not # want a None in the URL we set the olo number to 0 - if not olo_liaan_nummer and wabo_tag == 'prewabo': + if not olo_liaan_nummer and wabo_tag == "prewabo": olo_liaan_nummer = 0 activiteiten = [] - for activiteit in get_list_items(x_dossier, 'activiteiten', 'activiteit'): + for activiteit in get_list_items(x_dossier, "activiteiten", "activiteit"): activiteiten.append(activiteit[:250]) if type(activiteit) is str and len(activiteit) > 250: - log.warning(f'The activiteit str "{activiteit}" is more than 250 characters') + log.warning( + f'The activiteit str "{activiteit}" is more than 250 characters' + ) bouwdossier = models.BouwDossier( importfile=import_file, @@ -179,10 +186,10 @@ def add_wabo_dossier(x_dossier, file_path, import_file, count, total_count): # datering=datering, dossier_type=dossier_type, olo_liaan_nummer=olo_liaan_nummer, - wabo_bron=x_dossier.get('bron'), + wabo_bron=x_dossier.get("bron"), access=get_access(x_dossier), source=models.SOURCE_WABO, - activiteiten=activiteiten + activiteiten=activiteiten, ) try: @@ -198,30 +205,32 @@ def add_wabo_dossier(x_dossier, file_path, import_file, count, total_count): # if total_count % 1000 == 0: log.info(f"Bouwdossiers count in file: {count}, total: {total_count}") - for x_adres in get_list_items(x_dossier, 'locaties', 'locatie'): - bag_id = x_adres.get('bag_id') + for x_adres in get_list_items(x_dossier, "locaties", "locatie"): + bag_id = x_adres.get("bag_id") panden = [] verblijfsobjecten = [] openbareruimte_id = None nummeraanduidingen = [] if bag_id: # if no bag_ids, locatie aanduiding is available. - panden.append(bag_id.get('pandidentificatie')) - verblijfsobjecten.append(bag_id.get('verblijfsobjectidentificatie')) - openbareruimte_id = bag_id.get('openbareruimteidentificatie') - nummeraanduidingen.append(bag_id.get('Nummeraanduidingidentificatie')) + panden.append(bag_id.get("pandidentificatie")) + verblijfsobjecten.append(bag_id.get("verblijfsobjectidentificatie")) + openbareruimte_id = bag_id.get("openbareruimteidentificatie") + nummeraanduidingen.append(bag_id.get("Nummeraanduidingidentificatie")) - locatie_aanduiding = x_adres.get('locatie_aanduiding') + locatie_aanduiding = x_adres.get("locatie_aanduiding") if type(locatie_aanduiding) is str and len(locatie_aanduiding) > 250: locatie_aanduiding = locatie_aanduiding[:250] - log.warning(f'The locatie_aanduiding str "{locatie_aanduiding}" is more than 250 characters') + log.warning( + f'The locatie_aanduiding str "{locatie_aanduiding}" is more than 250 characters' + ) adres = models.Adres( bouwdossier=bouwdossier, - straat=x_adres.get('straatnaam'), - huisnummer_van=x_adres.get('huisnummer').replace(",", ""), - huisnummer_toevoeging=x_adres.get('huisnummertoevoeging'), - huisnummer_letter=x_adres.get('huisletter'), + straat=x_adres.get("straatnaam"), + huisnummer_van=x_adres.get("huisnummer").replace(",", ""), + huisnummer_toevoeging=x_adres.get("huisnummertoevoeging"), + huisnummer_letter=x_adres.get("huisletter"), stadsdeel=stadsdeel, nummeraanduidingen=nummeraanduidingen, nummeraanduidingen_label=[], @@ -229,64 +238,74 @@ def add_wabo_dossier(x_dossier, file_path, import_file, count, total_count): # panden=panden, verblijfsobjecten=verblijfsobjecten if verblijfsobjecten else [], verblijfsobjecten_label=[], - locatie_aanduiding=locatie_aanduiding + locatie_aanduiding=locatie_aanduiding, ) adres.save() documenten = [] - for x_document in get_list_items(x_dossier, 'documenten', 'document'): + for x_document in get_list_items(x_dossier, "documenten", "document"): bestanden = [] bestanden_pads = [] - for bestand in get_list_items(x_document, 'bestanden', 'bestand'): + for bestand in get_list_items(x_document, "bestanden", "bestand"): # Each bestand has an oorspronkelijk_pad. # oorspronkelijke_pads are added in another list (with the same order as bestanden) # to keep the same structure as the pre_wabo dossiers. # The removed part below is because we want to be consistent with the pre-wabo urls # in that we only store a relave url, not the full url - bestand_str = bestand.get('URL').replace(settings.WABO_BASE_URL, '') + bestand_str = bestand.get("URL").replace(settings.WABO_BASE_URL, "") if type(bestand_str) is str and len(bestand_str) > 250: # Bestand urls longer than 250 characters are not supported by the DB. Since only one in about 200.000 # records had this problem we'll just cap that url on 250 chars. This means that url will not work, but # we'll accept that for now. - log.warning(f'The bestand str "{bestand_str}" is more than 250 characters') + log.warning( + f'The bestand str "{bestand_str}" is more than 250 characters' + ) bestand_str = bestand_str[:250] bestanden.append(bestand_str) - bestand_pad = bestand.get('oorspronkelijk_pad') + bestand_pad = bestand.get("oorspronkelijk_pad") if type(bestand_pad) is str and len(bestand_pad) > 250: # Bestand_pads longer than 250 characters are not supported by the DB. Since only one in about 200.000 # records had this problem we'll just cap that pad on 250 chars. This means that pad will not work, but # we'll accept that for now. - log.warning(f'The bestand_pad str "{bestand_pad}" is more than 250 characters') + log.warning( + f'The bestand_pad str "{bestand_pad}" is more than 250 characters' + ) bestand_pad = bestand_pad[:250] bestanden_pads.append(bestand_pad) - barcode = x_document.get('barcode') + barcode = x_document.get("barcode") if not barcode and bestanden: # This is the case with wabo dossiers, and since wabo dossiers only have # one bestand per document, we use the number of the bestand as the barcode - barcode = bestanden[0].split('/')[-1].split('.')[0] + barcode = bestanden[0].split("/")[-1].split(".")[0] if type(barcode) is str and len(barcode) > 250: log.error(f'The barcode str "{barcode}" is more than 250 characters') - document_omschrijving = x_document.get('document_omschrijving') + document_omschrijving = x_document.get("document_omschrijving") if type(document_omschrijving) is str and len(document_omschrijving) > 250: document_omschrijving = document_omschrijving[:250] - log.warning(f'The document_omschrijving str "{document_omschrijving}" is more than 250 characters') + log.warning( + f'The document_omschrijving str "{document_omschrijving}" is more than 250 characters' + ) # Do not include metadata for paspoort scans. Een kavel paspoort is not a ID passport. - if document_omschrijving and "paspoort" in document_omschrijving.lower() and "kavel" not in document_omschrijving.lower(): + if ( + document_omschrijving + and "paspoort" in document_omschrijving.lower() + and "kavel" not in document_omschrijving.lower() + ): continue document = models.Document( barcode=barcode, bouwdossier=bouwdossier, - subdossier_titel=x_document.get('document_type'), + subdossier_titel=x_document.get("document_type"), oorspronkelijk_pad=bestanden_pads, bestanden=bestanden, access=get_access(x_document), - document_omschrijving=document_omschrijving + document_omschrijving=document_omschrijving, ) documenten.append(document) @@ -298,27 +317,31 @@ def add_wabo_dossier(x_dossier, file_path, import_file, count, total_count): # return count, total_count -def add_pre_wabo_dossier(x_dossier, file_path, import_file, count, total_count): # noqa C901 +def add_pre_wabo_dossier( + x_dossier, file_path, import_file, count, total_count +): # noqa C901 """ For information about wabo and pre_wabo please check the README """ - dossiernr = x_dossier['dossierNr'] - titel = x_dossier['titel'] + dossiernr = x_dossier["dossierNr"] + titel = x_dossier["titel"] if not titel: - titel = '' + titel = "" log.warning(f"Missing titel for bouwdossier {dossiernr} in {file_path}") - datering = get_date_from_year(x_dossier.get('datering')) - dossier_type = x_dossier.get('dossierType') - gebruiksdoel = x_dossier.get('gebruiksdoel') - bwt_nummer = x_dossier.get('bwtNummer') - stadsdeel = x_dossier.get('stadsdeelcode') + datering = get_date_from_year(x_dossier.get("datering")) + dossier_type = x_dossier.get("dossierType") + gebruiksdoel = x_dossier.get("gebruiksdoel") + bwt_nummer = x_dossier.get("bwtNummer") + stadsdeel = x_dossier.get("stadsdeelcode") if not stadsdeel: - stadsdeel = '' + stadsdeel = "" log.warning(f"Missing stadsdeel for bouwdossier {dossiernr} in {file_path}") access = get_access(x_dossier) - access_restricted_until = get_date_from_year(x_dossier.get('openbaarheidsBeperkingTot')) + access_restricted_until = get_date_from_year( + x_dossier.get("openbaarheidsBeperkingTot") + ) bouwdossier = models.BouwDossier( importfile=import_file, @@ -330,7 +353,7 @@ def add_pre_wabo_dossier(x_dossier, file_path, import_file, count, total_count): gebruiksdoel=gebruiksdoel, bwt_nummer=bwt_nummer, access=access, - access_restricted_until=access_restricted_until + access_restricted_until=access_restricted_until, ) bouwdossier.save() count += 1 @@ -339,15 +362,15 @@ def add_pre_wabo_dossier(x_dossier, file_path, import_file, count, total_count): if total_count % 1000 == 0: log.info(f"Bouwdossiers count in file: {count}, total: {total_count}") - for x_adres in get_list_items(x_dossier, 'adressen', 'adres'): - huisnummer_van = x_adres.get('huisnummerVan') + for x_adres in get_list_items(x_dossier, "adressen", "adres"): + huisnummer_van = x_adres.get("huisnummerVan") huisnummer_van = int(huisnummer_van) if huisnummer_van else None - huisnummer_tot = x_adres.get('huisnummerTot') + huisnummer_tot = x_adres.get("huisnummerTot") huisnummer_tot = int(huisnummer_tot) if huisnummer_tot else None adres = models.Adres( bouwdossier=bouwdossier, - straat=x_adres['straat'], + straat=x_adres["straat"], huisnummer_van=huisnummer_van, huisnummer_tot=huisnummer_tot, stadsdeel=stadsdeel, @@ -355,44 +378,52 @@ def add_pre_wabo_dossier(x_dossier, file_path, import_file, count, total_count): nummeraanduidingen_label=[], panden=[], verblijfsobjecten=[], - verblijfsobjecten_label=[] + verblijfsobjecten_label=[], ) adres.save() - for x_sub_dossier in get_list_items(x_dossier, 'subDossiers', 'subDossier'): - titel = x_sub_dossier['titel'] + for x_sub_dossier in get_list_items(x_dossier, "subDossiers", "subDossier"): + titel = x_sub_dossier["titel"] if not titel: - titel = '' - log.warning(f"Missing titel for subdossier for {bouwdossier.dossiernr} in {file_path}") + titel = "" + log.warning( + f"Missing titel for subdossier for {bouwdossier.dossiernr} in {file_path}" + ) - subdossier_bestanden = get_list_items(x_sub_dossier, 'bestanden', 'url') + subdossier_bestanden = get_list_items(x_sub_dossier, "bestanden", "url") # This is to check if there are any bestanden added to a subdossier directly. They are skipped because it is not # known if they are public or not if len(subdossier_bestanden) > 0: - log.warning("bestanden in sub_dossier, unexpected, no way to determine if this is Public or Restricted") + log.warning( + "bestanden in sub_dossier, unexpected, no way to determine if this is Public or Restricted" + ) log.warning(subdossier_bestanden) subdossier_bestanden = [] # Documenten are now added with their public (access) flag. This way the serializer shows exactly which # group of bestanden (scans) are public or not. documenten = [] - for x_document in get_list_items(x_sub_dossier, 'documenten', 'document'): - bestanden = get_list_items(x_document, 'bestanden', 'url') + for x_document in get_list_items(x_sub_dossier, "documenten", "document"): + bestanden = get_list_items(x_document, "bestanden", "url") access = get_access(x_document) - access_restricted_until = get_date_from_year(x_document.get('openbaarheidsBeperkingTot')) - copyright = openbaar_to_copyright(x_document.get('auteursrechtBeperking')) - copyright_until = get_date_from_year(x_document.get('auteursrechtBeperkingTot')) - copyright_holders = x_document.get('auteursrechtHouders') - copyright_manufacturers = x_document.get('auteursrechtVervaardigers') + access_restricted_until = get_date_from_year( + x_document.get("openbaarheidsBeperkingTot") + ) + copyright = openbaar_to_copyright(x_document.get("auteursrechtBeperking")) + copyright_until = get_date_from_year( + x_document.get("auteursrechtBeperkingTot") + ) + copyright_holders = x_document.get("auteursrechtHouders") + copyright_manufacturers = x_document.get("auteursrechtVervaardigers") valid_bestanden = [] for bestand in bestanden: normalized_bestand = _normalize_bestand(bestand) if normalized_bestand: valid_bestanden.append(normalized_bestand) document = models.Document( - barcode=x_document.get('barcode'), + barcode=x_document.get("barcode"), bouwdossier=bouwdossier, subdossier_titel=titel, bestanden=valid_bestanden, @@ -401,7 +432,7 @@ def add_pre_wabo_dossier(x_dossier, file_path, import_file, count, total_count): copyright=copyright, copyright_until=copyright_until, copyright_holders=copyright_holders, - copyright_manufacturers=copyright_manufacturers + copyright_manufacturers=copyright_manufacturers, ) documenten.append(document) @@ -413,11 +444,12 @@ def add_pre_wabo_dossier(x_dossier, file_path, import_file, count, total_count): return count, total_count + def import_wabo_dossiers(root_dir=settings.DATA_DIR, max_file_count=None): # noqa C901 total_count = 0 file_count = 0 - for file_path in glob.iglob(root_dir + '/**/*.xml', recursive=True): - wabo = re.search('WABO_.+\\.xml$', file_path) + for file_path in glob.iglob(root_dir + "/**/*.xml", recursive=True): + wabo = re.search("WABO_.+\\.xml$", file_path) importfiles = models.ImportFile.objects.filter(name=file_path) if not wabo or len(importfiles) > 0: @@ -437,9 +469,10 @@ def import_wabo_dossiers(root_dir=settings.DATA_DIR, max_file_count=None): # no xml = xmltodict.parse(fd.read()) with transaction.atomic(): - for x_dossier in get_list_items(xml, 'dossiers', 'dossier'): + for x_dossier in get_list_items(xml, "dossiers", "dossier"): (count, total_count) = add_wabo_dossier( - x_dossier, file_path, import_file, count, total_count) + x_dossier, file_path, import_file, count, total_count + ) import_file.status = models.IMPORT_FINISHED import_file.save() @@ -452,15 +485,19 @@ def import_wabo_dossiers(root_dir=settings.DATA_DIR, max_file_count=None): # no import_file.status = models.IMPORT_ERROR import_file.save() - log.info(f"Import finished. Bouwdossiers total: {total_count}. Bouwdossiers count query: {models.BouwDossier.objects.count()}") + log.info( + f"Import finished. Bouwdossiers total: {total_count}. Bouwdossiers count query: {models.BouwDossier.objects.count()}" + ) -def import_pre_wabo_dossiers(root_dir=settings.DATA_DIR, max_file_count=None): # noqa C901 +def import_pre_wabo_dossiers( + root_dir=settings.DATA_DIR, max_file_count=None +): # noqa C901 total_count = 0 file_count = 0 - for file_path in glob.iglob(root_dir + '/**/*.xml', recursive=True): + for file_path in glob.iglob(root_dir + "/**/*.xml", recursive=True): # SAA_BWT_02.xml - pre_wabo = re.search(r'SAA_BWT_[A-Za-z-_0-9]+\.xml$', file_path) + pre_wabo = re.search(r"SAA_BWT_[A-Za-z-_0-9]+\.xml$", file_path) importfiles = models.ImportFile.objects.filter(name=file_path) if not pre_wabo or len(importfiles) > 0: @@ -477,9 +514,10 @@ def import_pre_wabo_dossiers(root_dir=settings.DATA_DIR, max_file_count=None): xml = xmltodict.parse(fd.read()) with transaction.atomic(): - for x_dossier in get_list_items(xml, 'bwtDossiers', 'dossier'): + for x_dossier in get_list_items(xml, "bwtDossiers", "dossier"): (count, total_count) = add_pre_wabo_dossier( - x_dossier, file_path, import_file, count, total_count) + x_dossier, file_path, import_file, count, total_count + ) import_file.status = models.IMPORT_FINISHED import_file.save() @@ -501,7 +539,8 @@ def add_bag_ids_to_wabo(): log.info("Add nummeraanduidingen to wabo dossiers") with connection.cursor() as cursor: try: - cursor.execute(""" + cursor.execute( + """ WITH adres_nummeraanduiding AS ( SELECT ba.id AS id, @@ -521,7 +560,8 @@ def add_bag_ids_to_wabo(): nummeraanduidingen_label = adres_nummeraanduiding.nummeraanduidingen_label FROM adres_nummeraanduiding WHERE importer_adres.id = adres_nummeraanduiding.id - """) + """ + ) except Exception: log.exception("An error occurred while adding the nummeraanduidingen.") log.info("Finished adding nummeraanduidingen to wabo dossiers") @@ -547,7 +587,8 @@ def add_bag_ids_to_pre_wabo(): # sides of the street use even resp odd numbers. But sometimes we have to use all the # numbers in the range. Therefore we use bouwblokken to select all numbers in the range # that also are in the same bouwblok as the start or the end of the range - cursor.execute(""" + cursor.execute( + """ WITH adres_start_end_bouwblok AS ( SELECT iadre.id, ARRAY_AGG(DISTINCT bpand.ligtinbouwblokid) AS bouwblokken FROM importer_adres iadre @@ -604,13 +645,17 @@ def add_bag_ids_to_pre_wabo(): nummeraanduidingen_label = adres_pand.nummeraanduidingen_label FROM adres_pand WHERE importer_adres.id = adres_pand.id - """) - log.info("Finished adding nummeraanduidingen, verblijfsobjecten and panden to pre-wabo dossiers") + """ + ) + log.info( + "Finished adding nummeraanduidingen, verblijfsobjecten and panden to pre-wabo dossiers" + ) # First we try to match with openbare ruimtes that are streets 01 log.info("Add openbare ruimtes") with connection.cursor() as cursor: - cursor.execute(""" + cursor.execute( + """ UPDATE importer_adres iadre SET openbareruimte_id = bopen.identificatie FROM bag_openbareruimte bopen @@ -619,24 +664,28 @@ def add_bag_ids_to_pre_wabo(): OR bopen.eindgeldigheid >= NOW()) AND bopen.typecode = '1' AND bopen.identificatie like '0363%' -- Only match Amsterdam streets - """) + """ + ) # If the openbareruimte was not yet found we try to match with other openbare ruimtes with connection.cursor() as cursor: - cursor.execute(""" + cursor.execute( + """ UPDATE importer_adres iadre SET openbareruimte_id = bopen.identificatie FROM bag_openbareruimte bopen WHERE iadre.straat = bopen.naam AND bopen.identificatie LIKE '0363%' -- Only match Amsterdam streets AND (iadre.openbareruimte_id IS NULL OR iadre.openbareruimte_id = '') - """) + """ + ) log.info("Finished adding openbare ruimtes") def validate_import(min_bouwdossiers_count): with connection.cursor() as cursor: - cursor.execute(""" + cursor.execute( + """ SELECT COUNT(*), array_length(panden, 1) IS NOT NULL AS has_panden, @@ -644,35 +693,38 @@ def validate_import(min_bouwdossiers_count): openbareruimte_id IS NOT NULL AND openbareruimte_id <> '' AS has_openbareruimte_id FROM importer_adres GROUP BY has_openbareruimte_id, has_panden, has_nummeraanduidingen - """) + """ + ) rows = cursor.fetchall() result = { - 'total': 0, - 'has_panden': 0, - 'has_nummeraanduidingen': 0, - 'has_openbareruimte_id': 0, + "total": 0, + "has_panden": 0, + "has_nummeraanduidingen": 0, + "has_openbareruimte_id": 0, } for row in rows: - result['total'] += row[0] + result["total"] += row[0] if row[1]: - result['has_panden'] += row[0] + result["has_panden"] += row[0] if row[2]: - result['has_nummeraanduidingen'] += row[0] + result["has_nummeraanduidingen"] += row[0] if row[3]: - result['has_openbareruimte_id'] += row[0] - log.info('Validation import result: ' + str(result)) + result["has_openbareruimte_id"] += row[0] + log.info("Validation import result: " + str(result)) log.info( f"{result['has_panden']} number of records of a total of {result['total']} records" f" ({result['has_panden'] / result['total'] * 100}%) has one or more panden." f" The required minimum is {0.8 * result['total']} (80%)." ) - assert result['total'] >= min_bouwdossiers_count, \ - f'Imported total of {result["total"]} bouwdossiers is less than the required number {min_bouwdossiers_count}' - assert result['has_panden'] > 0.8 * result['total'], \ - f"{result['has_panden']} number of records of a total of {result['total']} records " \ - f"({result['has_panden'] / result['total'] * 100}%) has one or more panden, " \ + assert ( + result["total"] >= min_bouwdossiers_count + ), f'Imported total of {result["total"]} bouwdossiers is less than the required number {min_bouwdossiers_count}' + assert result["has_panden"] > 0.8 * result["total"], ( + f"{result['has_panden']} number of records of a total of {result['total']} records " + f"({result['has_panden'] / result['total'] * 100}%) has one or more panden, " f"which is less than the required minimum of {0.8 * result['total']} (80%)" - assert result['has_nummeraanduidingen'] > 0.8 * result['total'] - assert result['has_openbareruimte_id'] > 0.95 * result['total'] + ) + assert result["has_nummeraanduidingen"] > 0.8 * result["total"] + assert result["has_openbareruimte_id"] > 0.95 * result["total"] diff --git a/src/importer/management/commands/run_import.py b/src/importer/management/commands/run_import.py index 60c75a3..61fa49d 100644 --- a/src/importer/management/commands/run_import.py +++ b/src/importer/management/commands/run_import.py @@ -1,15 +1,23 @@ import logging -from django.conf import settings +from django.conf import settings from django.core.management.base import BaseCommand from bag.bag_loader import BagLoader from bag.koppeltabel_loader import KoppeltabelLoader -from importer.batch import (add_bag_ids_to_pre_wabo, add_bag_ids_to_wabo, - import_pre_wabo_dossiers, import_wabo_dossiers, - validate_import) -from importer.util_azure import download_all_files_from_container, download_blob_to_file, remove_directory -from importer.util_db import (swap_tables_between_apps, truncate_tables) +from importer.batch import ( + add_bag_ids_to_pre_wabo, + add_bag_ids_to_wabo, + import_pre_wabo_dossiers, + import_wabo_dossiers, + validate_import, +) +from importer.util_azure import ( + download_all_files_from_container, + download_blob_to_file, + remove_directory, +) +from importer.util_db import swap_tables_between_apps, truncate_tables BAG_SOURCE_SKIP = 0 BAG_SOURCE_API = 1 @@ -17,13 +25,14 @@ log = logging.getLogger(__name__) + class Command(BaseCommand): help = "Import (pre)WABO dossiers and combine with BAG data from datadienst export" def import_bag_from_api(self): - log.info('Importing bag data from api') + log.info("Importing bag data from api") bag = BagLoader() - truncate_tables(['bag']) + truncate_tables(["bag"]) bag.import_tables_from_endpoint() @@ -33,74 +42,78 @@ def import_bag_from_api(self): koppeltabel.load() def import_bag_from_azure_storage(self): - log.info('Importing bag data from azure storage') + log.info("Importing bag data from azure storage") bag = BagLoader() - truncate_tables(['bag']) + truncate_tables(["bag"]) output_dir = f"{settings.DATA_DIR}/bag" tables = bag.tables - tables['bag_verblijfsobjectpandrelatie'] = '' + tables["bag_verblijfsobjectpandrelatie"] = "" files = {} for table, _ in tables.items(): - files[table] = download_blob_to_file(settings.AZURE_CONTAINER_NAME_BAG, f"{table}.csv", output_dir) + files[table] = download_blob_to_file( + settings.AZURE_CONTAINER_NAME_BAG, f"{table}.csv", output_dir + ) bag.load_tables_from_csv(files) def import_dossiers(self, dossier_path): - log.info('Importing pre wabo dossiers') + log.info("Importing pre wabo dossiers") import_pre_wabo_dossiers(dossier_path) add_bag_ids_to_pre_wabo() - - log.info('Importing wabo dossiers') + + log.info("Importing wabo dossiers") import_wabo_dossiers(dossier_path) add_bag_ids_to_wabo() def add_arguments(self, parser): parser.add_argument( - '--bag_source', - dest='bag_source', + "--bag_source", + dest="bag_source", type=int, default=BAG_SOURCE_AZURE_STORAGE, - help='Bag data source') - + help="Bag data source", + ) + parser.add_argument( - '--skipgetfiles', - action='store_true', - dest='skipgetfiles', + "--skipgetfiles", + action="store_true", + dest="skipgetfiles", default=False, - help='Skip getting files from objectstore') - + help="Skip getting files from objectstore", + ) + parser.add_argument( - '--min_bouwdossiers_count', - dest='min_bouwdossiers_count', + "--min_bouwdossiers_count", + dest="min_bouwdossiers_count", type=int, default=settings.MIN_BOUWDOSSIERS_COUNT, - help='Minimum amount of bouwdossiers to be added') - + help="Minimum amount of bouwdossiers to be added", + ) + def handle(self, *args, **options): - log.info('Metadata import started') + log.info("Metadata import started") try: - if options['bag_source'] == BAG_SOURCE_API: + if options["bag_source"] == BAG_SOURCE_API: self.import_bag_from_api() - elif options['bag_source'] == BAG_SOURCE_AZURE_STORAGE: + elif options["bag_source"] == BAG_SOURCE_AZURE_STORAGE: self.import_bag_from_azure_storage() dossier_path = settings.DATA_DIR - if not options['skipgetfiles']: + if not options["skipgetfiles"]: dossier_path = f"{settings.DATA_DIR}/dossiers" remove_directory(dossier_path) - download_all_files_from_container(settings.AZURE_CONTAINER_NAME_DOSSIERS, dossier_path) + download_all_files_from_container( + settings.AZURE_CONTAINER_NAME_DOSSIERS, dossier_path + ) - truncate_tables(['importer']) + truncate_tables(["importer"]) self.import_dossiers(dossier_path) - validate_import(options['min_bouwdossiers_count']) - - swap_tables_between_apps('importer', 'bouwdossiers') + validate_import(options["min_bouwdossiers_count"]) + swap_tables_between_apps("importer", "bouwdossiers") except Exception as e: - raise Exception( - "An exception occurred importing the metadata." - ) from e \ No newline at end of file + raise Exception("An exception occurred importing the metadata.") from e diff --git a/src/importer/migrations/0001_initial.py b/src/importer/migrations/0001_initial.py index 7c6a50c..b074aec 100644 --- a/src/importer/migrations/0001_initial.py +++ b/src/importer/migrations/0001_initial.py @@ -10,102 +10,246 @@ class Migration(migrations.Migration): initial = True - dependencies = [ - ] + dependencies = [] operations = [ migrations.CreateModel( - name='BouwDossier', + name="BouwDossier", fields=[ - ('dossiernr', models.IntegerField()), - ('stadsdeel', models.CharField(db_index=True, max_length=10)), - ('titel', models.CharField(db_index=True, max_length=512)), - ('datering', models.DateField(null=True)), - ('dossier_type', models.CharField(max_length=255, null=True)), - ('gebruiksdoel', models.CharField(max_length=255, null=True)), - ('bwt_nummer', models.CharField(max_length=127, null=True)), - ('dossier_status', models.CharField(choices=[('A', 'Aanvraag'), ('B', 'Behandeling')], max_length=1, null=True)), - ('access', models.CharField(choices=[('PUBLIC', 'Public'), ('RESTRICTED', 'Restricted')], max_length=20, null=True)), - ('access_restricted_until', models.DateField(null=True)), - ('source', models.CharField(choices=[('EDEPOT', 'edepot'), ('WABO', 'wabo')], default='EDEPOT', help_text='Field that defines wabo and pre_wabo dossier', max_length=20)), - ('olo_liaan_nummer', models.IntegerField(default=None, null=True)), - ('wabo_bron', models.CharField(default=None, help_text='Should contain the origin of the dossier. Can be for example digital or paper dossier.', max_length=30, null=True)), - ('activiteiten', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=250), blank=True, default=list, size=None)), - ('id', models.AutoField(primary_key=True, serialize=False)), + ("dossiernr", models.IntegerField()), + ("stadsdeel", models.CharField(db_index=True, max_length=10)), + ("titel", models.CharField(db_index=True, max_length=512)), + ("datering", models.DateField(null=True)), + ("dossier_type", models.CharField(max_length=255, null=True)), + ("gebruiksdoel", models.CharField(max_length=255, null=True)), + ("bwt_nummer", models.CharField(max_length=127, null=True)), + ( + "dossier_status", + models.CharField( + choices=[("A", "Aanvraag"), ("B", "Behandeling")], + max_length=1, + null=True, + ), + ), + ( + "access", + models.CharField( + choices=[("PUBLIC", "Public"), ("RESTRICTED", "Restricted")], + max_length=20, + null=True, + ), + ), + ("access_restricted_until", models.DateField(null=True)), + ( + "source", + models.CharField( + choices=[("EDEPOT", "edepot"), ("WABO", "wabo")], + default="EDEPOT", + help_text="Field that defines wabo and pre_wabo dossier", + max_length=20, + ), + ), + ("olo_liaan_nummer", models.IntegerField(default=None, null=True)), + ( + "wabo_bron", + models.CharField( + default=None, + help_text="Should contain the origin of the dossier. Can be for example digital or paper dossier.", + max_length=30, + null=True, + ), + ), + ( + "activiteiten", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=250), + blank=True, + default=list, + size=None, + ), + ), + ("id", models.AutoField(primary_key=True, serialize=False)), ], ), migrations.CreateModel( - name='ImportFile', + name="ImportFile", fields=[ - ('name', models.CharField(max_length=512, unique=True)), - ('status', models.CharField(choices=[('B', 'Busy'), ('F', 'Finished'), ('E', 'Error')], max_length=1)), - ('last_import', models.DateTimeField(auto_now=True)), - ('id', models.AutoField(primary_key=True, serialize=False)), + ("name", models.CharField(max_length=512, unique=True)), + ( + "status", + models.CharField( + choices=[("B", "Busy"), ("F", "Finished"), ("E", "Error")], + max_length=1, + ), + ), + ("last_import", models.DateTimeField(auto_now=True)), + ("id", models.AutoField(primary_key=True, serialize=False)), ], options={ - 'ordering': ('name',), - 'abstract': False, + "ordering": ("name",), + "abstract": False, }, ), migrations.CreateModel( - name='Document', + name="Document", fields=[ - ('subdossier_titel', models.TextField(blank=True, null=True)), - ('document_omschrijving', models.CharField(blank=True, max_length=250, null=True)), - ('barcode', models.CharField(db_index=True, max_length=250, null=True)), - ('bestanden', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=250), blank=True, size=None)), - ('access', models.CharField(choices=[('PUBLIC', 'Public'), ('RESTRICTED', 'Restricted')], max_length=20, null=True)), - ('access_restricted_until', models.DateField(null=True)), - ('copyright', models.CharField(choices=[('Y', 'Yes'), ('N', 'No')], max_length=1, null=True)), - ('copyright_until', models.DateField(null=True)), - ('copyright_holders', models.CharField(max_length=512, null=True)), - ('copyright_manufacturers', models.CharField(max_length=512, null=True)), - ('oorspronkelijk_pad', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=250), blank=True, default=list, size=None)), - ('id', models.AutoField(primary_key=True, serialize=False)), - ('bouwdossier', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='bagimporter_documenten', to='importer.bouwdossier')), + ("subdossier_titel", models.TextField(blank=True, null=True)), + ( + "document_omschrijving", + models.CharField(blank=True, max_length=250, null=True), + ), + ("barcode", models.CharField(db_index=True, max_length=250, null=True)), + ( + "bestanden", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=250), + blank=True, + size=None, + ), + ), + ( + "access", + models.CharField( + choices=[("PUBLIC", "Public"), ("RESTRICTED", "Restricted")], + max_length=20, + null=True, + ), + ), + ("access_restricted_until", models.DateField(null=True)), + ( + "copyright", + models.CharField( + choices=[("Y", "Yes"), ("N", "No")], max_length=1, null=True + ), + ), + ("copyright_until", models.DateField(null=True)), + ("copyright_holders", models.CharField(max_length=512, null=True)), + ( + "copyright_manufacturers", + models.CharField(max_length=512, null=True), + ), + ( + "oorspronkelijk_pad", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=250), + blank=True, + default=list, + size=None, + ), + ), + ("id", models.AutoField(primary_key=True, serialize=False)), + ( + "bouwdossier", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="bagimporter_documenten", + to="importer.bouwdossier", + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, ), migrations.AddField( - model_name='bouwdossier', - name='importfile', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='bagimporter_bouwdossiers', to='importer.importfile'), + model_name="bouwdossier", + name="importfile", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="bagimporter_bouwdossiers", + to="importer.importfile", + ), ), migrations.CreateModel( - name='Adres', + name="Adres", fields=[ - ('straat', models.CharField(max_length=150, null=True)), - ('huisnummer_van', models.IntegerField(null=True)), - ('huisnummer_tot', models.IntegerField(null=True)), - ('openbareruimte_id', models.CharField(db_index=True, max_length=16, null=True)), - ('stadsdeel', models.CharField(db_index=True, max_length=10)), - ('nummeraanduidingen', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=16), blank=True, size=None)), - ('nummeraanduidingen_label', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=256), blank=True, size=None)), - ('panden', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=16), blank=True, size=None)), - ('verblijfsobjecten', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=16), blank=True, size=None)), - ('verblijfsobjecten_label', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=256), blank=True, size=None)), - ('locatie_aanduiding', models.CharField(max_length=250, null=True)), - ('huisnummer_toevoeging', models.CharField(default=None, max_length=10, null=True)), - ('huisnummer_letter', models.CharField(default=None, max_length=10, null=True)), - ('id', models.AutoField(primary_key=True, serialize=False)), - ('bouwdossier', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='bagimporter_adressen', to='importer.bouwdossier')), + ("straat", models.CharField(max_length=150, null=True)), + ("huisnummer_van", models.IntegerField(null=True)), + ("huisnummer_tot", models.IntegerField(null=True)), + ( + "openbareruimte_id", + models.CharField(db_index=True, max_length=16, null=True), + ), + ("stadsdeel", models.CharField(db_index=True, max_length=10)), + ( + "nummeraanduidingen", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=16), + blank=True, + size=None, + ), + ), + ( + "nummeraanduidingen_label", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=256), + blank=True, + size=None, + ), + ), + ( + "panden", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=16), + blank=True, + size=None, + ), + ), + ( + "verblijfsobjecten", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=16), + blank=True, + size=None, + ), + ), + ( + "verblijfsobjecten_label", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=256), + blank=True, + size=None, + ), + ), + ("locatie_aanduiding", models.CharField(max_length=250, null=True)), + ( + "huisnummer_toevoeging", + models.CharField(default=None, max_length=10, null=True), + ), + ( + "huisnummer_letter", + models.CharField(default=None, max_length=10, null=True), + ), + ("id", models.AutoField(primary_key=True, serialize=False)), + ( + "bouwdossier", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="bagimporter_adressen", + to="importer.bouwdossier", + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, ), migrations.AddConstraint( - model_name='bouwdossier', - constraint=models.UniqueConstraint(fields=('stadsdeel', 'dossiernr'), name='unique_bagimporter_bouwdossier'), + model_name="bouwdossier", + constraint=models.UniqueConstraint( + fields=("stadsdeel", "dossiernr"), name="unique_bagimporter_bouwdossier" + ), ), migrations.AddIndex( - model_name='adres', - index=django.contrib.postgres.indexes.GinIndex(fields=['nummeraanduidingen'], name='importer_ad_nummera_da4542_gin'), + model_name="adres", + index=django.contrib.postgres.indexes.GinIndex( + fields=["nummeraanduidingen"], name="importer_ad_nummera_da4542_gin" + ), ), migrations.AddIndex( - model_name='adres', - index=django.contrib.postgres.indexes.GinIndex(fields=['panden'], name='importer_ad_panden_ec3846_gin'), + model_name="adres", + index=django.contrib.postgres.indexes.GinIndex( + fields=["panden"], name="importer_ad_panden_ec3846_gin" + ), ), ] diff --git a/src/importer/models.py b/src/importer/models.py index 8ae4d5e..c41d544 100644 --- a/src/importer/models.py +++ b/src/importer/models.py @@ -6,21 +6,30 @@ class ImportFile(ImportFileBase): id = models.AutoField(primary_key=True) + class BouwDossier(BouwDossierBase): id = models.AutoField(primary_key=True) - importfile = models.ForeignKey(ImportFile, related_name='bagimporter_bouwdossiers', on_delete=CASCADE) + importfile = models.ForeignKey( + ImportFile, related_name="bagimporter_bouwdossiers", on_delete=CASCADE + ) class Meta: constraints = [ - models.UniqueConstraint(fields=['stadsdeel', 'dossiernr'], name='unique_bagimporter_bouwdossier'), + models.UniqueConstraint( + fields=["stadsdeel", "dossiernr"], name="unique_bagimporter_bouwdossier" + ), ] + class Adres(AdresBase): id = models.AutoField(primary_key=True) - bouwdossier = models.ForeignKey(BouwDossier, - related_name='bagimporter_adressen', - on_delete=CASCADE) + bouwdossier = models.ForeignKey( + BouwDossier, related_name="bagimporter_adressen", on_delete=CASCADE + ) + class Document(DocumentBase): id = models.AutoField(primary_key=True) - bouwdossier = models.ForeignKey(BouwDossier, related_name='bagimporter_documenten', on_delete=CASCADE) \ No newline at end of file + bouwdossier = models.ForeignKey( + BouwDossier, related_name="bagimporter_documenten", on_delete=CASCADE + ) diff --git a/src/importer/tests/test_import.py b/src/importer/tests/test_import.py index 4f7195c..79a32ca 100644 --- a/src/importer/tests/test_import.py +++ b/src/importer/tests/test_import.py @@ -1,10 +1,11 @@ -from django.conf import settings from django.db import connection from django.test import TestCase from importer import batch, models DATA_DIR = "importer/tests/data/" + + class APITest(TestCase): @classmethod def setUpTestData(cls): @@ -21,7 +22,7 @@ def test_prewabo_import(self): batch.add_bag_ids_to_pre_wabo() bd3 = models.BouwDossier.objects.get(dossiernr=3) - self.assertEqual(bd3.stadsdeel, 'SA') + self.assertEqual(bd3.stadsdeel, "SA") self.assertEqual(bd3.titel, "Hoogte Kadijk 40") self.assertEqual(bd3.datering.strftime("%Y"), "2003") self.assertEqual(bd3.dossier_type, "verbouwing") @@ -33,21 +34,36 @@ def test_prewabo_import(self): self.assertEqual(adres0.straat, "Hoogte Kadijk") self.assertEqual(adres0.huisnummer_van, 40) self.assertEqual(adres0.huisnummer_tot, 40) - self.assertEqual(adres0.nummeraanduidingen_label, ['Hoogte Kadijk 40-1','Hoogte Kadijk 40-2','Hoogte Kadijk 40-3','Hoogte Kadijk 40-H']) - - bd3_documenten = models.Document.objects.filter(bouwdossier_id=bd3.id).order_by('id').all() + self.assertEqual( + adres0.nummeraanduidingen_label, + [ + "Hoogte Kadijk 40-1", + "Hoogte Kadijk 40-2", + "Hoogte Kadijk 40-3", + "Hoogte Kadijk 40-H", + ], + ) + + bd3_documenten = ( + models.Document.objects.filter(bouwdossier_id=bd3.id).order_by("id").all() + ) document0 = bd3_documenten.first() self.assertEqual(document0.barcode, "SA00000007") self.assertEqual(document0.subdossier_titel, "Aanvraag en behandeling") - self.assertEqual(document0.bestanden, [ - "SA/00003/SA00000007_00001.jpg", - "SA/00003/SA00000007_00002.jpg", - "SA/00003/SA00000007_00003.jpg" - ]) + self.assertEqual( + document0.bestanden, + [ + "SA/00003/SA00000007_00001.jpg", + "SA/00003/SA00000007_00002.jpg", + "SA/00003/SA00000007_00003.jpg", + ], + ) self.assertEqual(document0.access, "PUBLIC") bd123 = models.BouwDossier.objects.get(dossiernr=123) - bd123_documenten = models.Document.objects.filter(bouwdossier_id=bd123.id).order_by('id').all() + bd123_documenten = ( + models.Document.objects.filter(bouwdossier_id=bd123.id).order_by("id").all() + ) for document123 in bd123_documenten.all(): if document123.barcode == "SA00001038": self.assertEqual(document123.access, "RESTRICTED") @@ -56,22 +72,25 @@ def test_prewabo_import(self): bd21388 = models.BouwDossier.objects.get(dossiernr=21388) bd21388_addressen = models.Adres.objects.filter(bouwdossier_id=bd21388.id) - fdb = bd21388_addressen.get(straat='Feike de Boerlaan') + fdb = bd21388_addressen.get(straat="Feike de Boerlaan") # 0363010000959579 Feike de Boerlaan 29 - self.assertTrue('0363010000959579' in fdb.verblijfsobjecten) + self.assertTrue("0363010000959579" in fdb.verblijfsobjecten) # 0363010000998545 Feike de Boerlaan 14 - self.assertFalse('0363010000998545' in fdb.verblijfsobjecten) + self.assertFalse("0363010000998545" in fdb.verblijfsobjecten) # 0363200000461980 Feike de Boerlaan 83 nummeraanduiding - self.assertTrue('0363200000461980' in fdb.nummeraanduidingen) + self.assertTrue("0363200000461980" in fdb.nummeraanduidingen) # 0363200000470955 Feike de Boerlaan 290 nummeraanduiding - self.assertFalse('0363200000470955' in fdb.nummeraanduidingen) + self.assertFalse("0363200000470955" in fdb.nummeraanduidingen) def test_wabo_import(self): batch.import_wabo_dossiers(DATA_DIR) bd1 = models.BouwDossier.objects.get(dossiernr=189) - self.assertEqual(bd1.stadsdeel, 'SDC') - self.assertEqual(bd1.titel, "het herstellen van de fundering en het veranderen en vernieuwen van de rechterzijvleugel van het gebouwencomplex Lauriergracht 1161 met bestemming daarvan tot kantoor") + self.assertEqual(bd1.stadsdeel, "SDC") + self.assertEqual( + bd1.titel, + "het herstellen van de fundering en het veranderen en vernieuwen van de rechterzijvleugel van het gebouwencomplex Lauriergracht 1161 met bestemming daarvan tot kantoor", + ) self.assertEqual(bd1.datering.strftime("%Y"), "2010") self.assertEqual(bd1.dossier_type, "omgevingsvergunning") self.assertEqual(bd1.access, "PUBLIC") @@ -88,36 +107,57 @@ def test_wabo_import(self): self.assertEqual(adres1.huisnummer_toevoeging, "H") self.assertEqual(adres1.huisnummer_tot, None) self.assertEqual(adres1.openbareruimte_id, "0363300000004136") - self.assertEqual(adres1.verblijfsobjecten, ['0363010000719556']) - self.assertEqual(adres1.panden, ['0363100012168986']) + self.assertEqual(adres1.verblijfsobjecten, ["0363010000719556"]) + self.assertEqual(adres1.panden, ["0363100012168986"]) - bd1_documenten = models.Document.objects.filter(bouwdossier_id=bd1.id).order_by('id').all() + bd1_documenten = ( + models.Document.objects.filter(bouwdossier_id=bd1.id).order_by("id").all() + ) self.assertEqual(len(bd1_documenten), 20) document5 = bd1_documenten[5] - self.assertEqual(document5.document_omschrijving, "Bij besluit behorende gewaarmerkte bescheiden - vergunningset189_bijlage16.pdf | Bij besluit behorende gewaarmerkte bescheiden: vergunningset") - self.assertEqual(document5.bestanden, ['SDC/KEY2Vergunning_33/Documentum/0901b69980335dcb.pdf']) - self.assertEqual(document5.oorspronkelijk_pad, ['G:\\Export files\\documentum\\primary/25/0901b69980335dcb']) + self.assertEqual( + document5.document_omschrijving, + "Bij besluit behorende gewaarmerkte bescheiden - vergunningset189_bijlage16.pdf | Bij besluit behorende gewaarmerkte bescheiden: vergunningset", + ) + self.assertEqual( + document5.bestanden, + ["SDC/KEY2Vergunning_33/Documentum/0901b69980335dcb.pdf"], + ) + self.assertEqual( + document5.oorspronkelijk_pad, + ["G:\\Export files\\documentum\\primary/25/0901b69980335dcb"], + ) self.assertEqual(document5.access, models.ACCESS_PUBLIC) document6 = bd1_documenten[6] - self.assertEqual(document6.document_omschrijving, "Bij besluit behorende gewaarmerkte bescheiden - vergunningset189_bijlage01.pdf | Bij besluit behorende gewaarmerkte bescheiden: vergunningset") - self.assertEqual(document6.bestanden, ['SDC/KEY2Vergunning_33/Documentum/0901b69980335dcd.pdf']) - self.assertEqual(document6.oorspronkelijk_pad, ['G:\\Export files\\documentum\\primary/25/0901b69980335dcd']) + self.assertEqual( + document6.document_omschrijving, + "Bij besluit behorende gewaarmerkte bescheiden - vergunningset189_bijlage01.pdf | Bij besluit behorende gewaarmerkte bescheiden: vergunningset", + ) + self.assertEqual( + document6.bestanden, + ["SDC/KEY2Vergunning_33/Documentum/0901b69980335dcd.pdf"], + ) + self.assertEqual( + document6.oorspronkelijk_pad, + ["G:\\Export files\\documentum\\primary/25/0901b69980335dcd"], + ) self.assertEqual(document6.access, models.ACCESS_RESTRICTED) # Test whether the dossier and document with no accesibility keys are set to restricted bd2 = models.BouwDossier.objects.get(dossiernr=233) self.assertEqual(bd2.access, models.ACCESS_RESTRICTED) - bd2_documenten = models.Document.objects.filter(bouwdossier_id=bd2.id).order_by('id').all() + bd2_documenten = ( + models.Document.objects.filter(bouwdossier_id=bd2.id).order_by("id").all() + ) document1 = bd2_documenten[0] self.assertEqual(document1.access, models.ACCESS_RESTRICTED) batch.add_bag_ids_to_wabo() self.assertEqual(bd1_addressen.count(), 25) adres1_new = bd1_addressen.first() - self.assertEqual(adres1_new.nummeraanduidingen_label, ['Lauriergracht 116-H']) - + self.assertEqual(adres1_new.nummeraanduidingen_label, ["Lauriergracht 116-H"]) def test_validate_import(self): batch.import_pre_wabo_dossiers(DATA_DIR) @@ -125,4 +165,3 @@ def test_validate_import(self): batch.import_wabo_dossiers(DATA_DIR) batch.add_bag_ids_to_wabo() batch.validate_import(min_bouwdossiers_count=38) - diff --git a/src/importer/tests/test_util_azure.py b/src/importer/tests/test_util_azure.py index c27ef58..fd2f6e8 100644 --- a/src/importer/tests/test_util_azure.py +++ b/src/importer/tests/test_util_azure.py @@ -7,22 +7,28 @@ from django.test import TestCase from importer.util_azure import ( - get_container_client, - get_blob_container_client, get_blob_client, + get_blob_container_client, + get_container_client, ) log = logging.getLogger(__name__) + def create_blob_container(container_name): blob_service_client = get_container_client() try: - container_client = blob_service_client.create_container(container_name, public_access=None) + container_client = blob_service_client.create_container( + container_name, public_access=None + ) except ResourceExistsError: pass - container_client = blob_service_client.get_container_client(container=container_name) + container_client = blob_service_client.get_container_client( + container=container_name + ) return container_client + def store_blob_in_container(container, file_path, filename): blob_client = get_blob_client(container, filename) with open(file=file_path, mode="rb") as data: @@ -35,14 +41,26 @@ class APITest(TestCase): @classmethod def setUpTestData(cls): create_blob_container(settings.AZURE_CONTAINER_NAME_BAG) - + dir_path = Path("importer/tests/data/bag") for file_path in os.listdir(dir_path): file = Path(dir_path, file_path) - store_blob_in_container(settings.AZURE_CONTAINER_NAME_BAG, file, file.name) + store_blob_in_container(settings.AZURE_CONTAINER_NAME_BAG, file, file.name) def test_retrieve_blob(self): client = get_blob_container_client(settings.AZURE_CONTAINER_NAME_BAG) names = [blob for blob in client.list_blob_names()] - self.assertTrue(all(item in names for item in ['bag_ligplaats.csv', 'bag_nummeraanduiding.csv', 'bag_openbareruimte.csv', 'bag_pand.csv', 'bag_standplaats.csv', 'bag_verblijfsobject.csv', 'bag_verblijfsobjectpandrelatie.csv'])) - \ No newline at end of file + self.assertTrue( + all( + item in names + for item in [ + "bag_ligplaats.csv", + "bag_nummeraanduiding.csv", + "bag_openbareruimte.csv", + "bag_pand.csv", + "bag_standplaats.csv", + "bag_verblijfsobject.csv", + "bag_verblijfsobjectpandrelatie.csv", + ] + ) + ) diff --git a/src/importer/util_azure.py b/src/importer/util_azure.py index 576e5c9..33fbf05 100644 --- a/src/importer/util_azure.py +++ b/src/importer/util_azure.py @@ -1,43 +1,56 @@ import logging import os import shutil -from django.conf import settings from azure.identity import DefaultAzureCredential from azure.storage.blob import BlobServiceClient +from django.conf import settings log = logging.getLogger(__name__) + def get_container_client(): if settings.AZURITE_STORAGE_CONNECTION_STRING: - blob_service_client = BlobServiceClient.from_connection_string(settings.AZURITE_STORAGE_CONNECTION_STRING) + blob_service_client = BlobServiceClient.from_connection_string( + settings.AZURITE_STORAGE_CONNECTION_STRING + ) else: default_credential = DefaultAzureCredential() - blob_service_client = BlobServiceClient(settings.STORAGE_ACCOUNT_URL, credential=default_credential) + blob_service_client = BlobServiceClient( + settings.STORAGE_ACCOUNT_URL, credential=default_credential + ) return blob_service_client + def get_blob_container_client(container_name): blob_service_client = get_container_client() - container_client = blob_service_client.get_container_client(container=container_name) + container_client = blob_service_client.get_container_client( + container=container_name + ) return container_client + def get_blob_client(container_name, blob_name): blob_service_client = get_container_client() - blob_client = blob_service_client.get_blob_client(container=container_name, blob=blob_name) + blob_client = blob_service_client.get_blob_client( + container=container_name, blob=blob_name + ) return blob_client + def download_blob_to_file(container_name, blob_name, output_dir): if not os.path.exists(output_dir): os.makedirs(output_dir) - + blob_client = get_blob_client(container_name=container_name, blob_name=blob_name) - file_path = f'{output_dir}/{blob_name}' + file_path = f"{output_dir}/{blob_name}" with open(file_path, "wb") as file: blob = blob_client.download_blob().readall() file.write(blob) return file_path + def download_all_files_from_container(container_name, output_dir=settings.DATA_DIR): if not os.path.exists(output_dir): os.makedirs(output_dir) @@ -48,12 +61,13 @@ def download_all_files_from_container(container_name, output_dir=settings.DATA_D files = [] for blob in blob_list: - file_path = f'{output_dir}/{blob.name}' - with open(f'{output_dir}/{blob.name}', "wb") as file: + file_path = f"{output_dir}/{blob.name}" + with open(f"{output_dir}/{blob.name}", "wb") as file: file.write(container_client.download_blob(blob.name).readall()) files.append(file_path) return files + def remove_directory(dir): if os.path.exists(dir): - shutil.rmtree(dir) \ No newline at end of file + shutil.rmtree(dir) diff --git a/src/importer/util_db.py b/src/importer/util_db.py index c6063f2..d8d6a9f 100644 --- a/src/importer/util_db.py +++ b/src/importer/util_db.py @@ -1,11 +1,12 @@ +from django.apps import apps from django.db import connection, transaction -from django.apps import apps def get_app_model_names(model_name): app_models = apps.get_app_config(model_name).get_models() return [model._meta.db_table for model in app_models] + @transaction.atomic def truncate_tables(apps): models = [model for app in apps for model in get_app_model_names(app)] @@ -15,22 +16,23 @@ def truncate_tables(apps): with connection.cursor() as cursor: cursor.execute(query) + @transaction.atomic def swap_tables_between_apps(app1, app2): - importer_models = get_app_model_names(app1) - bouwdossier_models = get_app_model_names(app2) - - query = "" - for model in importer_models: - new_model_name = model.replace(app1, app2) - temp_model_name = model.replace(app1, 'temp') - if (new_model_name not in bouwdossier_models): - raise Exception(f"Model {model} of {app1} not found in {app2} app") - - query += f""" + importer_models = get_app_model_names(app1) + bouwdossier_models = get_app_model_names(app2) + + query = "" + for model in importer_models: + new_model_name = model.replace(app1, app2) + temp_model_name = model.replace(app1, "temp") + if new_model_name not in bouwdossier_models: + raise Exception(f"Model {model} of {app1} not found in {app2} app") + + query += f""" ALTER TABLE {new_model_name} RENAME TO {temp_model_name}; ALTER TABLE {model} RENAME TO {new_model_name}; ALTER TABLE {temp_model_name} RENAME TO {model};""" - - with connection.cursor() as cursor: - cursor.execute(query) \ No newline at end of file + + with connection.cursor() as cursor: + cursor.execute(query) diff --git a/src/main/settings.py b/src/main/settings.py index 0b859cb..4fbba1f 100644 --- a/src/main/settings.py +++ b/src/main/settings.py @@ -10,45 +10,51 @@ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = os.getenv('SECRET_KEY') -DEBUG = os.getenv('DEBUG', 'false').lower() == 'true' +SECRET_KEY = os.getenv("SECRET_KEY") +DEBUG = os.getenv("DEBUG", "false").lower() == "true" -BOUWDOSSIER_PUBLIC_SCOPE = 'BD/P' # BouwDossiers_Public_Read. Access to anybody with e-mail link -BOUWDOSSIER_READ_SCOPE = 'BD/R' # BouwDossiers_Read. Access to civil servants of Amsterdam Municipality -BOUWDOSSIER_EXTENDED_SCOPE = 'BD/X' # BouwDossiers_eXtended. Access civil servants of Amsterdam Municipality with special rights. +BOUWDOSSIER_PUBLIC_SCOPE = ( + "BD/P" # BouwDossiers_Public_Read. Access to anybody with e-mail link +) +BOUWDOSSIER_READ_SCOPE = ( + "BD/R" # BouwDossiers_Read. Access to civil servants of Amsterdam Municipality +) +BOUWDOSSIER_EXTENDED_SCOPE = "BD/X" # BouwDossiers_eXtended. Access civil servants of Amsterdam Municipality with special rights. -WABO_BASE_URL = os.getenv('WABO_BASE_URL', 'https://bwt.hs3-saa-bwt.shcp04.archivingondemand.nl/') +WABO_BASE_URL = os.getenv( + "WABO_BASE_URL", "https://bwt.hs3-saa-bwt.shcp04.archivingondemand.nl/" +) -ALLOWED_HOSTS = ['*'] +ALLOWED_HOSTS = ["*"] -INTERNAL_IPS = ('127.0.0.1', '0.0.0.0') +INTERNAL_IPS = ("127.0.0.1", "0.0.0.0") INSTALLED_APPS = [ - 'django.contrib.contenttypes', - 'django.contrib.staticfiles', - 'django_filters', - 'django_extensions', - 'django.contrib.gis', - 'rest_framework', - 'rest_framework_gis', - 'drf_yasg', - 'bouwdossiers', - 'bag', - 'importer', - 'health', + "django.contrib.contenttypes", + "django.contrib.staticfiles", + "django_filters", + "django_extensions", + "django.contrib.gis", + "rest_framework", + "rest_framework_gis", + "drf_yasg", + "bouwdossiers", + "bag", + "importer", + "health", ] if DEBUG: - INSTALLED_APPS += ('debug_toolbar',) + INSTALLED_APPS += ("debug_toolbar",) MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'debug_toolbar.middleware.DebugToolbarMiddleware', - 'authorization_django.authorization_middleware', + "django.middleware.security.SecurityMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "debug_toolbar.middleware.DebugToolbarMiddleware", + "authorization_django.authorization_middleware", ] @@ -78,31 +84,31 @@ DATAPUNT_AUTHZ = { - 'ALWAYS_OK': False, - 'JWKS': PUB_JWKS, + "ALWAYS_OK": False, + "JWKS": PUB_JWKS, "JWKS_URL": os.getenv("KEYCLOAK_JWKS_URL"), - 'FORCED_ANONYMOUS_ROUTES': ['/status/health'] + "FORCED_ANONYMOUS_ROUTES": ["/status/health"], } -ROOT_URLCONF = 'main.urls' +ROOT_URLCONF = "main.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.messages.context_processors.messages", ], }, }, ] -WSGI_APPLICATION = 'main.wsgi.application' +WSGI_APPLICATION = "main.wsgi.application" DATABASE_HOST = os.getenv("DATABASE_HOST", "database") @@ -113,13 +119,13 @@ DATABASE_OPTIONS["sslmode"] = "require" DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': os.getenv('DATABASE_NAME', 'dev'), - 'USER': os.getenv('DATABASE_USER', 'dev'), - 'PASSWORD': DATABASE_PASSWORD, - 'HOST': DATABASE_HOST, - 'PORT': os.getenv('DATABASE_PORT', '5432'), + "default": { + "ENGINE": "django.contrib.gis.db.backends.postgis", + "NAME": os.getenv("DATABASE_NAME", "dev"), + "USER": os.getenv("DATABASE_USER", "dev"), + "PASSWORD": DATABASE_PASSWORD, + "HOST": DATABASE_HOST, + "PORT": os.getenv("DATABASE_PORT", "5432"), "OPTIONS": DATABASE_OPTIONS, } } @@ -127,140 +133,127 @@ # Internationalization # https://docs.djangoproject.com/en/1.9/topics/i18n/ -LANGUAGE_CODE = 'en-us' -TIME_ZONE = 'UTC' +LANGUAGE_CODE = "en-us" +TIME_ZONE = "UTC" USE_I18N = True USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.10/howto/static-files/ -STATIC_URL = '/iiif-metadata/static/' -STATIC_ROOT = os.path.abspath(os.path.join(BASE_DIR, '..', 'static')) +STATIC_URL = "/iiif-metadata/static/" +STATIC_ROOT = os.path.abspath(os.path.join(BASE_DIR, "..", "static")) # Django Logging settings LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'console': { - 'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s' - } + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "console": {"format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s"} }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'console' - }, - + "handlers": { + "console": {"class": "logging.StreamHandler", "formatter": "console"}, }, - 'root': { - 'handlers': ['console'], - 'level': os.getenv('LOG_LEVEL', 'INFO'), + "root": { + "handlers": ["console"], + "level": os.getenv("LOG_LEVEL", "INFO"), }, } OBJECTSTORE = dict( - VERSION='2.0', - AUTHURL='https://identity.stack.cloudvps.com/v2.0', - TENANT_NAME='BGE000081_BOUWDOSSIERS', - TENANT_ID='9d078258c1a547c09e0b5f88834554f1', - USER=os.getenv('OBJECTSTORE_USER', 'bouwdossiers'), - PASSWORD=os.getenv('BOUWDOSSIERS_OBJECTSTORE_PASSWORD'), - REGION_NAME='NL', + VERSION="2.0", + AUTHURL="https://identity.stack.cloudvps.com/v2.0", + TENANT_NAME="BGE000081_BOUWDOSSIERS", + TENANT_ID="9d078258c1a547c09e0b5f88834554f1", + USER=os.getenv("OBJECTSTORE_USER", "bouwdossiers"), + PASSWORD=os.getenv("BOUWDOSSIERS_OBJECTSTORE_PASSWORD"), + REGION_NAME="NL", ) BOUWDOSSIERS_OBJECTSTORE_CONTAINER = os.getenv( - 'BOUWDOSSIERS_OBJECTSTORE_CONTAINER', 'dossiers_acceptance' + "BOUWDOSSIERS_OBJECTSTORE_CONTAINER", "dossiers_acceptance" ) -STORAGE_ACCOUNT_URL = os.getenv('STORAGE_ACCOUNT_URL') +STORAGE_ACCOUNT_URL = os.getenv("STORAGE_ACCOUNT_URL") -PROJECT_DIR = os.path.abspath(os.path.join(BASE_DIR, '..')) -DATA_DIR = '/tmp/bouwdossiers' +PROJECT_DIR = os.path.abspath(os.path.join(BASE_DIR, "..")) +DATA_DIR = "/tmp/bouwdossiers" # SWAGGER -SWAG_PATH = 'acc.bouwdossiers.amsterdam.nl/iiif-metadata-server/docs' +SWAG_PATH = "acc.bouwdossiers.amsterdam.nl/iiif-metadata-server/docs" if DEBUG: - SWAG_PATH = '127.0.0.1:8000/iiif-metadata-server/docs' + SWAG_PATH = "127.0.0.1:8000/iiif-metadata-server/docs" SWAGGER_SETTINGS = { - 'exclude_namespaces': [], - 'api_version': '0.1', - 'api_path': '/', - - 'enabled_methods': [ - 'get', + "exclude_namespaces": [], + "api_version": "0.1", + "api_path": "/", + "enabled_methods": [ + "get", ], - - 'api_key': '', - 'USE_SESSION_AUTH': False, - 'VALIDATOR_URL': None, - - 'is_authenticated': False, - 'is_superuser': False, - - 'unauthenticated_user': 'django.contrib.auth.models.AnonymousUser', - 'permission_denied_handler': None, - 'resource_access_handler': None, - - 'protocol': 'https' if not DEBUG else '', - 'base_path': SWAG_PATH, - - 'info': { - 'contact': 'atlas.basisinformatie@amsterdam.nl', - 'description': 'This is the Bouwdossiers API server.', - 'license': 'Not known yet', - 'termsOfServiceUrl': 'https://data.amsterdam.nl/terms/', - 'title': 'Bouwdossiers', + "api_key": "", + "USE_SESSION_AUTH": False, + "VALIDATOR_URL": None, + "is_authenticated": False, + "is_superuser": False, + "unauthenticated_user": "django.contrib.auth.models.AnonymousUser", + "permission_denied_handler": None, + "resource_access_handler": None, + "protocol": "https" if not DEBUG else "", + "base_path": SWAG_PATH, + "info": { + "contact": "atlas.basisinformatie@amsterdam.nl", + "description": "This is the Bouwdossiers API server.", + "license": "Not known yet", + "termsOfServiceUrl": "https://data.amsterdam.nl/terms/", + "title": "Bouwdossiers", }, - - 'doc_expansion': 'list' + "doc_expansion": "list", } -HEALTH_MODEL = 'bouwdossiers.Bouwdossier' +HEALTH_MODEL = "bouwdossiers.Bouwdossier" -DUMP_DIR = 'mks-dump' +DUMP_DIR = "mks-dump" -TESTING = len(sys.argv) > 1 and sys.argv[1] == 'test' +TESTING = len(sys.argv) > 1 and sys.argv[1] == "test" REST_FRAMEWORK = dict( PAGE_SIZE=100, MAX_PAGINATE_BY=100, - DEFAULT_PAGINATION_CLASS='rest_framework.pagination.PageNumberPagination', - + DEFAULT_PAGINATION_CLASS="rest_framework.pagination.PageNumberPagination", UNAUTHENTICATED_USER={}, UNAUTHENTICATED_TOKEN={}, - DEFAULT_AUTHENTICATION_CLASSES=( # 'rest_framework.authentication.BasicAuthentication', # 'rest_framework.authentication.SessionAuthentication', ), - DEFAULT_RENDERER_CLASSES=( - 'rest_framework.renderers.JSONRenderer', - 'rest_framework.renderers.BrowsableAPIRenderer' + "rest_framework.renderers.JSONRenderer", + "rest_framework.renderers.BrowsableAPIRenderer", ), DEFAULT_FILTER_BACKENDS=( # 'rest_framework.filters.DjangoFilterBackend', - 'django_filters.rest_framework.DjangoFilterBackend', + "django_filters.rest_framework.DjangoFilterBackend", # 'rest_framework.filters.OrderingFilter', - ), COERCE_DECIMAL_TO_STRING=True, ) -IIIF_BASE_URL = os.getenv('IIIF_BASE_URL', 'https://bouwdossiers.amsterdam.nl/iiif/2/') +IIIF_BASE_URL = os.getenv("IIIF_BASE_URL", "https://bouwdossiers.amsterdam.nl/iiif/2/") -DATADIENSTEN_API_BASE_URL = os.getenv("DATADIENSTEN_API_BASE_URL", "https://api.data.amsterdam.nl") -BAG_CSV_BASE_URL = os.getenv("BAG_CSV_BASE_URL", "https://amsterdamdadipub.blob.core.windows.net/bulk-data/csv") +DATADIENSTEN_API_BASE_URL = os.getenv( + "DATADIENSTEN_API_BASE_URL", "https://api.data.amsterdam.nl" +) +BAG_CSV_BASE_URL = os.getenv( + "BAG_CSV_BASE_URL", "https://amsterdamdadipub.blob.core.windows.net/bulk-data/csv" +) -AZURITE_STORAGE_CONNECTION_STRING = os.getenv('AZURITE_STORAGE_CONNECTION_STRING') +AZURITE_STORAGE_CONNECTION_STRING = os.getenv("AZURITE_STORAGE_CONNECTION_STRING") -AZURE_CONTAINER_NAME_BAG = 'bag' -AZURE_CONTAINER_NAME_DOSSIERS = 'dossiers' +AZURE_CONTAINER_NAME_BAG = "bag" +AZURE_CONTAINER_NAME_DOSSIERS = "dossiers" -MIN_BOUWDOSSIERS_COUNT = os.getenv('MIN_BOUWDOSSIERS_COUNT', 10000) +MIN_BOUWDOSSIERS_COUNT = os.getenv("MIN_BOUWDOSSIERS_COUNT", 10000) diff --git a/src/main/urls.py b/src/main/urls.py index b7c28d3..f79dffe 100644 --- a/src/main/urls.py +++ b/src/main/urls.py @@ -8,18 +8,18 @@ from bouwdossiers import urls as bouwdossiers_urls grouped_url_patterns = { - 'base_patterns': [ - re_path(r'^status/', include('health.urls')), + "base_patterns": [ + re_path(r"^status/", include("health.urls")), ], - 'bouwdossiers_patterns': [ - re_path(r'^iiif-metadata/', include(bouwdossiers_urls.urls)), + "bouwdossiers_patterns": [ + re_path(r"^iiif-metadata/", include(bouwdossiers_urls.urls)), ], } schema_view = get_schema_view( openapi.Info( title="Bouwdossiers API", - default_version='v1', + default_version="v1", description="Bouwdossiers API", terms_of_service="https://data.amsterdam.nl/", contact=openapi.Contact(email="datapunt@amsterdam.nl"), @@ -31,19 +31,19 @@ urlpatterns = [ re_path( - r'^iiif-metadata/docs/swagger(?P\.json|\.yaml)$', + r"^iiif-metadata/docs/swagger(?P\.json|\.yaml)$", schema_view.without_ui(cache_timeout=None), - name='schema-json' + name="schema-json", ), re_path( - r'^iiif-metadata/docs/swagger/$', - schema_view.with_ui('swagger', cache_timeout=None), - name='schema-swagger-ui' + r"^iiif-metadata/docs/swagger/$", + schema_view.with_ui("swagger", cache_timeout=None), + name="schema-swagger-ui", ), re_path( - r'^iiif-metadata/docs/redoc/$', - schema_view.with_ui('redoc', cache_timeout=None), - name='schema-redoc' + r"^iiif-metadata/docs/redoc/$", + schema_view.with_ui("redoc", cache_timeout=None), + name="schema-redoc", ), ] @@ -54,6 +54,8 @@ if settings.DEBUG: import debug_toolbar - urlpatterns.extend([ - re_path(r'^__debug__/', include(debug_toolbar.urls)), - ]) + urlpatterns.extend( + [ + re_path(r"^__debug__/", include(debug_toolbar.urls)), + ] + ) diff --git a/src/manage.py b/src/manage.py index ad5d3e7..953a11f 100755 --- a/src/manage.py +++ b/src/manage.py @@ -11,7 +11,7 @@ # issue is really that Django is missing to avoid masking other # exceptions on Python 2. try: - import django + pass except ImportError: raise ImportError( "Couldn't import Django. Are you sure it's installed and " diff --git a/src/objectstore_utils.py b/src/objectstore_utils.py index fa243a7..4683ad7 100644 --- a/src/objectstore_utils.py +++ b/src/objectstore_utils.py @@ -9,12 +9,12 @@ log = logging.getLogger(__name__) -DIR_CONTENT_TYPE = 'application/directory' +DIR_CONTENT_TYPE = "application/directory" DocumentList = List[Tuple[str, str]] def get_objectstore_connection(): - assert os.getenv('BOUWDOSSIERS_OBJECTSTORE_PASSWORD') + assert os.getenv("BOUWDOSSIERS_OBJECTSTORE_PASSWORD") connection = objectstore.get_connection(settings.OBJECTSTORE) return connection @@ -25,19 +25,21 @@ def get_all_files(): os.makedirs(settings.DATA_DIR, exist_ok=True) documents_meta = get_full_container_list(connection, container_name) for meta in documents_meta: - if meta.get('content_type') != DIR_CONTENT_TYPE: - name = meta.get('name') - last_modified = meta.get('last_modified') + if meta.get("content_type") != DIR_CONTENT_TYPE: + name = meta.get("name") + last_modified = meta.get("last_modified") dt = time.strptime(last_modified, "%Y-%m-%dT%H:%M:%S.%f") epoch_dt = timegm(dt) output_path = os.path.join(settings.DATA_DIR, name) dirname = os.path.dirname(output_path) os.makedirs(dirname, exist_ok=True) - if os.path.isfile(output_path) and epoch_dt == os.path.getmtime(output_path): + if os.path.isfile(output_path) and epoch_dt == os.path.getmtime( + output_path + ): log.info(f"Using cached file: {output_path}") else: log.info(f"Fetching file: {output_path}") new_data = connection.get_object(container_name, name)[1] - with open(output_path, 'wb') as file: + with open(output_path, "wb") as file: file.write(new_data) os.utime(output_path, (epoch_dt, epoch_dt))