Skip to content

Commit

Permalink
Fix all remaining linting problems
Browse files Browse the repository at this point in the history
  • Loading branch information
TvanWalen committed Mar 18, 2024
1 parent beafe4c commit adf8133
Show file tree
Hide file tree
Showing 40 changed files with 2,267 additions and 1,175 deletions.
28 changes: 16 additions & 12 deletions src/bag/bag_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

class BagLoader:
tmp_folder = "/tmp"

tables = {
"bag_ligplaats": "bag_ligplaatsen.csv.zip",
"bag_openbareruimte": "bag_openbareruimtes.csv.zip",
Expand Down Expand Up @@ -77,19 +77,21 @@ def unpack_zip(self, *, table_name: str, endpoint: str):
def preprocess_csv_rows(self, reader: csv.DictReader, writer: csv.DictWriter):
seen = {}
for row in reader:
key = row['Identificatie']
if (key not in seen):
key = row["Identificatie"]
if key not in seen:
seen[key] = row
elif row['Volgnummer'] > seen[key]['Volgnummer']:
elif row["Volgnummer"] > seen[key]["Volgnummer"]:
seen[key] = row

filtered_rows = list(seen.values())
writer.writeheader()
for row in filtered_rows:
writer.writerow(row)

def preprocess_csv_files(self, csv_path_in, csv_path_out):
with open(csv_path_in, 'r') as infile, open(csv_path_out, 'w', newline='') as outfile:
with open(csv_path_in, "r") as infile, open(
csv_path_out, "w", newline=""
) as outfile:
reader = csv.DictReader(infile)
writer = csv.DictWriter(outfile, fieldnames=reader.fieldnames)
self.preprocess_csv_rows(reader, writer)
Expand All @@ -98,9 +100,7 @@ def preprocess_csv_files(self, csv_path_in, csv_path_out):
def download_zip(self, *, table_name: str, endpoint: str, path_base: str):
base_url = settings.BAG_CSV_BASE_URL
url = f"{base_url}/{endpoint}"
logger.info(
f"Downloading {table_name} from {url}"
)
logger.info(f"Downloading {table_name} from {url}")
response = requests.get(
url,
timeout=300,
Expand All @@ -125,9 +125,13 @@ def import_tables_from_endpoint(self):
for table, endpoint in self.tables.items():
logger.info(f"Loading table {table}")
path_base = f"{self.tmp_folder}"
path_zip = self.download_zip(table_name=table, endpoint=endpoint, path_base=path_base)
path_zip = self.download_zip(
table_name=table, endpoint=endpoint, path_base=path_base
)
path_csv = self.unpack_zip(table_name=table, endpoint=path_zip)
path_csv_processed = f"{path_csv}-processed.csv"
self.preprocess_csv_files(csv_path_in=path_csv, csv_path_out=path_csv_processed)
self.preprocess_csv_files(
csv_path_in=path_csv, csv_path_out=path_csv_processed
)
csv_files[table] = path_csv_processed
self.load_tables_from_csv(csv_files)
self.load_tables_from_csv(csv_files)
822 changes: 657 additions & 165 deletions src/bag/migrations/0001_initial.py

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -1,56 +1,56 @@
# Generated by Django 4.1.13 on 2024-02-19 12:16

from django.db import migrations, models
import django.db.models.manager
from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('bag', '0001_initial'),
("bag", "0001_initial"),
]

operations = [
migrations.AlterModelManagers(
name='pand',
name="pand",
managers=[
('transformed_geo_objects', django.db.models.manager.Manager()),
("transformed_geo_objects", django.db.models.manager.Manager()),
],
),
migrations.AlterModelManagers(
name='verblijfsobject',
name="verblijfsobject",
managers=[
('transformed_geo_objects', django.db.models.manager.Manager()),
("transformed_geo_objects", django.db.models.manager.Manager()),
],
),
migrations.AddField(
model_name='ligplaats',
name='plusvolgnummer',
model_name="ligplaats",
name="plusvolgnummer",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name='nummeraanduiding',
name='plusvolgnummer',
model_name="nummeraanduiding",
name="plusvolgnummer",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name='openbareruimte',
name='plusvolgnummer',
model_name="openbareruimte",
name="plusvolgnummer",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name='pand',
name='plusvolgnummer',
model_name="pand",
name="plusvolgnummer",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name='standplaats',
name='plusvolgnummer',
model_name="standplaats",
name="plusvolgnummer",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name='verblijfsobject',
name='plusvolgnummer',
model_name="verblijfsobject",
name="plusvolgnummer",
field=models.IntegerField(null=True),
),
]
10 changes: 6 additions & 4 deletions src/bag/migrations/0003_alter_pand_bouwblok.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@
class Migration(migrations.Migration):

dependencies = [
('bag', '0002_alter_pand_managers_alter_verblijfsobject_managers_and_more'),
("bag", "0002_alter_pand_managers_alter_verblijfsobject_managers_and_more"),
]

operations = [
migrations.AlterField(
model_name='pand',
name='bouwblok',
field=models.CharField(db_column='ligtinbouwblokid', db_index=True, max_length=16, null=True),
model_name="pand",
name="bouwblok",
field=models.CharField(
db_column="ligtinbouwblokid", db_index=True, max_length=16, null=True
),
),
]
8 changes: 5 additions & 3 deletions src/bag/models.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from django.conf import settings
from django.contrib.gis.db import models as gis_models
from django.contrib.gis.db.models.functions import Centroid, Transform
from django.contrib.postgres.fields import ArrayField
from django.db import models


Expand Down Expand Up @@ -40,6 +39,7 @@ class GeoModel(models.Model):
class Meta:
abstract = True


class Ligplaats(GeoModel, BagObject):
id = models.CharField(max_length=16, primary_key=True, db_column="identificatie")
document_mutatie = models.DateField(db_column="documentdatum")
Expand Down Expand Up @@ -179,7 +179,9 @@ class Pand(GeoModel):
hoogste_bouwlaag = models.IntegerField(null=True, db_column="hoogstebouwlaag")
geometrie = gis_models.PolygonField(srid=28992)
pandnaam = models.CharField(max_length=80, null=True, db_column="naam")
bouwblok = models.CharField(max_length=16, db_column="ligtinbouwblokid", db_index=True, null=True)
bouwblok = models.CharField(
max_length=16, db_column="ligtinbouwblokid", db_index=True, null=True
)

bouwlagen = models.IntegerField(null=True, db_column="aantalbouwlagen")
ligging = models.CharField(
Expand Down Expand Up @@ -381,4 +383,4 @@ def __str__(self):
return f"{self.openbare_ruimte} {self.huisnummer}{huisletter}{huisnr_toevoeging} ({self.type_adres})"

class Meta:
db_table = "bag_nummeraanduiding"
db_table = "bag_nummeraanduiding"
21 changes: 12 additions & 9 deletions src/bag/tests/test_bag_loader.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
import logging

from django.conf import settings
from django.test import TestCase

from bag.bag_loader import BagLoader

log = logging.getLogger(__name__)


class MockWriter:
content = []

def writeheader(self):
pass

def writerow(self, row):
self.content.append(row)

Expand All @@ -22,24 +22,27 @@ def __str__(self):

class APITest(TestCase):
@classmethod

def setUp(self):
pass

def test_preprocess_csv(self):
csv_data = [
{"Identificatie": "0457100000068588", "Volgnummer": "3"},
{"Identificatie": "0457100000068549", "Volgnummer": "3"},
{"Identificatie": "0457100000068588", "Volgnummer": "1"}
{"Identificatie": "0457100000068588", "Volgnummer": "1"},
]

reader = iter(csv_data)
writer = MockWriter()
bag = BagLoader()
bag.preprocess_csv_rows(reader, writer)

self.assertNotIn({'Identificatie': '0457100000068588', 'Volgnummer': '1'}, writer.content)
self.assertIn({'Identificatie': '0457100000068588', 'Volgnummer': '3'}, writer.content)
self.assertIn({'Identificatie': '0457100000068549', 'Volgnummer': '3'}, writer.content)


self.assertNotIn(
{"Identificatie": "0457100000068588", "Volgnummer": "1"}, writer.content
)
self.assertIn(
{"Identificatie": "0457100000068588", "Volgnummer": "3"}, writer.content
)
self.assertIn(
{"Identificatie": "0457100000068549", "Volgnummer": "3"}, writer.content
)
25 changes: 15 additions & 10 deletions src/bag/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,20 @@
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist

from bag.constants import (BAG_TYPE_LIGPLAATS, BAG_TYPE_NUMMERAANDUIDING,
BAG_TYPE_OPENBARE_RUIMTE, BAG_TYPE_PAND,
BAG_TYPE_STANDPLAATS, BAG_TYPE_VERBLIJFSOBJECT)
from bag.exceptions import (BagIdException, IncorrectBagIdLengthException,
IncorrectGemeenteCodeException,
IncorrectObjectTypeException)
from bag.constants import (
BAG_TYPE_LIGPLAATS,
BAG_TYPE_NUMMERAANDUIDING,
BAG_TYPE_OPENBARE_RUIMTE,
BAG_TYPE_PAND,
BAG_TYPE_STANDPLAATS,
BAG_TYPE_VERBLIJFSOBJECT,
)
from bag.exceptions import (
BagIdException,
IncorrectBagIdLengthException,
IncorrectGemeenteCodeException,
IncorrectObjectTypeException,
)
from bag.models import Ligplaats, Standplaats, Verblijfsobject

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -83,10 +91,7 @@ def get_object_from_id(id):
)

try:
bag_object = (
bag_model.objects.filter(id=id)
.get()
)
bag_object = bag_model.objects.filter(id=id).get()
except ObjectDoesNotExist:
raise bag_model.DoesNotExist(
f"{bag_model.__name__} met landelijk id {id} bestaat niet."
Expand Down
2 changes: 1 addition & 1 deletion src/bouwdossiers/apps.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@


class BouwDossiersConfig(AppConfig):
name = 'bouwdossiers'
name = "bouwdossiers"
Loading

0 comments on commit adf8133

Please sign in to comment.