From 50d89a7e274f095c64ba262a4516f93cc92cd3ec Mon Sep 17 00:00:00 2001 From: Rob Gries Date: Mon, 11 Nov 2024 14:04:36 -0500 Subject: [PATCH] Run pre-commit --- dear_petition/petition/admin.py | 2 + dear_petition/petition/api/serializers.py | 21 +- .../petition/api/tests/test_batch.py | 29 +- dear_petition/petition/api/viewsets.py | 47 +- dear_petition/petition/conftest.py | 14 +- dear_petition/petition/etl/load.py | 10 +- dear_petition/petition/etl/refresh.py | 4 +- .../petition/etl/tests/test_transform.py | 21 +- dear_petition/petition/etl/transform.py | 48 +- .../commands/convert_agency_table.py | 26 +- ...act_county_remove_contact_user_and_more.py | 9 +- ..._alter_ciprsrecord_arrest_date_and_more.py | 24 +- ...lter_offense_plea_alter_offense_verdict.py | 8 +- dear_petition/petition/models.py | 78 ++-- dear_petition/petition/resources.py | 429 +++++++++++------- dear_petition/petition/tests/factories.py | 35 +- dear_petition/petition/tests/test_models.py | 6 +- .../petition/tests/test_resources.py | 20 +- dear_petition/petition/tests/test_utils.py | 8 +- .../petition/types/adult_felonies.py | 1 - dear_petition/petition/types/dismissed.py | 7 +- .../types/tests/test_adult_felonies.py | 57 ++- .../types/tests/test_adult_misdemeanors.py | 57 ++- .../petition/types/tests/test_dismissed.py | 17 +- .../petition/types/tests/test_not_guilty.py | 33 +- .../types/tests/test_underaged_convictions.py | 19 +- .../petition/types/underaged_convictions.py | 4 +- dear_petition/petition/utils.py | 2 +- dear_petition/portal/etl/extract.py | 2 +- dear_petition/portal/etl/models.py | 6 +- dear_petition/portal/etl/parsers/case_info.py | 16 +- .../portal/etl/parsers/party_info.py | 8 +- .../portal/tests/parsers/test_case_info.py | 3 +- .../portal/tests/transform/test_record.py | 2 +- 34 files changed, 687 insertions(+), 386 deletions(-) diff --git a/dear_petition/petition/admin.py b/dear_petition/petition/admin.py index 26c66948..ccdacab4 100644 --- a/dear_petition/petition/admin.py +++ b/dear_petition/petition/admin.py @@ -159,6 +159,7 @@ class ContactAdmin(admin.ModelAdmin): list_filter = ("category",) ordering = ("category", "name") + @admin.register(models.Agency) class AgencyAdmin(ImportExportModelAdmin): resource_classes = [resources.AgencyResource] @@ -167,6 +168,7 @@ class AgencyAdmin(ImportExportModelAdmin): list_filter = ("category",) ordering = ("category", "name") + @admin.register(models.Client) class ClientAdmin(admin.ModelAdmin): list_display = ("pk", "name", "category", "address1") diff --git a/dear_petition/petition/api/serializers.py b/dear_petition/petition/api/serializers.py index 4b9da7c4..3c522ae4 100644 --- a/dear_petition/petition/api/serializers.py +++ b/dear_petition/petition/api/serializers.py @@ -139,6 +139,7 @@ class Meta: "zipcode", ] + class AgencySerializer(ContactSerializer): class Meta: model = Agency @@ -320,9 +321,13 @@ def validate_batch(self, value): class BatchSerializer(serializers.ModelSerializer): records = CIPRSRecordSerializer(many=True, read_only=True) petitions = PetitionSerializer(many=True, read_only=True) - generate_letter_errors = ValidationField(method_name='get_generate_errors_data', serializer=GenerateDocumentSerializer) - generate_summary_errors = ValidationField(method_name='get_generate_errors_data', serializer=GenerateDocumentSerializer) - + generate_letter_errors = ValidationField( + method_name="get_generate_errors_data", serializer=GenerateDocumentSerializer + ) + generate_summary_errors = ValidationField( + method_name="get_generate_errors_data", serializer=GenerateDocumentSerializer + ) + def get_generate_errors_data(self, obj): return {"batch": obj.pk} @@ -353,7 +358,7 @@ class BatchDetailSerializer(serializers.ModelSerializer): ) attorney = ContactSerializer(read_only=True) client_id = serializers.PrimaryKeyRelatedField( - source='client', queryset=Client.objects.all(), write_only=True, required=False + source="client", queryset=Client.objects.all(), write_only=True, required=False ) client = ClientSerializer(read_only=True) client_errors = serializers.SerializerMethodField() @@ -383,7 +388,7 @@ class Meta: "client_id", "generate_letter_errors", "generate_summary_errors", - "client_errors" + "client_errors", ] read_only_fields = ["user", "pk", "date_uploaded", "records", "petitions"] @@ -393,13 +398,15 @@ def get_petitions(self, instance): batch=instance.pk, ).order_by("county", "jurisdiction") return ParentPetitionSerializer(parent_petitions, many=True).data - + def get_client_errors(self, instance): errors = [] if not instance.client: return errors if not instance.client.dob: - errors.append("Date of birth missing. The petition generator will try its best to identify a date of birth from the records at time of petition creation.") + errors.append( + "Date of birth missing. The petition generator will try its best to identify a date of birth from the records at time of petition creation." + ) return errors diff --git a/dear_petition/petition/api/tests/test_batch.py b/dear_petition/petition/api/tests/test_batch.py index cc87ac71..26c9004d 100644 --- a/dear_petition/petition/api/tests/test_batch.py +++ b/dear_petition/petition/api/tests/test_batch.py @@ -7,7 +7,13 @@ from rest_framework import status -from dear_petition.petition.tests.factories import BatchFactory, CIPRSRecordFactory, OffenseFactory, ClientFactory, OffenseRecordFactory +from dear_petition.petition.tests.factories import ( + BatchFactory, + CIPRSRecordFactory, + OffenseFactory, + ClientFactory, + OffenseRecordFactory, +) pytestmark = pytest.mark.django_db @@ -34,37 +40,36 @@ def test_batch_post_multiple_files(api_client, fake_pdf, fake_pdf2, mock_import) assert mock_import.assert_called_once assert "id" in response.data + def test_adjust_for_new_client_dob(): """Test that when a DOB is added or updated on the client, the batch is updated accordingly.""" batch = BatchFactory() - record = CIPRSRecordFactory(batch=batch, offense_date=datetime.date(2000,1,1), dob=None) - offense = OffenseFactory(ciprs_record=record, disposition_method="OTHER") # Conviction charge - offense_record = OffenseRecordFactory( - action="CONVICTED", offense=offense - ) + record = CIPRSRecordFactory(batch=batch, offense_date=datetime.date(2000, 1, 1), dob=None) + offense = OffenseFactory(ciprs_record=record, disposition_method="OTHER") # Conviction charge + offense_record = OffenseRecordFactory(action="CONVICTED", offense=offense) - client = ClientFactory(dob=timezone.now().date()) # Create a youngster + client = ClientFactory(dob=timezone.now().date()) # Create a youngster batch.client = client batch.save() batch.adjust_for_new_client_dob() assert offense_record in batch.underaged_conviction_records() - batch.client.dob = datetime.date(1800,1,1) # Update the youngster to be an elder + batch.client.dob = datetime.date(1800, 1, 1) # Update the youngster to be an elder batch.client.save() - batch.refresh_from_db() # adjust_for_new_client_dob should get automatically called in Client save + batch.refresh_from_db() # adjust_for_new_client_dob should get automatically called in Client save assert offense_record not in batch.underaged_conviction_records() - client = ClientFactory(dob=datetime.date(1800,1,1)) # Create an elder + client = ClientFactory(dob=datetime.date(1800, 1, 1)) # Create an elder batch.client = client batch.save() batch.adjust_for_new_client_dob() assert offense_record not in batch.underaged_conviction_records() - batch.client.dob = timezone.now().date() # Update the elder to be a youngster + batch.client.dob = timezone.now().date() # Update the elder to be a youngster batch.client.save() - batch.refresh_from_db() # adjust_for_new_client_dob should get automatically called in Client save + batch.refresh_from_db() # adjust_for_new_client_dob should get automatically called in Client save assert offense_record in batch.underaged_conviction_records() # try un-setting client to test default behavior when no DOB known diff --git a/dear_petition/petition/api/viewsets.py b/dear_petition/petition/api/viewsets.py index 272b6507..16dae2da 100644 --- a/dear_petition/petition/api/viewsets.py +++ b/dear_petition/petition/api/viewsets.py @@ -224,18 +224,26 @@ def preview_import_agencies(self, request): else row_result.instance.address1 ) row_diff = { - 'name': row_result.instance.name, - 'address': address, - 'city': row_result.instance.city, - 'state': row_result.instance.state, - 'zipcode': row_result.instance.zipcode, - 'county': row_result.instance.county, - 'is_sheriff': row_result.instance.is_sheriff, + "name": row_result.instance.name, + "address": address, + "city": row_result.instance.city, + "state": row_result.instance.state, + "zipcode": row_result.instance.zipcode, + "county": row_result.instance.county, + "is_sheriff": row_result.instance.is_sheriff, } if row_result.import_type == RowResult.IMPORT_TYPE_SKIP: continue elif row_result.import_type == RowResult.IMPORT_TYPE_NEW: - row_diff['new_fields'] = ['name', 'address', 'city', 'state', 'zipcode', 'county', 'is_sheriff'] + row_diff["new_fields"] = [ + "name", + "address", + "city", + "state", + "zipcode", + "county", + "is_sheriff", + ] else: original_address = ( f"{row_result.original.address1}, {row_result.original.address2}" @@ -247,7 +255,7 @@ def preview_import_agencies(self, request): if original_address != address: row_diff["new_fields"].append("address") - for field in ['name', 'city', 'state', 'zipcode', 'county', 'is_sheriff']: + for field in ["name", "city", "state", "zipcode", "county", "is_sheriff"]: if getattr(row_result.original, field) != getattr(row_result.instance, field): row_diff["new_fields"].append(field) @@ -370,7 +378,7 @@ def generate_summary(self, request, pk): ] = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" resp["Content-Disposition"] = 'attachment; filename="Records Summary.docx"' return resp - + @action( detail=True, methods=[ @@ -398,8 +406,8 @@ def generate_spreadsheet(self, request, pk): ], ) def combine_batches(self, request): - batch_ids = request.data['batchIds'] - label = request.data['label'] + batch_ids = request.data["batchIds"] + label = request.data["label"] user_id = request.user.id if not batch_ids: @@ -408,12 +416,13 @@ def combine_batches(self, request): ) if not label: return Response( - "A new label needs to be included for the client.", status=status.HTTP_400_BAD_REQUEST + "A new label needs to be included for the client.", + status=status.HTTP_400_BAD_REQUEST, ) new_batch = combine_batches(batch_ids, label, user_id) return Response(self.get_serializer(new_batch).data) - + @action( detail=True, methods=[ @@ -421,20 +430,18 @@ def combine_batches(self, request): ], ) def assign_client_to_batch(self, request, pk): - client_id = request.data['client_id'] + client_id = request.data["client_id"] try: client = pm.Client.objects.get(pk=client_id) except pm.Client.DoesNotExist: - return Response( - "Unknown client.", status=status.HTTP_400_BAD_REQUEST - ) + return Response("Unknown client.", status=status.HTTP_400_BAD_REQUEST) batch = self.get_object() batch.client = client batch.save() batch.adjust_for_new_client_dob() return Response({"batch_id": batch.pk}) - + @action( detail=False, methods=[ @@ -448,7 +455,7 @@ def import_spreadsheet(self, request): for file in files: label = os.path.basename(file.name) batch = pm.Batch.objects.create(label=label, user=user) - batch.files.create(file=file) + batch.files.create(file=file) file.seek(0) workbook = load_workbook(filename=file) resource.import_data(workbook, batch) diff --git a/dear_petition/petition/conftest.py b/dear_petition/petition/conftest.py index af4088ee..71b76c80 100644 --- a/dear_petition/petition/conftest.py +++ b/dear_petition/petition/conftest.py @@ -161,13 +161,13 @@ def contact3(): @pytest.fixture def client(): return ClientFactory( - name='Test Name', - address1='123 Test Ct', - address2='Apt A', - city='Durham', - state='NC', - zipcode='27701', - dob = timezone.now().date() # I wasn't born yesterday + name="Test Name", + address1="123 Test Ct", + address2="Apt A", + city="Durham", + state="NC", + zipcode="27701", + dob=timezone.now().date(), # I wasn't born yesterday ) diff --git a/dear_petition/petition/etl/load.py b/dear_petition/petition/etl/load.py index 2a37322b..b7a914c9 100644 --- a/dear_petition/petition/etl/load.py +++ b/dear_petition/petition/etl/load.py @@ -79,17 +79,17 @@ def create_petitions_from_records(batch, form_type): jurisdiction=petition_type["jurisdiction"], county=petition_type["county"], ) - sheriff_agency = pm.Agency.get_sherriff_office_by_county( - petition_type["county"] - ) - + sheriff_agency = pm.Agency.get_sherriff_office_by_county(petition_type["county"]) + # petition arresting agencies - add agencies parsed from portal and assign sheriff's office for county if sheriff_agency is not None: logger.info( f"Detected {sheriff_agency.name} as {petition_type['county']} county's sherrif's office. Adding as default agency." ) petition.agencies.add(sheriff_agency) - offense_record_agencies = pm.Agency.objects.filter(pk__in=record_set.exclude(agency__isnull=True).values_list('agency')) + offense_record_agencies = pm.Agency.objects.filter( + pk__in=record_set.exclude(agency__isnull=True).values_list("agency") + ) petition.agencies.add(*offense_record_agencies) link_offense_records(petition) diff --git a/dear_petition/petition/etl/refresh.py b/dear_petition/petition/etl/refresh.py index dfb4af70..a091c8ed 100644 --- a/dear_petition/petition/etl/refresh.py +++ b/dear_petition/petition/etl/refresh.py @@ -74,7 +74,9 @@ def refresh_offenses(record): agency = Agency.objects.none() agency_name = data_offense_record.get("Agency", "") if agency_name: - agency = Agency.agencies_with_clean_name.filter(clean_name__icontains=agency_name) + agency = Agency.agencies_with_clean_name.filter( + clean_name__icontains=agency_name + ) if agency.exists(): logger.info(f"Matched agency '{agency.first().name}' to offense") diff --git a/dear_petition/petition/etl/tests/test_transform.py b/dear_petition/petition/etl/tests/test_transform.py index 1637d7b8..2109268f 100644 --- a/dear_petition/petition/etl/tests/test_transform.py +++ b/dear_petition/petition/etl/tests/test_transform.py @@ -42,12 +42,8 @@ def test_recalculate_petitions(petition): assert not petition.has_attachments() - - def test_combine_batches(batch, batch_file, fake_pdf): - record = CIPRSRecordFactory( - batch=batch, jurisdiction=constants.DISTRICT_COURT, county="DURHAM" - ) + record = CIPRSRecordFactory(batch=batch, jurisdiction=constants.DISTRICT_COURT, county="DURHAM") record.refresh_record_from_data() second_batch = BatchFactory() @@ -90,17 +86,21 @@ def test_combine_batches(batch, batch_file, fake_pdf): ], "Disposed On": "2018-02-01", "Disposition Method": "DISPOSED BY JUDGE", - } + }, ], "Superior Court Offense Information": [], } second_batch_file = BatchFileFactory(batch=second_batch, file=fake_pdf) second_record = CIPRSRecordFactory( - batch=second_batch, data = second_record_data, batch_file=second_batch_file, jurisdiction=constants.DISTRICT_COURT, county="DURHAM" + batch=second_batch, + data=second_record_data, + batch_file=second_batch_file, + jurisdiction=constants.DISTRICT_COURT, + county="DURHAM", ) second_record.refresh_record_from_data() - + assert batch.records.count() == 1 assert pm.Offense.objects.filter(ciprs_record__batch__id=batch.id).count() == 1 assert pm.Offense.objects.filter(ciprs_record__batch__id=second_batch.id).count() == 2 @@ -112,8 +112,9 @@ def test_combine_batches(batch, batch_file, fake_pdf): assert new_batch.records.count() == 2 assert pm.Offense.objects.filter(ciprs_record__batch__id=new_batch.id).count() == 3 - assert pm.OffenseRecord.objects.filter(offense__ciprs_record__batch__id=new_batch.id).count() == 4 + assert ( + pm.OffenseRecord.objects.filter(offense__ciprs_record__batch__id=new_batch.id).count() == 4 + ) assert new_batch.files.count() == 2 assert new_batch.label == new_label assert new_batch.user_id == user_id - \ No newline at end of file diff --git a/dear_petition/petition/etl/transform.py b/dear_petition/petition/etl/transform.py index 2e419903..cbe01b22 100644 --- a/dear_petition/petition/etl/transform.py +++ b/dear_petition/petition/etl/transform.py @@ -24,7 +24,6 @@ def recalculate_petitions(petition_id, offense_record_ids): def combine_batches(batch_ids: List[int], label: str, user_id: int): - client_resource = pr.ClientResource() record_resource = pr.RecordResource() offense_resource = pr.OffenseResource() @@ -38,14 +37,13 @@ def combine_batches(batch_ids: List[int], label: str, user_id: int): saved_batch_files = {} for batch in batches: for record in batch.records.iterator(): - if record.file_no in saved_file_nos: - continue # Duplicate record of one already imported + continue # Duplicate record of one already imported if record.batch_file: file = record.batch_file.file.file file_name = os.path.basename(file.name) - + if saved_batch_files.get(file_name): new_batch_file = saved_batch_files[file_name] else: @@ -53,12 +51,19 @@ def combine_batches(batch_ids: List[int], label: str, user_id: int): new_batch_file = new_batch.files.create(file=file) saved_batch_files[file_name] = new_batch_file else: - new_batch_file=None + new_batch_file = None - export_record_dataset = record_resource.export(queryset=pm.CIPRSRecord.objects.filter(id=record.id)) + export_record_dataset = record_resource.export( + queryset=pm.CIPRSRecord.objects.filter(id=record.id) + ) import_record_dataset = tablib.Dataset() - import_record_dataset.headers = [col.column_name for col in record_resource.get_import_fields()] - import_record_dataset.append([new_batch.id] + [value if value else None for value in export_record_dataset[0]]) + import_record_dataset.headers = [ + col.column_name for col in record_resource.get_import_fields() + ] + import_record_dataset.append( + [new_batch.id] + + [value if value else None for value in export_record_dataset[0]] + ) record_resource.import_data(import_record_dataset) record_id = record_resource.saved_instance_ids[-1] @@ -68,23 +73,36 @@ def combine_batches(batch_ids: List[int], label: str, user_id: int): new_record.save() for offense in record.offenses.all(): - export_offense_dataset = offense_resource.export(queryset=pm.Offense.objects.filter(id=offense.id)) + export_offense_dataset = offense_resource.export( + queryset=pm.Offense.objects.filter(id=offense.id) + ) import_offense_dataset = tablib.Dataset() - import_offense_dataset.headers = [col.column_name for col in offense_resource.get_import_fields()] - import_offense_dataset.append([record_id] + [value if value else None for value in export_offense_dataset[0]]) + import_offense_dataset.headers = [ + col.column_name for col in offense_resource.get_import_fields() + ] + import_offense_dataset.append( + [record_id] + + [value if value else None for value in export_offense_dataset[0]] + ) offense_resource.import_data(import_offense_dataset) offense_id = offense_resource.saved_instance_ids[-1] - export_offense_record_dataset = offense_record_resource.export(queryset=pm.OffenseRecord.objects.filter(offense_id=offense.id)) + export_offense_record_dataset = offense_record_resource.export( + queryset=pm.OffenseRecord.objects.filter(offense_id=offense.id) + ) import_offense_record_dataset = tablib.Dataset() - import_offense_record_dataset.headers = [col.column_name for col in offense_record_resource.get_import_fields()] + import_offense_record_dataset.headers = [ + col.column_name for col in offense_record_resource.get_import_fields() + ] for row in export_offense_record_dataset.dict: - import_offense_record_dataset.append([offense_id] + [value if value else None for value in row.values()]) + import_offense_record_dataset.append( + [offense_id] + [value if value else None for value in row.values()] + ) offense_record_resource.import_data(import_offense_record_dataset) saved_file_nos.append(record.file_no) - + create_batch_petitions(new_batch) return new_batch diff --git a/dear_petition/petition/management/commands/convert_agency_table.py b/dear_petition/petition/management/commands/convert_agency_table.py index 9adc2740..357cf67b 100644 --- a/dear_petition/petition/management/commands/convert_agency_table.py +++ b/dear_petition/petition/management/commands/convert_agency_table.py @@ -5,15 +5,18 @@ from dear_petition.petition import models + def convert_contacts_to_agency_objects(current_cls, target_cls): # Migrate from Contact model to Agency model - agency_contacts = current_cls.objects.filter(category='agency') + agency_contacts = current_cls.objects.filter(category="agency") print() for agency_contact in agency_contacts.values(): with transaction.atomic(): - id = agency_contact.pop('id') + id = agency_contact.pop("id") agency = target_cls(**agency_contact) - if re.search(models.AGENCY_SHERRIFF_OFFICE_PATTERN, agency.name, re.IGNORECASE) or re.search(models.AGENCY_SHERRIFF_DEPARTMENT_PATTERN, agency.name, re.IGNORECASE): + if re.search( + models.AGENCY_SHERRIFF_OFFICE_PATTERN, agency.name, re.IGNORECASE + ) or re.search(models.AGENCY_SHERRIFF_DEPARTMENT_PATTERN, agency.name, re.IGNORECASE): agency.is_sheriff = True else: agency.is_sheriff = False @@ -23,6 +26,7 @@ def convert_contacts_to_agency_objects(current_cls, target_cls): print(f"Migrated Contact '{agency.name}' to Agency table") print() + """ # attempt to provide backwards migration for agencies, but it doesn't seem worth the effort def convert_agencies_to_contact_objects(current_cls, target_cls): @@ -45,8 +49,16 @@ def convert_agencies_to_contact_objects(current_cls, target_cls): class Command(BaseCommand): def add_arguments(self, parser): - parser.add_argument("--convert_contacts", action="store_true", help="Convert existing agencies in Contact table to Agency table") - parser.add_argument("--convert_agencies", action="store_true", help="Convert existing agencies in Agency table to Contact table") + parser.add_argument( + "--convert_contacts", + action="store_true", + help="Convert existing agencies in Contact table to Agency table", + ) + parser.add_argument( + "--convert_agencies", + action="store_true", + help="Convert existing agencies in Agency table to Contact table", + ) def handle(self, *args, **options): if options["convert_contacts"]: @@ -56,4 +68,6 @@ def handle(self, *args, **options): print("This operation is currently not supported") pass else: - raise CommandError('Did not provide one of `convert_contacts` or `convert_agencies` arguments') \ No newline at end of file + raise CommandError( + "Did not provide one of `convert_contacts` or `convert_agencies` arguments" + ) diff --git a/dear_petition/petition/migrations/0064_remove_contact_county_remove_contact_user_and_more.py b/dear_petition/petition/migrations/0064_remove_contact_county_remove_contact_user_and_more.py index 8e43aaff..d1155720 100644 --- a/dear_petition/petition/migrations/0064_remove_contact_county_remove_contact_user_and_more.py +++ b/dear_petition/petition/migrations/0064_remove_contact_county_remove_contact_user_and_more.py @@ -5,14 +5,17 @@ import django.db.models.deletion import django.db.models.manager -from dear_petition.petition.management.commands.convert_agency_table import convert_contacts_to_agency_objects +from dear_petition.petition.management.commands.convert_agency_table import ( + convert_contacts_to_agency_objects, +) + def forwards(apps, schema_editor): if schema_editor.connection.alias != "default": return - ContactModel = apps.get_model('petition', 'Contact') - AgencyModel = apps.get_model('petition', 'Agency') + ContactModel = apps.get_model("petition", "Contact") + AgencyModel = apps.get_model("petition", "Agency") convert_contacts_to_agency_objects(ContactModel, AgencyModel) diff --git a/dear_petition/petition/migrations/0065_alter_ciprsrecord_arrest_date_and_more.py b/dear_petition/petition/migrations/0065_alter_ciprsrecord_arrest_date_and_more.py index e54a02c4..07602ef0 100644 --- a/dear_petition/petition/migrations/0065_alter_ciprsrecord_arrest_date_and_more.py +++ b/dear_petition/petition/migrations/0065_alter_ciprsrecord_arrest_date_and_more.py @@ -18,9 +18,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="ciprsrecord", name="case_status", - field=models.CharField( - blank=True, max_length=256, verbose_name="Case Status" - ), + field=models.CharField(blank=True, max_length=256, verbose_name="Case Status"), ), migrations.AlterField( model_name="ciprsrecord", @@ -45,16 +43,12 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="ciprsrecord", name="file_no", - field=models.CharField( - blank=True, max_length=256, verbose_name="File Number" - ), + field=models.CharField(blank=True, max_length=256, verbose_name="File Number"), ), migrations.AlterField( model_name="ciprsrecord", name="has_additional_offenses", - field=models.BooleanField( - default=False, verbose_name="Has Additional Offenses" - ), + field=models.BooleanField(default=False, verbose_name="Has Additional Offenses"), ), migrations.AlterField( model_name="ciprsrecord", @@ -78,9 +72,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="ciprsrecord", name="offense_date", - field=models.DateTimeField( - blank=True, null=True, verbose_name="Offense Date" - ), + field=models.DateTimeField(blank=True, null=True, verbose_name="Offense Date"), ), migrations.AlterField( model_name="ciprsrecord", @@ -115,9 +107,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="contact", name="county", - field=models.CharField( - blank=True, max_length=100, null=True, verbose_name="County" - ), + field=models.CharField(blank=True, max_length=100, null=True, verbose_name="County"), ), migrations.AlterField( model_name="contact", @@ -207,9 +197,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="offense", name="disposed_on", - field=models.DateField( - blank=True, null=True, verbose_name="Disposed On Date" - ), + field=models.DateField(blank=True, null=True, verbose_name="Disposed On Date"), ), migrations.AlterField( model_name="offense", diff --git a/dear_petition/petition/migrations/0066_alter_offense_plea_alter_offense_verdict.py b/dear_petition/petition/migrations/0066_alter_offense_plea_alter_offense_verdict.py index e1bfc42d..dbe3137d 100644 --- a/dear_petition/petition/migrations/0066_alter_offense_plea_alter_offense_verdict.py +++ b/dear_petition/petition/migrations/0066_alter_offense_plea_alter_offense_verdict.py @@ -12,15 +12,11 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="offense", name="plea", - field=models.CharField( - blank=True, max_length=256, null=True, verbose_name="Plea" - ), + field=models.CharField(blank=True, max_length=256, null=True, verbose_name="Plea"), ), migrations.AlterField( model_name="offense", name="verdict", - field=models.CharField( - blank=True, max_length=256, null=True, verbose_name="Verdict" - ), + field=models.CharField(blank=True, max_length=256, null=True, verbose_name="Verdict"), ), ] diff --git a/dear_petition/petition/models.py b/dear_petition/petition/models.py index c5bc38f0..22ec4b81 100644 --- a/dear_petition/petition/models.py +++ b/dear_petition/petition/models.py @@ -57,15 +57,22 @@ class CIPRSRecord(PrintableModelMixin, models.Model): file_no = models.CharField(max_length=256, blank=True, verbose_name="File Number") county = models.CharField(max_length=256, blank=True, verbose_name="County") dob = models.DateField(null=True, blank=True, verbose_name="Date of Birth") - sex = models.CharField(max_length=6, choices=SEX_CHOICES, default=NOT_AVAILABLE, verbose_name="Sex") + sex = models.CharField( + max_length=6, choices=SEX_CHOICES, default=NOT_AVAILABLE, verbose_name="Sex" + ) race = models.CharField(max_length=256, blank=True, verbose_name="Race") case_status = models.CharField(max_length=256, blank=True, verbose_name="Case Status") offense_date = models.DateTimeField(null=True, blank=True, verbose_name="Offense Date") arrest_date = models.DateField(null=True, blank=True, verbose_name="Arrest Date") jurisdiction = models.CharField( - max_length=16, choices=JURISDICTION_CHOICES, default=NOT_AVAILABLE, verbose_name="Jurisdiction" + max_length=16, + choices=JURISDICTION_CHOICES, + default=NOT_AVAILABLE, + verbose_name="Jurisdiction", + ) + has_additional_offenses = models.BooleanField( + default=False, verbose_name="Has Additional Offenses" ) - has_additional_offenses = models.BooleanField(default=False, verbose_name="Has Additional Offenses") def __str__(self): return f"{self.label} {self.file_no} ({self.pk})" @@ -106,7 +113,10 @@ class Offense(PrintableModelMixin, models.Model): "CIPRSRecord", related_name="offenses", on_delete=models.CASCADE ) jurisdiction = models.CharField( - choices=JURISDICTION_CHOICES, max_length=255, default=pc.DISTRICT_COURT, verbose_name="Jurisdiction" + choices=JURISDICTION_CHOICES, + max_length=255, + default=pc.DISTRICT_COURT, + verbose_name="Jurisdiction", ) disposed_on = models.DateField(blank=True, null=True, verbose_name="Disposed On Date") disposition_method = models.CharField(max_length=256, verbose_name="Disposition Method") @@ -162,16 +172,16 @@ def has_equivalent_offense_records(self): class OffenseRecord(PrintableModelMixin, models.Model): - offense = models.ForeignKey( - "Offense", related_name="offense_records", on_delete=models.CASCADE - ) + offense = models.ForeignKey("Offense", related_name="offense_records", on_delete=models.CASCADE) count = models.IntegerField(blank=True, null=True, verbose_name="Count") law = models.CharField(max_length=256, blank=True, verbose_name="Law") code = models.IntegerField(blank=True, null=True, verbose_name="Code") action = models.CharField(max_length=256, null=True, verbose_name="Action") severity = models.CharField(max_length=256, verbose_name="Severity") description = models.CharField(max_length=256, verbose_name="Description") - agency = models.ForeignKey("Contact", related_name="+", null=True, blank=True, on_delete=models.SET_NULL) + agency = models.ForeignKey( + "Contact", related_name="+", null=True, blank=True, on_delete=models.SET_NULL + ) def __str__(self): ciprs_record = self.offense.ciprs_record @@ -264,39 +274,45 @@ def get_queryset(self): return ( super(AgencyWithSherriffOfficeManager, self) .get_queryset() - .annotate(is_sheriff_office=Case( - When(name__iregex=AGENCY_SHERRIFF_OFFICE_PATTERN, then=Value(True)), - When(name__iregex=AGENCY_SHERRIFF_DEPARTMENT_PATTERN, then=Value(True)), - default=Value(False), - output_field=models.BooleanField(), - )) + .annotate( + is_sheriff_office=Case( + When(name__iregex=AGENCY_SHERRIFF_OFFICE_PATTERN, then=Value(True)), + When(name__iregex=AGENCY_SHERRIFF_DEPARTMENT_PATTERN, then=Value(True)), + default=Value(False), + output_field=models.BooleanField(), + ) + ) ) - + + class AgencyWithCleanNameManager(models.Manager): def get_queryset(self): """Annotation version of is_sheriff_office to be used from database""" return ( super(AgencyWithCleanNameManager, self) .get_queryset() - .annotate(clean_name=models.Func( - models.F('name'), - Value(r'[\'\-\"\(\)&:\/\.]'), # special characters to remove - Value(r''), - Value('g'), # regex flag - function='REGEXP_REPLACE', - output_field=models.TextField(), - )) + .annotate( + clean_name=models.Func( + models.F("name"), + Value(r"[\'\-\"\(\)&:\/\.]"), # special characters to remove + Value(r""), + Value("g"), # regex flag + function="REGEXP_REPLACE", + output_field=models.TextField(), + ) + ) ) - class Contact(PrintableModelMixin, models.Model): name = models.CharField(max_length=512, verbose_name="Name") category = models.CharField(max_length=16, choices=CONTACT_CATEGORIES) address1 = models.CharField("Address (Line 1)", max_length=512, blank=True) - address2 = models.CharField("Address (Line 2)", max_length=512, default='', blank=True) + address2 = models.CharField("Address (Line 2)", max_length=512, default="", blank=True) city = models.CharField(max_length=64, blank=True, verbose_name="City") - state = models.CharField(choices=us_states.US_STATES, max_length=64, blank=True, verbose_name="State") + state = models.CharField( + choices=us_states.US_STATES, max_length=64, blank=True, verbose_name="State" + ) zipcode = models.CharField("ZIP Code", max_length=16, blank=True) phone_number = PhoneNumberField(null=True, blank=True, verbose_name="Phone Number") email = models.EmailField(max_length=254, null=True, blank=True, verbose_name="Email Address") @@ -305,8 +321,8 @@ class Contact(PrintableModelMixin, models.Model): objects = models.Manager() def __str__(self): - return self.name if self.name else '' - + return self.name if self.name else "" + class Client(Contact): dob = models.DateField(null=True, blank=True, verbose_name="Date of Birth") @@ -355,7 +371,6 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) - @classmethod def get_sherriff_office_by_county(cls, county: str): qs = cls.agencies_with_sherriff_office.filter(county__iexact=county, is_sheriff_office=True) @@ -366,8 +381,6 @@ def get_sherriff_office_by_county(cls, county: str): return qs.first() if qs.exists() else None -class Batch(PrintableModelMixin, models.Model): - class Batch(PrintableModelMixin, models.Model): label = models.CharField(max_length=2048, blank=True) date_uploaded = models.DateTimeField(auto_now_add=True) @@ -436,12 +449,13 @@ def adult_felony_records(self, jurisdiction=""): def adult_misdemeanor_records(self, jurisdiction=""): return self.petition_offense_records(pc.ADULT_MISDEMEANORS, jurisdiction) - + def adjust_for_new_client_dob(self): """ Called when a new date of birth is added to a batch's client to adjust the petitions accordingly. """ from dear_petition.petition.etl.load import create_petitions_from_records + Petition.objects.filter(batch=self, form_type=pc.UNDERAGED_CONVICTIONS).delete() create_petitions_from_records(self, pc.UNDERAGED_CONVICTIONS) diff --git a/dear_petition/petition/resources.py b/dear_petition/petition/resources.py index 615e0b26..30fc4ddb 100644 --- a/dear_petition/petition/resources.py +++ b/dear_petition/petition/resources.py @@ -21,7 +21,7 @@ def titlecase(string): - string = string.replace('_',' ') + string = string.replace("_", " ") return string.title() @@ -49,11 +49,11 @@ def is_empty_row(row) -> bool: class ExcelField(fields.Field): - def __init__(self, **kwargs): - self.dropdown = kwargs.pop('dropdown', None) + self.dropdown = kwargs.pop("dropdown", None) super().__init__(**kwargs) + class ExcelDataset: def __init__(self): self.wb = Workbook() @@ -72,8 +72,15 @@ def create_new_worksheet(self, title=None): self.wb.create_sheet(title=title) self.change_worksheet(sheet_name=title) - def append(self, values, fields, is_header=False, num_indent=0, bold=False, color=None,): - + def append( + self, + values, + fields, + is_header=False, + num_indent=0, + bold=False, + color=None, + ): if len(values) != len(fields): raise ValueError("The values and fields passed to append should match.") @@ -81,23 +88,22 @@ def append(self, values, fields, is_header=False, num_indent=0, bold=False, colo self.ws.append(row) row_number = self.ws.max_row - fill=None + fill = None if color: fill = PatternFill(start_color=color, end_color=color, fill_type="solid") - + # Apply font and alignment to each cell in the row if specified - for col_number, field in enumerate(fields, num_indent+1): + for col_number, field in enumerate(fields, num_indent + 1): cell = self.ws.cell(row=row_number, column=col_number) if bold: cell.font = bold_font if fill: - cell.fill = fill - if not is_header and getattr(field, 'dropdown', None): + cell.fill = fill + if not is_header and getattr(field, "dropdown", None): self.ws.add_data_validation(field.dropdown) dropdown = field.dropdown dropdown.add(cell) - def append_separator(self): """ Appends a blank row @@ -109,13 +115,13 @@ def resize_columns(self): for column_cells in sheet.columns: max_length = 0 column = column_cells[0].column_letter - + for cell in column_cells: if cell.value: max_length = max(max_length, len(str(cell.value))) if max_length > 0: - adjusted_width = max_length + 2 + adjusted_width = max_length + 2 sheet.column_dimensions[column].width = adjusted_width def save(self, filename): @@ -126,9 +132,9 @@ def save(self, filename): class AgencyResource(resources.ModelResource): - county = ExcelField(attribute='county', column_name='County') - name = ExcelField(attribute='name', column_name='Arresting Agency') - address = ExcelField(column_name='Address') + county = ExcelField(attribute="county", column_name="County") + name = ExcelField(attribute="name", column_name="Arresting Agency") + address = ExcelField(column_name="Address") def init_instance(self, row=None): instance = super().init_instance(row) @@ -142,30 +148,34 @@ def get_instance(self, instance_loader, row): try: return super().get_instance(instance_loader, row) except models.Agency.MultipleObjectsReturned as e: - raise models.Agency.MultipleObjectsReturned(f"There are multiple agencies named '{row['Arresting Agency']}' in county '{row['County']}. Ensure there is only 1.") + raise models.Agency.MultipleObjectsReturned( + f"There are multiple agencies named '{row['Arresting Agency']}' in county '{row['County']}. Ensure there is only 1." + ) def before_import_row(self, row, **kwargs): if is_empty_row(row): return # found a bug where leading newlines were causing false negative matches for existing agencies - row['Arresting Agency'] = row['Arresting Agency'].strip() - row['name'] = row['Arresting Agency'] - row['County'] = row['County'].strip() - row['county'] = row['County'].strip() - - (address1, address2, city, state, zipcode) = parse_agency_full_address(row['Address']) - row['city'] = city.strip() - row['state'] = state.strip() - row['zipcode'] = zipcode.strip() - row['address1'] = address1.strip() - row['address2'] = address2.strip() if address2 else None - - name = row['name'] - if re.search(models.AGENCY_SHERRIFF_OFFICE_PATTERN, name, re.IGNORECASE) or re.search(models.AGENCY_SHERRIFF_DEPARTMENT_PATTERN, name, re.IGNORECASE): - row['is_sheriff'] = True + row["Arresting Agency"] = row["Arresting Agency"].strip() + row["name"] = row["Arresting Agency"] + row["County"] = row["County"].strip() + row["county"] = row["County"].strip() + + (address1, address2, city, state, zipcode) = parse_agency_full_address(row["Address"]) + row["city"] = city.strip() + row["state"] = state.strip() + row["zipcode"] = zipcode.strip() + row["address1"] = address1.strip() + row["address2"] = address2.strip() if address2 else None + + name = row["name"] + if re.search(models.AGENCY_SHERRIFF_OFFICE_PATTERN, name, re.IGNORECASE) or re.search( + models.AGENCY_SHERRIFF_DEPARTMENT_PATTERN, name, re.IGNORECASE + ): + row["is_sheriff"] = True else: - row['is_sheriff'] = False + row["is_sheriff"] = False def skip_row(self, instance, original, row, import_validation_errors=None): if is_empty_row(row): @@ -174,138 +184,206 @@ def skip_row(self, instance, original, row, import_validation_errors=None): instance, original, row, import_validation_errors=import_validation_errors ) - class Meta: model = models.Agency - import_id_fields = ('name', 'county') + import_id_fields = ("name", "county") store_instance = True + class MultiModelResource(resources.ModelResource): - - PARENT_OBJECT_FIELD = None - NUM_INDENT=0 - - def __init__(self): - super().__init__() - self.saved_instance_ids=[] - - def after_save_instance(self, instance, using_transactions, dry_run): - if not dry_run: - self.saved_instance_ids.append(instance.id) - - def export_resource(self, obj): - return [self.export_field(field, obj) for field in self.get_export_fields()] - - def get_export_headers(self): - headers = [titlecase(field.column_name) for field in self.get_fields()] - return headers - - def export_field(self, field, obj): - field_name = self.get_field_name(field) - dehydrate_method = field.get_dehydrate_method(field_name) - - if isinstance(self._meta.model._meta.get_field(field.attribute), BooleanField): - field.dropdown = DataValidation(type="list", formula1=f'"True,False"', showDropDown=True) - - method = getattr(self, dehydrate_method, None) - if method is not None: - return method(obj) - - return field.export(obj) - - def import_field(self, field, obj, data, is_m2m=False, **kwargs): - hydrate_method = getattr(self, f"hydrate_{field.attribute}", None) - if hydrate_method is not None: - hydrate_method(field, data) - - if field.attribute and field.column_name in data: - field.save(obj, data, is_m2m, **kwargs) - - def export_excel(self, data, dataset: ExcelDataset, color=None, **kwargs): - headers = self.get_export_headers() - fields = self.get_export_fields() - - dataset.append(headers, fields, num_indent=self.NUM_INDENT, is_header=True, bold=True, color=color) - - if isinstance(data, QuerySet): - for obj in data: - dataset.append(self.export_resource(obj), fields, num_indent=self.NUM_INDENT, color=color) - else: - dataset.append(self.export_resource(data), fields, num_indent=self.NUM_INDENT, color=color) - - return dataset - - - + PARENT_OBJECT_FIELD = None + NUM_INDENT = 0 + + def __init__(self): + super().__init__() + self.saved_instance_ids = [] + + def after_save_instance(self, instance, using_transactions, dry_run): + if not dry_run: + self.saved_instance_ids.append(instance.id) + + def export_resource(self, obj): + return [self.export_field(field, obj) for field in self.get_export_fields()] + + def get_export_headers(self): + headers = [titlecase(field.column_name) for field in self.get_fields()] + return headers + + def export_field(self, field, obj): + field_name = self.get_field_name(field) + dehydrate_method = field.get_dehydrate_method(field_name) + + if isinstance(self._meta.model._meta.get_field(field.attribute), BooleanField): + field.dropdown = DataValidation( + type="list", formula1=f'"True,False"', showDropDown=True + ) + + method = getattr(self, dehydrate_method, None) + if method is not None: + return method(obj) + + return field.export(obj) + + def import_field(self, field, obj, data, is_m2m=False, **kwargs): + hydrate_method = getattr(self, f"hydrate_{field.attribute}", None) + if hydrate_method is not None: + hydrate_method(field, data) + + if field.attribute and field.column_name in data: + field.save(obj, data, is_m2m, **kwargs) + + def export_excel(self, data, dataset: ExcelDataset, color=None, **kwargs): + headers = self.get_export_headers() + fields = self.get_export_fields() + + dataset.append( + headers, fields, num_indent=self.NUM_INDENT, is_header=True, bold=True, color=color + ) + + if isinstance(data, QuerySet): + for obj in data: + dataset.append( + self.export_resource(obj), fields, num_indent=self.NUM_INDENT, color=color + ) + else: + dataset.append( + self.export_resource(data), fields, num_indent=self.NUM_INDENT, color=color + ) + + return dataset + + class BatchResource(MultiModelResource): class Meta: model = models.Batch - include = ('label',) + include = ("label",) + class RecordResource(MultiModelResource): batch_id = fields.Field(attribute="batch_id") - label = ExcelField(attribute='label', column_name='Defendent Name') - dob = ExcelField(attribute='dob', column_name='Date of Birth') - has_additional_offenses = ExcelField(attribute='has_additional_offenses', column_name='Additional Offenses Exist') - jurisdiction = ExcelField(attribute='jurisdiction', dropdown=DataValidation(type="list", formula1=f'"{",".join(constants.JURISDICTION_MAP.values())}"', showDropDown=True)) - sex = ExcelField(attribute='sex', dropdown=DataValidation(type="list", formula1=f'"{",".join(constants.SEX_MAP.keys())}"', showDropDown=True)) - + label = ExcelField(attribute="label", column_name="Defendent Name") + dob = ExcelField(attribute="dob", column_name="Date of Birth") + has_additional_offenses = ExcelField( + attribute="has_additional_offenses", column_name="Additional Offenses Exist" + ) + jurisdiction = ExcelField( + attribute="jurisdiction", + dropdown=DataValidation( + type="list", + formula1=f'"{",".join(constants.JURISDICTION_MAP.values())}"', + showDropDown=True, + ), + ) + sex = ExcelField( + attribute="sex", + dropdown=DataValidation( + type="list", formula1=f'"{",".join(constants.SEX_MAP.keys())}"', showDropDown=True + ), + ) + def dehydrate_jurisdiction(self, record): - jurisdiction = getattr(record, 'jurisdiction') + jurisdiction = getattr(record, "jurisdiction") return constants.JURISDICTION_MAP[jurisdiction] - + def hydrate_jurisdiction(self, field, data): attribute = field.attribute - for k,v in constants.JURISDICTION_MAP.items(): + for k, v in constants.JURISDICTION_MAP.items(): if v == data[attribute]: data[attribute] = k break return data - + def dehydrate_sex(self, record): - sex = getattr(record, 'sex') + sex = getattr(record, "sex") return constants.SEX_CHOICES[sex] - + def hydrate_sex(self, field, data): attribute = field.attribute data[attribute] = constants.SEX_MAP[data[attribute]] return data - + def get_export_fields(self): - return [field for field in super().get_export_fields() if field.column_name not in ('batch_id',)] - + return [ + field for field in super().get_export_fields() if field.column_name not in ("batch_id",) + ] + def get_export_headers(self): - return [header for header in super().get_export_headers() if header not in ('Batch Id',)] + return [header for header in super().get_export_headers() if header not in ("Batch Id",)] class Meta: model = models.CIPRSRecord - exclude = ('id', 'batch', 'batch_file', 'date_uploaded', 'data',) - export_order = ('batch_id','label','file_no','dob', 'jurisdiction', 'county', 'has_additional_offenses') + exclude = ( + "id", + "batch", + "batch_file", + "date_uploaded", + "data", + ) + export_order = ( + "batch_id", + "label", + "file_no", + "dob", + "jurisdiction", + "county", + "has_additional_offenses", + ) force_init_instance = True + class OffenseResource(MultiModelResource): ciprs_record_id = fields.Field(attribute="ciprs_record_id") - disposed_on = ExcelField(attribute='disposed_on', column_name='Disposition Date') - jurisdiction = ExcelField(attribute='jurisdiction', dropdown=DataValidation(type="list", formula1=f'"{",".join(constants.JURISDICTION_MAP.values())}"', showDropDown=True)) - plea = ExcelField(attribute='plea', dropdown=DataValidation(type="list", formula1=f'"{",".join(constants.VERDICT_CODE_MAP.keys())}"', showDropDown=True)) - verdict = ExcelField(attribute='verdict', dropdown=DataValidation(type="list", formula1=f'"{",".join(constants.VERDICT_CODE_MAP.keys())}"', showDropDown=True)) - disposition_method = ExcelField(attribute='disposition_method', dropdown=DataValidation(type="list", formula1=f'"{",".join(constants.DISPOSITION_METHOD_CODE_MAP.keys())}"', showDropDown=True)) + disposed_on = ExcelField(attribute="disposed_on", column_name="Disposition Date") + jurisdiction = ExcelField( + attribute="jurisdiction", + dropdown=DataValidation( + type="list", + formula1=f'"{",".join(constants.JURISDICTION_MAP.values())}"', + showDropDown=True, + ), + ) + plea = ExcelField( + attribute="plea", + dropdown=DataValidation( + type="list", + formula1=f'"{",".join(constants.VERDICT_CODE_MAP.keys())}"', + showDropDown=True, + ), + ) + verdict = ExcelField( + attribute="verdict", + dropdown=DataValidation( + type="list", + formula1=f'"{",".join(constants.VERDICT_CODE_MAP.keys())}"', + showDropDown=True, + ), + ) + disposition_method = ExcelField( + attribute="disposition_method", + dropdown=DataValidation( + type="list", + formula1=f'"{",".join(constants.DISPOSITION_METHOD_CODE_MAP.keys())}"', + showDropDown=True, + ), + ) def get_export_fields(self): - return [field for field in super().get_export_fields() if field.column_name != 'ciprs_record_id'] - + return [ + field for field in super().get_export_fields() if field.column_name != "ciprs_record_id" + ] + def get_export_headers(self): - return [header for header in super().get_export_headers() if header != 'Ciprs Record Id'] + return [header for header in super().get_export_headers() if header != "Ciprs Record Id"] def dehydrate_jurisdiction(self, record): - jurisdiction = getattr(record, 'jurisdiction') + jurisdiction = getattr(record, "jurisdiction") return constants.JURISDICTION_MAP[jurisdiction] - + def hydrate_jurisdiction(self, field, data): attribute = field.attribute - for k,v in constants.JURISDICTION_MAP.items(): + for k, v in constants.JURISDICTION_MAP.items(): if v == data[attribute]: data[attribute] = k break @@ -314,45 +392,57 @@ def hydrate_jurisdiction(self, field, data): class Meta: model = models.Offense - exclude = ('id', 'ciprs_record') - export_order = ('ciprs_record_id',) + exclude = ("id", "ciprs_record") + export_order = ("ciprs_record_id",) force_init_instance = True class OffenseRecordResource(MultiModelResource): - - NUM_INDENT=1 + NUM_INDENT = 1 offense_id = fields.Field(attribute="offense_id") - count = ExcelField(attribute='count', column_name='#') - severity = ExcelField(attribute='severity', dropdown=DataValidation(type="list", formula1=f'"{",".join(constants.SEVERITIES._db_values)}"', showDropDown=True)) - action = ExcelField(attribute='action', dropdown=DataValidation(type="list", formula1=f'"{",".join(constants.ACTIONS._db_values)}"', showDropDown=True)) + count = ExcelField(attribute="count", column_name="#") + severity = ExcelField( + attribute="severity", + dropdown=DataValidation( + type="list", + formula1=f'"{",".join(constants.SEVERITIES._db_values)}"', + showDropDown=True, + ), + ) + action = ExcelField( + attribute="action", + dropdown=DataValidation( + type="list", formula1=f'"{",".join(constants.ACTIONS._db_values)}"', showDropDown=True + ), + ) def get_export_fields(self): - return [field for field in super().get_export_fields() if field.column_name != 'offense_id'] - + return [field for field in super().get_export_fields() if field.column_name != "offense_id"] + def get_export_headers(self): - return [header for header in super().get_export_headers() if header not in ('Offense Id')] + return [header for header in super().get_export_headers() if header not in ("Offense Id")] class Meta: model = models.OffenseRecord - exclude = ('id', 'agency', 'offense') - export_order = ('offense_id',) + exclude = ("id", "agency", "offense") + export_order = ("offense_id",) force_init_instance = True + class ClientResource(MultiModelResource): - dob = ExcelField(attribute='dob', column_name='Date of Birth') + dob = ExcelField(attribute="dob", column_name="Date of Birth") class Meta: model = models.Client - exclude = ('id', 'contact_ptr', 'category','user') + exclude = ("id", "contact_ptr", "category", "user") force_init_instance = True def get_import_fields(self): return ["batch_id"] + super().get_import_fields() -class RecordSummaryResource(resources.ModelResource): +class RecordSummaryResource(resources.ModelResource): class Meta: model = models.Batch @@ -371,8 +461,7 @@ def __init__(self): self.first_offense_record_header = self.offense_record_headers[0] def export(self, batch_object=None, *args, **kwargs): - - if not isinstance(batch_object,models.Batch): + if not isinstance(batch_object, models.Batch): raise ValueError("Queryset must be a Batch object") dataset = ExcelDataset() @@ -383,27 +472,32 @@ def export(self, batch_object=None, *args, **kwargs): self.client_resource.export_excel(batch_object.client, dataset) # Record - for record_object in batch_object.records.order_by('file_no').prefetch_related('offenses','offenses__offense_records').all(): + for record_object in ( + batch_object.records.order_by("file_no") + .prefetch_related("offenses", "offenses__offense_records") + .all() + ): dataset.create_new_worksheet(title=record_object.file_no) - self.record_resource.export_excel(record_object, dataset, color='E0FFFF') + self.record_resource.export_excel(record_object, dataset, color="E0FFFF") dataset.append_separator() # Offense - i=0 + i = 0 for offense_object in record_object.offenses.all(): - color = '90EE90' if i % 2 == 1 else 'FFFFE0' + color = "90EE90" if i % 2 == 1 else "FFFFE0" self.offense_resource.export_excel(offense_object, dataset, color=color) # Offense Record - self.offense_record_resource.export_excel(offense_object.offense_records.all(), dataset, color=color) + self.offense_record_resource.export_excel( + offense_object.offense_records.all(), dataset, color=color + ) dataset.append_separator() - i+=1 + i += 1 dataset.resize_columns() return dataset - - + def import_data(self, workbook: Workbook, batch: models.Batch, *args, **kwargs): with transaction.atomic(): if CLIENT_SHEET_TITLE in workbook: @@ -424,39 +518,56 @@ def import_data(self, workbook: Workbook, batch: models.Batch, *args, **kwargs): current_resource = None current_dataset = tablib.Dataset() for row_number, row in enumerate(worksheet.iter_rows(), 1): - - if all(cell.value in (None, '') for cell in row): - continue # This is an empty row. + if all(cell.value in (None, "") for cell in row): + continue # This is an empty row. for cell_number, cell in enumerate(worksheet[row_number]): if cell.value is not None: - break # Get first non-null value - - if cell.value in (self.first_record_header, self.first_offense_header, self.first_offense_record_header): + break # Get first non-null value + if cell.value in ( + self.first_record_header, + self.first_offense_header, + self.first_offense_record_header, + ): if current_resource and current_dataset: result = current_resource.import_data(current_dataset) current_dataset = tablib.Dataset() if cell.value == self.first_record_header: current_resource = self.record_resource - current_dataset.headers = [col.column_name for col in current_resource.get_import_fields()] + current_dataset.headers = [ + col.column_name for col in current_resource.get_import_fields() + ] elif cell.value == self.first_offense_header: current_resource = self.offense_resource - current_dataset.headers = [col.column_name for col in current_resource.get_import_fields()] + current_dataset.headers = [ + col.column_name for col in current_resource.get_import_fields() + ] elif cell.value == self.first_offense_record_header: current_resource = self.offense_record_resource - current_dataset.headers = [col.column_name for col in current_resource.get_import_fields()] - else: + current_dataset.headers = [ + col.column_name for col in current_resource.get_import_fields() + ] + else: if current_resource == self.record_resource: - row = list(row)[current_resource.NUM_INDENT:len(self.record_headers) + current_resource.NUM_INDENT] + row = list(row)[ + current_resource.NUM_INDENT : len(self.record_headers) + + current_resource.NUM_INDENT + ] current_dataset.append([batch.id] + [cell.value for cell in row]) elif current_resource == self.offense_resource: - row = list(row)[current_resource.NUM_INDENT:len(self.offense_headers) + current_resource.NUM_INDENT] + row = list(row)[ + current_resource.NUM_INDENT : len(self.offense_headers) + + current_resource.NUM_INDENT + ] record_id = self.record_resource.saved_instance_ids[-1] current_dataset.append([record_id] + [cell.value for cell in row]) elif current_resource == self.offense_record_resource: - row = list(row)[current_resource.NUM_INDENT:len(self.offense_record_headers) + current_resource.NUM_INDENT] + row = list(row)[ + current_resource.NUM_INDENT : len(self.offense_record_headers) + + current_resource.NUM_INDENT + ] offense_id = self.offense_resource.saved_instance_ids[-1] current_dataset.append([offense_id] + [cell.value for cell in row]) else: diff --git a/dear_petition/petition/tests/factories.py b/dear_petition/petition/tests/factories.py index 5f1e0f67..3634971c 100644 --- a/dear_petition/petition/tests/factories.py +++ b/dear_petition/petition/tests/factories.py @@ -2,18 +2,37 @@ import random import factory -from dear_petition.petition.models import (Batch, BatchFile, CIPRSRecord, - Contact, Agency, Client, GeneratedPetition, Offense, - OffenseRecord, Petition, - PetitionDocument, - PetitionOffenseRecord) +from dear_petition.petition.models import ( + Batch, + BatchFile, + CIPRSRecord, + Contact, + Agency, + Client, + GeneratedPetition, + Offense, + OffenseRecord, + Petition, + PetitionDocument, + PetitionOffenseRecord, +) from dear_petition.users.tests.factories import UserFactory from django.core.files.uploadedfile import InMemoryUploadedFile from pytz import timezone -from ..constants import (CHARGED, CONVICTED, DISMISSED, DISTRICT_COURT, - DISTRICT_COURT_WITHOUT_DA_LEAVE, DURHAM_COUNTY, - FEMALE, MALE, NOT_AVAILABLE, SUPERIOR_COURT, UNKNOWN) +from ..constants import ( + CHARGED, + CONVICTED, + DISMISSED, + DISTRICT_COURT, + DISTRICT_COURT_WITHOUT_DA_LEAVE, + DURHAM_COUNTY, + FEMALE, + MALE, + NOT_AVAILABLE, + SUPERIOR_COURT, + UNKNOWN, +) class ContactFactory(factory.django.DjangoModelFactory): diff --git a/dear_petition/petition/tests/test_models.py b/dear_petition/petition/tests/test_models.py index 1cfa5abb..e4e89c5b 100644 --- a/dear_petition/petition/tests/test_models.py +++ b/dear_petition/petition/tests/test_models.py @@ -23,7 +23,11 @@ MALE, ) from dear_petition.petition.models import GeneratedPetition -from dear_petition.petition.tests.factories import AgencyFactory, OffenseFactory, OffenseRecordFactory +from dear_petition.petition.tests.factories import ( + AgencyFactory, + OffenseFactory, + OffenseRecordFactory, +) from dear_petition.users.tests.factories import UserFactory pytestmark = pytest.mark.django_db diff --git a/dear_petition/petition/tests/test_resources.py b/dear_petition/petition/tests/test_resources.py index 4d41e0c8..c5092061 100644 --- a/dear_petition/petition/tests/test_resources.py +++ b/dear_petition/petition/tests/test_resources.py @@ -5,7 +5,6 @@ pytestmark = pytest.mark.django_db - @pytest.mark.parametrize( "full_address, expected_result", [ @@ -46,14 +45,13 @@ def test_record_summary_resource(batch, record1, client, dismissed_offense): client_worksheet = dataset.wb.worksheets[0] record_worksheet = dataset.wb.worksheets[1] - assert client_worksheet.title == 'Client Information' - assert client_worksheet['B1'].value == "Name" - assert client_worksheet['B2'].value == "Test Client" + assert client_worksheet.title == "Client Information" + assert client_worksheet["B1"].value == "Name" + assert client_worksheet["B2"].value == "Test Client" assert record_worksheet.title == "99CRAAAAAAAAAAAA" - assert record_worksheet['A1'].value == 'Defendent Name' - assert record_worksheet['A2'].value == 'Test Record' - assert record_worksheet['B1'].value == 'File No' - assert record_worksheet['B2'].value == '99CRAAAAAAAAAAAA' - assert record_worksheet['C1'].value == 'Date Of Birth' - assert record_worksheet['C2'].value == '1980-07-07' - + assert record_worksheet["A1"].value == "Defendent Name" + assert record_worksheet["A2"].value == "Test Record" + assert record_worksheet["B1"].value == "File No" + assert record_worksheet["B2"].value == "99CRAAAAAAAAAAAA" + assert record_worksheet["C1"].value == "Date Of Birth" + assert record_worksheet["C2"].value == "1980-07-07" diff --git a/dear_petition/petition/tests/test_utils.py b/dear_petition/petition/tests/test_utils.py index 273e9856..75441bcd 100644 --- a/dear_petition/petition/tests/test_utils.py +++ b/dear_petition/petition/tests/test_utils.py @@ -83,7 +83,7 @@ def test_make_datetime_aware_ambiguous(settings): expected_datetime_obj = make_aware( datetime(year=2011, month=11, day=6, hour=1, minute=46, second=0), timezone=pytz.timezone("America/New_York"), - is_dst=False + is_dst=False, ) assert pu.make_datetime_aware(naive_datetime_str) == expected_datetime_obj @@ -112,8 +112,12 @@ def test_get_truncation_point_of_short_text_by_pixel_size(): truncation_point = pu.get_truncation_point_of_text_by_pixel_size(text, 20000) assert truncation_point == len(text) + def test_get_petition_filename(petition): petition.created = datetime(2024, 7, 28, 0, 0) petition.save() petitioner_name = "Test" - assert pu.get_petition_filename(petitioner_name, petition, 'pdf') == '07-28-2024 DURHAM DC 146(a) Test.pdf' \ No newline at end of file + assert ( + pu.get_petition_filename(petitioner_name, petition, "pdf") + == "07-28-2024 DURHAM DC 146(a) Test.pdf" + ) diff --git a/dear_petition/petition/types/adult_felonies.py b/dear_petition/petition/types/adult_felonies.py index 59ad2741..928c633e 100644 --- a/dear_petition/petition/types/adult_felonies.py +++ b/dear_petition/petition/types/adult_felonies.py @@ -34,4 +34,3 @@ def build_query(): adult_felony_portal = methods & severity & waiting_period return adult_felony_ciprs | adult_felony_portal - diff --git a/dear_petition/petition/types/dismissed.py b/dear_petition/petition/types/dismissed.py index f3403c4d..738ecff2 100644 --- a/dear_petition/petition/types/dismissed.py +++ b/dear_petition/petition/types/dismissed.py @@ -1,7 +1,10 @@ from django.db.models import Q from dear_petition.petition.models import OffenseRecord -from dear_petition.petition.constants import CIPRS_DISPOSITION_METHODS_DISMISSED, PORTAL_DISPOSITION_METHODS_DISMISSED +from dear_petition.petition.constants import ( + CIPRS_DISPOSITION_METHODS_DISMISSED, + PORTAL_DISPOSITION_METHODS_DISMISSED, +) def get_offense_records(batch, jurisdiction=""): @@ -14,7 +17,7 @@ def get_offense_records(batch, jurisdiction=""): def build_query(): action = Q(action="CHARGED") - + methods_ciprs = Q() for method in CIPRS_DISPOSITION_METHODS_DISMISSED: methods_ciprs |= Q(offense__disposition_method__iexact=method) diff --git a/dear_petition/petition/types/tests/test_adult_felonies.py b/dear_petition/petition/types/tests/test_adult_felonies.py index e81e780f..f356ddac 100644 --- a/dear_petition/petition/types/tests/test_adult_felonies.py +++ b/dear_petition/petition/types/tests/test_adult_felonies.py @@ -9,20 +9,59 @@ @pytest.mark.parametrize( - "action, disposition_method, severity, date, should_be_included", [ + "action, disposition_method, severity, date, should_be_included", + [ # records that have data as they would from Portal (no action) ("", "District Guilty - Judge", SEVERITIES.FELONY, datetime(2014, 8, 1), True), - ("", "District Not Guilty - Judge", SEVERITIES.FELONY, datetime(2014, 8, 1), False), # exclude because not Portal guilty disposition method - ("", "District Guilty - Judge", SEVERITIES.MISDEMEANOR, datetime(2014, 8, 1), False), # exclude because misdemeanor severity - ("", "District Guilty - Judge", SEVERITIES.FELONY, datetime(2024, 8, 1), False), # exclude because too recent + ( + "", + "District Not Guilty - Judge", + SEVERITIES.FELONY, + datetime(2014, 8, 1), + False, + ), # exclude because not Portal guilty disposition method + ( + "", + "District Guilty - Judge", + SEVERITIES.MISDEMEANOR, + datetime(2014, 8, 1), + False, + ), # exclude because misdemeanor severity + ( + "", + "District Guilty - Judge", + SEVERITIES.FELONY, + datetime(2024, 8, 1), + False, + ), # exclude because too recent # records that have data as they would from CIPRS (disposition_method not one seen in Portal) (CONVICTED, "Disposed By Judge", SEVERITIES.FELONY, datetime(2014, 8, 1), True), - (CHARGED, "Disposed By Judge", SEVERITIES.FELONY, datetime(2014, 8, 1), False), # exclude because not convicted action - (CONVICTED, "Disposed By Judge", SEVERITIES.MISDEMEANOR, datetime(2014, 8, 1), False), # exclude because misdemeanor severity - (CONVICTED, "Disposed By Judge", SEVERITIES.FELONY, datetime(2024, 8, 1), False), # exclude because too recent - ] + ( + CHARGED, + "Disposed By Judge", + SEVERITIES.FELONY, + datetime(2014, 8, 1), + False, + ), # exclude because not convicted action + ( + CONVICTED, + "Disposed By Judge", + SEVERITIES.MISDEMEANOR, + datetime(2014, 8, 1), + False, + ), # exclude because misdemeanor severity + ( + CONVICTED, + "Disposed By Judge", + SEVERITIES.FELONY, + datetime(2024, 8, 1), + False, + ), # exclude because too recent + ], ) -def test_adult_felonies(action, disposition_method, severity, date, should_be_included, batch, record1): +def test_adult_felonies( + action, disposition_method, severity, date, should_be_included, batch, record1 +): offense = Offense.objects.create( ciprs_record=record1, disposition_method=disposition_method, diff --git a/dear_petition/petition/types/tests/test_adult_misdemeanors.py b/dear_petition/petition/types/tests/test_adult_misdemeanors.py index a22a2d3b..29538c7f 100644 --- a/dear_petition/petition/types/tests/test_adult_misdemeanors.py +++ b/dear_petition/petition/types/tests/test_adult_misdemeanors.py @@ -9,20 +9,59 @@ @pytest.mark.parametrize( - "action, disposition_method, severity, date, should_be_included", [ + "action, disposition_method, severity, date, should_be_included", + [ # records that have data as they would from Portal (no action) ("", "District Guilty - Judge", SEVERITIES.MISDEMEANOR, datetime(2019, 8, 1), True), - ("", "District Not Guilty - Judge", SEVERITIES.MISDEMEANOR, datetime(2019, 8, 1), False), # exclude because not Portal guilty disposition method - ("", "District Guilty - Judge", SEVERITIES.FELONY, datetime(2019, 8, 1), False), # exclude because felony severity - ("", "District Guilty - Judge", SEVERITIES.MISDEMEANOR, datetime(2024, 8, 1), False), # exclude because too recent + ( + "", + "District Not Guilty - Judge", + SEVERITIES.MISDEMEANOR, + datetime(2019, 8, 1), + False, + ), # exclude because not Portal guilty disposition method + ( + "", + "District Guilty - Judge", + SEVERITIES.FELONY, + datetime(2019, 8, 1), + False, + ), # exclude because felony severity + ( + "", + "District Guilty - Judge", + SEVERITIES.MISDEMEANOR, + datetime(2024, 8, 1), + False, + ), # exclude because too recent # records that have data as they would from CIPRS (disposition_method not one seen in Portal) (CONVICTED, "Disposed By Judge", SEVERITIES.MISDEMEANOR, datetime(2019, 8, 1), True), - (CHARGED, "Disposed By Judge", SEVERITIES.MISDEMEANOR, datetime(2019, 8, 1), False), # exclude because not convicted action - (CONVICTED, "Disposed By Judge", SEVERITIES.FELONY, datetime(2019, 8, 1), False), # exclude because felony severity - (CONVICTED, "Disposed By Judge", SEVERITIES.MISDEMEANOR, datetime(2024, 8, 1), False), # exclude because too recent - ] + ( + CHARGED, + "Disposed By Judge", + SEVERITIES.MISDEMEANOR, + datetime(2019, 8, 1), + False, + ), # exclude because not convicted action + ( + CONVICTED, + "Disposed By Judge", + SEVERITIES.FELONY, + datetime(2019, 8, 1), + False, + ), # exclude because felony severity + ( + CONVICTED, + "Disposed By Judge", + SEVERITIES.MISDEMEANOR, + datetime(2024, 8, 1), + False, + ), # exclude because too recent + ], ) -def test_adult_misdemeanors(action, disposition_method, severity, date, should_be_included, batch, record1): +def test_adult_misdemeanors( + action, disposition_method, severity, date, should_be_included, batch, record1 +): offense = Offense.objects.create( ciprs_record=record1, disposition_method=disposition_method, diff --git a/dear_petition/petition/types/tests/test_dismissed.py b/dear_petition/petition/types/tests/test_dismissed.py index 5b4fd87a..a3a3d24e 100644 --- a/dear_petition/petition/types/tests/test_dismissed.py +++ b/dear_petition/petition/types/tests/test_dismissed.py @@ -14,15 +14,24 @@ @pytest.mark.parametrize( - "action, disposition_method, should_be_included", [ + "action, disposition_method, should_be_included", + [ # records that have data as they would from Portal (no action) ("", "No Probable Cause Found", True), - ("", "District Guilty - Judge", False), # exclude because not Portal dismissed disposition method + ( + "", + "District Guilty - Judge", + False, + ), # exclude because not Portal dismissed disposition method # records that have data as they would from CIPRS (disposition_method not one seen in Portal) (CHARGED, "No Probable Cause", True), (CONVICTED, "No Probable Cause", False), # exclude because not charged action - (CHARGED, "Disposed By Judge", False), # exclude because not CIPRS dismissed disposition method - ] + ( + CHARGED, + "Disposed By Judge", + False, + ), # exclude because not CIPRS dismissed disposition method + ], ) def test_dismissed(action, disposition_method, should_be_included, batch, record1): offense = Offense.objects.create( diff --git a/dear_petition/petition/types/tests/test_not_guilty.py b/dear_petition/petition/types/tests/test_not_guilty.py index 9fb30256..11eb1d5e 100644 --- a/dear_petition/petition/types/tests/test_not_guilty.py +++ b/dear_petition/petition/types/tests/test_not_guilty.py @@ -12,16 +12,37 @@ @pytest.mark.parametrize( - "action, verdict, disposition_method, should_be_included", [ + "action, verdict, disposition_method, should_be_included", + [ # records that have data as they would from Portal (no action or verdict) ("", "", "District Not Guilty - Judge", True), - ("", "", "District Guilty - Judge", False), # exclude because not Portal not-guilty disposition method + ( + "", + "", + "District Guilty - Judge", + False, + ), # exclude because not Portal not-guilty disposition method # records that have data as they would from CIPRS (disposition_method not one seen in Portal) (CHARGED, VERDICT_NOT_GUILTY, "Disposed By Judge", True), - (CONVICTED, VERDICT_NOT_GUILTY, "Disposed By Judge", False), # exclude because not charged action - (CHARGED, VERDICT_GUILTY, "Disposed By Judge", False), # exclude because not not-guilty verdict - (CHARGED, VERDICT_NOT_GUILTY, "Dismissed by Court", False), # exclude because meets dismissed criteria - ] + ( + CONVICTED, + VERDICT_NOT_GUILTY, + "Disposed By Judge", + False, + ), # exclude because not charged action + ( + CHARGED, + VERDICT_GUILTY, + "Disposed By Judge", + False, + ), # exclude because not not-guilty verdict + ( + CHARGED, + VERDICT_NOT_GUILTY, + "Dismissed by Court", + False, + ), # exclude because meets dismissed criteria + ], ) def test_not_guilty(action, verdict, disposition_method, should_be_included, batch, record1): offense = Offense.objects.create( diff --git a/dear_petition/petition/types/tests/test_underaged_convictions.py b/dear_petition/petition/types/tests/test_underaged_convictions.py index 0717e480..2a7ae833 100644 --- a/dear_petition/petition/types/tests/test_underaged_convictions.py +++ b/dear_petition/petition/types/tests/test_underaged_convictions.py @@ -12,9 +12,7 @@ def test_no_dob_conviction_not_included(batch, record1, non_dismissed_offense): # This tests the scenario where the record has no DOB (aka Portal), nor has the user added a DOB for the client. record1.dob = None record1.save() - offense_record = OffenseRecordFactory( - action="CONVICTED", offense=non_dismissed_offense - ) + offense_record = OffenseRecordFactory(action="CONVICTED", offense=non_dismissed_offense) assert offense_record not in batch.underaged_conviction_records() @@ -36,14 +34,12 @@ def test_overaged_conviction_not_included(batch, record1, non_dismissed_offense) def test_underaged_conviction_using_client_dob_included(batch, record1, non_dismissed_offense): # This tests the scenario where the record has no DOB (aka Portal), but the user has included a DOB for the client - + record1.dob = None record1.offense_date = datetime(2018, 1, 1) record1.save() - offense_record = OffenseRecordFactory( - action="CONVICTED", offense=non_dismissed_offense - ) + offense_record = OffenseRecordFactory(action="CONVICTED", offense=non_dismissed_offense) batch.client.dob = datetime(2000, 1, 2) batch.client.save() @@ -51,14 +47,14 @@ def test_underaged_conviction_using_client_dob_included(batch, record1, non_dism assert offense_record in batch.underaged_conviction_records() -def test_saving_client_dob_recalculates_underaged_convictions(batch, record1, non_dismissed_offense): +def test_saving_client_dob_recalculates_underaged_convictions( + batch, record1, non_dismissed_offense +): record1.dob = None record1.offense_date = datetime(2018, 1, 1) record1.save() - offense_record = OffenseRecordFactory( - action="CONVICTED", offense=non_dismissed_offense - ) + offense_record = OffenseRecordFactory(action="CONVICTED", offense=non_dismissed_offense) batch.client.dob = datetime(2000, 1, 2) batch.client.save() @@ -69,4 +65,3 @@ def test_saving_client_dob_recalculates_underaged_convictions(batch, record1, no batch.client.save() assert offense_record not in batch.underaged_conviction_records() - diff --git a/dear_petition/petition/types/underaged_convictions.py b/dear_petition/petition/types/underaged_convictions.py index bf4b1045..6fcbdaa1 100644 --- a/dear_petition/petition/types/underaged_convictions.py +++ b/dear_petition/petition/types/underaged_convictions.py @@ -23,7 +23,9 @@ def get_offense_records(batch, jurisdiction=""): else: dob = resolve_dob_from_offense_records(qs) if not dob: - return OffenseRecord.objects.none() # We can't determine this petition type without the date of birth + return ( + OffenseRecord.objects.none() + ) # We can't determine this petition type without the date of birth if jurisdiction: qs = qs.filter(offense__ciprs_record__jurisdiction=jurisdiction) diff --git a/dear_petition/petition/utils.py b/dear_petition/petition/utils.py index 0ba00f57..c7886fdf 100644 --- a/dear_petition/petition/utils.py +++ b/dear_petition/petition/utils.py @@ -212,4 +212,4 @@ def resolve_dob_from_offense_records(qs): f"This batch has multiple birthdates. Using the earliest birthdate {earliest_dob}" ) - return earliest_dob \ No newline at end of file + return earliest_dob diff --git a/dear_petition/portal/etl/extract.py b/dear_petition/portal/etl/extract.py index eda03c26..13ab4611 100644 --- a/dear_petition/portal/etl/extract.py +++ b/dear_petition/portal/etl/extract.py @@ -29,5 +29,5 @@ def parse_party_information(soup): return PartyInfo( defendant_name=party_info.parse_defendant_name(soup), defendant_race=party_info.parse_defendant_race(soup) or "", - defendant_sex=party_info.parse_defendant_sex(soup) or "" + defendant_sex=party_info.parse_defendant_sex(soup) or "", ) diff --git a/dear_petition/portal/etl/models.py b/dear_petition/portal/etl/models.py index 351469fc..bf01a607 100644 --- a/dear_petition/portal/etl/models.py +++ b/dear_petition/portal/etl/models.py @@ -32,7 +32,7 @@ class Charge(BaseModel): @field_validator("offense_date", "filed_date", "arrest_date", mode="before") @classmethod def parse_date(cls, v): - return parse_date(v); + return parse_date(v) def transform_severity(self): """Attempt to convert Portal's degree to CIPRS severity""" @@ -53,7 +53,7 @@ class CaseInfo(BaseModel): @field_validator("case_status_date", mode="before") @classmethod def parse_date(cls, v): - return parse_date(v); + return parse_date(v) class PartyInfo(BaseModel): @@ -72,7 +72,7 @@ class Disposition(BaseModel): @field_validator("event_date", mode="before") @classmethod def parse_date(cls, v): - return parse_date(v); + return parse_date(v) def transform_disposition_method(self) -> str: return self.criminal_disposition diff --git a/dear_petition/portal/etl/parsers/case_info.py b/dear_petition/portal/etl/parsers/case_info.py index 60969643..a1288ea9 100644 --- a/dear_petition/portal/etl/parsers/case_info.py +++ b/dear_petition/portal/etl/parsers/case_info.py @@ -23,8 +23,8 @@ def parse_case_information(soup): degree=parse_charge_degree(tr=tr) or "", offense_date=parse_charge_offense_date(tr=tr) or None, filed_date=parse_charge_filed_date(tr=tr) or None, - agency=parse_charge_agency(tr.findNext('tr')) or '', - arrest_date = parse_arrest_date(soup) or None, + agency=parse_charge_agency(tr.findNext("tr")) or "", + arrest_date=parse_arrest_date(soup) or None, ) ) ci = CaseInfo( @@ -84,6 +84,7 @@ def parse_case_status(soup): # status is always last, so select last one return soup.select("tr[ng-if*=caseInfo\\.CaseStatuses] span")[-1].text.strip() + @catch_parse_error def parse_charge_agency(tr): """ @@ -103,8 +104,9 @@ def parse_charge_agency(tr): """ - return tr.select_one("div[ng-if*='::charge.FilingAgencyDescription'] > div.roa-value > div:first-of-type").text.strip() - + return tr.select_one( + "div[ng-if*='::charge.FilingAgencyDescription'] > div.roa-value > div:first-of-type" + ).text.strip() @catch_parse_error @@ -244,7 +246,7 @@ def parse_arrest_date(soup): """ - arrest_date_div = soup.find('div', {'ng-if': '::arrest.ArrestDate'}) + arrest_date_div = soup.find("div", {"ng-if": "::arrest.ArrestDate"}) if not arrest_date_div: - return None; - return arrest_date_div.find('span', class_='ng-binding ng-scope').get_text(strip=True) + return None + return arrest_date_div.find("span", class_="ng-binding ng-scope").get_text(strip=True) diff --git a/dear_petition/portal/etl/parsers/party_info.py b/dear_petition/portal/etl/parsers/party_info.py index 10e6298b..20b7ee8f 100644 --- a/dear_petition/portal/etl/parsers/party_info.py +++ b/dear_petition/portal/etl/parsers/party_info.py @@ -40,8 +40,8 @@ def parse_defendant_race(soup):
White
- """ # noqa - race_div = soup.find('div', {'ng-if': '::party.Race'}) + """ # noqa + race_div = soup.find("div", {"ng-if": "::party.Race"}) if not race_div: return "" @@ -57,8 +57,8 @@ def parse_defendant_sex(soup):
Female
- """ # noqa - sex_div = soup.find('div', {'ng-if': '::party.Gender'}) + """ # noqa + sex_div = soup.find("div", {"ng-if": "::party.Gender"}) if not sex_div: return "" diff --git a/dear_petition/portal/tests/parsers/test_case_info.py b/dear_petition/portal/tests/parsers/test_case_info.py index f40513f7..dec7cafd 100644 --- a/dear_petition/portal/tests/parsers/test_case_info.py +++ b/dear_petition/portal/tests/parsers/test_case_info.py @@ -41,10 +41,9 @@ def test_parse_charge_file_date(self, soup): tr = soup.select_one(case_info.SELECT_OFFENSES) assert case_info.parse_charge_filed_date(tr) == "01/09/2001" - def test_parse_charge_agency(self, soup): tr = soup.select_one(case_info.SELECT_OFFENSES) - agency_tr = tr.findNext('tr') + agency_tr = tr.findNext("tr") assert agency_tr is not None assert case_info.parse_charge_agency(agency_tr) == "Creedmoor Police Department" diff --git a/dear_petition/portal/tests/transform/test_record.py b/dear_petition/portal/tests/transform/test_record.py index 371b8be6..6ceb5ccd 100644 --- a/dear_petition/portal/tests/transform/test_record.py +++ b/dear_petition/portal/tests/transform/test_record.py @@ -6,7 +6,7 @@ def test_transform_full_record(sample_record): "Case Information": { "Case Status": "Disposed", "Offense Date": "2001-01-01", - "Arrest Date": "2001-01-03" + "Arrest Date": "2001-01-03", }, "Defendant": { "Name": "DOE, JANE EMMA",