eCourts Portal import proof of concept

config/urls.py

@@ -37,6 +37,7 @@
         name="password_reset_complete",
     ),
     path("sendgrid/", include("dear_petition.sendgrid.urls")),
+    path("portal/", include("dear_petition.portal.urls")),
     # React SPA:
     path(r"", index, name="index"),
     re_path(r"^(?:.*)/?$", index, name="index-others"),

dear_petition/petition/constants.py

@@ -97,6 +97,9 @@
     "No Probable Cause",
     "Never To Be Served",
     "Deferred Proceeding or Deferred Prosecution Dismissal",
+    # Portal:
+    "VD-District Dismissals w/o Leave by DA - No Plea Agreement",
+    "VD-Superior Dismissals w/o Leave by DA - No Plea Agreement",
 )
 
 DISP_METHOD_SUPERSEDING_INDICTMENT = "SUPERSEDING INDICTMENT OR PROCESS"
@@ -167,3 +170,8 @@
     "number_of_charges",
     "created",
 )
+
+SEVERITY_FELONY = "FELONY"
+SEVERITY_MISDEMEANOR = "MISDEMEANOR"
+CHARGED_DEGREE_FELONY = ("FH", "FNC")
+CHARGED_DEGREE_MISDEMEANOR = ("MNC",)

dear_petition/petition/etl/load.py

@@ -56,6 +56,8 @@ def import_ciprs_records(files, user, parser_mode, batch_label=""):
 
 
 def create_batch_petitions(batch):
+    # Start with a fresh petition list
+    batch.petitions.all().delete()
     # Dismissed
     create_petitions_from_records(batch, DISMISSED)
     # Not guilty
@@ -78,9 +80,13 @@ def create_petitions_from_records(batch, form_type):
             jurisdiction=petition_type["jurisdiction"],
             county=petition_type["county"],
         )
-        sheriff_agency = pm.Contact.get_sherriff_office_by_county(petition_type["county"])
+        sheriff_agency = pm.Contact.get_sherriff_office_by_county(
+            petition_type["county"]
+        )
         if sheriff_agency is not None:
-            logger.info(f"Detected {sheriff_agency.name} as {petition_type['county']} county's sherrif's office. Adding as default agency.")
+            logger.info(
+                f"Detected {sheriff_agency.name} as {petition_type['county']} county's sherrif's office. Adding as default agency."
+            )
             petition.agencies.add(sheriff_agency)
         link_offense_records(petition)
         logger.info(
@@ -143,7 +149,6 @@ def assign_agencies_to_documents(petition):
     first_iteration = True
     i = 0
     while True:
-
         current_document_agencies = agencies[
             i : (i + 3)
         ]  # 3 boxes for agencies per document

dear_petition/portal/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class PortalConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "dear_petition.portal"

dear_petition/portal/etl/extract.py

@@ -0,0 +1,29 @@
+from bs4 import BeautifulSoup
+
+from .models import CaseSummary, PartyInfo, PortalRecord
+from .parsers import case_summary, dispositions, case_info, party_info
+
+
+def extract_portal_record(source):
+    """Parse HTML source to extract eCourts Portal record"""
+    soup = BeautifulSoup(source, features="html.parser")
+    return PortalRecord(
+        case_summary=parse_case_summary(soup),
+        case_info=case_info.parse_case_information(soup),
+        party_info=parse_party_information(soup),
+        dispositions=dispositions.parse_dispositions(soup),
+    )
+
+
+def parse_case_summary(soup):
+    """Case Summary section"""
+    return CaseSummary(
+        case_number=case_summary.parse_case_number(soup) or "",
+        county=case_summary.parse_county(soup) or "",
+        court=case_summary.parse_court(soup) or "",
+    )
+
+
+def parse_party_information(soup):
+    """Party Information section"""
+    return PartyInfo(defendant_name=party_info.parse_defendant_name(soup))

dear_petition/portal/etl/load.py

@@ -0,0 +1,21 @@
+import logging
+
+from dear_petition.petition.models import Batch, CIPRSRecord
+from dear_petition.petition.etl.load import create_batch_petitions
+
+from .transform import transform_portal_record
+
+__all__ = ("import_portal_record",)
+
+logger = logging.getLogger(__name__)
+
+
+def import_portal_record(user, source: str, location: str):
+    """Import eCourts Portal records into models."""
+    logger.info("Importing Portal record")
+    data = transform_portal_record(source, location)
+    batch, _ = Batch.objects.get_or_create(user=user, label=data["Defendant"]["Name"])
+    record = CIPRSRecord(batch=batch, data=data)
+    record.refresh_record_from_data()
+    record.save()
+    create_batch_petitions(batch)

dear_petition/portal/etl/models.py

@@ -0,0 +1,101 @@
+import datetime as dt
+from typing import List, Union
+
+from pydantic import BaseModel, field_validator
+
+from dear_petition.petition import constants
+
+
+class CaseSummary(BaseModel):
+    case_number: str
+    county: str
+    court: str
+
+
+class Charge(BaseModel):
+    number: Union[int, None]
+    offense: str
+    statute: str
+    degree: str
+    offense_date: Union[dt.date, None]
+    filed_date: Union[dt.date, None]
+
+    @field_validator("offense_date", "filed_date", mode="before")
+    @classmethod
+    def parse_date(cls, v):
+        if isinstance(v, str):
+            return dt.datetime.strptime(v, "%m/%d/%Y")
+        return v
+
+    def transform_severity(self):
+        """Attempt to convert Portal's degree to CIPRS severity"""
+        severity = self.degree
+        if self.degree in constants.CHARGED_DEGREE_FELONY:
+            severity = constants.SEVERITY_FELONY
+        elif self.degree in constants.CHARGED_DEGREE_MISDEMEANOR:
+            severity = constants.SEVERITY_MISDEMEANOR
+        return severity
+
+
+class CaseInfo(BaseModel):
+    case_type: str
+    case_status: str
+    case_status_date: Union[dt.date, None]
+    charges: List[Charge]
+
+    @field_validator("case_status_date", mode="before")
+    @classmethod
+    def parse_date(cls, v):
+        if isinstance(v, str):
+            return dt.datetime.strptime(v, "%m/%d/%Y")
+        return v
+
+
+class PartyInfo(BaseModel):
+    defendant_name: str
+
+
+class Disposition(BaseModel):
+    event_date: Union[dt.date, None]
+    event: str
+    charge_number: int
+    charge_offense: str
+    criminal_disposition: str
+
+    @field_validator("event_date", mode="before")
+    @classmethod
+    def parse_date(cls, v):
+        if isinstance(v, str):
+            return dt.datetime.strptime(v, "%m/%d/%Y")
+        return v
+
+    def is_dismissed(self) -> bool:
+        return self.criminal_disposition in constants.DISMISSED_DISPOSITION_METHODS
+
+    def transform_action(self) -> str:
+        action = self.event
+        if self.is_dismissed():
+            action = constants.CHARGED
+        return action
+
+    def transform_disposition_method(self) -> str:
+        if self.is_dismissed():
+            return constants.DISTRICT_COURT_WITHOUT_DA_LEAVE
+        return self.criminal_disposition
+
+
+class PortalRecord(BaseModel):
+    case_summary: CaseSummary
+    case_info: CaseInfo
+    party_info: PartyInfo
+    dispositions: List[Disposition]
+
+    def get_charge_by_number(self, charge_number: int):
+        """Return matching CaseInfo.charges Charge by charge_number"""
+        for charge in self.case_info.charges:
+            if charge.number == charge_number:
+                return charge
+
+    def transform_offense_date(self) -> dt.date:
+        offense_dates = [c.offense_date for c in self.case_info.charges]
+        return min(offense_dates).isoformat()