Merge pull request #1 from SEED-platform/add-public-disclosure

Add example of parsing public disclosure and uploading monthly readings as meters

.github/workflows/build.yml

@@ -0,0 +1,21 @@
+name: Test Installing Python Packages
+
+on: push
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/[email protected]
+      with:
+        python-version: '3.x'
+    - name: Install Poetry
+      uses: snok/[email protected]
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        poetry install

.gitignore

@@ -9,6 +9,9 @@ __pycache__/
 *.py[cod]
 *$py.class
 
+# to_upload files that are dynamically generated
+examples/data/public_disclosure/to_upload
+
 # C extensions
 *.so
 
@@ -21,8 +24,6 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
-lib64/
 parts/
 sdist/
 var/

.pre-commit-config.yaml

@@ -1,51 +1,42 @@
-exclude: |
-  (?x)(
-    ^docs/conf.py|
-    ^docs/license.rst
-  )
-
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
-      - id: trailing-whitespace
-      - id: check-added-large-files
-        args: ["--maxkb=50000"]
       - id: check-ast
+      - id: check-added-large-files
+        args: ['--maxkb=2000']
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-toml
+      - id: check-yaml
       - id: check-json
+      - id: name-tests-test
+        args: ["--pytest-test-first"]
+      - id: fix-byte-order-marker
+      - id: check-case-conflict
+      - id: check-docstring-first
+      - id: check-executables-have-shebangs
       - id: check-merge-conflict
       - id: check-xml
       - id: check-yaml
       - id: debug-statements
       - id: end-of-file-fixer
-      - id: requirements-txt-fixer
       - id: mixed-line-ending
-        args: ["--fix=auto"]
-  - repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v2.0.2
-    hooks:
-      - id: autopep8
-        args:
-          [
-            "--in-place",
-            "--aggressive",
-            "--aggressive",
-            "--recursive",
-            "--max-line-length=100",
-            "--ignore=E501,E402,W503,W504,E731",
-          ]
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
-        args: ["--ignore=E501,E402,W503,W504,E731,F401"]
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.0.0-alpha.9-for-vscode
+      - id: pretty-format-json
+        args: ['--autofix', '--no-sort-keys']
+        exclude: examples/.*.ipynb
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.6.1
     hooks:
-      - id: prettier
-        types_or: [css, yaml, markdown, html, scss, javascript]
-  - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.10.1
+      - id: nbstripout
+  # https://docs.astral.sh/ruff/integrations/#pre-commit
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.4
     hooks:
-      - id: isort
-        args: ["-m=VERTICAL_HANGING_INDENT"] # vertical hanging
+      # Run the linter
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix, --output-format=full]
+        types_or: [python, pyi, jupyter]
+      # Run the formatter
+      - id: ruff-format
+        types_or: [python, pyi, jupyter]

README.md

@@ -28,7 +28,7 @@ SEED needs to be configured for accessing its web application.
 Copy `seed-config-example.json` to `seed-config-dev.json` and fill
 in with the correct credentials. The format of the SEED configuration should contain the following:
 
-```
+```json
 {
     "name": "seed_api_test",
     "base_url": "https://dev1.seed-platform.org",
@@ -41,7 +41,7 @@ in with the correct credentials. The format of the SEED configuration should con
 
 ### Running Examples
 
-There are two options for running the examples, either open the `ipynb` files or execute the scripts in the 
+There are two options for running the examples, either open the `ipynb` files or execute the scripts in the
 examples directory. The `ipynb` files provide more flexibility to understand how the py-seed library works.
 
 ### TODO

examples/data/public_disclosure/mappings-base-dc-data.csv

@@ -0,0 +1,53 @@
+Raw Columns,units,SEED Table,SEED Columns
+X,,PropertyState,Unused X
+Y,,PropertyState,Unused Y
+OBJECTID,,PropertyState,Object ID
+PID,,PropertyState,pm_property_id
+SSL,,TaxLotState,jurisdiction_tax_lot_id
+PMPROPERTYID,,PropertyState,PM Property Old
+PROPERTYNAME,,PropertyState,property_name
+PMPARENTPROPERTYID,,PropertyState,pm_parent_property_id
+PARENTPROPERTYNAME,,PropertyState,parent_property_name
+REPORTINGYEAR,,PropertyState,Year Ending
+REPORTSTATUS,,PropertyState,Reporting Status
+ADDRESSOFRECORD,,PropertyState,address_line_1
+OWNEROFRECORD,,PropertyState,owner
+WARD,,PropertyState,Ward
+REPORTEDADDRESS,,PropertyState,Reported Address
+CITY,,PropertyState,city
+STATE,,PropertyState,state
+POSTALCODE,,PropertyState,postal_code
+YEARBUILT,,PropertyState,year_built
+PRIMARYPROPERTYTYPE_SELFSELECT,,PropertyState,property_type
+PRIMARYPROPERTYTYPE_EPACALC,,PropertyState,Property Tyoe (EPA)
+TAXRECORDFLOORAREA,ft**2,PropertyState,gross_floor_area
+REPORTEDBUILDINGGROSSFLOORAREA,ft**2,PropertyState,gross_floor_area_reported
+ENERGYSTARSCORE,,PropertyState,energy_score
+SITEEUI_KBTU_FT,kBtu/ft**2/year,PropertyState,site_eui
+WEATHERNORMALZEDSITEEUI_KBTUFT,kBtu/ft**2/year,PropertyState,site_eui_weather_normalized
+SOURCEEUI_KBTU_FT,kBtu/ft**2/year,PropertyState,source_eui
+WEATHERNORMALZEDSOUREUI_KBTUFT,kBtu/ft**2/year,PropertyState,source_eui_weather_normalized
+TOTGHGEMISSIONS_METRICTONSCO2E,MtCO2e/year,PropertyState,total_ghg_emissions
+TOTGHGEMISSINTENSITY_KGCO2EFT,kgCO2e/ft**2/year,PropertyState,total_ghg_emissions_intensity
+WATERSCORE_MFPROPERTIES,,PropertyState,Water Score (Multifamily)
+WATERUSE_ALLWATERSOURCES_KGAL,kGal,PropertyState,Water Use
+NATURALGASUSE_THERMS,therms,PropertyState,Natural Gas Use
+FUELOILANDDIESELFUELUSEKBTU,kBtu,PropertyState,Fuel Oil and Diesel
+METEREDAREAS_ENERGY,,PropertyState,Metered Areas (Energy)
+METEREDAREAS_WATER,,PropertyState,Metered Areas (Water)
+ELECTRICITYUSE_RENEWABLE_KWH,kWh,PropertyState,Electricity Use (Renewable)
+ELECTRICITYUSE_GRID_KWH,kWh,PropertyState,Electricity Use (Grid)
+DISTRCHILLEDWATER_KBTU,kBtu,PropertyState,Annual District Chilled Water [kBtu]
+DISTRHOTWATER_KBTU,kBtu,PropertyState,Annual District Hot Water [kBtu]
+DISTRSTEAM_KBTU,kBtu,PropertyState,Annual Steam [kBtu]
+LATITUDE,,PropertyState,latitude
+LONGITUDE,,PropertyState,longitude
+ADDRESSID,,PropertyState,Address ID
+XCOORD,,PropertyState,Unused XCOORD
+YCOORD,,PropertyState,Unused YCOORD
+GIS_LAST_MOD_DTTM,,PropertyState,Unused GIS_LAST_MOD_DTTM
+LASTUPDATE,,PropertyState,Unused LASTUPDATE
+BEPS_METRIC_TYPE,,PropertyState,BPS Metric Type
+BEPS_ENERGYSTAR,,PropertyState,BPS Target Energy Star Score
+BEPS_SOURCEEUI,kBtu/ft**2/year,PropertyState,BPS Target Normalized Source EUI
+PROPERTY_BEPS_METRIC_YEAR,,PropertyState,BPS Metric Year

examples/lib/helpers.py

@@ -0,0 +1,55 @@
+from itertools import islice
+
+
+# Helper functions
+def clean_up_name(name):
+    return name.replace("/", "_").replace(" ", "_").lower()
+
+
+def chunks(data, size):
+    it = iter(data)
+    for i in range(0, len(data), size):
+        yield {k: data[k] for k in islice(it, size)}
+
+
+def map_months_to_date(year, values=None):
+    month_map = {
+        "JANUARY": {"start_time": f"{year}-01-01", "end_time": f"{year}-01-31"},
+        "FEBRUARY": {"start_time": f"{year}-02-01", "end_time": f"{year}-02-{29 if (year % 4) == 0 else 28}"},
+        "MARCH": {"start_time": f"{year}-03-01", "end_time": f"{year}-03-31"},
+        "APRIL": {"start_time": f"{year}-04-01", "end_time": f"{year}-04-30"},
+        "MAY": {"start_time": f"{year}-05-01", "end_time": f"{year}-05-31"},
+        "JUNE": {"start_time": f"{year}-06-01", "end_time": f"{year}-06-30"},
+        "JULY": {"start_time": f"{year}-07-01", "end_time": f"{year}-07-31"},
+        "AUGUST": {"start_time": f"{year}-08-01", "end_time": f"{year}-08-31"},
+        "SEPTEMBER": {"start_time": f"{year}-09-01", "end_time": f"{year}-09-30"},
+        "OCTOBER": {"start_time": f"{year}-10-01", "end_time": f"{year}-10-31"},
+        "NOVEMBER": {"start_time": f"{year}-11-01", "end_time": f"{year}-11-30"},
+        "DECEMBER": {"start_time": f"{year}-12-01", "end_time": f"{year}-12-31"},
+    }
+
+    results = []
+
+    if values:
+        # Check if all the values are zero, if so then do not add them to the payload
+        if all(value == 0 for value in values.values()):
+            return []
+
+        # add in the values to the dictionary if they match the key
+        # Values can be in the form of the following:
+        # "ELECTRICITYUSE_KBTU_JANUARY": 3017508.8,
+        # "ELECTRICITYUSE_KBTU_FEBRUARY": 2722907.3,
+        # "ELECTRICITYUSE_KBTU_MARCH": 2728590.7,
+
+        for key, value in values.items():
+            for month in month_map:
+                if month in key:
+                    month_map[month]["reading"] = value
+                    month_map[month]["source_unit"] = "kBtu (Thousand BTU)"
+                    month_map[month]["conversion_factor"] = 1
+                    results.append(month_map[month])
+    else:
+        for key, value in month_map.items():
+            results.append(value)
+
+    return results

examples/seed_better.py

@@ -7,20 +7,20 @@
 
 from pyseed.seed_client import SeedClient
 
-seed_config = Path('seed-config-dev.json')
-data_dir = Path(__file__).parent.absolute() / 'data' / 'better'
+seed_config = Path("seed-config-dev.json")
+data_dir = Path(__file__).parent.absolute() / "data" / "better"
 
 client = SeedClient(None, connection_config_filepath=seed_config)
-client.get_org_by_name('better', set_org_id=True)
+client.get_org_by_name("better", set_org_id=True)
 
 # get or create the cycle
-cycle = client.get_or_create_cycle('BETTER Cycle', date(2020, 1, 1), date(2020, 12, 31), set_cycle_id=True)
+cycle = client.get_or_create_cycle("BETTER Cycle", date(2020, 1, 1), date(2020, 12, 31), set_cycle_id=True)
 
 # upload the data to seed
 client.upload_and_match_datafile(
-    'better-data',
-    str(data_dir / 'better-test-data.xlsx'),
-    'better mappings',
-    str(data_dir / 'better-mappings.csv'),
+    "better-data",
+    str(data_dir / "better-test-data.xlsx"),
+    "better mappings",
+    str(data_dir / "better-mappings.csv"),
     import_meters_if_exist=True,
 )

examples/seed_bps.py

@@ -1,18 +1,19 @@
 # # ./manage.py create_test_user_json --username [email protected] --file ../py-seed/seed-config.json --pyseed
 
+import sys
 from datetime import date
 from pathlib import Path
 
 from pyseed.seed_client import SeedClient
 
 # Location to SEED BPS Data -- this will be a checkout of the SEED repo and point to the {seed_repo}/seed/data/bps directory.
 
-seed_config = Path('seed-config-local.json')
-org_name = 'Program Tracking 1'
+seed_config = Path("seed-config-local.json")
+org_name = "Program Tracking 1"
 # path to seed repo checkout at the same level as pyseed-examples
-data_dir = Path(__file__).parent.parent.parent.absolute() / 'seed' / 'seed' / 'tests' / 'data' / 'bps'
+data_dir = Path(__file__).parent.parent.parent.absolute() / "seed" / "seed" / "tests" / "data" / "bps"
 # path to where the mapping files exist
-mappings_dir = Path(__file__).parent.absolute() / 'data' / 'bps'
+mappings_dir = Path(__file__).parent.absolute() / "data" / "bps"
 
 client = SeedClient(
     None,
@@ -30,59 +31,59 @@
 print()
 print("If this is not the correct instance, then verify the seed config file.")
 cont_resp = input("Continue [y/n]: ")
-if cont_resp.lower() == 'y':
+if cont_resp.lower() == "y":
     print("Continuing...")
 else:
     print("Exiting...")
-    exit()
+    sys.exit()
 
 # verify if they have the correct organization
 orgs = client.get_organizations()
-if org_name not in [org['name'] for org in orgs]:
+if org_name not in [org["name"] for org in orgs]:
     print(f"Organization '{org_name}' not found. Please create the organization and try again. You belong to the following organizations:")
     for org in orgs:
         print(f"\t{org['name']}")
-    exit()
+    sys.exit()
 
 # You will need to make sure that the org name is already created on SEED.
 client.get_org_by_name(org_name, set_org_id=True)
 
 # upload mapping profile for the ESPM webservice
-client.create_or_update_column_mapping_profile_from_file('ESPM Webservice', mappings_dir / 'espm-webservice-mappings.csv')
+client.create_or_update_column_mapping_profile_from_file("ESPM Webservice", mappings_dir / "espm-webservice-mappings.csv")
 
 # for cycle_year in range(2019, 2024):
 for cycle_year in range(2019, 2020):
-    cycle = client.get_or_create_cycle(f"{str(cycle_year)}", date(cycle_year, 1, 1), date(cycle_year, 12, 31), set_cycle_id=True)
+    cycle = client.get_or_create_cycle(f"{cycle_year!s}", date(cycle_year, 1, 1), date(cycle_year, 12, 31), set_cycle_id=True)
 
     # upload CBL data
-    upload_file_name = f'CBL-building-performance-standards-sample-{cycle_year}.xlsx'
+    upload_file_name = f"CBL-building-performance-standards-sample-{cycle_year}.xlsx"
     if (data_dir / upload_file_name).exists():
-        print(f'uploading {upload_file_name}')
+        print(f"uploading {upload_file_name}")
         client.upload_and_match_datafile(
-            'cbl-data',
+            "cbl-data",
             str(data_dir / upload_file_name),
-            'cbl mappings',
-            str(mappings_dir / 'cbl-mappings.csv'),
+            "cbl mappings",
+            str(mappings_dir / "cbl-mappings.csv"),
             import_meters_if_exist=False,
         )
     else:
-        print(f'Could not find {data_dir / upload_file_name}, exiting...')
-        exit()
+        print(f"Could not find {data_dir / upload_file_name}, exiting...")
+        sys.exit()
 
     # upload targets
-    upload_file_name = f'BPS-sample-Targets-{cycle_year}.xlsx'
+    upload_file_name = f"BPS-sample-Targets-{cycle_year}.xlsx"
     if (data_dir / upload_file_name).exists():
-        print(f'uploading {upload_file_name}')
+        print(f"uploading {upload_file_name}")
         client.upload_and_match_datafile(
-            'bps-target-data',
+            "bps-target-data",
             str(data_dir / upload_file_name),
-            'bps target mappings',
-            str(mappings_dir / 'bps-targets-mappings.csv'),
+            "bps target mappings",
+            str(mappings_dir / "bps-targets-mappings.csv"),
             import_meters_if_exist=False,
         )
     else:
-        print(f'Could not find {data_dir / upload_file_name}, exiting...')
-        exit()
+        print(f"Could not find {data_dir / upload_file_name}, exiting...")
+        sys.exit()
 
     # upload performance data
     # upload_file_name = f'BPS-sample-Targets-{cycle_year}.xlsx'
@@ -97,4 +98,4 @@
     #     )
     # else:
     #     print(f'Could not find {data_dir / upload_file_name}, exiting...')
-    #     exit()
+    #     sys.exit()