Merge pull request #57 from rickecon/demog

Merging

.github/workflows/build_and_test.yml

@@ -1,4 +1,4 @@
-name: Build and test [Python 3.7]
+name: Build and test [Python 3.9]
 
 on: [push, pull_request]
 
@@ -7,36 +7,34 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest]
-        python-version: [3.7]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: [3.9]
+      fail-fast: false
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           persist-credentials: false
-
-      - name: Setup Miniconda using Python ${{ matrix.python-version }}
-        uses: conda-incubator/setup-miniconda@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
         with:
-          activate-environment: oguk-dev
-          environment-file: environment.yml
-          python-version: ${{ matrix.python-version }}
-          auto-activate-base: false
-
-      - name: Build
-        shell: bash -l {0}
+          python-version: 3.9
+      - name: Install
+        run: make install
+      - name: Conda info and list
+        shell: bash -el {0}
+        # Don't run this on Windows, as it fails
+        if: matrix.os != 'windows-latest'
         run: |
-          pip install -e . --user
-          pip install pytest-cov --user
-          pip install pytest-pycodestyle --user
+          conda info
+          conda list
       - name: Test
-        shell: bash -l {0}
         working-directory: ./
         run: |
-          pytest -m 'not local and not regression' --cov=./ --cov-report=xml
+          pytest -m 'not local' --cov=./ --cov-report=xml --maxfail=0
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3
         with:
           files: ./coverage.xml
           flags: unittests

.github/workflows/check_black.yml

@@ -10,4 +10,5 @@ jobs:
       - uses: actions/setup-python@v2
       - uses: psf/black@stable
         with:
-          args: ". -l 79 --check"
+          options: "-l 79 --check"
+          src: "."

.github/workflows/deploy_docs.yml

@@ -16,17 +16,22 @@ jobs:
       - name: Setup Miniconda
         uses: conda-incubator/setup-miniconda@v2
         with:
-          activate-environment: ogusa-dev
+          activate-environment: oguk-dev
           environment-file: environment.yml
-          python-version: 3.7
+          python-version: 3.9
           auto-activate-base: false
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
 
       - name: Build  # Build Jupyter Book
         shell: bash -l {0}
         run: |
-          pip install jupyter-book
-          pip install sphinxcontrib-bibtex==1.0.0
           pip install -e .
+          pip install jupyter-book>=0.11.3
+          pip install sphinxcontrib-bibtex>=2.0.0
+          pip install policyengine-uk
           cd docs
           jb build ./book
 

.github/workflows/docs_check.yml

@@ -9,20 +9,24 @@ jobs:
         uses: actions/checkout@v2 # If you're using actions/checkout@v2 you must set persist-credentials to false in most cases for the deployment to work correctly.
         with:
           persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
 
       - name: Setup Miniconda
         uses: conda-incubator/setup-miniconda@v2
         with:
           activate-environment: oguk-dev
           environment-file: environment.yml
-          python-version: 3.7
+          python-version: 3.9
           auto-activate-base: false
 
       - name: Build # Build Jupyter Book
         shell: bash -l {0}
         run: |
-          pip install jupyter-book
-          pip install sphinxcontrib-bibtex==1.0.0
-          pip install -e .
+          make install
+          pip install jupyter-book>=0.11.3
+          pip install sphinxcontrib-bibtex>=2.0.0
           cd docs
           jb build ./book

.gitignore

@@ -30,7 +30,7 @@ htmlcov/*
 *.fls
 *.m~
 *.sublime*
-.DS_Store
+*.DS_Store*
 *puf.csv
 *tax_func_loop_inputs_large.pkl
 */OUTPUT/*
@@ -40,7 +40,6 @@ envs/
 .ipynb_checkpoints*
 */__pycache__/*
 docs/book/_build/*
-docs/build*
-examples/OG-UK_example_plots/*
-examples/oguk_example_output.csv
-examples/OG-UK-Example/*
+*/OG-UK_example_plots/*
+*oguk_example_output.csv
+*/OG-UK-Example/*

CHANGELOG.md

@@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0).
 
+## [0.2.0] - 2023-01-16
+
+### Updated
+
+* Updated all OpenFisca references to PolicyEngine. This represents a big change in the underlying microsimulation dependency structure
+* Updated `get_micro_data.py` and `test_get_micro_data.py` with `PolicyEngine` references instead of `OpenFisca`.
+* Updated `demographics.py` to use the UN data portal. Also updated corresponding structures in `calibrate.py`
+* Added four `.csv` files to `oguk/data/demographic/`. This files allow for `demographics.py` to have the option to not download the data from the UN data portal.
+* Updated `environment.yml` and `setup.py`.
+* Updated `run_oguk.py` with more consistent specification, and updated references from `openfisca` to `policyengine`.
+* Small updates to `.gitignore`, `README.md`, `demographics.rst`, `get_micro_data.rst`, and `tax_functions.md`.
+* Deleted `pyproject.toml` which was just a reference to the black package.
+* Updates the `CHANGELOG.md` and updates the version number in `setup.py`.
+* Updates the GitHub Actions `deploy_docs.yml`, `docs_check.yml`, and `build_and_test.yml`.
+
 ## [0.1.2] - 2022-07-29
 
 ### Fixed

Makefile

@@ -3,3 +3,7 @@ format:
 	black . -l 79
 test:
 	pytest .
+install:
+	pip install policyengine-uk
+	pip install -e .[dev]
+	pip install --upgrade jsonschema[format-nongpl]

README.md

@@ -1,5 +1,7 @@
-[![Python 3.7+](https://img.shields.io/badge/python-3.7%2B-blue.svg)](https://www.python.org/downloads/release/python-377/)
-[![Build Status](https://travis-ci.com/PSLmodels/OG-UK.svg?branch=master)](https://travis-ci.com/PSLmodels/OG-UK)
+[![Python 3.9](https://img.shields.io/badge/python-3.9-blue.svg)](https://www.python.org/downloads/release/python-3916/)
+![example event parameter](https://github.com/PSLmodels/OG-Core/actions/workflows/build_and_test.yml/badge.svg?branch=main)
+![example event parameter](https://github.com/PSLmodels/OG-Core/actions/workflows/deploy_docs.yml/badge.svg?branch=main)
+![example event parameter](https://github.com/PSLmodels/OG-Core/actions/workflows/check_black.yml/badge.svg?branch=main)
 [![Codecov](https://codecov.io/gh/PSLmodels/OG-UK/branch/main/graph/badge.svg)](https://codecov.io/gh/PSLmodels/OG-UK)
 
 # OG-UK
@@ -9,7 +11,7 @@ OG-UK is an overlapping-generations (OG) model of the economy of the United King
 
 ## Disclaimer
 
-The model is currently under development.  PLEASE DO NOT USE OR CITE THE MODEL'S OUTPUT FOR ANY PURPOSE.
+The model is currently under development. PLEASE DO NOT USE OR CITE THE MODEL'S OUTPUT FOR ANY PURPOSE.
 
 
 ## Using/contributing to OG-UK
@@ -21,7 +23,7 @@ The model is currently under development.  PLEASE DO NOT USE OR CITE THE MODEL'S
 * Then install by `pip install -e .`
 * Navigate to `./examples`
 * Run the model with an example reform from terminal/command prompt by typing `python run_og_uk.py examples.small_ubi_reform.ubi_reform`
-* You can adjust the `./run_examples/run_og_uk.py` by adjusting the individual income tax reform (using an OpenFisca `Reform` object) or other model parameters specified in a dictionary and passed to the `Specifications.update_specification()` method.
+* You can adjust the `./run_examples/run_og_uk.py` by adjusting the individual income tax reform (using a PolicyEngine-Core `Reform` object) or other model parameters specified in a dictionary and passed to the `Specifications.update_specification()` method.
 * Model outputs will be saved in the following files:
   * `./examples/OG-UK_example_plots`
     * This folder will contain a number of plots generated from OG-UK to help you visualize the output from your run

docs/book/content/api/demographics.rst

@@ -0,0 +1,15 @@
+.. _demographics:
+
+Demographics Functions
+========================
+
+**demographics.py modules**
+
+oguk.demographics
+------------------------------------------
+
+.. automodule:: oguk.demographics
+  :members: get_un_fert_data, get_un_mort_data, get_wb_infmort_rate,
+    get_un_pop_data, get_fert, get_mort, pop_rebin, get_imm_resid,
+    immsolve, get_pop_objs, extrap_exp_3, extrap_arctan_3,
+    ab_zero_eqs_exp_func, b_zero_eq_arctan_func

docs/book/content/api/get_micro_data.rst

@@ -1,6 +1,6 @@
 .. _get_micro_data:
 
-Functions to Get Micro-data from OpenFisca-UK
+Functions to Get Micro-data from PolicyEngine-UK
 =================================================
 
 **get_micro_data.py classes, methods, and modules**

docs/book/content/calibration/tax_functions.md

@@ -67,9 +67,9 @@ The second difficulty in modeling realistic tax and incentive detail is the need
 (SecTaxCalcMicro)=
 ## Microeconomic Data
 
-  For `OG-UK`, we use an open-source microsimulation model called [`OpenFisca-UK`](https://policyengine.github.io/openfisca-uk//) that uses microeconomic data on U.K. households from the [Family Resources Survey (FRS)](https://www.gov.uk/government/collections/family-resources-survey--2), [HMRC Survey of Personal Incomes (SPI)](https://data.gov.uk/dataset/5908f917-7dfa-451d-81ac-e942e00884a2/hmrc-survey-of-personal-incomes-spi), [Living Costs and Food Survey (LCFS)](https://www.ons.gov.uk/peoplepopulationandcommunity/personalandhouseholdfinances/incomeandwealth/methodologies/livingcostsandfoodsurvey), and [Wealth and Assets Survey QMI (WAS)](https://www.ons.gov.uk/peoplepopulationandcommunity/personalandhouseholdfinances/debt/methodologies/wealthandassetssurveyqmi).[^openfisca_note] All of these data are public, except for the FRS. For users that do not have access to the FRS, `OpenFisca-UK` includes a synthetic FRS option.
+  For `OG-UK`, we use an open-source microsimulation model called [`PolicyEngine-UK`](https://github.com/PolicyEngine/policyengine-uk) that uses microeconomic data on U.K. households from the [Family Resources Survey (FRS)](https://www.gov.uk/government/collections/family-resources-survey--2), [HMRC Survey of Personal Incomes (SPI)](https://data.gov.uk/dataset/5908f917-7dfa-451d-81ac-e942e00884a2/hmrc-survey-of-personal-incomes-spi), [Living Costs and Food Survey (LCFS)](https://www.ons.gov.uk/peoplepopulationandcommunity/personalandhouseholdfinances/incomeandwealth/methodologies/livingcostsandfoodsurvey), and [Wealth and Assets Survey QMI (WAS)](https://www.ons.gov.uk/peoplepopulationandcommunity/personalandhouseholdfinances/debt/methodologies/wealthandassetssurveyqmi).[^policyengine_note] All of these data are public, except for the FRS. For users that do not have access to the FRS, `PolicyEngine-UK` includes a synthetic FRS option.
 
-  `OpenFisca-UK` starts with the underlying population microeconomic data, in which each observation is a filer with a population weight that renders the sample representative. It then processes the relevant income and demographic characteristics in order to calculate the tax liability of each individual, according to the tax and benefit law of the U.K.. `OpenFisca-UK` can then calculate effective tax rates for all of these individuals, thereby creating a sample of how ETR's are related to other variables in our `OG-UK` model, such as total income $x + y$, labor income $x$, and capital income $y$. `OpenFisca-UK` can also generate marginal tax rates by adding a dollar to each filer's income of a particular type and calculate how the filer's tax liability changes. This is a finite difference calculation of a derivative.
+  `PolicyEngine-UK` starts with the underlying population microeconomic data, in which each observation is a filer with a population weight that renders the sample representative. It then processes the relevant income and demographic characteristics in order to calculate the tax liability of each individual, according to the tax and benefit law of the U.K.. `PolicyEngine-UK` can then calculate effective tax rates for all of these individuals, thereby creating a sample of how ETR's are related to other variables in our `OG-UK` model, such as total income $x + y$, labor income $x$, and capital income $y$. `PolicyEngine-UK` can also generate marginal tax rates by adding a dollar to each filer's income of a particular type and calculate how the filer's tax liability changes. This is a finite difference calculation of a derivative.
 
   [TODO: Update this paragraph and associated figure, which currently come from `OG-USA`.] Figure {numref}`FigTaxCalcETRtotinc` shows a scatter plot of $ETR$'s for 43-year-olds in 2017 and unadjusted gross income $x + y$. It is clear that $ETR$ is positively related to income. It is also clear that a significant number of filers have a negative $ETR$. We will discuss in Section {ref}`SecTaxCalcFuncs` the functional form `OG-UK` uses to best capture the main characteristics of these ETR data.
 
@@ -81,7 +81,7 @@ The second difficulty in modeling realistic tax and incentive detail is the need
   Plot of estimated $ETR$ functions: $t=2017$ and $s=43$ under current law
   ```
 
-  [TODO: Update this paragraph and associated figure, which currently come from `OG-USA`.] Figure {numref}`FigTaxCalc3Dtaxrates` shows 3D scatter plots of $ETR$, $MTRx$, and $MTRy$ (and a histogram of the data) with the labor income and capital income, separately, of each age-42 filer in 2017, generated by `OpenFisca-UK`. This figure presents the main visual evidence for the functional form we use to fit tax functions to these data in Section {ref}`SecTaxCalcFuncs`. Figure {numref}`FigTaxCalc3Dtaxrates` presents strong evidence that the tax rate---$ETR$, $MTRx$, and $MTRy$---is most accurately modeled as a function of labor income and capital income, separately: $\tau(x,y)$.
+  [TODO: Update this paragraph and associated figure, which currently come from `OG-USA`.] Figure {numref}`FigTaxCalc3Dtaxrates` shows 3D scatter plots of $ETR$, $MTRx$, and $MTRy$ (and a histogram of the data) with the labor income and capital income, separately, of each age-42 filer in 2017, generated by `PolicyEngine-UK`. This figure presents the main visual evidence for the functional form we use to fit tax functions to these data in Section {ref}`SecTaxCalcFuncs`. Figure {numref}`FigTaxCalc3Dtaxrates` presents strong evidence that the tax rate---$ETR$, $MTRx$, and $MTRy$---is most accurately modeled as a function of labor income and capital income, separately: $\tau(x,y)$.
 
 
   ```{figure} ./images/Age42_2017_scatters.png
@@ -293,7 +293,7 @@ The second difficulty in modeling realistic tax and incentive detail is the need
 (SecTaxfootnotes)=
 ## Footnotes
 
-  [^openfisca_note]:`OpenFisca-UK` is available through an open-source repository [https://github.com/PolicyEngine/openfisca-uk](https://github.com/PolicyEngine/openfisca-uk), with online documentation available at [https://policyengine.github.io/openfisca-uk](https://policyengine.github.io/openfisca-uk).
+  [^policyengine_note]:`PolicyEngine-UK` is available through an open-source repository https://github.com/PolicyEngine/policyengine-uk, with online documentation available at https://policyengine.github.io/policyengine-uk/.
 
   [^interpolation_note]: We use two criterion to determine whether the function should be interpolated. First, we require a minimum number of observations of filers of that age and in that tax year. Second, we require that that sum of squared errors meet a predefined threshold.
 

docs/book/content/intro/intro.md

@@ -13,7 +13,7 @@
 (Sec_CoreMaintainers)=
 ## Core Maintainers
 
-[Jason DeBacker](https://www.jasondebacker.com/) (GitHub handle [@jdebacker](https://github.com/jdebacker)), [Richard W. Evans](https://sites.google.com/site/rickecon/) (GitHub handle [@rickecon](https://github.com/rickecon)), and [Jonahtan Pycroft](https://www.linkedin.com/in/jonathan-pycroft-9839756/) (GitHub handle [@jpycroft](https://github.com/jpycroft/)) are the core maintainers of `OG-UK`. If you have questions about or contributions to the model or repository, please submit a GitHub "Issue" described in the {ref}`Sec_GitHubIssue` subsection or "Pull Request" as described in the {ref}`Sec_GitHubPR` subsection of the {ref}`Sec_Workflow` section of the `OG-UK` {ref}`Chap_ContribGuide`.
+[Jason DeBacker](https://www.jasondebacker.com/) (GitHub handle [@jdebacker](https://github.com/jdebacker)), [Richard W. Evans](https://sites.google.com/site/rickecon/) (GitHub handle [@rickecon](https://github.com/rickecon)), and [Jonathan Pycroft](https://www.linkedin.com/in/jonathan-pycroft-9839756/) (GitHub handle [@jpycroft](https://github.com/jpycroft/)) are the core maintainers of `OG-UK`. If you have questions about or contributions to the model or repository, please submit a GitHub "Issue" described in the {ref}`Sec_GitHubIssue` subsection or "Pull Request" as described in the {ref}`Sec_GitHubPR` subsection of the {ref}`Sec_Workflow` section of the `OG-UK` {ref}`Chap_ContribGuide`.
 
 
 (Sec_Disclaimer)=

environment.yml

@@ -2,10 +2,18 @@ name: oguk-dev
 channels:
 - conda-forge
 dependencies:
-- python<=3.7.11
-- dask==2.30.0
-- dask-core==2.30.0
-- distributed==2.30.1
+- python>=3.7.7, <=3.11  # This restriction can be removed as soon as these packages support Python 3.10
+- ipython
+- setuptools
+- psutil
+- openssl=1.1.1  # This is a requirement of the UN data portal which we are using in demographics.py
+- numpy<=1.21.2  # This restriction can be removed as soon as Numba supports NumPy 1.22
+- scipy>=1.7.1
+- pandas>=1.2.5
+- matplotlib
+- dask>=2.30.0
+- dask-core>=2.30.0
+- distributed>=2.30.1
 - paramtools>=0.15.0
 - pip
 - pytest>=6.0
@@ -15,9 +23,14 @@ dependencies:
 - pylint
 - coverage
 - requests
+- xlwt
+- openpyxl
+- statsmodels
+- linearmodels
 - black
 - pre-commit
+- jupyter
 - pip:
   - jupyter-book>=0.8.0
   - ogcore
-  - openfisca-uk>=0.21.0
+  - PolicyEngine-UK>=0.38.6