diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 7d6b03a..1f549bb 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -13,11 +13,11 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@master with: persist-credentials: false - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install @@ -34,8 +34,9 @@ jobs: run: | pytest -m 'not local' --cov=./ --cov-report=xml --maxfail=0 - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: + token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml flags: unittests name: codecov-umbrella diff --git a/.github/workflows/check_black.yml b/.github/workflows/check_black.yml index 6a0aa63..09c4d90 100644 --- a/.github/workflows/check_black.yml +++ b/.github/workflows/check_black.yml @@ -6,8 +6,8 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@master + - uses: actions/setup-python@v5 - uses: psf/black@stable with: options: "-l 79 --check" diff --git a/.github/workflows/deploy_docs.yml b/.github/workflows/deploy_docs.yml index 4e03b49..cf47180 100644 --- a/.github/workflows/deploy_docs.yml +++ b/.github/workflows/deploy_docs.yml @@ -9,19 +9,19 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 # If you're using actions/checkout@v2 you must set persist-credentials to false in most cases for the deployment to work correctly. + uses: actions/checkout@master with: persist-credentials: false - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: activate-environment: oguk-dev environment-file: environment.yml python-version: 3.9 auto-activate-base: false - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.9 diff --git a/.github/workflows/docs_check.yml b/.github/workflows/docs_check.yml index e80987b..437092f 100644 --- a/.github/workflows/docs_check.yml +++ b/.github/workflows/docs_check.yml @@ -6,11 +6,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 # If you're using actions/checkout@v2 you must set persist-credentials to false in most cases for the deployment to work correctly. + uses: actions/checkout@master with: persist-credentials: false - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.9 diff --git a/.github/workflows/version_check.yml b/.github/workflows/version_check.yml index 6647c8e..b9d6da5 100644 --- a/.github/workflows/version_check.yml +++ b/.github/workflows/version_check.yml @@ -7,12 +7,12 @@ jobs: name: Check versioning runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@master with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: - python-version: 3.7 + python-version: 3.9 - name: Check version number has been properly updated run: ".github/is-version-number-acceptable.sh" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index b2ed11b..0000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 - hooks: - - id: check-yaml - - id: end-of-file-fixer - - id: trailing-whitespace -- repo: https://github.com/psf/black - rev: 19.3b0 - hooks: - - id: black \ No newline at end of file diff --git a/PSL_catalog.json b/PSL_catalog.json new file mode 100644 index 0000000..5e4eea8 --- /dev/null +++ b/PSL_catalog.json @@ -0,0 +1,34 @@ +{ + "name": "OG-UK", + "img": "https://raw.githubusercontent.com/PSLmodels/OG-UK/main/docs/OG-UK_logo.png", + "banner_title": "OG-UK", + "banner_subtitle": "Large-scale open source overlapping generations model of UK fiscal policy", + "detailed_description": "https://raw.githubusercontent.com/PSLmodels/OG-UK/main/README.md", + "policy_area": "Policy analysis, policy reform, fiscal policy, tax, benefits, social security, public finance, macroeconomic analysis, general equilibrium, dynamic scoring, overlapping generations, OG, OLG", + "geography": "USA", + "language": "Python", + "maintainers": [ + { + "name": "Richard W. Evans", + "image": "https://github.com/rickecon.png", + "link": "https://github.com/rickecon" + }, + { + "name": "Jason DeBacker", + "image": "https://github.com/jdebacker.png", + "link": "https://github.com/jdebacker" + }, + { + "name": "Jon Pycroft", + "image": "https://avatars.githubusercontent.com/u/65815956?v=4", + "link": "https://github.com/jpycroft/" + } + ], + "links": { + "code_repository": "https://github.com/PSLmodels/OG-UK", + "user_documentation": "https://pslmodels.github.io/OG-UK", + "contributor_documentation": "https://pslmodels.github.io/OG-UK/content/contributing/contributor_guide.html", + "webapp": "", + "recent_changes": "https://github.com/PSLmodels/OG-UK/blob/main/CHANGELOG.md" + } +} diff --git a/environment.yml b/environment.yml index 9e3e565..4e545e2 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: oguk-dev channels: - conda-forge dependencies: -- python>=3.7.7, <=3.11 # This restriction can be removed as soon as these packages support Python 3.10 +- python>=3.7.7, <3.12 # This restriction can be removed as soon as these packages support Python 3.10 - ipython - setuptools - psutil @@ -24,13 +24,12 @@ dependencies: - coverage - requests - xlwt -- openpyxl - statsmodels - linearmodels - black -- pre-commit - jupyter - pip: - jupyter-book>=0.8.0 + - openpyxl>=3.1.2 - ogcore - PolicyEngine-UK>=0.38.6 diff --git a/examples/run_oguk.py b/examples/run_oguk.py index c4cfa3b..ca7f1be 100644 --- a/examples/run_oguk.py +++ b/examples/run_oguk.py @@ -5,7 +5,7 @@ import os import copy from policyengine_core.reforms import Reform -from policyengine_uk.api import * +from policyengine_uk.model_api import * from oguk.calibrate import Calibration from ogcore.parameters import Specifications from ogcore import output_tables as ot diff --git a/oguk/calibrate.py b/oguk/calibrate.py index 6114571..d445d33 100644 --- a/oguk/calibrate.py +++ b/oguk/calibrate.py @@ -26,11 +26,14 @@ def __init__( client=None, num_workers=1, ): - self.estimate_tax_functions = estimate_tax_functions self.estimate_beta = estimate_beta self.estimate_chi_n = estimate_chi_n if estimate_tax_functions: + if tax_func_path is not None: + run_micro = False + else: + run_micro = True self.tax_function_params = self.get_tax_function_parameters( p, iit_reform, @@ -38,7 +41,7 @@ def __init__( data, client, num_workers, - run_micro=True, + run_micro=run_micro, tax_func_path=tax_func_path, ) # if estimate_beta: @@ -69,7 +72,7 @@ def __init__( def get_tax_function_parameters( self, p, - iit_reform={}, + pit_reform={}, guid="", data="", client=None, @@ -109,12 +112,15 @@ def get_tax_function_parameters( # If run_micro is false, check to see if parameters file exists # and if it is consistent with Specifications instance if not run_micro: - dict_params, run_micro = self.read_tax_func_estimate(tax_func_path) + dict_params, run_micro = self.read_tax_func_estimate( + p, tax_func_path + ) + taxcalc_version = "Cached tax parameters, no taxcalc version" if run_micro: micro_data, taxcalc_version = get_micro_data.get_data( baseline=p.baseline, start_year=p.start_year, - reform=iit_reform, + reform=pit_reform, data=data, path=p.output_base, client=client, @@ -132,12 +138,11 @@ def get_tax_function_parameters( analytical_mtrs=p.analytical_mtrs, tax_func_type=p.tax_func_type, age_specific=p.age_specific, - reform=iit_reform, + reform=pit_reform, data=data, client=client, num_workers=num_workers, tax_func_path=tax_func_path, - graph_est=True, ) mean_income_data = dict_params["tfunc_avginc"][0] frac_tax_payroll = np.append( @@ -145,109 +150,75 @@ def get_tax_function_parameters( np.ones(p.T + p.S - p.BW) * dict_params["tfunc_frac_tax_payroll"][-1], ) - for key in dict_params: - try: - dict_params[key] = np.array(dict_params[key]) - except TypeError: - pass - # Reorder indices of tax function and tile for all years after - # budget window ends - num_etr_params = dict_params["tfunc_etr_params_S"].shape[2] - num_mtrx_params = dict_params["tfunc_mtrx_params_S"].shape[2] - num_mtry_params = dict_params["tfunc_mtry_params_S"].shape[2] - # First check to see if tax parameters that are used were - # estimated with a budget window and ages that are as long as - # the those implied based on the start year and model age. - # N.B. the tax parameters dictionary does not save the years - # that correspond to the parameter estimates, so the start year - # used there may name match what is used in a run that reads in - # some cached tax function parameters. Likewise for age. + # Conduct checks to be sure tax function params are consistent + # with the model run params_list = ["etr", "mtrx", "mtry"] - BW_in_tax_params = dict_params["tfunc_etr_params_S"].shape[0] - S_in_tax_params = dict_params["tfunc_etr_params_S"].shape[1] - if p.BW != BW_in_tax_params: + BW_in_tax_params = dict_params["BW"] + start_year_in_tax_params = dict_params["start_year"] + S_in_tax_params = len(dict_params["tfunc_etr_params_S"][0]) + # Check that start years are consistent in model and cached tax functions + if p.start_year != start_year_in_tax_params: print( - "Warning: There is a discrepency between the start" + "Input Error: There is a discrepancy between the start" + " year of the model and that of the tax functions!!" - + f"p.BW = {p.BW}, BW_in_tax_params = {BW_in_tax_params}" ) - # After printing warning, make it work by tiling - if p.BW > BW_in_tax_params: - for item in params_list: - dict_params["tfunc_" + item + "_params_S"] = np.concatenate( - ( - dict_params["tfunc_" + item + "_params_S"], - np.tile( - dict_params["tfunc_" + item + "_params_S"][ - :, -1, : - ].reshape(S_in_tax_params, 1, num_etr_params), - (1, p.BW - BW_in_tax_params, 1), - ), - ), - axis=1, - ) - dict_params["tfunc_avg_" + item] = np.append( - dict_params["tfunc_avg_" + item], - np.tile( - dict_params["tfunc_avg_" + item][-1], - (p.BW - BW_in_tax_params), - ), - ) + assert False + # Check that S is consistent in model and cached tax functions + # Note: even if p.age_specific = False, the arrays coming from + # ogcore.txfunc_est should be of length S if p.S != S_in_tax_params: print( - "Warning: There is a discrepency between the ages" + "Input Error: There is a discrepancy between the ages" + " used in the model and those in the tax functions!!" - + f"p.S = {p.S}, S_in_tax_params = {S_in_tax_params}" ) - # After printing warning, make it work by tiling - if p.S > S_in_tax_params: - for item in params_list: - dict_params["tfunc_" + item + "_params_S"] = np.concatenate( - ( - dict_params["tfunc_" + item + "_params_S"], - np.tile( - dict_params["tfunc_" + item + "_params_S"][ - -1, :, : - ].reshape(1, p.BW, num_etr_params), - (p.S - S_in_tax_params, 1, 1), - ), - ), - axis=0, - ) - etr_params = dict_params["tfunc_etr_params_S"] - mtrx_params = dict_params["tfunc_mtrx_params_S"] - mtry_params = dict_params["tfunc_mtry_params_S"] + assert False + + # Extrapolate tax function parameters for years after budget window + # list of list: BW x S - either an array of function at that element... + etr_params = [[None] * p.S] * p.T + mtrx_params = [[None] * p.S] * p.T + mtry_params = [[None] * p.S] * p.T + for s in range(p.S): + for t in range(p.T): + if t < p.BW: + etr_params[t][s] = dict_params["tfunc_etr_params_S"][t][s] + mtrx_params[t][s] = dict_params["tfunc_mtrx_params_S"][t][ + s + ] + mtry_params[t][s] = dict_params["tfunc_mtry_params_S"][t][ + s + ] + else: + etr_params[t][s] = dict_params["tfunc_etr_params_S"][-1][s] + mtrx_params[t][s] = dict_params["tfunc_mtrx_params_S"][-1][ + s + ] + mtry_params[t][s] = dict_params["tfunc_mtry_params_S"][-1][ + s + ] if p.constant_rates: print("Using constant rates!") - # Make all ETRs equal the average - etr_params = np.zeros(etr_params.shape) - # set shift to average rate - etr_params[: p.BW, :, 10] = np.tile( - dict_params["tfunc_avg_etr"].reshape(p.BW, 1), (1, p.S) - ) - etr_params[p.BW :, :, 10] = dict_params["tfunc_avg_etr"][-1] - - # # Make all MTRx equal the average - mtrx_params = np.zeros(mtrx_params.shape) - # set shift to average rate - mtrx_params[: p.BW, :, 10] = np.tile( - dict_params["tfunc_avg_mtrx"].reshape(p.BW, 1), (1, p.S) - ) - mtrx_params[p.BW :, :, 10] = dict_params["tfunc_avg_mtrx"][-1] - - # # Make all MTRy equal the average - mtry_params = np.zeros(mtry_params.shape) - # set shift to average rate - mtry_params[: p.BW, :, 10] = np.tile( - dict_params["tfunc_avg_mtry"].reshape(p.BW, 1), (1, p.S) - ) - mtry_params[p.BW :, :, 10] = dict_params["tfunc_avg_mtry"][-1] + # Make all tax rates equal the average + p.tax_func_type = "linear" + etr_params = [[None] * p.S] * p.T + mtrx_params = [[None] * p.S] * p.T + mtry_params = [[None] * p.S] * p.T + for s in range(p.S): + for t in range(p.T): + if t < p.BW: + etr_params[t][s] = dict_params["tfunc_avg_etr"][t] + mtrx_params[t][s] = dict_params["tfunc_avg_mtrx"][t] + mtry_params[t][s] = dict_params["tfunc_avg_mtry"][t] + else: + etr_params[t][s] = dict_params["tfunc_avg_etr"][-1] + mtrx_params[t][s] = dict_params["tfunc_avg_mtrx"][-1] + mtry_params[t][s] = dict_params["tfunc_avg_mtry"][-1] if p.zero_taxes: print("Zero taxes!") - etr_params = np.zeros(etr_params.shape) - mtrx_params = np.zeros(mtrx_params.shape) - mtry_params = np.zeros(mtry_params.shape) + etr_params = [[0] * p.S] * p.T + mtrx_params = [[0] * p.S] * p.T + mtry_params = [[0] * p.S] * p.T tax_param_dict = { "etr_params": etr_params, "mtrx_params": mtrx_params, @@ -279,23 +250,6 @@ def read_tax_func_estimate(self, p, tax_func_path): print("Tax Function Path Exists") dict_params = safe_read_pickle(tax_func_path) # check to see if tax_functions compatible - current_taxcalc = pkg_resources.get_distribution("taxcalc").version - try: - if current_taxcalc != dict_params["tax_calc_version"]: - print( - "WARNING: Tax function parameters estimated" - + " from Tax Calculator version that is not " - + " the one currently installed on this machine." - ) - print( - "Current TC version is ", - current_taxcalc, - ", Estimated tax functions from version ", - dict_params.get("tax_calc_version", None), - ) - flag = 1 - except KeyError: - pass try: if p.start_year != dict_params["start_year"]: print( @@ -306,11 +260,14 @@ def read_tax_func_estimate(self, p, tax_func_path): except KeyError: pass try: + p.BW = dict_params["BW"] # QUICK FIX if p.BW != dict_params["BW"]: print( - "Model budget window length is not " - + "consistent with tax function parameter " - + "estimates" + "Model budget window length is " + + str(p.BW) + + " but the tax function parameter " + + "estimates have a budget window length of " + + str(dict_params["BW"]) ) flag = 1 except KeyError: diff --git a/oguk/demographics.py b/oguk/demographics.py index d28e0e6..ffd35fb 100755 --- a/oguk/demographics.py +++ b/oguk/demographics.py @@ -19,6 +19,7 @@ b_zero_eq_artctan_func() ------------------------------------------------------------------------------- """ + # Import packages import os import numpy as np diff --git a/oguk/get_micro_data.py b/oguk/get_micro_data.py index 67bbccc..5abe36d 100644 --- a/oguk/get_micro_data.py +++ b/oguk/get_micro_data.py @@ -4,6 +4,7 @@ model (PolicyEngine-UK). ------------------------------------------------------------------------ """ + from dask import delayed, compute import dask.multiprocessing import numpy as np @@ -12,7 +13,7 @@ from policyengine_uk import Microsimulation import pandas as pd import warnings -from policyengine_uk.api import * +from policyengine_uk.model_api import * from policyengine_uk.data import EnhancedFRS, SynthFRS import logging @@ -48,7 +49,7 @@ def get_household_mtrs( """Calculates household MTRs with respect to a given variable. Args: - reform (ReformType): The reform to apply to the simulation. + reform (Reform): The reform to apply to the simulation. variable (str): The variable to increase. period (int): The period (year) to calculate the MTRs for. kwargs (dict): Additional arguments to pass to the simulation. diff --git a/oguk/tests/test_run_example.py b/oguk/tests/test_run_example.py index dfa7372..a367a41 100644 --- a/oguk/tests/test_run_example.py +++ b/oguk/tests/test_run_example.py @@ -3,6 +3,7 @@ work by making sure that it does not break (is still running) after 5 minutes (300 seconds). """ + import multiprocessing import time import os