rmh-classifier-metrics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This document trains a classifier on the Rate My Hydrograph ratings.\n",
    "\n",
    "The classifier gets as input the metrics of model a and b."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import torch\n",
    "import itertools\n",
    "from collections import defaultdict\n",
    "from datetime import datetime\n",
    "\n",
    "from torch import nn\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import xarray\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import KFold\n",
    "from tqdm import tqdm\n",
    "from pathlib import Path\n",
    "from neuralhydrology.evaluation.metrics import calculate_all_metrics, nse, kge\n",
    "from neuralhydrology.evaluation.signatures import calculate_all_signatures\n",
    "\n",
    "plt.rc('font', **{'family':'serif','serif': ['Computer Modern']})\n",
    "plt.rcParams.update({\"text.usetex\": True,})\n",
    "plt.rc('font', size=8)\n",
    "plt.rc('axes', labelsize=8)\n",
    "plt.rc('legend', fontsize=8)\n",
    "\n",
    "df = pd.read_csv('data/rmh-stage1.csv', index_col=0)\n",
    "\n",
    "mm_to_inch = lambda mm: mm * 0.03937008"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>focus_flood-modeling</th>\n",
       "      <th>focus_water-resources</th>\n",
       "      <th>focus_drought-modeling</th>\n",
       "      <th>focus_waterquality</th>\n",
       "      <th>focus_hydropower</th>\n",
       "      <th>focus_other</th>\n",
       "      <th>focus_consulting</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   focus_flood-modeling  focus_water-resources  focus_drought-modeling  \\\n",
       "0                  True                   True                    True   \n",
       "1                  True                   True                    True   \n",
       "2                 False                  False                   False   \n",
       "3                 False                  False                   False   \n",
       "4                  True                   True                    True   \n",
       "\n",
       "   focus_waterquality  focus_hydropower  focus_other  focus_consulting  \n",
       "0                True             False        False             False  \n",
       "1                True             False        False             False  \n",
       "2               False             False         True             False  \n",
       "3               False             False         True             False  \n",
       "4                True             False        False             False  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "areas = ['flood-modeling',\n",
    "         'water-resources',\n",
    "         'drought-modeling',\n",
    "         'waterquality',\n",
    "         'hydropower',\n",
    "         'other',\n",
    "         'consulting']  # 'social-sciences', 'insurance' have very few participants, so we merge them into 'other'.\n",
    "areas_df = pd.DataFrame(columns=[f'focus_{a}' for a in areas], index=df.index, dtype=bool)\n",
    "areas_df.loc[:, :] = False\n",
    "for idx, vals in df['focus_areas'].str.split(',').iteritems():\n",
    "    for v in vals:\n",
    "        if v.lower() in areas:\n",
    "            areas_df.loc[idx, f'focus_{v}'] = True\n",
    "        else:\n",
    "            areas_df.loc[idx, 'focus_other'] = True\n",
    "areas_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load observations and predictions from the GRIP-GL study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(base_dir):\n",
    "    obs_file = base_dir / 'all_gauges.nc'\n",
    "    if not obs_file.exists():\n",
    "        raise ValueError(f'Observations netCDF file not found at {obs_file}')\n",
    "    # netcdfs only have a numeric dimension \"nstations\" that maps to the station_id variable.\n",
    "    # for easier processing, we directly make the station_id the dimension.\n",
    "    obs = xarray.load_dataset(obs_file).swap_dims({'nstations': 'station_id'})\n",
    "\n",
    "    # load hydrographs from individual models\n",
    "    hydrographs = {'Q': obs['Q']}\n",
    "    model_dirs = [d for d in base_dir.glob('model/*') if d.is_dir()]\n",
    "    for model_dir in model_dirs:\n",
    "        model_name = model_dir.name\n",
    "        model_nc = list(model_dir.glob('*.nc'))\n",
    "        if len(model_nc) != 1:\n",
    "            if len(model_nc) == 0:\n",
    "                continue\n",
    "        hydrographs[model_name] = xarray.open_dataset(model_nc[0]).swap_dims({'nstations': 'station_id'})['Q']\n",
    "\n",
    "    hydrograph_xr = xarray.concat(hydrographs.values(), dim='model')\n",
    "    hydrograph_xr['model'] = list(hydrographs.keys())\n",
    "    return hydrograph_xr\n",
    "\n",
    "OBJECTIVES = ['objective_1/great-lakes/validation-temporal', 'objective_2/great-lakes/validation-temporal']\n",
    "XR = {obj: load_data(Path(f'data/{obj.split(\"/\")[0]}')) for obj in OBJECTIVES}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n"
     ]
    }
   ],
   "source": [
    "metric_vals = defaultdict(dict)\n",
    "for obj, model in itertools.product(OBJECTIVES, df['model_a'].unique()):\n",
    "    sub_df = df[(df['objective']==obj) & ((df['model_a']==model) | (df['model_b']==model))]\n",
    "    basins_dates = sub_df[['basin', 'start_date', 'end_date']].drop_duplicates()\n",
    "    for _, (basin, start_date, end_date) in basins_dates.iterrows():\n",
    "        sub_xr = XR[obj].sel(station_id=basin, time=slice(start_date, end_date))\n",
    "        setting_metrics = calculate_all_metrics(sub_xr.sel(model='Q'), sub_xr.sel(model=model), datetime_coord='time')\n",
    "        setting_metrics['logNSE'] = nse(np.log(sub_xr.sel(model='Q') + 1e-5), np.log(sub_xr.sel(model=model) + 1e-5))\n",
    "        setting_metrics['logKGE'] = kge(np.log(sub_xr.sel(model='Q') + 1e-5), np.log(sub_xr.sel(model=model) + 1e-5))\n",
    "        for metric, val in setting_metrics.items():\n",
    "            metric_vals[metric][(obj, model, basin, start_date)] = val\n",
    "\n",
    "metric_vals = pd.DataFrame(metric_vals)\n",
    "metric_vals.index = metric_vals.index.set_names(('objective', 'model', 'basin', 'start_date'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add metrics to the ratings dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "metric_df = df.copy()\n",
    "input_cols_metrics = []\n",
    "for ab in ['a', 'b']:\n",
    "    for metric in metric_vals.columns:\n",
    "        input_cols_metrics.append(f'model_{ab}_{metric}')\n",
    "        metric_df[input_cols_metrics[-1]] = np.nan\n",
    "for idx in metric_df.index:\n",
    "    obj = metric_df.loc[idx, 'objective']\n",
    "    basin = metric_df.loc[idx, 'basin']\n",
    "    start_date = metric_df.loc[idx, 'start_date']\n",
    "    for metric in metric_vals.columns:\n",
    "        for ab in ['a', 'b']:\n",
    "            model = metric_df.loc[idx, f'model_{ab}']\n",
    "            metric_df.loc[idx, f'model_{ab}_{metric}'] = metric_vals.loc[(obj, model, basin, start_date), metric]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prepare input and target arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dropping columns with NaNs: \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((14586, 33), (14586,))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_names = ['num_a_wins', 'num_b_wins', 'num_equal_good', 'num_equal_bad']\n",
    "\n",
    "print(f'Dropping columns with NaNs: {\", \".join(metric_df[input_cols_metrics].columns[pd.isna(metric_df[input_cols_metrics]).any(axis=0)])}')\n",
    "input_cols_metrics = [c for c in input_cols_metrics if not pd.isna(metric_df[input_cols_metrics]).any(axis=0)[c]]\n",
    "input_cols = input_cols_metrics\n",
    "\n",
    "x_in = pd.concat([metric_df[input_cols_metrics], pd.get_dummies(df['task'])], axis=1)\n",
    "y = pd.Series(np.argmax(metric_df[target_names].astype(int).values, axis=1), index=x_in.index)\n",
    "\n",
    "x_in.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run(n_estimators=1000, max_depth=10, run_test=False):\n",
    "    models = {}\n",
    "    targets = target_names\n",
    "    for task in df['task'].unique():\n",
    "        np.random.seed(0)\n",
    "        torch.manual_seed(0)\n",
    "\n",
    "        print(f'\\n####################################################################################\\n{task.upper()}')\n",
    "        model = RandomForestClassifier(n_estimators=n_estimators, max_depth=max_depth, random_state=0, n_jobs=-1)\n",
    "\n",
    "        # Select ratings from the current task\n",
    "        drop_vars = df['task'].unique().tolist()\n",
    "        y_sub = y[x_in[task] == 1]\n",
    "        x_sub = x_in[x_in[task] == 1].drop(drop_vars, axis=1)\n",
    "        \n",
    "        # Run 5-fold CV\n",
    "        shuffled_indices = np.random.permutation(x_sub.index.values)\n",
    "        kf = KFold(n_splits=5, shuffle=False)\n",
    "        confusions = []\n",
    "        reports = []\n",
    "        test_reports = []\n",
    "        for i, (idx_train, idx_test) in enumerate(kf.split(x_sub.loc[shuffled_indices].values)):\n",
    "            \n",
    "            # Further split train set into train & val\n",
    "            n_train = int(idx_train.shape[0] * 0.8)\n",
    "            idx_train, idx_val = idx_train[:n_train], idx_train[n_train:]\n",
    "\n",
    "            scale = x_sub.iloc[idx_train].std(axis=0)\n",
    "            loc = x_sub.iloc[idx_train].mean(axis=0)\n",
    "\n",
    "            if i == 0:\n",
    "                print(f'{len(idx_train)} training samples, {len(idx_val)} validation, {len(idx_test)} test samples, {x_sub.shape[1]} features.')\n",
    "\n",
    "            model.fit((x_sub.iloc[idx_train] - loc) / scale, y_sub.iloc[idx_train])\n",
    "            models[(i, task)] = model\n",
    "\n",
    "            y_hat_val = model.predict((x_sub.iloc[idx_val] - loc) / scale)\n",
    "            y_hat_test = model.predict((x_sub.iloc[idx_test] - loc) / scale)\n",
    "            \n",
    "            # Generate classification metrics\n",
    "            confusions.append(confusion_matrix(y_sub.iloc[idx_val], y_hat_val, normalize='pred', labels=list(range(len(targets)))))\n",
    "            reports.append(classification_report(y_sub.iloc[idx_val], y_hat_val, target_names=targets, output_dict=True, zero_division=0))\n",
    "            if run_test:\n",
    "                test_reports.append(classification_report(y_sub.iloc[idx_test], y_hat_test, target_names=targets, output_dict=True, zero_division=0))\n",
    "        \n",
    "        print(f'Avg validation accuracy: {np.mean([rep[\"accuracy\"] for rep in reports]):.3f}')\n",
    "        display(sum(pd.DataFrame({t: rep[t] for t in targets}) for rep in reports) / len(reports))\n",
    "\n",
    "        if run_test:\n",
    "            print(f'Avg test accuracy: {np.mean([rep[\"accuracy\"] for rep in test_reports]):.3f}')\n",
    "            display(sum(pd.DataFrame({t: rep[t] for t in targets}) for rep in test_reports) / len(test_reports))            \n",
    "\n",
    "    return models, x_sub.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit model, print results and plot feature importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "####################################################################################\n",
      "OVERALL\n",
      "3716 training samples, 929 validation, 1162 test samples, 30 features.\n",
      "Avg validation accuracy: 0.577\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num_a_wins</th>\n",
       "      <th>num_b_wins</th>\n",
       "      <th>num_equal_good</th>\n",
       "      <th>num_equal_bad</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.581592</td>\n",
       "      <td>0.604854</td>\n",
       "      <td>0.275024</td>\n",
       "      <td>0.512248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.737242</td>\n",
       "      <td>0.797318</td>\n",
       "      <td>0.054983</td>\n",
       "      <td>0.274581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1-score</th>\n",
       "      <td>0.649910</td>\n",
       "      <td>0.687760</td>\n",
       "      <td>0.090785</td>\n",
       "      <td>0.356823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>support</th>\n",
       "      <td>276.600000</td>\n",
       "      <td>342.400000</td>\n",
       "      <td>114.400000</td>\n",
       "      <td>196.200000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           num_a_wins  num_b_wins  num_equal_good  num_equal_bad\n",
       "precision    0.581592    0.604854        0.275024       0.512248\n",
       "recall       0.737242    0.797318        0.054983       0.274581\n",
       "f1-score     0.649910    0.687760        0.090785       0.356823\n",
       "support    276.600000  342.400000      114.400000     196.200000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg test accuracy: 0.538\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num_a_wins</th>\n",
       "      <th>num_b_wins</th>\n",
       "      <th>num_equal_good</th>\n",
       "      <th>num_equal_bad</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.557096</td>\n",
       "      <td>0.560343</td>\n",
       "      <td>0.302521</td>\n",
       "      <td>0.432765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.675986</td>\n",
       "      <td>0.775769</td>\n",
       "      <td>0.053584</td>\n",
       "      <td>0.240968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1-score</th>\n",
       "      <td>0.610284</td>\n",
       "      <td>0.650173</td>\n",
       "      <td>0.090743</td>\n",
       "      <td>0.307228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>support</th>\n",
       "      <td>354.000000</td>\n",
       "      <td>412.400000</td>\n",
       "      <td>155.200000</td>\n",
       "      <td>239.800000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           num_a_wins  num_b_wins  num_equal_good  num_equal_bad\n",
       "precision    0.557096    0.560343        0.302521       0.432765\n",
       "recall       0.675986    0.775769        0.053584       0.240968\n",
       "f1-score     0.610284    0.650173        0.090743       0.307228\n",
       "support    354.000000  412.400000      155.200000     239.800000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "####################################################################################\n",
      "HIGH-FLOW\n",
      "2937 training samples, 735 validation, 918 test samples, 30 features.\n",
      "Avg validation accuracy: 0.508\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num_a_wins</th>\n",
       "      <th>num_b_wins</th>\n",
       "      <th>num_equal_good</th>\n",
       "      <th>num_equal_bad</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.551610</td>\n",
       "      <td>0.506867</td>\n",
       "      <td>0.344912</td>\n",
       "      <td>0.479267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.625250</td>\n",
       "      <td>0.706148</td>\n",
       "      <td>0.091603</td>\n",
       "      <td>0.373575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1-score</th>\n",
       "      <td>0.586007</td>\n",
       "      <td>0.589950</td>\n",
       "      <td>0.143279</td>\n",
       "      <td>0.418878</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>support</th>\n",
       "      <td>193.200000</td>\n",
       "      <td>241.000000</td>\n",
       "      <td>107.000000</td>\n",
       "      <td>193.800000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           num_a_wins  num_b_wins  num_equal_good  num_equal_bad\n",
       "precision    0.551610    0.506867        0.344912       0.479267\n",
       "recall       0.625250    0.706148        0.091603       0.373575\n",
       "f1-score     0.586007    0.589950        0.143279       0.418878\n",
       "support    193.200000  241.000000      107.000000     193.800000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg test accuracy: 0.484\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num_a_wins</th>\n",
       "      <th>num_b_wins</th>\n",
       "      <th>num_equal_good</th>\n",
       "      <th>num_equal_bad</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.523605</td>\n",
       "      <td>0.494492</td>\n",
       "      <td>0.404041</td>\n",
       "      <td>0.415570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.619939</td>\n",
       "      <td>0.690498</td>\n",
       "      <td>0.084262</td>\n",
       "      <td>0.317492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1-score</th>\n",
       "      <td>0.565972</td>\n",
       "      <td>0.574620</td>\n",
       "      <td>0.136107</td>\n",
       "      <td>0.358010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>support</th>\n",
       "      <td>246.600000</td>\n",
       "      <td>302.800000</td>\n",
       "      <td>144.000000</td>\n",
       "      <td>224.600000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           num_a_wins  num_b_wins  num_equal_good  num_equal_bad\n",
       "precision    0.523605    0.494492        0.404041       0.415570\n",
       "recall       0.619939    0.690498        0.084262       0.317492\n",
       "f1-score     0.565972    0.574620        0.136107       0.358010\n",
       "support    246.600000  302.800000      144.000000     224.600000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "####################################################################################\n",
      "LOW-FLOW\n",
      "2680 training samples, 671 validation, 838 test samples, 30 features.\n",
      "Avg validation accuracy: 0.537\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num_a_wins</th>\n",
       "      <th>num_b_wins</th>\n",
       "      <th>num_equal_good</th>\n",
       "      <th>num_equal_bad</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.551886</td>\n",
       "      <td>0.562847</td>\n",
       "      <td>0.297226</td>\n",
       "      <td>0.468092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.705029</td>\n",
       "      <td>0.717477</td>\n",
       "      <td>0.032353</td>\n",
       "      <td>0.377392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1-score</th>\n",
       "      <td>0.618956</td>\n",
       "      <td>0.630458</td>\n",
       "      <td>0.057106</td>\n",
       "      <td>0.416993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>support</th>\n",
       "      <td>199.400000</td>\n",
       "      <td>222.000000</td>\n",
       "      <td>97.000000</td>\n",
       "      <td>152.600000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           num_a_wins  num_b_wins  num_equal_good  num_equal_bad\n",
       "precision    0.551886    0.562847        0.297226       0.468092\n",
       "recall       0.705029    0.717477        0.032353       0.377392\n",
       "f1-score     0.618956    0.630458        0.057106       0.416993\n",
       "support    199.400000  222.000000       97.000000     152.600000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg test accuracy: 0.525\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num_a_wins</th>\n",
       "      <th>num_b_wins</th>\n",
       "      <th>num_equal_good</th>\n",
       "      <th>num_equal_bad</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.542301</td>\n",
       "      <td>0.563524</td>\n",
       "      <td>0.268831</td>\n",
       "      <td>0.433269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.676891</td>\n",
       "      <td>0.697348</td>\n",
       "      <td>0.027553</td>\n",
       "      <td>0.382228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1-score</th>\n",
       "      <td>0.600919</td>\n",
       "      <td>0.622910</td>\n",
       "      <td>0.048651</td>\n",
       "      <td>0.405570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>support</th>\n",
       "      <td>250.600000</td>\n",
       "      <td>277.400000</td>\n",
       "      <td>116.000000</td>\n",
       "      <td>193.800000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           num_a_wins  num_b_wins  num_equal_good  num_equal_bad\n",
       "precision    0.542301    0.563524        0.268831       0.433269\n",
       "recall       0.676891    0.697348        0.027553       0.382228\n",
       "f1-score     0.600919    0.622910        0.048651       0.405570\n",
       "support    250.600000  277.400000      116.000000     193.800000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "models, input_vars = run(run_test=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "findfont: Font family ['serif'] not found. Falling back to DejaVu Sans.\n",
      "findfont: Generic family 'serif' not found because none of the following families were found: Computer Modern\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAADDCAYAAABpoN2wAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAnDUlEQVR4nO2dd7hkVZW33x9JUFCQpKiIymfAIaggoJJbaEZkQEAGJIwIqASbcRgVdaAbQT7RETAyCEgD0oOAxCFJ0zRZQiMN46gfCDooSlAaUHL/vj/2ru5z87n3nFPprvd56qlTJ6xaVXVqr73XWnsv2SYIgiAIiizRaQWCIAiC7iOMQxAEQTCEMA5BEATBEMI4BEEQBEMI4xAEQRAMIYxDEARBMIQwDkEQBMEQwjgEQRAEQwjjEARBEAwhjENQCUnbSbpB0nWSvilpyRplXydpKUnTJU2pS24VJG0p6ZjC6zMkrS1pqqQPjXLdjaMce72kOyR9uyWvbr2bZvD3UlHWDEm3SHqPpLPrkBmMn6U6rUDQLNpy70rro/i6szSibGkV4EvAVNt/lXQEcABw8rh0lJawvbCKnoOZvfNJlT73NhdOG/FzD4ftKyu83ebAqbZPlnRGBTljM0vV1svZw+P6XibI1rY3lbRWG94rGIEYOQRV+BBwlu2/5tcnALtI+hqApFdLuihvH5lHAtdKWis/5kg6H/gnSUdImivpZ5Le1ZFPUwFJ/yRp/7z9Q0nXSDpd0vR8ygqSzpR0t6QNCtctDxwJ/LOkQwr7XyXpMknXS/pW3ndu3n+gpAvzvv+qc7RWJ5I+L+mm/JuvKelDkg6R9HJJz+X7Yz9JHy1ccwiwnqTrgOUL+/eSdGuWt76kd0r6Rj72e0kbSNpa0ufa/kH7lDAOQRVeC/yh9cL2s6TR6CZ5147AxZLWBV5ne0vgYOCIfHw1YHfbpwMn2d4C+BhweHvUnzB7Z0N3HTC1eEDSxsBztqcAvyoceg1wIHAQsG9rp+2ngf8LfN32dwrnHwica3tz4OVZ7q2k73ZD4AVJSwMv2X6p7g9YFUmvIY0A3k8yfkcANwObAu8Frsvbm+b9AOTv4J58rzydZS0JfAbYjHR/HAv8Algnjy7uBd4PvK8oK6hGGIegCg8Da7ReSFoWeAGYn3v/OwIXA+8AtsyN6feBV+ZL7i40bHtLuh44tSizSznL9pa5ARvsTnoTMD9v/7yw/75sPH8PrFjiPd4CzMvbdwBrAzeRGsDlgLuB3YG7xq9+W1iLxd/DHcDatv8CrEz6DMfn5zfYfmgMWasCv7X9gu0HgVc5LSf9HLA18B1gA+A9+b2CGgjjEFThCmAfSa/Ir/8ZuAg4H/g4sKTtP5N60FcXGtR98vnFOMNBwJakmEU7/NpN8QCwbt5er7C/6Osv8/l+Q2rsII0U7icZgm2BP5EMxeF0b0/5QWD9vN3SH+B/SQ36tcDfAY+XkPUosJakpfNIYUHefydwCHA9sAzwsmyAgxqIgHQwYWw/Iuk44EpJC0mN19dJjf6PgBn5vLsl/TGPHAzMAq4eJO420p/8+jap3wi2fybpU5Jmk1xuv5ygqFOAcyQdAMy3fSuApOdJhuE24O0kV1O38DFJLZfiMcAcSTcDz7PYlXYzsJptS3qaEvrbfknSd4AbSPfWwfnQTcButhdIegh4rMbPMulRFPsJgnqRtJTtFyV9Hvid7Vmd1ikIxkuMHIKgfk6T9CbgSWDXTisTBBMhRg5BEATBECIgHQRBEAwhjEMQBEEwhDAOQRAEwRDCOARBEARDCOMQBEEQDCGMQxAEQTCEMec5SFrFdlfNPJw6daqvvLLKCslBEAQBoyzlUmYS3MmSlgIuBH5i+6na1Jogjz3WVbYqCIKg7xjTrWR7V2Bv0po450i6QNIu3bqGfBAEQVCdMY2DJAEbAR8gDUFmk9bhv7xZ1YIgCIJOUcatNBu4BjjO9gOtnclmBEEQBP1ImWyly21/tWUYJB0IYPv7jWoWBEHQJUyfPh1JSGL69OmdVqctjLjwnqQVSVWbzgb2JLmUlgK+a/uD7VJwODbccEPfcUcUfAqCoH1Iog8XKp1QttIWwE6kcn//loW8AJxco2JBEARBFzKicbB9Mak4/Bts/28bdQqCIAg6zIjGQdJJtqeR0lcHjKVsb964ZkEQBEHHGG3kMC2nsR5rO6YjB0EQTCJGzVZyir58uk26BEEQBF1CqRrSki4H5gELAWwf2aRSQRAEQWcpYxy+2bgWQRAEQVdRZhKch3mMiaQTJN0g6aRB+/eT9ICkswv7VpB0qaSbJO0zDv2DIAiCBihjHLbKj62BacBnxrpA0ruBV9jeDFhG0kaFw5cAgyfRHQDMAjYH9pe0TAm9giAIgoYY061ke0bxtaSLSsjdlLQeE/l5E+D2LO8xScsPc/7Btl+SdDfwNuCeEu8TBEEQNECZYj/7FV6uAaxSQu6KwP15ewHwzhLnP1k4f6Vh9DgQOBBgzTXXLKFCEARBMFHKuJVeyo8XSb35qSWueQJ4Zd5+ZX5d6Xzbp9je0PaGq666agkVgiAIgolSxjj8CHgWeAOwLPBciWtuAbbJ21OAW8ucnwsIbQD8qsR7BEEQBA1RxjicB6xJihmsmV+Piu15wLOSbgAW2r5N0rcBJO1AWul1G0kX5EtOBT4G3ACcbruMAQqCIAgaYsQluxedIF1pe2rh9VW2t2tcs1GIJbuDIGg3sWT3UBZI+g/gLmB94MlWkNr26fXoFwRBEHQTZYxDsVb0WLGDIAiCoA8oM89hZq4KtxJ5CGL7Nw3rFQRBEHSQMQPSkk4B/pNUDe7fgC83rVQQBMFEmIy1npuiTEB6ru0t2qRPKSIgHQTBSDQVOI6A9FBmSZoG3EtedM/2tTUpFgRBEHQhZYzD9sAfSUtcQDIQYRyCIAj6mDKT4Ja0/UnbM/Lj6Ma1CoIgCMakyRhLmZjDj4BHGOhW6uj8hog5BEEwEpMt5lBRr0oxh6sn+q5BEARBbzKiW0nSa/PmDcM8xmSUSnBrSLpW0s2SpuR96+QqcDdJ+sqEPkkQBMEkp04302gxh73zc2tuQ+l5DmNUgvtClrFtQdangCNsvx/YJE+6C4Ig6HnaOfeiJd925fcaM+YwIaHSwcCjtn8saRdgDdutVVmvA7aybUmXAnvmxx9IS3VcCexo+5mR5EfMIQiCkejGmEOT8YrBssf5XiPGHMpkK02EFRm5stuSXqx569i1wDdIdRxuGc4wSDpQ0h2S7nj00UcbUjsIgmAQs5Qere1JQlPG4QlGruz2UmG7dewYYC/grcC6ktYaLDAqwQVBELSPMjWk1wB2IY0GWgvvjTXX4Rbgk8CPSZXgzigcmy9pU2A+8ErbT0oS8GfbCyUtAFYY5+cIgiAIaqTMyOFi4DHgemBufozKaJXggOOBY4FrgK/mfV8DzsrnP2/7nvF9jCAIgmq0K3DcK4sDlpkEd57t3dqkTykiIB0EwUjUHjjOcQbtCT4H2GP8sisGjZuUXWkS3Ksl3Qn8Ir+27X3KvnMQBEHQe5QxDvs1rkUQBEHQVYxoHCRtb/sKYJthDkft6CAIgj5mtJHDwvz80ijnBEEQBH3IiNlKtq/KzzMHP9qnXhAEvU6vZOcEA2lk+YymiWylIOgt2rncdWQrdffyGUEQBEEPM6ZxkLStpGsk3SNpycJktiAIgqBPKTNyOJJUR/ox2y8B72xWpSAIep2m4gwRv2gfZWZIzwW2Ii13sQ1wre2t2qDbiETMIQi6n6Lvu0kf+3iPj1t2xBxG5HhgNrAOcFV+PfY7jq8S3BKSviFptqTzysgPgiAYwCRdWrspxpwhbfu/JF0OrGK7VCGFYiU4Sd+XtJHt2/PhViW4+cBlpBHJrsD/2D58Qp8iCIIgqJUyAekTc3Ge7STNk/S1EnI3JTX65OdNCsfWIxX0eRp4StIKwA7AOpKuk3TA+D5CEARBUDdl3Err5ueptt8NbF7imhUZXyW41UlV4KYAH5O0+mCBUQkuCJolgr29hbbcG22596LtuiljHJaQdCRwf379YolrnmB8leAWAHNtv0gqFLT2YIFRCS4ImqXO4vRB71PGOHwEuAk4WtIywBdLXHMLixfsmwLcWjg2X9Kmkl5BrgQH3ExyN5Gff1tG+SCY7ERvP2iKMkt2vwC8CXgfi9OebhjtAtvzJLUqwd3dqgRn+1BSttOZwHLAUfmS04CZkg4DrrL90Pg/ShBMPqZPn86MGTPatjRF0L3M3vmkAdvbXDitkrwyxuF84DxgZ+B7wEZlBNueNuj1ofn5IWDrQceeIo1QgiAIgi6gjFvpZbZPA56yfSrwhoZ1CoIg6AuaDho3SRnj8IikZYF7JJ0OrNCwTkEQBEGHKTMJbncASZ8BNiClnAZBEPQNxV69ttwbX3dWI+9Td1ygScpMgttA0oXAT4G7gcOaVioIgmCy0K0ZZ2XcSicB+wFL5FVZtx7j/CCY9HTrHz7oPrp1fkkZ4yDbfwFauXJlMpyCYFLTrX/4djN755MWuVJaz2E4e4MyxmGmpIuBN0v6MfDDhnUKgrYQjVRnCMPZG4xqHCQJeIjkVtoNOMj2GW3QKwgaJxqpoCzTz0/P2pOeuVeqdn5GdRHZtqSDbP8D8PgEdQyCIOhppu+aHgDsMb2TqpSm6uz5Mm4lJF0u6RhJR0s6ekLvFARBV9FrbrVenlDWi5QxDt8EvkZKZZ2dH2MynkpwhWOXSDqmpO5BEFSgKbdasRHvVQYYzvM7rU1nKGMcVrM91/Zc4HpgtbEuKFaCA5aRVFyPqVUJbtv83LpmfWDZ8SgfTD56rbfbKwyXVdQLNNWIDzCcu45+br9Sxjh8qrWRi/R8apRzW4y3EhzAZ0gL+wXBiEQQOSgSjXhzlFp4T9JKAJJeTbne/YqMoxKcpLcDjzCwKNAAohJcMBkZdaQ0S+nR2g6CGiljHD4PXCzpeuAnwL+WuOYJxlcJ7rOkmdgjEpXguptw9zRDjJSCTjGmcbB9E6mq2z/a3tL2zSXkjrcS3BuBM0iFgPaQtMU4PkNQkiYb8F5oxJr+/O0wjhEoDdpFmYX39gUuB66QtKSk88a6xvY8oFUJbmGrElw+fDxwLCkW8dV8/na2pwKfA2bl4HdQM73QgA+mzka3yc9flL3Z3Ss1FtQNH3vQLsq4lfa3/UHgz3nhvZXLCLY9zfZmtg/JrxdVgrO9te1NbV896JrrbH95OHlBd9HOnjL0lkHrB3rVTdiLM5m7lTLG4UVJywOWtBywsGGdgjZRpQEYrdGu2rD0asM0Er34eXrVKE/fFXxOevSE3l2cVFA2IH0+8I78fESjGvUo3dgAjKVTUw1AVbntaJjauVporza0jdDFjWFP8uDd6Xnu2cz8nysGHqv4PZcJSN9me6rt19r+kO3bJ/ROfU6TDW0TvfuepOGGpe++r1GIpSj6hLXWhy32gi32Yt93bF+r6BGNg6Q5eZmLIY9aNehR2pX50mIyNFhB0C304mxxYNHoYcpFh1XOZhvRONjeyvbWwM+ALwFTgS8Ct1V7y/6gXZkvYRAmL9G7H51eXfKjSfZ9x/Zcs9OJXLPTiZWz2crEHLYAbrX9PMkwxByEmunGeEUQTDpG899PQsoYhzOAWyT9J3BTft0TDG50q/rve3kCWZO9rF7owTXZC29XD78XvufhqNPV0ShN+e8HGZ1e+T7GrAdt+xRJpwGrAI/luQ49wfRhil1MtPjFcLKC+ik2rsUGcPbOJ7HNRzugUFCZfd+x/aLGdptdD+usMp1grfXTA9h3pQ3Tcw98H6WK/dh+yfafeskwBGNQIfMnfOH9yYAe7S6RbjrZKWUcJsJ4iv1IOkDSrfmxZ5X3Df99oqMNeC/msTeVJttDef11BjMbI+ICbaNsmdB1JW0maXNJm5c4f7zFfn5qexNgM+BfxvUJBtGRTJ8aG4CiX7mXGpagM0y65SIazOsPBjJmzEHSZcDvgT/kXSZVhBuN4Yr9tCbPrQdMs21JT0lawfaD+diLDFzSuysJX3gwFq1erfaEoz7S3Pt0beH73MOfctFh7P227aIh70HGNA6AbH9ynHJXBO7P2wuAdxaODSn2AzyVX38KuGhYJaQDgQMB1lxzzXGqU42iW0Zb7s01OajU7bL7kWJP+ahfTe/a3nIrCLvNRw8DYMZPOqtP28lB2Life5cyxmGBpBOAe0mjBmyfPsY1TzC+Yj9I2hj4e2Cn4QTaPgU4BWDDDTccM2VoaO/+sMUHZwn2iKyjXqTOnnIxAHvUEnSvnz0IOkCZmMNVwM9Z7PIp4/YZV7EfSa8D/h3YNzKignbRiwHYWnPkOxTcrTNO0tScgV6ZizAaVb/nMvMcZkp6GbAaUCoqanuepFaxn7tbxX5yTYfjgTOB5YCj8iVHAqsDP8nrCW1v+5nxfphIqxydJnvKg2VXouCvXm/ltwyQ2xON+Nserc/H3mSO/DCy20Gdo7+m5lC0c25GU67Sqt9zmYD0F0iZRW8nxRGeI40GRsX2tEGvFxX7AbYedGy8MY1gAtR6ww/TGBZlV/Kxj+CvbkLn2uhQQ9sUvRLb6Qe6NamgTB9vx7wA369zaurjDesU1MUYboNKw85RUgq7Nr2yQ2mQXft9jELPFc0JaqdMQPr5/Py3PMdhnQb16RlGcqPU0dMqpkFu8fYKcsfozTbVY+nWntBYdOvwPgg6QRnjMC3HHP4F+DRweLMqdSljuFGgQgMwguwBGVYTkRuMi2jEg2AxZYzDvcBuwKrAZ4F3NapRjdTau2/Sp9xn/uogCHqfMsbhbGAusIftb0s6jhIB6Y4xaGbmNTudCNTQuw+CIJhElDEOq9o+WVJvLBIRMzODIAgqUyZb6RFJuwPLSdoZeLhhnYIgCIIOU8Y47EeKN8wDXg8c0KhGQRAEQccZ0a0kqbi63SX5AWmm9O+aVCoIgiDoLKPFHB4kLbP934V9Ii2+t1+DOgVBEAQdZjS30sakWgyvBv4EfM/2x22XMgzjrAS3gqRLJd0kaZ8JfpYgCIKgJkY0DrZvt/0l2zuRaiz8QNIPygidQCW4A4BZwObA/pKWGe8HCYIgCOpjROMgaRVJn5B0DvBJ4Fhg2kjnD2K4SnAt1gNusf008JSkFVrn5+W67wbeNr6PEQRBENSJFhdlG3RAegG4C5hDWl9p0Ym2jxxVqPQl4E7bV2bX0ftsH52P3ZBHFEg6G/gicBrwYdvPSjoGuNr29YNkLqoERzIevyr5GVcBHit57njpRdmhc+/LDp3bI7sXdR6v7MdsTx3uwGgB6SqzoJ9gfJXgWuc/O8z5wMBKcONB0h22G5kR14uyQ+felx06t0d2L+pcp+wRjYPtuRXk3kJyRf2YZGTOKBybL2lTYD6LK8HdAmwj6cfABpQfFQRBEAQNULVm17DYnge0KsEtbFWCy4ePJ8UvrgG+mvedCnwMuAE43fZzTegVBEEQlKPM2koTYpyV4J4EdmhIlXG7ovpcdujc+7JD5/bI7kWda5M9YkA6CIIgmLw04lYKgiAIepswDkEQBMEQwjgEQRAEQwjjEHQdkj4gSZ3WI6iGEut1Wo9gYvStcZA0Jy/w13pcJen7ktaeoLzzC9vfKWz/tAZddyhsb1jY/lhFuU3qXNTz9YXtbarKBnYFbpV0iqTtJS1dVeAgHZcqbL+zquws5zRJH6p7XTBJR0p6Vd7eUdIdedHKypUZJX25sL1/YfvrVWUDOGW7HFuHrBZN/o6D/iPF7+asinIb+X9nGY3oDH1sHICfkRb5mwp8HrgX+CEwc4LyVi5sr1PYriMd+LOF7eML25+oKLdJnYt6nlnY/lJVwbYPs70x8H3SulvXSZolabcKYos6Xl3Y/vbgEyfIt4GNgDmSzpX0j3ndsKpMsb0gbx8PfBDYAvhMDbKLKeV7FrbfU4PsFn/NnbL9Je0nqepy/03+jkUDU/xuXj/4xHHS1P8bmtO5r43D5rZvs/08cAewqe3bgBcmKjAPk5dcvKkl61K2KXpR5xa27wKOIU2WXJ1qvVCNsF0Ltn9ue7rt9wNHAK8DrpB0eUXRSwFk98xvbf/F9gvAwopy28UVwK2k/91LDFw+ZyI0+TuOlNffzfn+jenc2CS4LuDMvCzH70hWdGZuGC+qIHN2YfvaCnIG82ZJR5Nu9uL2m2qQ3ZTOa+ReoAZtv7aqYEmvBD4M/AOwBvBT4DDb8yuIXUnS1lnHAdtV9S2Sdd+EVA9lIXBzRZEXSJpDMo7T8nusQVoMsyrvlnQ96XtYt7D9dzXIBsD2TABJ+9g+c6zzS9Dk77hG4TtYtbC9SkW5Tf6/m9K5vyfBZWOwMvB4Xg68K5G0xUjHKq5x1RiS9h3pWKtBqCB7LnApcKHt+wv7T7W9/8hXjirzqJGO2Z4xEZmD5B8E7AgsTdL9ItsPVpWbZa8AvGT7b/n1y4CX5ZUFegJJ19reeuwzx5TT6O/YBL34/4Y+HjlI2hb4FPCq9FJUuTklHQmcZHuBpB2BI0m9txNsn1dR3fuAR20/L2k5YB+S9a8aCGtS53OBpW0/Jem9wLJ5/z0V5WJ7pD/TGyvIHNBwSHorsITtX05U5iBeBPax/UhN8gCQdHphe9EmNZTrlfQPpHjcM8CSwBds31pF5ihUda8BA3/HPIJqtWF/rCo7B+XPy/+XTUjuzIXAVyo24o38v7POOwCP5PXrzgFekw991/YFlWT368hB0jxgB9t/qEne9bY3z9u/JAVKnwbm2P5ARdk3Alvnm+dc4E7Seuw75kp83ajzTODfbc/P3/VFpF7zmrZHHFWUlD1cLEzAlbY/OEGZF5Ma7wXZaG4E/Bn4o+3PT1zbRfLPYqifV6SknQmXvpV0GWkZ+//Kj6dax2z/dqJys+xbSffd3yStCpwz0e93lPd4ObANsCI5RlDFvaS0gOe5tm+UNJ8UT1waWGD7kIq63tj6X2TZHyaVD7i09T+aqFwa+H9n2deRkhZelDTH9lZKWVxXVP0t+3bkANwE1Lm665DAYH5dR2DwhXzjrACsb3v3LLtqqluTOq9ViAHcUyjmdFUNsu8g/SlbPWQGbU+EFbNhWIrUc3ur7YXZhVUHrTRCkUre/mMdQm3vIGl54EPA4cAywFm2r6hB/DPAM9kYPw4spTQ8ke26At5XkUYNtXTSSP+PQ/P2w8417Wu675xlvQV4omV8a/i/NPX/htT5eDFvH513vDhCB2tc9LNxeA9ws6TWMN9VrD/NBgZfkrQVsCVwWWH/8hXlNqpza2PQSGHZYc4dLwtaLkBJp7cagIosnXuxWwM/KzR+tfwHir14Sc9U7dUPkv20pAeAh0mpi6vWJZuUsFDM+rmW1EhWjg9k/mL7uJpkQXLfAWB7u8L+OuaX3C7pTOCtwEkAklakelZnU/9vSKURVrP9iO05AJJeS/WssP51KzVBU4FBSZsBhwJ/A75Och0sSRou/qABnZct5M5PVO4M0sjsG7lXtAzwuSz7y6NfPabsawvGoa5A5hRS734hcIDt+5UmRH7O9oGjX11KfqsBEalWSSuLhiq9cEknAWsCtwEXAr9uHauxd198vyWAD3hQmd4K8i4iGZt78/OYZYbHkPcd4Gbb5xT27QW83/anq2m7aDLdM7Z/k1+vCKxcTIyYgMzNSPNSngOOY3GMZ8sa/t/rkeZ4zCV1Hl5HmgdzqO2fV5Ldb8ZB0hG2jxvOB1zR93v6cLuT2Go92zxsvR3474JcqsoewQ/eEjzh7yLLXgLYnzSb+WWkEq8XAv/hijeVpAdJk50E7E1h4lOVhqVJJN0JLKDw27H4/qiSCDEvy23JpA65g95jaVLFxh1Jqci/sP3FmmQPSS6oEtyVtCwpseL9hd03AzNsPztRuVn2Vxj5/1LFoP2wKCo/bwy8zXalkauk80iJCRuQ0r4fJrlkT7X93kqy+9A4rG77T5KGZLZUGeo3HBjcCNiJ5DL4H+AC23dUkZnlDv4OtgUOIsUIqhqHTW3fMsz+g21/t6Ls2lP/lKoSDr7ZlwNWt73mRGQOkj/H9lZ5+zTbdcx+HSy3LhdbS/ZHSUHXl5NiA7vbrmP5k+J7LAHsAqwN3E+6tyfs8pC0tu37htn/YduXTlzTIfedgXeQevyPV3RJt+QvAXwEOIBUCvlbw32WccrcBfg0sJvtv0jaDvgKKfmiUiZe38UcbP8pbz4CbEeaHNPqzQ3X+y8rt7HAoO3bSSMHJG0M/ECpSPgBFeX+Nrt79mFxGdbtbVdO+wOOkfQt2xcDrTkllYxCiyo9y1FkbtbaznGXg4H3ASfX9RaF7TomNw0nd60a5UKacX49cKLtOyXtVLN8SOma/03K0NkQOBvYo4o8SdOcVjsAQNIXSCOJSsahdd9lF+RBwJOkRvbOKnKVJkbuTzLEl5Ia8lrmqNi+QNLDwGU5KP8+YFvbT1SV3XcjhxZKMwUvJw2zgOqTs7Lc9wI7k3r557uGWZ+SViHNBt6G5J65HLi8FSeoIHcGyf94DulPuWjYXdVfrZSvfTYpgHkO8GNSyt+3qshtEknvJgXmXwF8z3ZtM8abcoU17WLLPuuPAO8mzYz+BMmnX0umn6S5LsxbGfx6AvJWJc2xOYE02jkNeBQ4vIZ7+uPAXqRMx+8WOpqVkPQkaaWGS0iB4kWNbtXfsOAKezvJM/B98hJBlWX3sXG4alA2Q1V5jQUGJb0A3AXMIWUS1XLz5EylRaJau6nBX52HyEuSgmFTgEPIC6E1ESitSu4svJqk730M/I4rG4kmXGFNyh3hvd5McgH9fcuVVYPMc0gjh3mkkcM7bVdK880dkx8BbyaNes6oqmeW+wDJ0LQMYzFuVGWeQ2O/YaOy+9g47Exyp9SVJTFn0K7WF7e67UrLBberAWjFY2qSNYfFf54itQVK60QjL7tg5zkakw1JIzV4quu+y+7GnYG3kGIOF3lxXv5E5LViR0uRJtY9ng9VasCDofSzcbgT+AaFyTcVe3CvJvlKTfKjvhH4IrCqa55V2hR1pYWOILs2w9MOJG1r++qxz+xf8oh1Hin19jkGZsnVYjAlTbV9ZU4b/iwwy/YNdchuimEyE18AfkPKAHp8mEv6kr4LSBe4z/asGuXNItWCWAm4kTSLd7rtX496VXfRpLtnFvVNnGoHX2BgPYDJyOtIsa4ppNz7y6gh1jWIw4ErSR2pHwDfIi1dUomGG/CnSf/xnwPrkWKBD5DiarVmc3Uz/WwcVsmjh1/k166YvrmU88QbSQcA+1fN5283tqc0KL7r4gxjMK/TCnQap0UCf0DKjnsv8B+kDL9KWXKDWEHSmqSJmLdI+mtNcptswNez3Sqm9GtJh9r+tKSDK8rtKfrZONSWD555ixavw74iMEN5lcxunZjVIqfm/SuproVJSxpsNvpV46Nhw1MreZb4SbnRwvbvOqxSR8jZWzuT8vl/SZo5Xnl+zSCOIxVsOiZPYKtr1dcmG/Ab8ryme0lZiXNz7GTIvJ5+pu9iDpK2t32FhilHaHvC8xzamTVSN5JuI7kOLiG5EY51xRUsC7IbNzx1knPityWl/t0PPNdLhq1O8sz8O0gTsmBgBlelSZJZvoCZdcgaRvZXgHexuAG/C5hBure/UIP815CyE39X07ygnqMfRw5r5+dai/t0uwEYg7/aflKSSXMdKk2rH8RXGWR4apTdBDvafp+k62xvqbSE8mSlzsl6Q7BtSQ9nl9U8suuxjlRn2/9WaMBPLDTgdRiG15NiJGsD90k6zvb/VpXba/SjcdgZ+HYdE976iJl5SP8t0ozYS2qU3aThaYLWirR/y6mc63RSmU7iQcu+SPqm7c/W/DbvZeA9UcuKrw034GcAR5FGVRuRElF6KdmiFvrROCyXJ/MMwXmlxUnIC06Lkl0IXJjngNRFk4anCablmMO/kNakObzD+nQT76pbYF2T6YbhDJprwJe1fVPevjHfL5OOfjQOa5GWZh4yOYv6g9RdTw6k7Z9nqoo0q3l/kqGogyYNT+3YvlvSuqSkgguoVkCo3zgW0uz3uma5S1qfFJN6LSxawryORrzJBvwSSVcA84H16f4OTyP0o3H4pWtcubKXkbQv8E+k5XxbRV2ep6Z6vm0wPLWTs1B+z+LJkSaNeCYtkk6w/c+2r8mjwJnA7jWJP5m0XtGppBTZj9ckt/YGvLBOkYCHSOtNPUhai2vS0Y/GYUC6WUN+1J4gx11mKi0Jfhewmuurqd2o4WkQ2f5kp5XoMm6TdCLJTfMjUsGpunjOqbDSErbvUyp8M2EabsCvqUFG39B3xsFDi5TU7kftQdYhZRWtlnPb/9P2blUENml4mqCQ2rxA0gkMXHNrwinOvYwWV687l1QK80ZgX9LEsrq4Mo9GzpH0c9LClVVorAHv8YzE2um7eQ6DkTQlD5dr86P2GpJusL2ZcuGYOtdYyiOIvYDVSL24yoanCbKewzJZM9sKiyfCwBhdrYsnSlqKtAbZw2OeHHQNfTdyaNGwH7XXeFGpUJHzcsd1Gsn9C4bnJUkr1yi7NooGIOfdrw3cb/tnndOqszSYSbSIYuehrlFr0B6WGPuUnuU2SSdKehVwPvC9TivUQT5P+g7enp8rTxQq0KThqR1Jp5JcJ8sD+wyzgNukQ9L6ks6WNFvStZJqK4JE6jx8EPizU3nQruw8BEPpu5FDm/yoPYGkt5Fy+ReQ8vm/SqqDvVqNb9Ok4WmCNxaXWJcUQcjmMoqgxzoPwWL6zjiQMmeKftTHgH+nppmZPcappBzzFUklFTcA/koK6lXKKmqT4WmChyQdQQqir59fbw31VITrUWrNKAJQql2+K6m64aOkesznA0dUlR20h74zDu3wo/YQL9i+FUDSfbYfzdvPjn5ZKRozPA3zILAMsHF+/QCwGanzMFmNQ90ZRZBG7jeTVmH9JvBK24fWIDdoE32brdTgzMyeQcMXpxewl+1Ki64VM55UKBpfZyZUnRTcjUOYrFlsRerOKBp8H0iabXvSFMrpB/pu5FCgST9qr1BM37xmhO2J8mYtrm/xxsJ2oyt9VqDobmyxJPCB/DxpaSijqHV/QLovWvVQur7+SZDoZ+NQux+112h4Uk+Thqd2Wu7G7AufAuxIGlUe30m9uoQm0pEHzyvpyvsiGJl+Ng5N+FGDTK/NJpX0UeDDwMtJMZL/E26ORdSeUdRr90cwlL6NOUDMzAwWI+n/kRbY+57tOyVdbvvvO61XJylkFK1NyiJ6krQE9nTbt3dSt6Dz9K1x6JVlHYL2IWk94COk++HvgE8AN9t+rqOKdQhJF5Iyiu4GtiAyioIC/exW6ollHYL2YXs+aYlnckGoXYAjgcma/vwq260VWK+WNLuj2gRdRT8bh5iZGYxIrgr4depdnrrXiIyiYET6zjjEzMwgKE1kFAUj0ncxh/CjBkEQVKfvRg6EHzUIgqAy/ThyeJC0TAQMXDYi/KhBEAQl6UfjsMVIx2JiThAEQTn6zjgEQRAE1ennSnBBEATBBAnjEARBEAwhjEMQBEEwhDAOQRAEwRDCOARBEARD+P8StPkiCkxFcwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 394.016x198.425 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Take mean feature importance across the folds\n",
    "forest_importances = pd.DataFrame({(i, task): pd.Series(model.feature_importances_, index=input_vars) \n",
    "                                   for (i, task), model in models.items()}).groupby(axis=1, level=1).mean()\n",
    "std = pd.DataFrame({(i, task): pd.Series(np.std([tree.feature_importances_ \n",
    "                                            for tree in model.estimators_], axis=0), index=input_vars) \n",
    "                    for (i, task), model in models.items()}).groupby(axis=1, level=1).mean()\n",
    "\n",
    "# Each metric appears twice in the importances (once for each involved model). For the plot, we reduce it to one bar\n",
    "# per metric by takeing the mean of the two values.\n",
    "single_metric_importances = pd.DataFrame(columns=forest_importances.columns, \n",
    "                                         index=list(set(m.replace('model_a_', '').replace('model_b_', '') \n",
    "                                                          for m in forest_importances.index)))\n",
    "single_metric_std = single_metric_importances.copy()\n",
    "for metric in single_metric_importances.index:\n",
    "    single_metric_importances.loc[metric] = (forest_importances.loc[f'model_a_{metric}'] + forest_importances.loc[f'model_b_{metric}']) / 2\n",
    "    single_metric_std.loc[metric] = (std.loc[f'model_a_{metric}'] + std.loc[f'model_b_{metric}']) / 2\n",
    "\n",
    "single_metric_importances = single_metric_importances[['overall', 'high-flow', 'low-flow']].sort_values(by=\"overall\")\n",
    "single_metric_std = single_metric_std[['overall', 'high-flow', 'low-flow']]\n",
    "rename = dict(zip(['overall', 'high-flow', 'low-flow'], ['Overall', 'High flow', 'Low flow']))\n",
    "single_metric_importances = single_metric_importances.rename(rename, axis=1)\n",
    "single_metric_std = single_metric_std.rename(rename, axis=1)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(mm_to_inch(139), mm_to_inch(70)))\n",
    "single_metric_importances.plot.bar(ax=ax, yerr=single_metric_std, error_kw={'alpha': 1, 'capsize': 1.5, 'elinewidth': 1}, \n",
    "                                   color=dict(zip(['Overall', 'High flow', 'Low flow'], ['#003f5c', '#bc5090', '#ffa600'])))\n",
    "ax.set_ylabel(\"Mean decrease in impurity\")\n",
    "\n",
    "ax.spines['right'].set_visible(False)\n",
    "ax.spines['top'].set_visible(False)\n",
    "ax.legend(frameon=False, ncol=3, bbox_to_anchor=(0.2, 1, 0.6, 0), loc=\"lower left\", mode=\"expand\")\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('figures/feature-importance.pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "maps",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a2a6f1a0764d418ed202c4d837ddf731754b5106b91308298f70f739593497bf"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}