rmh-metrics-vs-hydrographs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This document trains a RF on metrics and a GRU on hydrographs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T11:13:02.079905Z",
     "iopub.status.busy": "2022-07-31T11:13:02.079771Z",
     "iopub.status.idle": "2022-07-31T11:13:06.831820Z",
     "shell.execute_reply": "2022-07-31T11:13:06.831117Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import copy\n",
    "import torch\n",
    "import itertools\n",
    "from collections import defaultdict\n",
    "\n",
    "from torch import nn\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import xarray\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import KFold\n",
    "from tqdm import tqdm\n",
    "from pathlib import Path\n",
    "from neuralhydrology.evaluation.metrics import calculate_all_metrics, nse, kge\n",
    "\n",
    "df = pd.read_csv('data/rmh-stage1.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load simulations and observations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T11:13:06.835120Z",
     "iopub.status.busy": "2022-07-31T11:13:06.834986Z",
     "iopub.status.idle": "2022-07-31T11:13:11.637083Z",
     "shell.execute_reply": "2022-07-31T11:13:11.636357Z"
    }
   },
   "outputs": [],
   "source": [
    "def load_data(base_dir):\n",
    "    obs_file = base_dir / 'all_gauges.nc'\n",
    "    if not obs_file.exists():\n",
    "        raise ValueError(f'Observations netCDF file not found at {obs_file}')\n",
    "    # netcdfs only have a numeric dimension \"nstations\" that maps to the station_id variable.\n",
    "    # for easier processing, we directly make the station_id the dimension.\n",
    "    obs = xarray.load_dataset(obs_file).swap_dims({'nstations': 'station_id'})\n",
    "\n",
    "    # load hydrographs from individual models\n",
    "    hydrographs = {'Q': obs['Q']}\n",
    "    model_dirs = [d for d in base_dir.glob('model/*') if d.is_dir()]\n",
    "    for model_dir in model_dirs:\n",
    "        model_name = model_dir.name\n",
    "        model_nc = list(model_dir.glob('*.nc'))\n",
    "        if len(model_nc) != 1:\n",
    "            if len(model_nc) == 0:\n",
    "                continue\n",
    "        hydrographs[model_name] = xarray.open_dataset(model_nc[0]).swap_dims({'nstations': 'station_id'})['Q']\n",
    "\n",
    "    hydrograph_xr = xarray.concat(hydrographs.values(), dim='model')\n",
    "    hydrograph_xr['model'] = list(hydrographs.keys())\n",
    "    return hydrograph_xr\n",
    "\n",
    "OBJECTIVES = ['objective_1/great-lakes/validation-temporal', 'objective_2/great-lakes/validation-temporal']\n",
    "XR = {obj: load_data(Path(f'data/{obj.split(\"/\")[0]}')) for obj in OBJECTIVES}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate metrics for each model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T11:13:11.639899Z",
     "iopub.status.busy": "2022-07-31T11:13:11.639764Z",
     "iopub.status.idle": "2022-07-31T11:19:57.743749Z",
     "shell.execute_reply": "2022-07-31T11:19:57.743000Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n",
      "/system/apps/mlsoft/conda/group-bioinf/gauch/envs/cf/lib/python3.9/site-packages/xarray/core/computation.py:727: RuntimeWarning: invalid value encountered in log\n",
      "  result_data = func(*input_data)\n"
     ]
    }
   ],
   "source": [
    "metric_vals = defaultdict(dict)\n",
    "for obj, model in itertools.product(OBJECTIVES, df['model_a'].unique()):\n",
    "    sub_df = df[(df['objective']==obj) & ((df['model_a']==model) | (df['model_b']==model))]\n",
    "    basins_dates = sub_df[['basin', 'start_date', 'end_date']].drop_duplicates()\n",
    "    for _, (basin, start_date, end_date) in basins_dates.iterrows():\n",
    "        sub_xr = XR[obj].sel(station_id=basin, time=slice(start_date, end_date))\n",
    "        setting_metrics = calculate_all_metrics(sub_xr.sel(model='Q'), sub_xr.sel(model=model), datetime_coord='time')\n",
    "        setting_metrics['logNSE'] = nse(np.log(sub_xr.sel(model='Q') + 1e-5), np.log(sub_xr.sel(model=model) + 1e-5))\n",
    "        setting_metrics['logKGE'] = kge(np.log(sub_xr.sel(model='Q') + 1e-5), np.log(sub_xr.sel(model=model) + 1e-5))\n",
    "        for metric, val in setting_metrics.items():\n",
    "            metric_vals[metric][(obj, model, basin, start_date)] = val\n",
    "\n",
    "metric_vals = pd.DataFrame(metric_vals)\n",
    "metric_vals.index = metric_vals.index.set_names(('objective', 'model', 'basin', 'start_date'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prepare input data for the GRU and the RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T11:19:57.746493Z",
     "iopub.status.busy": "2022-07-31T11:19:57.746365Z",
     "iopub.status.idle": "2022-07-31T11:19:58.403835Z",
     "shell.execute_reply": "2022-07-31T11:19:58.403323Z"
    }
   },
   "outputs": [],
   "source": [
    "def prepare_data(df, targets):\n",
    "    metric_df = df.copy()\n",
    "    input_cols = []\n",
    "    # Prepare columns that will contain the metrics in metric_df\n",
    "    for ab in ['a', 'b']:\n",
    "        for metric in metric_vals.columns:\n",
    "            input_cols.append(f'model_{ab}_{metric}')\n",
    "            metric_df[input_cols[-1]] = np.nan\n",
    "\n",
    "    task_onehot = torch.from_numpy(pd.get_dummies(df['task']).values)\n",
    "    timeseries = []\n",
    "    y = []\n",
    "    dropped_idx = []\n",
    "    for i, (idx, rating) in enumerate(df.iterrows()):\n",
    "        obj = metric_df.loc[idx, 'objective']\n",
    "        basin = metric_df.loc[idx, 'basin']\n",
    "        start_date = metric_df.loc[idx, 'start_date']\n",
    "\n",
    "        # GRU inputs: observations & simulations from model a and b & task one-hot encoding\n",
    "        basin_xr = XR[obj].sel(time=slice(start_date, rating['end_date']), station_id=basin)\n",
    "        model_a_ts = basin_xr.sel(model=rating['model_a']).values\n",
    "        model_b_ts = basin_xr.sel(model=rating['model_b']).values\n",
    "        qobs = basin_xr.sel(model='Q').values\n",
    "        ts_inputs_list = [model_a_ts, model_b_ts, qobs]\n",
    "\n",
    "        if any(np.any(np.isnan(ts)) for ts in ts_inputs_list):\n",
    "            dropped_idx.append(idx)\n",
    "            continue\n",
    "        \n",
    "        # RF inputs: metrics from model a and b & task one-hot encoding\n",
    "        for metric in metric_vals.columns:\n",
    "            for ab in ['a', 'b']:\n",
    "                model = metric_df.loc[idx, f'model_{ab}']\n",
    "                metric_df.loc[idx, f'model_{ab}_{metric}'] = metric_vals.loc[(obj, model, basin, start_date), metric]\n",
    "\n",
    "        # some timeseries have 731 entries. for simplicity, we ignore that last entry where it exists.\n",
    "        ts_inputs = torch.stack([torch.from_numpy(x) for x in ts_inputs_list], dim=1)[:730]\n",
    "        ts_inputs = torch.cat([ts_inputs, task_onehot[i].unsqueeze(0).repeat(ts_inputs.shape[0], 1)], dim=1)\n",
    "        timeseries.append(ts_inputs)\n",
    "\n",
    "        # Target: rating encoded as integer\n",
    "        y.append(np.argmax(rating[targets].astype(int).values))\n",
    "\n",
    "    if len(dropped_idx) > 0:\n",
    "        metric_df = metric_df.loc[np.setdiff1d(metric_df.index, dropped_idx)]\n",
    "        print(f'Dropped {len(dropped_idx)} NaN samples, remaining {len(metric_df)}.')\n",
    "\n",
    "    # Convert to tensors that can be used with PyTorch\n",
    "    x_gru = torch.stack(timeseries)\n",
    "    # Add task encoding to the metrics df that will be the inputs to the RF\n",
    "    x_rf = pd.concat([metric_df[input_cols], pd.get_dummies(metric_df['task'])], axis=1).reset_index(drop=True)\n",
    "    y = torch.tensor(y).to(torch.long)\n",
    "\n",
    "    return x_rf, x_gru, y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prepare train/val/test splits for GRU and RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T11:19:58.406424Z",
     "iopub.status.busy": "2022-07-31T11:19:58.405999Z",
     "iopub.status.idle": "2022-07-31T11:19:58.430636Z",
     "shell.execute_reply": "2022-07-31T11:19:58.430129Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_data(df, target_names):\n",
    "\n",
    "    # Get input and target data\n",
    "    x_rf, x_gru, y = prepare_data(df, target_names)\n",
    "    \n",
    "    targets = target_names\n",
    "    np.random.seed(0)\n",
    "    torch.manual_seed(0)\n",
    "\n",
    "    x_sub_rf = x_rf\n",
    "    x_sub_gru = x_gru\n",
    "    y_sub = y\n",
    "\n",
    "    # 5-fold CV\n",
    "    shuffled_indices = np.random.permutation(x_sub_gru.shape[0])\n",
    "    kf = KFold(n_splits=5, shuffle=False)\n",
    "    idx_train, idx_val, idx_test = [], [], []\n",
    "    for i, (idx_a, idx_b) in enumerate(kf.split(x_sub_rf.iloc[shuffled_indices].values)):\n",
    "        # Further split the train fold into train and val\n",
    "        n_train = int(idx_a.shape[0] * 0.8)\n",
    "        idx_train.append(idx_a[:n_train])\n",
    "        idx_val.append(idx_a[n_train:])\n",
    "        idx_test.append(idx_b)\n",
    "\n",
    "    return x_sub_rf, x_sub_gru, y_sub, idx_train, idx_val, idx_test, targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T11:19:58.433359Z",
     "iopub.status.busy": "2022-07-31T11:19:58.432994Z",
     "iopub.status.idle": "2022-07-31T11:19:58.455983Z",
     "shell.execute_reply": "2022-07-31T11:19:58.455478Z"
    }
   },
   "outputs": [],
   "source": [
    "def evaluate(y_sub, y_hat, targets):\n",
    "    report = classification_report(y_sub, y_hat, target_names=targets, output_dict=True, zero_division=0)\n",
    "    confusion = confusion_matrix(y_sub, y_hat, normalize='pred', labels=list(range(len(targets))))\n",
    "\n",
    "    return report, confusion\n",
    "\n",
    "def display_eval(reports, confusions, targets):\n",
    "    print(f'Average accuracy: {np.mean([report[\"accuracy\"] for report in reports]):.3f}')\n",
    "    display(sum(pd.DataFrame({t: report[t] for t in targets}) for report in reports) / len(reports))\n",
    "\n",
    "    display((sum(pd.DataFrame(confusion,\n",
    "                            index=[f'true {t}' for t in targets],\n",
    "                            columns=[f'predicted {t}' for t in targets])\n",
    "                 for confusion in confusions) / len(confusions)).style.background_gradient(cmap='Greens', axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T11:19:58.458348Z",
     "iopub.status.busy": "2022-07-31T11:19:58.457983Z",
     "iopub.status.idle": "2022-07-31T11:19:58.479526Z",
     "shell.execute_reply": "2022-07-31T11:19:58.479085Z"
    }
   },
   "outputs": [],
   "source": [
    "def run_rf(x_sub, y_sub, idx_train, idx_val, idx_test, targets, test=False):\n",
    "\n",
    "    reports, confusions = [], []\n",
    "    val_indices = idx_val if not test else idx_test\n",
    "    for i, (i_train, i_val) in enumerate(zip(idx_train, val_indices)):\n",
    "        if i == 0:\n",
    "            print(f'{len(i_train)} training samples, {len(i_val)} validation/test samples, {x_sub.shape[1]} features.')\n",
    "\n",
    "        model = RandomForestClassifier(n_estimators=1000, max_depth=10, random_state=0, n_jobs=-1)\n",
    "        model.fit(x_sub.iloc[i_train], y_sub[i_train])\n",
    "\n",
    "        y_hat_val = model.predict(x_sub.iloc[i_val])\n",
    "\n",
    "        report, confusion = evaluate(y_sub[i_val], y_hat_val, targets)\n",
    "        reports.append(report)\n",
    "        confusions.append(confusion)\n",
    "    display_eval(reports, confusions, targets=targets)\n",
    "\n",
    "    return model, reports, confusions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-31T11:19:58.481842Z",
     "iopub.status.busy": "2022-07-31T11:19:58.481730Z",
     "iopub.status.idle": "2022-07-31T11:19:58.510041Z",
     "shell.execute_reply": "2022-07-31T11:19:58.509550Z"
    }
   },
   "outputs": [],
   "source": [
    "def run_gru(x_sub, y_sub, idx_train, idx_val, idx_test, targets, hidden_size, lr, bs, dropout, test=False):\n",
    "    ds = torch.utils.data.TensorDataset(x_sub, y_sub)\n",
    "\n",
    "    best_val_reports = []\n",
    "    best_val_confusions = []\n",
    "    test_reports = []\n",
    "    test_confusions = []\n",
    "    for i, (i_train, i_val, i_test) in enumerate(zip(idx_train, idx_val, idx_test)):\n",
    "        np.random.seed(0)\n",
    "        torch.manual_seed(0)\n",
    "    \n",
    "        # Initialize model and optimizer\n",
    "        device = 'cuda:0'\n",
    "        gru = nn.GRU(input_size=x_sub.shape[2], hidden_size=hidden_size, batch_first=True).to(device)\n",
    "        head = nn.Linear(gru.hidden_size, len(targets)).to(device)\n",
    "        dropout_layer = nn.Dropout(p=dropout)\n",
    "        loss_fn = nn.CrossEntropyLoss()\n",
    "        optimizer = torch.optim.Adam(params=list(gru.parameters()) + list(head.parameters()), lr=lr)\n",
    "        \n",
    "        # Create train/val/test dataloaders\n",
    "        train_ds = torch.utils.data.Subset(ds, indices=i_train)\n",
    "        val_ds = torch.utils.data.Subset(ds, indices=i_val)\n",
    "        test_ds = torch.utils.data.Subset(ds, indices=i_test)\n",
    "        train_loader = torch.utils.data.DataLoader(train_ds, batch_size=bs, shuffle=True)\n",
    "        val_loader = torch.utils.data.DataLoader(val_ds, batch_size=bs, shuffle=False)\n",
    "        test_loader = torch.utils.data.DataLoader(test_ds, batch_size=bs, shuffle=False)\n",
    "\n",
    "        if i == 0:\n",
    "            print(f'{len(train_ds)} training samples, {len(val_ds)} validation, {len(test_ds)} test samples.')\n",
    "\n",
    "        # Run training\n",
    "        train_losses = {}\n",
    "        val_reports = {}\n",
    "        prev_acc = -np.inf\n",
    "        best_ep = None\n",
    "        best_model = None\n",
    "        patience = 20\n",
    "        for epoch in range(500):\n",
    "            gru.train(), head.train()\n",
    "            total_loss = 0.0\n",
    "            for batch_x, batch_y in train_loader:\n",
    "                optimizer.zero_grad()\n",
    "                gru_out, _ = gru(batch_x.to(device))\n",
    "                head_out = head(dropout_layer(gru_out[:, -1]))\n",
    "                loss = loss_fn(head_out, batch_y.to(device))\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "\n",
    "                total_loss += loss.detach().cpu().item()\n",
    "            \n",
    "            train_losses[epoch] = total_loss / len(train_loader)\n",
    "            # Run evaluation every few epochs\n",
    "            if epoch % 5 == 0:\n",
    "                gru.eval(), head.eval()\n",
    "                y_val = []\n",
    "                y_hat = []\n",
    "                with torch.no_grad():\n",
    "                    for batch_x, batch_y in val_loader:\n",
    "                        gru_out, _ = gru(batch_x.to(device))\n",
    "                        head_out = head(gru_out[:, -1])\n",
    "                        y_hat.append(head_out.cpu())\n",
    "                        y_val.append(batch_y)\n",
    "                y_val = torch.cat(y_val)\n",
    "                y_hat = torch.cat(y_hat)\n",
    "                y_hat_cls = y_hat.argmax(dim=1)\n",
    "                \n",
    "                report, confusion = evaluate(y_val, y_hat_cls, targets)\n",
    "                val_reports[epoch] = report, confusion\n",
    "                print(f'Fold {i} Epoch {str(epoch).ljust(5)} train loss: {train_losses[epoch]:.3f}, validation accuracy: {report[\"accuracy\"]:.3f}')\n",
    "                # Check for early-stopping based on validation results\n",
    "                if report['accuracy'] > prev_acc:\n",
    "                    prev_acc = report['accuracy']\n",
    "                    best_ep = epoch\n",
    "                    best_model = copy.deepcopy(gru), copy.deepcopy(head)\n",
    "                    best_model[0].flatten_parameters()\n",
    "                if best_ep + patience < epoch:\n",
    "                    print('early stopping.')\n",
    "                    break\n",
    "        \n",
    "        best_val_reports.append(val_reports[best_ep][0])\n",
    "        best_val_confusions.append(val_reports[best_ep][1])\n",
    "\n",
    "        # Run final test set evaluation based on best validation epoch\n",
    "        if test:\n",
    "            gru, head = best_model\n",
    "            gru.eval(), head.eval()\n",
    "            y_test = []\n",
    "            y_hat = []\n",
    "            with torch.no_grad():\n",
    "                for batch_x, batch_y in test_loader:\n",
    "                    gru_out, _ = gru(batch_x.to(device))\n",
    "                    head_out = head(gru_out[:, -1])\n",
    "                    y_hat.append(head_out.cpu())\n",
    "                    y_test.append(batch_y)\n",
    "            y_test = torch.cat(y_test)\n",
    "            y_hat = torch.cat(y_hat)\n",
    "            y_hat_cls = y_hat.argmax(dim=1)\n",
    "            \n",
    "            report, confusion = evaluate(y_test, y_hat_cls, targets)\n",
    "            test_reports.append(report)\n",
    "            test_confusions.append(confusion)\n",
    "\n",
    "        torch.cuda.empty_cache()\n",
    "    \n",
    "    if test:\n",
    "        display_eval(test_reports, test_confusions, targets=targets)\n",
    "        return test_reports, test_confusions\n",
    "    return best_val_reports, best_val_confusions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dropped 572 NaN samples, remaining 14014.\n",
      "shapes  - x_rf: (14014, 33), x_gru: torch.Size([14014, 730, 6]), y: torch.Size([14014]). Targets: ['num_a_wins', 'num_b_wins', 'num_equal_good', 'num_equal_bad']\n",
      "samples - folds: 5. train: 8968, val: 2243, test: 2803\n"
     ]
    }
   ],
   "source": [
    "target_names = ['num_a_wins', 'num_b_wins', 'num_equal_good', 'num_equal_bad']\n",
    "\n",
    "x_rf, x_gru, y, idx_train, idx_val, idx_test, targets = get_data(df, target_names=target_names)\n",
    "print(f'shapes  - x_rf: {x_rf.shape}, x_gru: {x_gru.shape}, y: {y.shape}. Targets: {targets}')\n",
    "print(f'samples - folds: {len(idx_train)}. train: {len(idx_train[0])}, val: {len(idx_val[0])}, test: {len(idx_test[0])}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run the RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8968 training samples, 2803 validation/test samples, 33 features.\n",
      "Average accuracy: 0.508\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num_a_wins</th>\n",
       "      <th>num_b_wins</th>\n",
       "      <th>num_equal_good</th>\n",
       "      <th>num_equal_bad</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.533053</td>\n",
       "      <td>0.517356</td>\n",
       "      <td>0.326597</td>\n",
       "      <td>0.437722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.650604</td>\n",
       "      <td>0.739525</td>\n",
       "      <td>0.040190</td>\n",
       "      <td>0.274477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1-score</th>\n",
       "      <td>0.585404</td>\n",
       "      <td>0.608154</td>\n",
       "      <td>0.071338</td>\n",
       "      <td>0.335568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>support</th>\n",
       "      <td>816.400000</td>\n",
       "      <td>956.400000</td>\n",
       "      <td>401.600000</td>\n",
       "      <td>628.400000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           num_a_wins  num_b_wins  num_equal_good  num_equal_bad\n",
       "precision    0.533053    0.517356        0.326597       0.437722\n",
       "recall       0.650604    0.739525        0.040190       0.274477\n",
       "f1-score     0.585404    0.608154        0.071338       0.335568\n",
       "support    816.400000  956.400000      401.600000     628.400000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_6fbc5_row0_col0, #T_6fbc5_row1_col1, #T_6fbc5_row2_col2, #T_6fbc5_row3_col3 {\n",
       "  background-color: #00441b;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_6fbc5_row0_col1, #T_6fbc5_row2_col0, #T_6fbc5_row2_col3, #T_6fbc5_row3_col2 {\n",
       "  background-color: #f7fcf5;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_6fbc5_row0_col2 {\n",
       "  background-color: #2d954d;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_6fbc5_row0_col3 {\n",
       "  background-color: #9cd797;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_6fbc5_row1_col0 {\n",
       "  background-color: #f5fbf2;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_6fbc5_row1_col2 {\n",
       "  background-color: #005221;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_6fbc5_row1_col3 {\n",
       "  background-color: #95d391;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_6fbc5_row2_col1 {\n",
       "  background-color: #eff9ec;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_6fbc5_row3_col0 {\n",
       "  background-color: #e3f4de;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_6fbc5_row3_col1 {\n",
       "  background-color: #def2d9;\n",
       "  color: #000000;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_6fbc5\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_6fbc5_level0_col0\" class=\"col_heading level0 col0\" >predicted num_a_wins</th>\n",
       "      <th id=\"T_6fbc5_level0_col1\" class=\"col_heading level0 col1\" >predicted num_b_wins</th>\n",
       "      <th id=\"T_6fbc5_level0_col2\" class=\"col_heading level0 col2\" >predicted num_equal_good</th>\n",
       "      <th id=\"T_6fbc5_level0_col3\" class=\"col_heading level0 col3\" >predicted num_equal_bad</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_6fbc5_level0_row0\" class=\"row_heading level0 row0\" >true num_a_wins</th>\n",
       "      <td id=\"T_6fbc5_row0_col0\" class=\"data row0 col0\" >0.533053</td>\n",
       "      <td id=\"T_6fbc5_row0_col1\" class=\"data row0 col1\" >0.134071</td>\n",
       "      <td id=\"T_6fbc5_row0_col2\" class=\"data row0 col2\" >0.260060</td>\n",
       "      <td id=\"T_6fbc5_row0_col3\" class=\"data row0 col3\" >0.229443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6fbc5_level0_row1\" class=\"row_heading level0 row1\" >true num_b_wins</th>\n",
       "      <td id=\"T_6fbc5_row1_col0\" class=\"data row1 col0\" >0.142372</td>\n",
       "      <td id=\"T_6fbc5_row1_col1\" class=\"data row1 col1\" >0.517356</td>\n",
       "      <td id=\"T_6fbc5_row1_col2\" class=\"data row1 col2\" >0.315910</td>\n",
       "      <td id=\"T_6fbc5_row1_col3\" class=\"data row1 col3\" >0.236323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6fbc5_level0_row2\" class=\"row_heading level0 row2\" >true num_equal_good</th>\n",
       "      <td id=\"T_6fbc5_row2_col0\" class=\"data row2 col0\" >0.135755</td>\n",
       "      <td id=\"T_6fbc5_row2_col1\" class=\"data row2 col1\" >0.155612</td>\n",
       "      <td id=\"T_6fbc5_row2_col2\" class=\"data row2 col2\" >0.326597</td>\n",
       "      <td id=\"T_6fbc5_row2_col3\" class=\"data row2 col3\" >0.096512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6fbc5_level0_row3\" class=\"row_heading level0 row3\" >true num_equal_bad</th>\n",
       "      <td id=\"T_6fbc5_row3_col0\" class=\"data row3 col0\" >0.188821</td>\n",
       "      <td id=\"T_6fbc5_row3_col1\" class=\"data row3 col1\" >0.192961</td>\n",
       "      <td id=\"T_6fbc5_row3_col2\" class=\"data row3 col2\" >0.097432</td>\n",
       "      <td id=\"T_6fbc5_row3_col3\" class=\"data row3 col3\" >0.437722</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f24f22d2160>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "_ = run_rf(x_rf, y.numpy(), idx_train, idx_val, idx_test, targets=targets, test=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run the GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8968 training samples, 2243 validation, 2803 test samples.\n",
      "Fold 0 Epoch 0     train loss: 1.383, validation accuracy: 0.346\n",
      "Fold 0 Epoch 5     train loss: 1.330, validation accuracy: 0.381\n",
      "Fold 0 Epoch 10    train loss: 1.298, validation accuracy: 0.404\n",
      "Fold 0 Epoch 15    train loss: 1.271, validation accuracy: 0.438\n",
      "Fold 0 Epoch 20    train loss: 1.226, validation accuracy: 0.463\n",
      "Fold 0 Epoch 25    train loss: 1.201, validation accuracy: 0.477\n",
      "Fold 0 Epoch 30    train loss: 1.181, validation accuracy: 0.483\n",
      "Fold 0 Epoch 35    train loss: 1.172, validation accuracy: 0.495\n",
      "Fold 0 Epoch 40    train loss: 1.160, validation accuracy: 0.498\n",
      "Fold 0 Epoch 45    train loss: 1.155, validation accuracy: 0.504\n",
      "Fold 0 Epoch 50    train loss: 1.135, validation accuracy: 0.503\n",
      "Fold 0 Epoch 55    train loss: 1.121, validation accuracy: 0.506\n",
      "Fold 0 Epoch 60    train loss: 1.111, validation accuracy: 0.508\n",
      "Fold 0 Epoch 65    train loss: 1.094, validation accuracy: 0.504\n",
      "Fold 0 Epoch 70    train loss: 1.080, validation accuracy: 0.504\n",
      "Fold 0 Epoch 75    train loss: 1.056, validation accuracy: 0.486\n",
      "Fold 0 Epoch 80    train loss: 1.040, validation accuracy: 0.480\n",
      "Fold 0 Epoch 85    train loss: 1.018, validation accuracy: 0.493\n",
      "early stopping.\n",
      "Fold 1 Epoch 0     train loss: 1.386, validation accuracy: 0.346\n",
      "Fold 1 Epoch 5     train loss: 1.334, validation accuracy: 0.377\n",
      "Fold 1 Epoch 10    train loss: 1.303, validation accuracy: 0.400\n",
      "Fold 1 Epoch 15    train loss: 1.260, validation accuracy: 0.401\n",
      "Fold 1 Epoch 20    train loss: 1.233, validation accuracy: 0.462\n",
      "Fold 1 Epoch 25    train loss: 1.205, validation accuracy: 0.477\n",
      "Fold 1 Epoch 30    train loss: 1.182, validation accuracy: 0.473\n",
      "Fold 1 Epoch 35    train loss: 1.167, validation accuracy: 0.488\n",
      "Fold 1 Epoch 40    train loss: 1.161, validation accuracy: 0.494\n",
      "Fold 1 Epoch 45    train loss: 1.148, validation accuracy: 0.502\n",
      "Fold 1 Epoch 50    train loss: 1.132, validation accuracy: 0.495\n",
      "Fold 1 Epoch 55    train loss: 1.118, validation accuracy: 0.502\n",
      "Fold 1 Epoch 60    train loss: 1.108, validation accuracy: 0.499\n",
      "Fold 1 Epoch 65    train loss: 1.094, validation accuracy: 0.508\n",
      "Fold 1 Epoch 70    train loss: 1.080, validation accuracy: 0.507\n",
      "Fold 1 Epoch 75    train loss: 1.064, validation accuracy: 0.490\n",
      "Fold 1 Epoch 80    train loss: 1.040, validation accuracy: 0.498\n",
      "Fold 1 Epoch 85    train loss: 1.023, validation accuracy: 0.485\n",
      "Fold 1 Epoch 90    train loss: 0.994, validation accuracy: 0.491\n",
      "early stopping.\n",
      "Fold 2 Epoch 0     train loss: 1.381, validation accuracy: 0.354\n",
      "Fold 2 Epoch 5     train loss: 1.323, validation accuracy: 0.380\n",
      "Fold 2 Epoch 10    train loss: 1.291, validation accuracy: 0.391\n",
      "Fold 2 Epoch 15    train loss: 1.247, validation accuracy: 0.414\n",
      "Fold 2 Epoch 20    train loss: 1.201, validation accuracy: 0.449\n",
      "Fold 2 Epoch 25    train loss: 1.180, validation accuracy: 0.482\n",
      "Fold 2 Epoch 30    train loss: 1.156, validation accuracy: 0.485\n",
      "Fold 2 Epoch 35    train loss: 1.141, validation accuracy: 0.491\n",
      "Fold 2 Epoch 40    train loss: 1.134, validation accuracy: 0.494\n",
      "Fold 2 Epoch 45    train loss: 1.116, validation accuracy: 0.500\n",
      "Fold 2 Epoch 50    train loss: 1.117, validation accuracy: 0.509\n",
      "Fold 2 Epoch 55    train loss: 1.089, validation accuracy: 0.496\n",
      "Fold 2 Epoch 60    train loss: 1.073, validation accuracy: 0.506\n",
      "Fold 2 Epoch 65    train loss: 1.057, validation accuracy: 0.508\n",
      "Fold 2 Epoch 70    train loss: 1.035, validation accuracy: 0.498\n",
      "Fold 2 Epoch 75    train loss: 1.016, validation accuracy: 0.505\n",
      "early stopping.\n",
      "Fold 3 Epoch 0     train loss: 1.379, validation accuracy: 0.349\n",
      "Fold 3 Epoch 5     train loss: 1.320, validation accuracy: 0.379\n",
      "Fold 3 Epoch 10    train loss: 1.288, validation accuracy: 0.402\n",
      "Fold 3 Epoch 15    train loss: 1.245, validation accuracy: 0.432\n",
      "Fold 3 Epoch 20    train loss: 1.211, validation accuracy: 0.466\n",
      "Fold 3 Epoch 25    train loss: 1.187, validation accuracy: 0.481\n",
      "Fold 3 Epoch 30    train loss: 1.161, validation accuracy: 0.483\n",
      "Fold 3 Epoch 35    train loss: 1.156, validation accuracy: 0.489\n",
      "Fold 3 Epoch 40    train loss: 1.146, validation accuracy: 0.487\n",
      "Fold 3 Epoch 45    train loss: 1.125, validation accuracy: 0.497\n",
      "Fold 3 Epoch 50    train loss: 1.116, validation accuracy: 0.502\n",
      "Fold 3 Epoch 55    train loss: 1.100, validation accuracy: 0.505\n",
      "Fold 3 Epoch 60    train loss: 1.089, validation accuracy: 0.497\n",
      "Fold 3 Epoch 65    train loss: 1.071, validation accuracy: 0.498\n",
      "Fold 3 Epoch 70    train loss: 1.051, validation accuracy: 0.501\n",
      "Fold 3 Epoch 75    train loss: 1.038, validation accuracy: 0.497\n",
      "Fold 3 Epoch 80    train loss: 1.022, validation accuracy: 0.498\n",
      "early stopping.\n",
      "Fold 4 Epoch 0     train loss: 1.390, validation accuracy: 0.348\n",
      "Fold 4 Epoch 5     train loss: 1.334, validation accuracy: 0.393\n",
      "Fold 4 Epoch 10    train loss: 1.306, validation accuracy: 0.410\n",
      "Fold 4 Epoch 15    train loss: 1.278, validation accuracy: 0.436\n",
      "Fold 4 Epoch 20    train loss: 1.242, validation accuracy: 0.461\n",
      "Fold 4 Epoch 25    train loss: 1.209, validation accuracy: 0.487\n",
      "Fold 4 Epoch 30    train loss: 1.193, validation accuracy: 0.502\n",
      "Fold 4 Epoch 35    train loss: 1.176, validation accuracy: 0.510\n",
      "Fold 4 Epoch 40    train loss: 1.168, validation accuracy: 0.520\n",
      "Fold 4 Epoch 45    train loss: 1.156, validation accuracy: 0.522\n",
      "Fold 4 Epoch 50    train loss: 1.147, validation accuracy: 0.520\n",
      "Fold 4 Epoch 55    train loss: 1.129, validation accuracy: 0.530\n",
      "Fold 4 Epoch 60    train loss: 1.114, validation accuracy: 0.522\n",
      "Fold 4 Epoch 65    train loss: 1.109, validation accuracy: 0.516\n",
      "Fold 4 Epoch 70    train loss: 1.088, validation accuracy: 0.527\n",
      "Fold 4 Epoch 75    train loss: 1.073, validation accuracy: 0.525\n",
      "Fold 4 Epoch 80    train loss: 1.063, validation accuracy: 0.528\n",
      "early stopping.\n",
      "Average accuracy: 0.498\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num_a_wins</th>\n",
       "      <th>num_b_wins</th>\n",
       "      <th>num_equal_good</th>\n",
       "      <th>num_equal_bad</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.520403</td>\n",
       "      <td>0.526642</td>\n",
       "      <td>0.335079</td>\n",
       "      <td>0.383861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.662700</td>\n",
       "      <td>0.718184</td>\n",
       "      <td>0.074181</td>\n",
       "      <td>0.219543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1-score</th>\n",
       "      <td>0.581030</td>\n",
       "      <td>0.606967</td>\n",
       "      <td>0.115385</td>\n",
       "      <td>0.271769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>support</th>\n",
       "      <td>816.400000</td>\n",
       "      <td>956.400000</td>\n",
       "      <td>401.600000</td>\n",
       "      <td>628.400000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           num_a_wins  num_b_wins  num_equal_good  num_equal_bad\n",
       "precision    0.520403    0.526642        0.335079       0.383861\n",
       "recall       0.662700    0.718184        0.074181       0.219543\n",
       "f1-score     0.581030    0.606967        0.115385       0.271769\n",
       "support    816.400000  956.400000      401.600000     628.400000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_aa2dd_row0_col0, #T_aa2dd_row1_col1, #T_aa2dd_row2_col2, #T_aa2dd_row3_col3 {\n",
       "  background-color: #00441b;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_aa2dd_row0_col1, #T_aa2dd_row2_col0, #T_aa2dd_row2_col3, #T_aa2dd_row3_col2 {\n",
       "  background-color: #f7fcf5;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_aa2dd_row0_col2 {\n",
       "  background-color: #e0f3db;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_aa2dd_row0_col3 {\n",
       "  background-color: #73c476;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_aa2dd_row1_col0 {\n",
       "  background-color: #f4fbf1;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_aa2dd_row1_col2 {\n",
       "  background-color: #1c8540;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_aa2dd_row1_col3 {\n",
       "  background-color: #68be70;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_aa2dd_row2_col1 {\n",
       "  background-color: #f0f9ed;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_aa2dd_row3_col0, #T_aa2dd_row3_col1 {\n",
       "  background-color: #d9f0d3;\n",
       "  color: #000000;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_aa2dd\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_aa2dd_level0_col0\" class=\"col_heading level0 col0\" >predicted num_a_wins</th>\n",
       "      <th id=\"T_aa2dd_level0_col1\" class=\"col_heading level0 col1\" >predicted num_b_wins</th>\n",
       "      <th id=\"T_aa2dd_level0_col2\" class=\"col_heading level0 col2\" >predicted num_equal_good</th>\n",
       "      <th id=\"T_aa2dd_level0_col3\" class=\"col_heading level0 col3\" >predicted num_equal_bad</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_aa2dd_level0_row0\" class=\"row_heading level0 row0\" >true num_a_wins</th>\n",
       "      <td id=\"T_aa2dd_row0_col0\" class=\"data row0 col0\" >0.520403</td>\n",
       "      <td id=\"T_aa2dd_row0_col1\" class=\"data row0 col1\" >0.127784</td>\n",
       "      <td id=\"T_aa2dd_row0_col2\" class=\"data row0 col2\" >0.195308</td>\n",
       "      <td id=\"T_aa2dd_row0_col3\" class=\"data row0 col3\" >0.248437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_aa2dd_level0_row1\" class=\"row_heading level0 row1\" >true num_b_wins</th>\n",
       "      <td id=\"T_aa2dd_row1_col0\" class=\"data row1 col0\" >0.142867</td>\n",
       "      <td id=\"T_aa2dd_row1_col1\" class=\"data row1 col1\" >0.526642</td>\n",
       "      <td id=\"T_aa2dd_row1_col2\" class=\"data row1 col2\" >0.298440</td>\n",
       "      <td id=\"T_aa2dd_row1_col3\" class=\"data row1 col3\" >0.256073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_aa2dd_level0_row2\" class=\"row_heading level0 row2\" >true num_equal_good</th>\n",
       "      <td id=\"T_aa2dd_row2_col0\" class=\"data row2 col0\" >0.133748</td>\n",
       "      <td id=\"T_aa2dd_row2_col1\" class=\"data row2 col1\" >0.146834</td>\n",
       "      <td id=\"T_aa2dd_row2_col2\" class=\"data row2 col2\" >0.335079</td>\n",
       "      <td id=\"T_aa2dd_row2_col3\" class=\"data row2 col3\" >0.111629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_aa2dd_level0_row3\" class=\"row_heading level0 row3\" >true num_equal_bad</th>\n",
       "      <td id=\"T_aa2dd_row3_col0\" class=\"data row3 col0\" >0.202982</td>\n",
       "      <td id=\"T_aa2dd_row3_col1\" class=\"data row3 col1\" >0.198740</td>\n",
       "      <td id=\"T_aa2dd_row3_col2\" class=\"data row3 col2\" >0.171172</td>\n",
       "      <td id=\"T_aa2dd_row3_col3\" class=\"data row3 col3\" >0.383861</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7ff0f4715a60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h = 265\n",
    "lr = 5e-4\n",
    "bs = 1024\n",
    "dropout = 0.3\n",
    "reports, confusions = run_gru(x_gru, y, idx_train, idx_val, idx_test, targets=targets, hidden_size=h, lr=lr, bs=bs, dropout=dropout, test=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "maps",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "a2a6f1a0764d418ed202c4d837ddf731754b5106b91308298f70f739593497bf"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}