sec3_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Miniconda3\\envs\\myenv\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3049: DtypeWarning: Columns (7,8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Index(['YEAR', 'MONTH', 'DAY', 'DAY_OF_WEEK', 'AIRLINE', 'FLIGHT_NUMBER',\n",
       "       'TAIL_NUMBER', 'ORIGIN_AIRPORT', 'DESTINATION_AIRPORT',\n",
       "       'SCHEDULED_DEPARTURE', 'DEPARTURE_TIME', 'DEPARTURE_DELAY', 'TAXI_OUT',\n",
       "       'WHEELS_OFF', 'SCHEDULED_TIME', 'ELAPSED_TIME', 'AIR_TIME', 'DISTANCE',\n",
       "       'WHEELS_ON', 'TAXI_IN', 'SCHEDULED_ARRIVAL', 'ARRIVAL_TIME',\n",
       "       'ARRIVAL_DELAY', 'DIVERTED', 'CANCELLED', 'CANCELLATION_REASON',\n",
       "       'AIR_SYSTEM_DELAY', 'SECURITY_DELAY', 'AIRLINE_DELAY',\n",
       "       'LATE_AIRCRAFT_DELAY', 'WEATHER_DELAY'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"flights.csv\")\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "can_del=df['CANCELLED'] | (df['DEPARTURE_DELAY']>15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['CAN_DEL']=can_del"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2=df[['MONTH','DAY','DAY_OF_WEEK','AIRLINE','ORIGIN_AIRPORT','DESTINATION_AIRPORT','SCHEDULED_DEPARTURE','SCHEDULED_TIME','CAN_DEL']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MONTH</th>\n",
       "      <th>DAY</th>\n",
       "      <th>DAY_OF_WEEK</th>\n",
       "      <th>AIRLINE</th>\n",
       "      <th>ORIGIN_AIRPORT</th>\n",
       "      <th>DESTINATION_AIRPORT</th>\n",
       "      <th>SCHEDULED_DEPARTURE</th>\n",
       "      <th>SCHEDULED_TIME</th>\n",
       "      <th>CAN_DEL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>AS</td>\n",
       "      <td>ANC</td>\n",
       "      <td>SEA</td>\n",
       "      <td>5</td>\n",
       "      <td>205.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>AA</td>\n",
       "      <td>LAX</td>\n",
       "      <td>PBI</td>\n",
       "      <td>10</td>\n",
       "      <td>280.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>US</td>\n",
       "      <td>SFO</td>\n",
       "      <td>CLT</td>\n",
       "      <td>20</td>\n",
       "      <td>286.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>AA</td>\n",
       "      <td>LAX</td>\n",
       "      <td>MIA</td>\n",
       "      <td>20</td>\n",
       "      <td>285.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>AS</td>\n",
       "      <td>SEA</td>\n",
       "      <td>ANC</td>\n",
       "      <td>25</td>\n",
       "      <td>235.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   MONTH  DAY  DAY_OF_WEEK AIRLINE ORIGIN_AIRPORT DESTINATION_AIRPORT  \\\n",
       "0      1    1            4      AS            ANC                 SEA   \n",
       "1      1    1            4      AA            LAX                 PBI   \n",
       "2      1    1            4      US            SFO                 CLT   \n",
       "3      1    1            4      AA            LAX                 MIA   \n",
       "4      1    1            4      AS            SEA                 ANC   \n",
       "\n",
       "   SCHEDULED_DEPARTURE  SCHEDULED_TIME  CAN_DEL  \n",
       "0                    5           205.0    False  \n",
       "1                   10           280.0    False  \n",
       "2                   20           286.0    False  \n",
       "3                   20           285.0    False  \n",
       "4                   25           235.0    False  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5819079"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1106420"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(df2['CAN_DEL'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "clean data, drops airports with less than 5000 flights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MONTH</th>\n",
       "      <th>DAY</th>\n",
       "      <th>DAY_OF_WEEK</th>\n",
       "      <th>SCHEDULED_DEPARTURE</th>\n",
       "      <th>SCHEDULED_TIME</th>\n",
       "      <th>CAN_DEL</th>\n",
       "      <th>AIRLINE_AA</th>\n",
       "      <th>AIRLINE_AS</th>\n",
       "      <th>AIRLINE_B6</th>\n",
       "      <th>AIRLINE_DL</th>\n",
       "      <th>...</th>\n",
       "      <th>DESTINATION_AIRPORT_SLC</th>\n",
       "      <th>DESTINATION_AIRPORT_SMF</th>\n",
       "      <th>DESTINATION_AIRPORT_SNA</th>\n",
       "      <th>DESTINATION_AIRPORT_STL</th>\n",
       "      <th>DESTINATION_AIRPORT_SYR</th>\n",
       "      <th>DESTINATION_AIRPORT_TPA</th>\n",
       "      <th>DESTINATION_AIRPORT_TUL</th>\n",
       "      <th>DESTINATION_AIRPORT_TUS</th>\n",
       "      <th>DESTINATION_AIRPORT_TYS</th>\n",
       "      <th>DESTINATION_AIRPORT_XNA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>205.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>280.0</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>286.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>285.0</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>25</td>\n",
       "      <td>235.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 248 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   MONTH  DAY  DAY_OF_WEEK  SCHEDULED_DEPARTURE  SCHEDULED_TIME  CAN_DEL  \\\n",
       "0      1    1            4                    5           205.0    False   \n",
       "1      1    1            4                   10           280.0    False   \n",
       "2      1    1            4                   20           286.0    False   \n",
       "3      1    1            4                   20           285.0    False   \n",
       "4      1    1            4                   25           235.0    False   \n",
       "\n",
       "   AIRLINE_AA  AIRLINE_AS  AIRLINE_B6  AIRLINE_DL  ...  \\\n",
       "0           0           1           0           0  ...   \n",
       "1           1           0           0           0  ...   \n",
       "2           0           0           0           0  ...   \n",
       "3           1           0           0           0  ...   \n",
       "4           0           1           0           0  ...   \n",
       "\n",
       "   DESTINATION_AIRPORT_SLC  DESTINATION_AIRPORT_SMF  DESTINATION_AIRPORT_SNA  \\\n",
       "0                        0                        0                        0   \n",
       "1                        0                        0                        0   \n",
       "2                        0                        0                        0   \n",
       "3                        0                        0                        0   \n",
       "4                        0                        0                        0   \n",
       "\n",
       "   DESTINATION_AIRPORT_STL  DESTINATION_AIRPORT_SYR  DESTINATION_AIRPORT_TPA  \\\n",
       "0                        0                        0                        0   \n",
       "1                        0                        0                        0   \n",
       "2                        0                        0                        0   \n",
       "3                        0                        0                        0   \n",
       "4                        0                        0                        0   \n",
       "\n",
       "   DESTINATION_AIRPORT_TUL  DESTINATION_AIRPORT_TUS  DESTINATION_AIRPORT_TYS  \\\n",
       "0                        0                        0                        0   \n",
       "1                        0                        0                        0   \n",
       "2                        0                        0                        0   \n",
       "3                        0                        0                        0   \n",
       "4                        0                        0                        0   \n",
       "\n",
       "   DESTINATION_AIRPORT_XNA  \n",
       "0                        0  \n",
       "1                        0  \n",
       "2                        0  \n",
       "3                        0  \n",
       "4                        0  \n",
       "\n",
       "[5 rows x 248 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfopg=df2.groupby(by=['ORIGIN_AIRPORT'])\n",
    "airports=dfopg.groups.keys()\n",
    "dfop={op:dfopg.get_group(op) for op in airports}\n",
    "airports=[op for op in airports if type(op)==str and len(op)==3 and len(dfop[op])>5000]\n",
    "df3=df2[[x in airports for x in df2['ORIGIN_AIRPORT']]]\n",
    "df4=df3[[x in airports for x in df3['DESTINATION_AIRPORT']]]\n",
    "df5=df4[df4['SCHEDULED_TIME'].notnull()]\n",
    "dfv=pd.get_dummies(df5)\n",
    "dfv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'AIRLINE_WN'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfv.columns[-229]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['MONTH', 'DAY', 'DAY_OF_WEEK', 'SCHEDULED_DEPARTURE', 'SCHEDULED_TIME',\n",
       "       'CAN_DEL', 'AIRLINE_AA', 'AIRLINE_AS', 'AIRLINE_B6', 'AIRLINE_DL',\n",
       "       'AIRLINE_EV', 'AIRLINE_F9', 'AIRLINE_HA', 'AIRLINE_MQ', 'AIRLINE_NK',\n",
       "       'AIRLINE_OO', 'AIRLINE_UA', 'AIRLINE_US', 'AIRLINE_VX', 'AIRLINE_WN'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfv.columns[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "925031"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(dfv['CAN_DEL'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "pass to numpy array for modelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "xy_np=np.array(dfv)\n",
    "\n",
    "np.random.shuffle(xy_np)\n",
    "\n",
    "xy_npo=np.concatenate((xy_np[:,:5],xy_np[:,6:],xy_np[:,5,np.newaxis]),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('xyo.npy',xy_npo,allow_pickle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "xy_npo=np.load('xyo.npy',allow_pickle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "114"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(airports)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4653855, 248)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xy_npo.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "n=xy_npo.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Mini Data: only time and airline, 20 columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train=xy_npo[:round(n*0.8),:20]\n",
    "y_train=xy_npo[:round(n*0.8),-1]\n",
    "X_val=xy_npo[round(n*0.8):round(n*0.9),:20]\n",
    "y_val=xy_npo[round(n*0.8):round(n*0.9),-1]\n",
    "X_test=xy_npo[round(n*0.9):,:20]\n",
    "y_test=xy_npo[round(n*0.9):,-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Miniconda3\\envs\\myenv\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\myenv\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2053: FutureWarning: You should specify a value for 'cv' instead of relying on the default value. The default value will change from 3 to 5 in version 0.22.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\myenv\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\myenv\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\ProgramData\\Miniconda3\\envs\\myenv\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.80117467, 0.80118176, 0.80113325])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "cls = LogisticRegression()\n",
    "cls.fit(X_train, y_train)\n",
    "cross_val_score(cls, X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decision tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train=[int(y) for y in y_train]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Miniconda3\\envs\\myenv\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2053: FutureWarning: You should specify a value for 'cv' instead of relying on the default value. The default value will change from 3 to 5 in version 0.22.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.72317327, 0.72255421, 0.72327355])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cls=DecisionTreeClassifier()\n",
    "cls.fit(X_train,y_train)\n",
    "cross_val_score(cls, X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3 layers NN with full data (248 columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train=xy_npo[:round(n*0.8),:-1]\n",
    "y_train=xy_npo[:round(n*0.8),-1]\n",
    "X_val=xy_npo[round(n*0.8):round(n*0.9),:-1]\n",
    "y_val=xy_npo[round(n*0.8):round(n*0.9),-1]\n",
    "X_test=xy_npo[round(n*0.9):,:-1]\n",
    "y_test=xy_npo[round(n*0.9):,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset(object):\n",
    "    def __init__(self, X, y, batch_size, shuffle=False):\n",
    "        self.X, self.y = X, y\n",
    "        self.batch_size, self.shuffle = batch_size, shuffle\n",
    "    def __iter__(self):\n",
    "        N, B = self.X.shape[0], self.batch_size\n",
    "        idxs = np.arange(N)\n",
    "        if self.shuffle:\n",
    "            np.random.shuffle(idxs)\n",
    "        return iter((self.X[i:i+B], self.y[i:i+B]) for i in range(0, N, B))\n",
    "\n",
    "train_dset = Dataset(X_train, y_train, batch_size=64)\n",
    "val_dset = Dataset(X_val, y_val, batch_size=64)\n",
    "test_dset = Dataset(X_test, y_test, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using device:  /device:GPU:0\n"
     ]
    }
   ],
   "source": [
    "device = '/device:GPU:0'\n",
    "print_every = 100\n",
    "print('Using device: ', device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_accuracy(sess, dset, x, scores, is_training=None):\n",
    "    num_correct, num_samples = 0, 0\n",
    "    for x_batch, y_batch in dset:\n",
    "        feed_dict = {x: x_batch, is_training: 0}\n",
    "        scores_np = sess.run(scores, feed_dict=feed_dict)\n",
    "        y_pred = scores_np.argmax(axis=1)\n",
    "        num_samples += x_batch.shape[0]\n",
    "        num_correct += (y_pred == y_batch).sum()\n",
    "    acc = float(num_correct) / num_samples\n",
    "    print('Got %d / %d correct (%.2f%%)' % (num_correct, num_samples, 100 * acc))\n",
    "    acc_hist.append(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model_init_fn, optimizer_init_fn, num_epochs=1):\n",
    "    tf.reset_default_graph()    \n",
    "    with tf.device(device):\n",
    "        x = tf.placeholder(tf.float32, [None, 247])\n",
    "        y = tf.placeholder(tf.int32, [None])\n",
    "        \n",
    "        is_training = tf.placeholder(tf.bool, name='is_training')\n",
    "        scores = model_init_fn(x, is_training)\n",
    "        loss = tf.losses.sparse_softmax_cross_entropy(labels=y, logits=scores)\n",
    "        loss = tf.reduce_mean(loss)\n",
    "\n",
    "        optimizer = optimizer_init_fn()\n",
    "        update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "        with tf.control_dependencies(update_ops):\n",
    "            train_op = optimizer.minimize(loss)\n",
    "\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        t = 0\n",
    "        for epoch in range(num_epochs):\n",
    "            print('Starting epoch %d' % epoch)\n",
    "            for x_np, y_np in train_dset:\n",
    "                feed_dict = {x: x_np, y: y_np, is_training:1}\n",
    "                loss_np, _ = sess.run([loss, train_op], feed_dict=feed_dict)\n",
    "                if t % print_every == 0:\n",
    "                    print('Iteration %d, loss = %.4f' % (t, loss_np))\n",
    "                    loss_hist.append(loss_np)\n",
    "                    check_accuracy(sess, val_dset, x, scores, is_training=is_training)\n",
    "                    print()\n",
    "                t += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting epoch 0\n",
      "Iteration 0, loss = 27.4099\n",
      "Got 92621 / 465386 correct (19.90%)\n",
      "\n",
      "Iteration 100, loss = 10.2065\n",
      "Got 294043 / 465386 correct (63.18%)\n",
      "\n",
      "Iteration 200, loss = 4.5322\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 300, loss = 0.4528\n",
      "Got 372205 / 465386 correct (79.98%)\n",
      "\n",
      "Iteration 400, loss = 6.3260\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 500, loss = 0.8169\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 600, loss = 3.2701\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 700, loss = 0.6210\n",
      "Got 231820 / 465386 correct (49.81%)\n",
      "\n",
      "Iteration 800, loss = 0.5389\n",
      "Got 372760 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 900, loss = 0.9181\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 1000, loss = 0.9139\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 1100, loss = 0.6563\n",
      "Got 372766 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 1200, loss = 0.8951\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 1300, loss = 0.6279\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 1400, loss = 0.9230\n",
      "Got 371995 / 465386 correct (79.93%)\n",
      "\n",
      "Iteration 1500, loss = 0.5666\n",
      "Got 327332 / 465386 correct (70.34%)\n",
      "\n",
      "Iteration 1600, loss = 0.5209\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 1700, loss = 0.6025\n",
      "Got 372567 / 465386 correct (80.06%)\n",
      "\n",
      "Iteration 1800, loss = 0.4818\n",
      "Got 372235 / 465386 correct (79.98%)\n",
      "\n",
      "Iteration 1900, loss = 0.5493\n",
      "Got 290374 / 465386 correct (62.39%)\n",
      "\n",
      "Iteration 2000, loss = 0.4813\n",
      "Got 369496 / 465386 correct (79.40%)\n",
      "\n",
      "Iteration 2100, loss = 0.5816\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 2200, loss = 0.4753\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 2300, loss = 0.5285\n",
      "Got 372730 / 465386 correct (80.09%)\n",
      "\n",
      "Iteration 2400, loss = 0.5053\n",
      "Got 372761 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 2500, loss = 0.4456\n",
      "Got 370359 / 465386 correct (79.58%)\n",
      "\n",
      "Iteration 2600, loss = 0.4925\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 2700, loss = 0.3719\n",
      "Got 371860 / 465386 correct (79.90%)\n",
      "\n",
      "Iteration 2800, loss = 0.5571\n",
      "Got 308570 / 465386 correct (66.30%)\n",
      "\n",
      "Iteration 2900, loss = 0.5912\n",
      "Got 352857 / 465386 correct (75.82%)\n",
      "\n",
      "Iteration 3000, loss = 0.5800\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 3100, loss = 0.6017\n",
      "Got 372723 / 465386 correct (80.09%)\n",
      "\n",
      "Iteration 3200, loss = 0.4517\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 3300, loss = 0.5873\n",
      "Got 372700 / 465386 correct (80.08%)\n",
      "\n",
      "Iteration 3400, loss = 0.4255\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 3500, loss = 0.6036\n",
      "Got 372724 / 465386 correct (80.09%)\n",
      "\n",
      "Iteration 3600, loss = 0.4174\n",
      "Got 372754 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 3700, loss = 0.5033\n",
      "Got 372764 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 3800, loss = 0.4929\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 3900, loss = 0.4406\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 4000, loss = 0.4130\n",
      "Got 372764 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 4100, loss = 0.4936\n",
      "Got 372727 / 465386 correct (80.09%)\n",
      "\n",
      "Iteration 4200, loss = 0.5595\n",
      "Got 372556 / 465386 correct (80.05%)\n",
      "\n",
      "Iteration 4300, loss = 0.5358\n",
      "Got 320993 / 465386 correct (68.97%)\n",
      "\n",
      "Iteration 4400, loss = 0.4738\n",
      "Got 372763 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 4500, loss = 0.4771\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 4600, loss = 0.5571\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 4700, loss = 0.5021\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 4800, loss = 0.5389\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 4900, loss = 0.5210\n",
      "Got 372398 / 465386 correct (80.02%)\n",
      "\n",
      "Iteration 5000, loss = 0.5850\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 5100, loss = 0.4040\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 5200, loss = 0.6062\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 5300, loss = 0.5422\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 5400, loss = 0.5022\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 5500, loss = 0.5114\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 5600, loss = 0.5245\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 5700, loss = 0.5254\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 5800, loss = 0.4334\n",
      "Got 372751 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 5900, loss = 0.6091\n",
      "Got 372763 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 6000, loss = 0.4801\n",
      "Got 372028 / 465386 correct (79.94%)\n",
      "\n",
      "Iteration 6100, loss = 0.4357\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 6200, loss = 0.4996\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 6300, loss = 0.4296\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 6400, loss = 0.5553\n",
      "Got 372727 / 465386 correct (80.09%)\n",
      "\n",
      "Iteration 6500, loss = 0.4650\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 6600, loss = 0.4585\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 6700, loss = 0.4606\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 6800, loss = 0.4834\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 6900, loss = 0.4826\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 7000, loss = 0.5694\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 7100, loss = 0.3554\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 7200, loss = 0.4630\n",
      "Got 372678 / 465386 correct (80.08%)\n",
      "\n",
      "Iteration 7300, loss = 0.5710\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 7400, loss = 0.5057\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 7500, loss = 0.3943\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 7600, loss = 0.6555\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 7700, loss = 0.4638\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 7800, loss = 0.4611\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 7900, loss = 0.4830\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 8000, loss = 0.4989\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 8100, loss = 0.4385\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 8200, loss = 0.5932\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 8300, loss = 0.4613\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 8400, loss = 0.3712\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 8500, loss = 0.4611\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 8600, loss = 0.5055\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 8700, loss = 0.5690\n",
      "Got 372705 / 465386 correct (80.09%)\n",
      "\n",
      "Iteration 8800, loss = 0.5261\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 8900, loss = 0.4828\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 9000, loss = 0.5700\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 9100, loss = 0.5048\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 9200, loss = 0.5047\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 9300, loss = 0.4617\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 9400, loss = 0.5264\n",
      "Got 372502 / 465386 correct (80.04%)\n",
      "\n",
      "Iteration 9500, loss = 0.5525\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 9600, loss = 0.6132\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 9700, loss = 0.4182\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 9800, loss = 0.5876\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 9900, loss = 0.6127\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 10000, loss = 0.5053\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 10100, loss = 0.4795\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 10200, loss = 0.5048\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 10300, loss = 0.4832\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 10400, loss = 0.5048\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 10500, loss = 0.4616\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 10600, loss = 0.5709\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 10700, loss = 0.4392\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 10800, loss = 0.4184\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 10900, loss = 0.4827\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 11000, loss = 0.5047\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 11100, loss = 0.5043\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 11200, loss = 0.5048\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 11300, loss = 0.4830\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 11400, loss = 0.5479\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 11500, loss = 0.4615\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 11600, loss = 0.6980\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 11700, loss = 0.4619\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 11800, loss = 0.5273\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 11900, loss = 0.5276\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 12000, loss = 0.6144\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 12100, loss = 0.3913\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 12200, loss = 0.3804\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 12300, loss = 0.5472\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 12400, loss = 0.5047\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 12500, loss = 0.4404\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 12600, loss = 0.5690\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 12700, loss = 0.5051\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 12800, loss = 0.7478\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 12900, loss = 0.4629\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 13000, loss = 0.4833\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 13100, loss = 0.5048\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 13200, loss = 0.3680\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 13300, loss = 0.3954\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 13400, loss = 0.5936\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 13500, loss = 0.5055\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 13600, loss = 0.6335\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 13700, loss = 0.5049\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 13800, loss = 0.6115\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 13900, loss = 0.4385\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 14000, loss = 0.4838\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 14100, loss = 0.5260\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 14200, loss = 0.4826\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 14300, loss = 0.4381\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 14400, loss = 0.3950\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 14500, loss = 0.5953\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 14600, loss = 0.4604\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 14700, loss = 0.4840\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 14800, loss = 0.6839\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 14900, loss = 0.4383\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 15000, loss = 0.5474\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 15100, loss = 0.4380\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 15200, loss = 0.5675\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 15300, loss = 0.3967\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 15400, loss = 1.6510\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 15500, loss = 0.4638\n",
      "Got 372762 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 15600, loss = 0.5047\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 15700, loss = 0.5894\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 15800, loss = 0.6134\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 15900, loss = 0.4385\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 16000, loss = 0.5269\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 16100, loss = 0.4834\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 16200, loss = 0.4414\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 16300, loss = 0.5255\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 16400, loss = 0.3937\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 16500, loss = 0.3275\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 16600, loss = 0.6113\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 16700, loss = 0.5050\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 16800, loss = 0.4832\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 16900, loss = 0.3930\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 17000, loss = 0.5047\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 17100, loss = 0.4614\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 17200, loss = 0.6336\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 17300, loss = 0.4613\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 17400, loss = 0.4610\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 17500, loss = 0.3966\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 17600, loss = 0.4603\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 17700, loss = 0.5909\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 17800, loss = 0.5282\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 17900, loss = 0.4408\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 18000, loss = 0.5887\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 18100, loss = 0.4827\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 18200, loss = 0.7074\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 18300, loss = 0.4185\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 18400, loss = 0.4833\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 18500, loss = 0.4405\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 18600, loss = 0.5477\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 18700, loss = 0.5670\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 18800, loss = 0.4830\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 18900, loss = 0.3935\n",
      "Got 372765 / 465386 correct (80.10%)\n",
      "\n",
      "Iteration 19000, loss = 0.5048\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-72-0e8f13979776>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     21\u001b[0m \u001b[0mloss_hist\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m \u001b[0macc_hist\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m \u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel_init_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimizer_init_fn\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-70-de4a73cda131>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(model_init_fn, optimizer_init_fn, num_epochs)\u001b[0m\n\u001b[0;32m     37\u001b[0m                     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Iteration %d, loss = %.4f'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mloss_np\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m                     \u001b[0mloss_hist\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss_np\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 39\u001b[1;33m                     \u001b[0mcheck_accuracy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msess\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mval_dset\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscores\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mis_training\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mis_training\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     40\u001b[0m                     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     41\u001b[0m                 \u001b[0mt\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-56-370e84ecfa80>\u001b[0m in \u001b[0;36mcheck_accuracy\u001b[1;34m(sess, dset, x, scores, is_training)\u001b[0m\n\u001b[0;32m     15\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mx_batch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_batch\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdset\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m         \u001b[0mfeed_dict\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mx_batch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mis_training\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m         \u001b[0mscores_np\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     18\u001b[0m         \u001b[0my_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mscores_np\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m         \u001b[0mnum_samples\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mx_batch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Miniconda3\\envs\\myenv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    927\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    928\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 929\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    930\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    931\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Miniconda3\\envs\\myenv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1150\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1151\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1152\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1153\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1154\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Miniconda3\\envs\\myenv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1326\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1327\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[1;32m-> 1328\u001b[1;33m                            run_metadata)\n\u001b[0m\u001b[0;32m   1329\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1330\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Miniconda3\\envs\\myenv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1332\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1333\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1334\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1335\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1336\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Miniconda3\\envs\\myenv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1317\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1318\u001b[0m       return self._call_tf_sessionrun(\n\u001b[1;32m-> 1319\u001b[1;33m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[0;32m   1320\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1321\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Miniconda3\\envs\\myenv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[1;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[0;32m   1405\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[0;32m   1406\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1407\u001b[1;33m         run_metadata)\n\u001b[0m\u001b[0;32m   1408\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1409\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "def model_init_fn(inputs, is_training):\n",
    "    input_shape = (247,)\n",
    "    hidden_layer_size, num_classes = 200, 2\n",
    "    initializer = tf.variance_scaling_initializer(scale=2.0)\n",
    "    layers = [\n",
    "        tf.layers.Flatten(input_shape=input_shape),\n",
    "        tf.layers.Dense(hidden_layer_size, activation=tf.nn.relu,\n",
    "                        kernel_initializer=initializer),\n",
    "        tf.layers.Dense(hidden_layer_size, activation=tf.nn.relu,\n",
    "                        kernel_initializer=initializer),\n",
    "        tf.layers.Dense(hidden_layer_size, activation=tf.nn.relu,\n",
    "                        kernel_initializer=initializer),\n",
    "        tf.layers.Dense(num_classes, kernel_initializer=initializer),\n",
    "    ]\n",
    "    model = tf.keras.Sequential(layers)\n",
    "    return model(inputs)\n",
    "\n",
    "def optimizer_init_fn():\n",
    "    return tf.train.AdamOptimizer()\n",
    "\n",
    "loss_hist=[]\n",
    "acc_hist=[]\n",
    "train(model_init_fn, optimizer_init_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font size='20'>Plot 2</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XucW3Wd//HXN8nc26YXKC1tIQUqWFELlpt4wQuKRAHdFYHVxUVFd8G7rvGybn66+9t4X3blh1tBxVW5eUWDXBa5LMitQLkWEEqgV1raknamnUtyvr8/viczZ9LMJDOTTDKZ9/Px6KPJyTnJd06S88nn+/2c7zHWWkRERBpNqN4NEBERKUUBSkREGpIClIiINCQFKBERaUgKUCIi0pAUoEREpCEpQImISENSgBIRkYakACUiIg0pUu8GVCIUCtmOjo56N0NEZMrbs2ePtdZOieRkSgSojo4Oenp66t0MEZEpzxizt95tqNSUiKIiIjL9KECJiEhDUoASEZGGpAAlIiINSQFKREQakgKUiIg0pClRZi4iIpMjlkifAlwEhIFLM6l4qujxg4DLgdn+OolMKn5dLdrS3AEquwE2rYFDToK2GTy6McvszhYOmNXOIxuzvLi7b3DVg+d1cfiCmeQ9y13PbGdPf27Epw2HDEcfNIc5Xa0AWGtZ/dxO2iNhFs3pYE5nC8aYYdus3byL9Tv2DFvW1hLmtYfOoyUcYt22blojIRbP6Ry2Ti7vcecz2xnIeYRDBmPc64eMwbMWz4LnWTxrsdZtU3jpwf8xYCBsDHM6Wzls/gw6WsP0DuT58zMvksvbkn/n/FntLJrdwfqde4btq7FafuAsFs/p5Jlt3TyztXvcz9OIImHDglkdGANbsr0M5L16N0mmgbe8/ADCIVN+xTGKJdJh4GLgZGADcF8skb42k4o/HljtK8DVmVT8klgivRy4DohVvTE0e4B67s/w64/Axx8gm2/j9IvvJO9ZWsMh+osOJPO6Wrn/n07mvswO3n/ZPWWfOhIyvPNVC/ne+1bwxJbdvPcHdw0+1tka5l/OOJL3HL0YgHuf3cH7Vt01GECC3nHkAs573VI+cNk99OU83nLEfL7z3hVEO1sAuPmJrXz0v++fwE7Y1xtftj+Xn3csV69ez1d/91hVn3skB0bb2ZTtnZTXEml2T3z9FMKhcC2e+ljg6Uwqvg4glkhfCZwOBAOUBWb5t6PAplo0BJo9QBl/iM3LsbtvgLxnOeUVC1g0p4OVB89hyVyXrfz4zgx/eNjt4+5elzn9+/tWcNj8GSWfdu9Anh/c+gy/XbOJf333K9nR0w/AZ05+GV1tEb51wxM8+PxLvOfoxeztz/OPv3yIxXM6uPicowkFMqtbn9zKt298ihsff4Elczp43bL9+Nndz/Pg+p2cdPh8gMGs68rzj6e9JUzez5Y8z/oZlSHkZ1UGg8VFwUIwtLgMDyDnWb7xxyfY4geKQrt/f+HrKEr4sBY2ZfeycedelsztZGG0fVxvQd6z/PmZ7Tz4/E4+/PpDOCY2d5/Xmsr68x6bX+rFYlkY7aAtomFdqb3W8IQ+ZxFjzOrA/VXW2lX+7UXA+sBjG4DjirZPAjfGEumPA13AWyfSmFEbWqsnbgiFXxhenv6cy5hOOXIBZxy1aNhqC6Jt5L2hgzjAyw6YyfIDZzGSJ4/Yzc1PbKWnL0dPnwtqbz5iPkcuivJftz0z2NVzya1Pk9m+h1985DhetXj2sOc4clGUvpzH7x/axE/PO47uvhw/u/t5egeGsrvN2V46W8Mct3TuPt2G47F4Tgf3P78TcIG2NRLilYujJdcdaflYvXrJ7PIrTWUH1bsBImOSs9auHOGxUgeZ4r6fs4GfZFLx78QS6ROA/44l0kdmUvGq928398894wcomx/s0mst8Qs3HAqR8yzW2sFAFQmPHgy62txz9/Tn2dOfB1zXXuE1CgFx7ZbdHLFgJq89dL+Sz/PZtx3OLZ87iYPmddLe4trWl8sPPr4l28uCaHtVghNAR2uEvf2ubXv784NtFhHBZUxLAvcXs28X3oeAqwEyqfhdQDtQ+gA3QdMng/L8AFUiNY74g42ehZy/XqhMQOhqdbuupy9Ht59BzWiLDL5Gnx8Q+3MebS2jB4FC8Cms1zswFKA2Z/eOu3utlM7WMHv9ApA9/Xk6y7RNRKaV+4BlsUR6KbAROAs4p2id54G3AD+JJdIvxwWobbVozPTJoHKjZVAuQOQ8D88fr4mUqZDpahsKUIWKv85CgApkUP05j7YK+4vb/bYFu/i2ZHtZMKt6lxrpbA2zZyCPtZa9/XnalUGJiC+TiueAC4EbgLW4ar3HYon012KJ9Gn+ap8FPhJLpB8CrgA+mEnFS5cCT9A0yaC8igJU3rODJdflSjgLAWpPf56ePpfxdLTs28XXn/cGl5fT7q9X6OLLe5YXdvdVNYNqbwljLfTlPPYOqItPRIbzz2m6rmjZVwO3HwdOnIy2NHkG5f95Nj/Y5VYqQEUCAariMSj/wN7tZ1AdLeHBoNYaDg0WSfTnvJKvWUpbUQb1Yncfec+yoMpdfOAC657+HJ0tzf0bRUSmruYOUCH/4Buo4is1BjUsg/LGmkHl6O7LD96Hfbv4Ki0JjYRDREJmcAxqs18OvmBW9QPU3oE8e/vzdCiDEpEG1eQBqtDFlxsMGKXOU4kMjkEFMqjQ6LumUCTR3ecykUJVH0BL4ETg/nzlGRS4LrhCBrUl6y58Wc0MqsNv997+HHv68xV3P4qITLbmDlAVF0m4ZWPJoDr9gLSnL0dPX57O1lEyqDEFqBC9ueEZVFWr+FqGuvg0BiUijay5A1SwSKKCMSiXQXnDlo2kJRyiNRKiu9+NQXUFDvTBANU3xgDVFgkPdvFtyfbSGg4x15/zrxo6AmNQ6uITkUbW3AGqVAY12hhUvvIMClyhxJ6+PD39+cESc4C2QBdfXy4/pmlJ2ltC9Plt3Vzlk3RhKEDtHXAnGCuDEpFG1dwBqjCOFCiSKHXSbKFiL+d55POVnQcFrlCiMNXRjLbSGVR/zhvT/GxtkTB9gQyqmuNPEKji63NdfBqDEpFG1dwBKpBBFc4tqlYVH7hCiZ7+HHv6csPGoApFEtbacRRJhIaKJHb1VnX8CRgsK9+5x00U29GqMnMRaUzNHaBKTBbbUuL8puIqvsIs4eV0tYXdibr9+ZJjUG5+v7HNPOyq+Fww3bq7l/kz2yrethKFLr7t3S5AqYtPRBpV7X4+J6NLgJ8CCwAPWEUyexHJaBL4CENzN32JZLYmV2MclkH5mUypwFNcxVfphcC62iKDJ+p2ljgParTKwZG0t4TZ1TtALu/RO+Axs72l4m0rMRigevqG3RcRaTS17N/JAZ8lmX2AZHQmcD/J6E3+Y98jmf12DV/bKTpRd6Q58Yqr+CoZfwLXxbdh514G8nZwolhwGVPOs4PFDmOr4gvRN+ANTp8UPAG4GgpjTtv9a0FpDEpEGlUNM6jsZmCzf3s3yeha3MWwJk9RkcRIgWJoDMobUwbV2RZm6y53rlJnURcfMHidqDGfqJvL0+NPQNtV5QwnHDK0RULsUBefiDS4yRmDSkZjwFFA4VrqF5KMPkwy+iOS0TmlNjHGnG+MWW2MWZ3L5cb3ukVl5iMFisEMKu/GoCrNoGa0RejxrwXV1To8gwLY7V+dd6xl5r0D3mBwq3YGBS4oqYtPRBpd7QNUMjoD+BXwKZLZXcAlwKHAClyG9Z1Sm1lrV1lrV1prV0Yi4zxIB4skRqmmK67iC5eZ5qggWLnX2bZvBtU9jgyqcKLuYOBrq34A6WyNBIokVMUnIo2ptkenZLQFF5x+TjL7a7cs+0Lg8R8Cf6jZ6xdnUCONQYWHxqA8z1JpwhPsfusqmuoIoLtvACg9/99I2lvC/hhUoYuv+m9Re0uITf48fxqDEpFGVbsMKhk1wGXAWpLZ7waWLwys9W7g0Zq1oajMfOQMangVX7mJYguC3W9dbaN08Y2xSKI/7w1uW5suvgj+dRk1BiUiDauWGdSJwAeAR0hG1/jLvgScTTK6ArBABvhozVowmEF5o3bxlToPqhLB7rfggb5lsEiicHJw5UGgcNHCwom0tQhQwXEnjUGJSKOqZRXfHUCpI31tznkqJVDF1zdKF19xFV/FZeZlMqhCF99YZ5IA2N7tihiqXcUHw4OpMigRaVRNPpOEHzTKVPGFi86DqjiDCowPBQNJYcypexxdfO1F5ynVqopv8PUiClAi0piaO0AVuvj8CxaOVKwwrIovP7aZJErdLgSk3X1jLzMvtHF7dz/G1KaIoRAE21tChCr8W0VEJltzB6gKy8z3OQ+qxHx9pQQzkWAgaa1CBrWjp5/OlnBNAkih3SoxF5FG1twBKlAk0ZfL0zZCd9Z4z4MqTG/U2To8kLT4GVNhNoixlZn7GVRPf02692AoMKnEXEQaWXMHqOIy8xHn4vPLzO3YZpIonJxbnIlMpMy8MCa0vbuvZgGqEJhUwScijay5A5QxgBlTkURuHEUSxbM97DOTxFjGoAJdfLWYRQKCXXwKUCLSuJo7QIHLosqcqBsZvOS7N6YMqqMljDH7zvYwkSq+wrY5z9ZsjKiQOamLT0QaWfMHKBN2GdRoc/GFgxlU5VV8oZChsyVcPoMaR5EEMOwSHtVUCEzKoESkkTV/GVcojJfPM5C3o4xBDRVJjCWDAldeXpzptISHApQxjOn5CkUSULsAMlgkoQAlIkViifQpwEVAGLg0k4qnih7/HvAm/24nMD+Tis+uRVumQYCK0J8f/cKBw8ag8pVX8QHM7WplblfrsGXBDKo1XPoqviOZjAyqc7CLr/nffhGpXCyRDgMXAycDG4D7Yon0tZlU/PHCOplU/NOB9T+Ou5RSTTT/EcqE6M+7mVFHKveOBCaLHWsG9f1zjt4nkBQyNWvH1r0HwwNUrceg1MUnIkWOBZ7OpOLrAGKJ9JXA6cDjI6x/NvDPtWpM8weoUJj+nAtQIwWLQjwarOKr8ERdgMPmz9hnWUtg+5HOvRpJMIjOqFEVn8agRKa1iDFmdeD+KmvtKv/2ImB94LENwHGlniSWSB8MLAX+VJNWMh0ClAnT7/kBaoQxKGMMkZAh73l4dmxjRiM9X2skNOr0SiNpCYcIhwx5z9bwRN3CVEcKUCLTUM5au3KEx0od/OwI654F/DKTiuer06x9NX8VXyiMP6HDqN1t4ZAZyqDGMGY0kkIwHGsXH0C7v01nrar41MUnIqVtAJYE7i8GNo2w7lnAFbVszDTJoNzN0YJFJGTI5y35MUwWO5rWSAj6xnaSbkF7S5ie/nzNuvjmdLYysy3CQXM7a/L8IjJl3QcsiyXSS4GNuCB0TvFKsUT6cGAOcFctGzMNMqgQ/X4COlqwGMqgKp8sdjQTyaAK3YK1KpLoaotw75ffyilHLqjJ84vI1JRJxXPAhcANwFrg6kwq/lgskf5aLJE+LbDq2cCVmVR8pO6/qpgWGVRfJRlUODRYxVe1DKrMa46kMDZUqzJz0DlQIlJaJhW/jqILy2ZS8a8W3U9ORlumQQYVwb/y+qgVdcMyqDGcBzWSwQA1ji6+NlXZiYhMhwAVHjwPquwYlOdVLYNqmUiRhD+bRC0zKBGRRtf8AcqE6fdcwBmt5DtYxTfRMnOYYBefn+nVqopPRGQqaP4AFQrRn3cBp1yZeTXHoNomUiRRyKB0xVsRmcaaP0AFy8wrreKrYgbVNp4y88EMSmNQIjJ9NX+ACg118ZUbgxrIeVjLmCaLHUlhuqPxjkG1RkKD41giItNR8x8BTWUBKhwK0ZdzqVZVzoOawBhUV1uEWe0tE26DiMhU1vyDHGPIoPpyrh69OudBue658ZSZn/+GQzj1lQsn3AYRkams+QOUCdPnlT8nKRwyQxlUNQLUBIokDp7XxcHzuibcBhGRqaz5u/iCGdQoASoSMvT7AareM0mIiMi0CVAhWsKG0CiBp/oZ1PiLJEREZDoEKBOm34bKjgVFwsEMqr5THYmIyHQIUH4XX7lMxlXxuSKJqp4HpQxKRGRcalckkYwuAX4KLAA8YBXJ7EUko3OBq4AYkAHOJJndWbN2mDD9Nlw2QEUCXXyjdQVWqjXsV/EpQImIjEstj5454LMksy8HjgcuIBldDiSAm0lmlwE3+/drJxSizysfoMIhQ99AFcegVCQhIjIhtTt6JrObSWYf8G/vxl38ahFwOnC5v9blwBk1awMMjkGNdqkNcEGpt4rnQQ3OJBHWdEUiIuMxOT/vk9EYcBRwD3AAyexmtzy7GZhfahNjzPnGmNXGmNW5XG78rx0K0++FyxYrhEMG618bshoZVJsyKBGRCan90TMZnQH8CvgUyeyuSjez1q6y1q601q6MRCYwVBaK0G9DtFQwBlWg86BEROqvtkfPZLQFF5x+TjL7a3/pCySjC/3HFwJba9oGEyaPKZsVBUvLqzoXn8rMRUTGpYZjUFEDXAasJZn9buCRa4Fz/dvnAr+rWRsAQiHy1hA2owed4RlUFc6DUhWfiMiE1HIuvhOBDwCPkIyu8Zd9CUgBV5OMfgh4HnhvDdsAJoxnDeViTjiQNVVjDGpBtJ2Qgfkz2yb8XCIi01ENz4PK3gGMdKR/S81et1gojAe0lAk61R6Des3Bc1j9lZOZ29U64ecSEZmOmr//yYTJW0OoTBdfMChVI4MCFJxERCag+QNUKIxHqHyAMtXNoEREZGKaP0D5GVS5oDN8DKr5d4uISKNr/iNxKIxH+S6+ao9BiYjIxDT/FXVDfhVfmZhT7fOgRESmolgifQpwERAGLs2k4qkS65wJJAELPJRJxc+pRVuaP4PyT9QtlxUpgxKR6S6WSIeBi4F3AMuBs2OJ9PKidZYBXwROzKTirwA+Vav2NH+ACoXJV1IkUYMqPhGRKeZY4OlMKr4uk4r3A1fiJvgO+ghwcSYV3wmQScVrNhtQ83fxGVfFV67XLhiUygUzEZEpLGKMWR24v8pau8q/vQhYH3hsA3Bc0fYvA4gl0nfiugGTmVT8+po0tBZP2lBCIb9Iwo662rAMSmNQItK8ctbalSM8VurgV3zwjADLgJOAxcD/xhLpIzOp+EvVa6LT/F18JkzehsoWSWgMSkSEDcCSwP3FwKYS6/wuk4oPZFLxZ4EncQGr6qZBBlVZF184MOu4zoMSkWnqPmBZLJFeCmwEzgKKK/R+C5wN/CSWSO+H6/JbV4vGNP+R2LgiiXCZLj5lUCIy3WVS8RxwIXAD7iroV2dS8cdiifTXYon0af5qNwDbY4n048AtwOczqfj2WrRnGmRQETwMZp9u1OFUxSciAplU/DrguqJlXw3ctsBn/H811fwZVIVdfMqgREQaS/MHKBOqqItPGZSISGNp/gBVmM28TBdfsDBCGZSISP01f4Ay4TGdBxUOGYxO1BURqbvmD1D+VEfhMqtFAgFKRETqr/kDlClcsLBMBuVXUWj8SUSkMTR/gAqN7TwoZVAiIo1hWgQoD0O4wvOglEGJiDSGpg9QljCWEAZv1PXCppBBNf0uERGZEpp+Jom8ceURZbv4NAYlIlJ1sUT6V8CPgD9mUvHRM4UiTZ8u5P0/sXwXn7+eApSISDVdgptw9i+xRDoVS6SPqHTDps+grAkDXtkqPhVJiIhUXyYV/x/gf2KJdBQ3C/pNsUR6PfBD4GeZVHxgpG2nTQZVbiYJFUmIiNRGLJGeB3wQ+DDwIHARcDRw02jbNX0GNTgGVaZIQhmUiEj1xRLpXwNHAP8NvCuTim/2H7oqlkivHnnLaRCgPOsC1FimOhIRkar5fiYV/1OpBzKp+EiXngemQRefZwpFEuUyKLdepNx1OUREZCxeHkukZxfuxBLpObFE+h8q2bDpA1TeuCQxZCub6kjnQYmIVNVHMqn4S4U7mVR8J/CRSjasXRdfMvoj4J3AVpLZI/1lSb9h2/y1vkQye13J7avEs36RhKlsDEpFEiIiVRWKJdLGvxIvsUQ6DLRWsmEtx6B+Anwf+GnR8u+RzH67hq87TL7CLj6NQYmI1MQNwNWxRPoHgAU+BlxfyYa1689KZm8HdtTs+SvkDZaZK4MSEamDLwB/Av4euAC4GfjHSjasRxXfhSSjfwusBj5LMruz1ErGmPOB8wFaWyvKBkuqNEApgxIRqT5/eqNL/H9jMtkB6hLg67g07+vAd4DzSq1orV0FrALo6uoavcJhFENTHVVYxacAJSJSNbFEehnwb8ByoL2wPJOKH1Ju28oCVDL6SeDHwG7gUuAoIEEye+OYWprMvhB4zh8CfxjT9uOQ96+lW+lMEqriExGpqh8D/wx8D3gT8HdARZlApUfj80hmdwFvA/b3XyA15mYmowsD994NPDrm5xgj6++HsKr4RETqoSOTit8MmEwq/lwmFU8Cb65kw0q7+ApH7VOBH5PMPkQyOvqRPBm9AjgJ2I9kdAMugp5EMroC18WXAT5a4euPW2Gqo5DNj7peKGQwZuh8KBERqYreWCIdws1mfiGwEZhfyYaVBqj7SUZvBJYCXyQZnQllBnWS2bNLLL2swtermnyFRRLgsidlUCIiVfUpoBP4BK724E3AuZVsWGmA+hCwAlhHMruHZHQurpuv4RVO1C1XJAFuHEpVfCIyncUS6VNws42HgUszqXiq6PEPAt/CZULg5tq7dITnCgNnZlLxzwPdjDFuVDoGdQLwJMnsSySj7we+AmTH8kL14hUu5V6miw/cZd+VQYnIdOUHlIuBd+Cq7s6OJdLLS6x6VSYVX+H/KxmcADKpeB54TSyRHteBtdIM6hLg1SSjr8adYHUZboaIN47nRSdT3s+gTJkqPlAGJSLT3rHA05lUfB1ALJG+EjgdeHwCz/kg8LtYIn0N0FNYmEnFf11uw0ozqBzJrMU19CKS2YuAmeNp6WTzClV8lM+g5s1oY27X+E8KFhGZAiLGmNWBf+cHHlsErA/c3+AvK/ZXsUT64Vgi/ctYIr2kzOvNBbbjKvfe5f97Z0UNrWQlYDfJ6BeBDwCvJxkNAy0VbltX+cExqPIB6qqPHs+Mtqa/RJaITG85a+1I12Eq1YVU3P30e+CKTCreF0ukPwZczihl45lUfNz1CpUejd8HnIM7H2oLyehBuEGyhpf393fIli+SmD+zvew6IiJNbAMQzIgWA5uCK2RS8e2Buz8EvjHaE8YS6R+zb5Ajk4qXnEUoqLIuvmR2C/BzIEoy+k6gl2S2eJbyhmQHy8zLZ1AiItPcfcCyWCK9NJZItwJnAdcGV4gl0sEJF04D1pZ5zj8Aaf/fzcAsXEVfWZVOdXQmLmO6FZcC/ifJ6OdJZn9Z0fZ1NDQXnwKUiMhoMql4zj+Z9gZcmfmPMqn4Y7FE+mvA6kwqfi3wiVgifRqQw12x4oNlnvNXwfuxRPoK4H8qaU+lXXxfBo4hmd0KQDK6v/8CUyBAVd7FJyIy3WVS8euA64qWfTVw+4vAFyfwEsuAgypZsdIAFRoMTs52psjl4j2/51MZlIjI5Isl0rsZPga1BXeNqLIqDVDXk4zeAFzh338fRRG2URUCVLm5+EREpPoyqfi4T0mqtEji87hrM70KeDWwimS2oghYb3k/QqlIQkRk8sUS6XfHEulo4P7sWCJ9RiXbVn7STzL7K+BXZddrMJ51AaqSqY5ERKTq/jmTiv+mcCeTir8US6T/GfhtuQ1HD1DJaHHfYYEBLMnsrDE2dNIVMiiNQYmI1EWpnrqKkqMyASo7JaYzGk0hg9IYlIhIXayOJdLfxU1Ca4GPA/dXsuGUqMSbiKEAlatzS0REpqWPA/3AVcDVwF7ggko2bPqJ5/L+6U/q4hMRmXyZVLwHSIxn2+bPoDxlUCIi9RJLpG+KJdKzA/fnxBLpGyrZtvkD1GAVn2aSEBGpg/0yqfhLhTuZVHwnML+SDZs+QOU1BiUiUk9eLJEenNoolkjHKF0dvo+mH4PydKKuiEg9fRm4I5ZI3+bffwNw/ijrD2r+DKpwHpQyKBGRSZdJxa8HVgJP4ir5Pour5Cur+TOowmSxClAiIpMulkh/GPgk7uKHa4DjgbsY5Sq8BU2fQRWKJIxO1BURqYdPAscAz2VS8TcBRwHbKtmw6QPUYBefpwxKRKQOejOpeC9ALJFuy6TiTwCHV7Jh03fxFar4wihAiYjUwQb/PKjfAjfFEumdwKZKNmz6AGUL14NSBiUiMukyqfi7/ZvJWCJ9CxAFrq9k26YPUEPXg1KAEhGpp0wqflv5tYZoDEpERBpS0wcoz1oMFoOmOhIRmUpq18WXjP4IeCewlWT2SH/ZXNyJWjEgA5xJMruzZm3ABagwHngqMxcRmUpqmUH9BDilaFkCuJlkdhlwM+Ocgn0s8h6EDOAN1PqlRESkimoXoJLZ24EdRUtPBy73b18OnFGz1/d51hLCwkBvrV9KRESqaLLHoA4gmd0M4P8/4pTrxpjzjTGrjTGrc7nxFzjkPUvYWMhVNPWTiIg0iIYtkrDWrrLWrrTWroxExj9U5lnruvgGFKBERKaSyT4P6gWS0YUks5tJRhcCW2v9gp6nACUiUqlYIn0KcBEQBi7NpOKpEdb7a+Aa4JhMKr66Fm2Z7AzqWuBc//a5wO9q/YJ5awkrQImIlBVLpMPAxcA7gOXA2bFEenmJ9WYCnwDuqWV7algkEb0CN6X64SSjG0hGPwSkgJNJRv8CnOzfr6nBKr58H3g6F0pEZBTHAk9nUvF1mVS8H7gSV9xW7OvAN4GaVp/V8Dyo7NkjPPKWmr1mCZ5nCRvj7uT2QmvXZL68iEijiRhjgl1yq6y1q/zbi4D1gcc2AMcFN44l0kcBSzKp+B9iifTnatrQWj55I/CsJRQCLK7UXAFKRKa3nLV25QiPmRLLbOFGLJEOAd8DPliDdu2jYav4qiVvLaFCBjWwp76NERFpbBuAJYH7ixl+aYyZwJHArbFEOoO7Ou61sUR6pIA3Ic2fQXmWcMiAB+R0sq6IyCjuA5bFEumlwEbgLOCcwoOZVDwL7Fe4H0ukbwU+1yxVfJMub3EBCpRBiYiMIpOK54ALgRuAtcDVmVT8sVgi/bVYIn3aZLfHWGvLr1VnXV1dtqenZ1zbXvCLB1j73Bb+1Hc2nHcjHHQL64ZWAAAZPklEQVRc+Y1ERJqUMWaPtXZKDMY3fQY12MUHyqBERKaQpg9Qec8SDvl/psagRESmjKYPUK7M3P8zlUGJiEwZ0yBAQThcCFDKoEREpoqmD1B5zxIKhd0dZVAiIlNG0weoYV18GoMSEZkymj5A5T1LOFzIoDSjuYjIVNH0AWowgwpFFKBERKaQ5g9QhctttHQqQImITCFNH6Dy1j9RN9LuLrchIiJTQvMHKM+fzbylQxmUiMgU0vQByhYyKAUoEZEppekD1OD1oCLtClAiIlNI8wcoD7+Lr1PnQYmITCFNH6DcbOZAizIoEZGppPkD1OAYlMrMRUSmkqYPUHlrMaZEmXl/jyaPFRFpYE0foDzPEi5VZn7F2XD9F+rXMBERGVWk3g2otfxIZebZ9RBuqV/DRERkVNMgg6L0iboDvRqTEhFpYM0foKx1c/FFOiDf5yIWuPEoXR9KRKRhNX2AynuBLj4YKpQY6FWRhIhIA2v6AOUutxEIUAO9YK0yKBGRBjcNAhRDVXzgglKuz7+tMSgRkUbV9AHKzWaOG4MCN91RIXNSgBIRaVj1KTNPRjPAbiAP5EhmV9bqpTyvuItvD+S63G1dH0pEpGHV8zyoN5HMvljrF8nbwom67W5BsLzcy0F+QOdDiYj4Yon0KcBFQBi4NJOKp4oe/xhwAS7B6AbOz6Tij9eiLU3fxTdsLj7wM6hA9Z4KJUREAIgl0mHgYuAdwHLg7FgivbxotV9kUvFXZlLxFcA3ge/Wqj31ClAWuJFk9H6S0fNLrWCMOd8Ys9oYszqXy437hTyPobn4wB+DCgYodfOJiPiOBZ7OpOLrMql4P3AlcHpwhUwqvitwtwt3PK+JenXxnUgyu4lkdD5wE8noEySztwdXsNauAlYBdHV1jXsHuKmOKMqggjNKKECJyLQSMcasDtxf5R9vARYB6wOPbQCOK36CWCJ9AfAZoBV4c60aWp8MKpnd5P+/FfgNLmrXRN4bZQwKFKBEZLrJWWtXBv6tCjxmSqy/T4KQScUvzqTihwJfAL5Sq4ZOfoBKRrtIRmcO3oa3AY/W4qU8z+3XUPEYlAKUiEgpG4AlgfuLgU2jrH8lcEatGlOPDOoA4A6S0YeAe4E0yez1tXghz/oByhho9UvL+7tVJCEiUtp9wLJYIr00lki3AmcB1wZXiCXSywJ348BfatWYyR+DSmbXAa+ejJfK+wFqsIrPhKCve3jWlNN8fCIiAJlUPBdLpC8EbsCVmf8ok4o/FkukvwaszqTi1wIXxhLptwIDwE7g3Fq1p6mvB1WYuDxkDBgDrTOVQYmIjCKTil8HXFe07KuB25+crLY09XlQQxmUv6BtBvTt1hiUiMgU0NQBatgYFECrH6CUQYmINLzmDlBeUYBqm+G6+IJBSdeEEhFpSE0doPJeoEgC/Ayq2wWlFr+qTxmUiEhDau4AZQPnQQG0FYok9rrbJqwxKBGRBtXUAcqPT24mCXBBqW+3n0G1u9Lzgb3Q8yJsfKB+DRURkX00dYDKD45B+QsGiyT2ugsYtnS423f+O/z09KGIJiIidTc9AlSouEiikEG1uwxq1ybo26XxKBGRBtLUAapQZh4Olpl7Oeh9yXXvtXS6oNSzzT2+Z3udWioiIsWaPEC5/8PBIglwASnS7rr4CmNQMPS/iIjUXVMHqEIXXyGBGgxQ3dtccBoskihkUDsmv5EiIlJSUwcoz5Y4DwpgoMdlUJF2NyZV6NpTF5+ISMNo6gA1eKJucCaJgha/i2/XJrD+rLJ71MUnItIomns28+ITdVtnDj0Y6YCWfuh+YWiZMigRkYbR3AEqeLkNKMqgOiDfN3wDBSgRkYbR1AFqn8tttBYHqP6h+6GIqvhERBpIcweofWYzD3bxtbsuvoJ5h6mKT0SkgTR1kYQdqYoPhsrMwV0Kft5h6uITEWkgTR2g9smgwhFXHAFDZeYAnfOga38FKJGpbM0VcPu36t0KqaLmDlDFV9SFoUKJFn+yWHDBqWs/2LtjqLJCRKaW1ZfBn/9Tkz43kaYOUIVYM9jFB0PdfMEuvs557p/13Dx9IjK1WAvbnoTeLOzaWO/WSJU0d4AqruKDoQwqUpRBdc5zt9XNJzL1FK5IAPDC4/Vti1RNUweoQhefCXbxFU7WLcwkAX6Amutuq9RcZOrZtnbo9tbH6tcOqaqmDlBe8VRHMFRqvk8GtZ+7XcigPM8Fq95dI7/AQG+VWywi47L1Cfd/exReUIBqFk0doAbn4guVKpJoHxqD6tpv3y6+X38EvnUopJbA//wf8PLw2wvgdxe64HXjP8F3DoeX1k/SXyMiI9q21v3IXHK8uviaSFOfqFu4HtSwKr7WwBhUe9SdA7XfsuEB6vl74NFfwivf6y5weMd34fm74fk/u3V6s7D2Wnf7xq/AmZdPzh8k5XkehJr6d1d5256CUBjmHTrx59q1GW76KrzuU3DAKyb+fOMx0OsKH0b7e7Y+AfNfDgcsh2f+BLl+iLROXhsn4plbYO4hMOfgerek4TR5gCpMFhtYWOjia+mA6CL47FMwY39/WSc8fbMLPjMWwLsugnCr6+rL/C+89uOwews8cg3MXw4vezvc8T14+GqIvQ7W/t4Fr2M/Ai1dsDPjvlQDe+GJNCx4pfsCeXlXdRQeYffv3ena8eJTbr1IK3RvhbmHuufe/gw8exsc/bfQtxvu/Hc46gOw/+Hld8qLf3F/Q3QRzI4N7Zz8AGxa49rbMWdoLC4UhnW3QNssWHay+3X6wE9h0dFwyJuG9l2Bte7vC0fcgeXWf3PPueL9pQNH91b3I2DWgSO3OZ+DZ2+FA14JMw8YWvbMn9zfMX+5u+jX+nvhF2fC8f8Ab/h84EJgRXY8636AHP3BfdtfaNPzd8Hsg2D+K9z+37MDnvuz+7uL2+p50L0Ftj8N625178mbv+J+/Nz9A9j+F3fO3Ws/AfsdNvLfCe7Amvlf9zodc0Zft8Ba91mbdSBsXQs/ibupu867wX0mBva49/fJP8LTN8Gbvjx0sA9uG2lzy3q2w8bVLiBdcRZseQQ23g8fvW34bCxBL62HNb+A5ae757v9m7Ds7fCqM+G+S2H2we6zW3hPPM/dLn6PdjwLj/wSVp4HXfNg70vw8/e69px9lfsM7t4MMxe6bZ+/G+bEXAXfq89y75c34Pb57IPg0V/Dpgfdd+q4j8HBJwyVoY/0+QD32b3pn2DBq9zzhltKr/fUjXBbCk76omtbsd4sPPgzmLUIDn0ztM9yf9Ojv3LHjPX3wrUXwpyl8NHb3eMFXt61ffdmt/2BR43e5iqJJdKnABcBYeDSTCqeKnr8M8CHgRywDTgvk4o/V4u2GDsFzhno6uqyPT09Y97u9w9t4uNXPMhNn34Dyw7wv1i3/Jv7QP3js0OFEQU3fBnu/aGbRPZd/wGvOdct3/uSO/C8/DTI7YVbUy44RBfDJSfCjmeGP0/HHL9kPQtd890XZu9OwMDile7X3kCPC4IHHe8yuM0Pw/wj3Gtc80F46Tm3PgDWFXf073YH6RefdPMILnqNCyQvPeeywdd/Fp66wWWB7bPd44tXunY+f5cLrFseGWpn4fWNgcyd0LPVvWbHbL+9RY56P6z9g/u78D83Mw6AUIs7COZ6XTA2Bl72DjdT/MbVbr0Dj4Yj4i4YrbsN5h3iumTuvsTtn2Vvd78iQyF3YO/N+sErDy886n5Bz4nBuX9wf8utqaH9Pu8wF5Ru+4bbLtcLL3+X+9L37XYXpOze6g7Ai46G1T92FV/ts93BesU5sOkBeOw3sGG1v48Kl2Nuhf2PcD8Wcv6Y46zFrp1zlrq/f90tQxe9NGH3/wHLXRDd9gREl7hLueR63Y+Urv3dvsn1w4z50N/j/h2wHDbcD9nn3Tpv/IJ77a1r3ecv+7y/7xl6rRnz3QHspef9cVTrege8AfdetnbCjnWBN9G4z8N517sfHb/9e3jiD+49LHxe1vx86P03YRfsb/8mLD5mqO0De13AfdWZ7gfTT08bXt7dNd//PAW84t1u+aYH3Oe9baZ7Pw482v3fHoWrz3XBvnMeHPFO96NgZwZmL3Hv4QFHwvq7YeEKF6Se+qN7H3tfgvh34KDXwiUnuB8tLz3vrvfWMdd9Jnt3wave57bxcu6Af+BR0NcNj/3a3T712zB3KVz3ebh31dBnfO6hsOQYWH6G2z/r73FB94m0C15ezh0TOua493jhCvcD6vZvu78H3I+U5WfAc3dCdr17L4xx625e474zc2Lu+zvvMHjkave5K1j4alj5IXjlX0Nr177fzwoZY/ZYa0s+QSyRDgNPAScDG4D7gLMzqfjjgXXeBNyTScX3xBLpvwdOyqTi7xt3g0ZrazMHqN+t2cgnr1zDzZ99I4fu73ftPZF2XRb/cHfpX0V7drgv0KFvqezXSu8ud8Dc9iQccpLb5rZvuq7EJcfAs/8LWHjN38Gzt8MzN7svQud+7ov37G1+dnSIfyCx7rG/vgwOOsEdOLwBd6B8+Gr44+fdL7HDToY/fsFlgvFvw81fd78a5x3mujp6XoSNDwyfsX3RSvfh3v8IF9SeuQU2P+QCwvyXuy/PzmfdgWa/w92v8IEe145Hrhn6Jfy3v/O7Qu92B0/wzyvzx/X6uuGhK9wB+d3/5Q6+d3zX/7IZWPgq90u5bxe84j3uS/nQlS6Y2LwLSu2z/OAXgZkL4LC3ws1fc4HQy7lfym/8vPvxsPpHsOVhl7V++Cb3C/zeH7r91jrDHcS79nfv7QuPuAPtm77kDh7P3Tm0j1v9A2bsdbD0jbB7k9uHWx5278/y010m8eJfXBtffMrtq9jr4eDXuoC1eKULcle93+2P9/7EfS66t8E9P3D7u2er+9sibe69b53hbm951GWIK89z+3rj/UPv3dxDYN4ydwAsfC7zAy5YtM6AQ94ImTtcMD/7Kve+X/l+9+Pn4Ne6z8+BK1xQuPw0tx9DLe6Hzus+7f7+dbe69h30WjjxE+71DzgSXnEG3PX/3HvYOc+9H3u2u0A+Y4ELzh1z4L0/dttH2uHoc10m+Pzd7gfAI9fALf/qPh8LV7jvQG/Wvca2Jxj2g+fUb8Pd/899p2YfBG/5JxdwLj3ZvfcrznEZcPc216vx6C/dd+fvrnf7/9K3uB+Ii1a6dRcf4wLYNR9038HDT3XjzpsedAUVJuyyn3W3uf22+Fh47g44/gJY+nqX7WQ3uGzH5t1n8qATXGBe+gZ4+7/C9Ql4/Fr/85sbet8WvhpO/Y7bv49cAw9d5d7jU7/tvvvZjXDaf8Cfvw+3/l/3PnXMdUFt/5e792b/l7nP4X2XuQrFtihceK97H8ahTIA6AUhmUvG3+/e/CJBJxf9thPWPAr6fScVPHFdjyrW1LgEqGR2WQpLMpkZbfbwB6jcPbuDTVz3ErZ87idh+4//FUVOe5w7krZ3uC/Pgz+CEC9wBaaT1C11lu19wQbZzrvuyb3vKfUELB7CBXhe0shtcV89Iz1kJa12348JXuQN+OQN7XWDq2m9o2d6d7nk650Kuzx2cZy+pvA3r73XBf8U5LpgW9oPnwZPXuXYtOXb05+jNukAUCrm2rL/HBbQDV8CRfzVU2TlRO551B+NCl+RYeZ4LgN1bXPfOfsuq0y5wgeWRX7og9Yr3uG6vgr7dLuCV+3Hm5d0Pg0ImcdTflP987X3JBchQePjyvm4X2LathWVvc0GplP4eFxwibS479XLuR8DenS64Lj9j9HZ7/on4wZ6TgV73PG0zXLC462KXUc5a5H6IBcexshvgqevdD8QRv5959wPlhUfg4BPdj8Fgm3L97jtb3E7Pc/tywZFuH+3d6QJRsFvcWhfwn7nZdSGPkzGmHwh0pbDKWrsKIJZI/zVwSiYV/7B//wPAcZlU/MJSzxVLpL8PbMmk4v8y7gaN1tZJD1DJaMkUkmR2xNKb8QYocKXmrqu79n23IiKNrkwG9V7g7UUB6thMKv7xEuu+H7gQeGMmFe8rfrwa6lHudCzwNMnsOpLZfuBK4PRavVgoZBScREQqswEIdmssBjYVrxRLpN8KfBk4rVbBCepTxbcICJ48tAE4rnglY8z5wPkAra1TpFxURGRquw9YFkuklwIbgbOAc4Ir+ONO/4XrCty671NUTz0yqFLpzD79jNbaVdbaldbalZFIU1fDi4g0hEwqnsN1290ArAWuzqTij8US6a/FEunT/NW+BcwArokl0mtiifS1tWpPPY78FaWQIiIy+TKp+HXAdUXLvhq4/dbJaks9AtR9wDKS0RFTSBERkcnv4ktm90khSWY1u6OIiAzT1CfqiojIcKOVmTeaaT6rpoiINCoFKBERaUhToovPGOMBe8e5eQQ3624jUxurQ22sDrVx4hq5fR3W2imRnEyJADURxpjV1tqV9W7HaNTG6lAbq0NtnLhGb99UMSWiqIiITD8KUCIi0pCmQ4BaVe8GVEBtrA61sTrUxolr9PZNCU0/BiUiIlPTdMigRERkClKAEhGRhtTUAcoYc4ox5kljzNPGmES92wNgjFlijLnFGLPWGPOYMeaT/vKkMWajMWaN/+/UOrczY4x5xG/Lan/ZXGPMTcaYv/j/z6lT2w4P7Kc1xphdxphPNcI+NMb8yBiz1RjzaGBZyf1mnP/wP58PG2OOrlP7vmWMecJvw2+MMbP95TFjzN7A/vxBrds3ShtHfG+NMV/09+GTxpi317GNVwXalzHGrPGX12U/NgVrbVP+A8LAM8AhQCvwELC8Adq1EDjavz0TeApYDiSBz9W7fYF2ZoD9ipZ9E0j4txPANxqgnWFgC3BwI+xD4A3A0cCj5fYbcCrwR9w10o4H7qlT+94GRPzb3wi0LxZcr877sOR76393HgLagKX+dz5cjzYWPf4d4Kv13I/N8K+ZM6hjgaetteustTW/tHylrLWbrbUP+Ld342Z0X1TfVlXsdOBy//blwBl1bEvBW4BnrLXP1bshANba24EdRYtH2m+nAz+1zt3AbGPMwslun7X2RmttYdaDu3HXaKubEfbhSE4HrrTW9llrnwWexn33a2q0NhpjDHAmcEWt29HsmjlAlbq0fEMFAmNMDDgKuMdfdKHfzfKjenWfBVjgRmPM/caY8/1lB1hrN4MLtMD8urVuyFkMPxA00j4sGGm/NeJn9DxcVlew1BjzoDHmNmPM6+vVKF+p97YR9+HrgRestX8JLGuk/ThlNHOAqujS8vVijJkB/Ar4lLV2F3AJcCiwAtiM6yKopxOttUcD7wAuMMa8oc7t2YcxphU4DbjGX9Ro+7CchvqMGmO+jJs/7uf+os3AQdbao4DPAL8wxsyqU/NGem8bah/6zmb4j6ZG2o9TSjMHqIa9tLwxpgUXnH5urf01gLX2BWtt3lrrAT9kEropRmOt3eT/vxX4jd+eFwpdUP7/W+vXQsAFzwestS9A4+3DgJH2W8N8Ro0x5wLvBP7G+gMnfrfZdv/2/bjxnZfVo32jvLcNsw8BjDER4D3AVYVljbQfp5pmDlD3AcuMMUv9X9pnAdfWuU2F/unLgLXW2u8GlgfHHt4NPFq87WQxxnQZY2YWbuMG0R/F7b9z/dXOBX5XnxYOGvZLtZH2YZGR9tu1wN/61XzHA9lCV+BkMsacAnwBOM1auyewfH9jTNi/fQiwDFg32e3zX3+k9/Za4CxjTJsxZimujfdOdvsC3go8Ya3dUFjQSPtxyql3lUYt/+GqpJ7C/WL5cr3b47fpdbguiIeBNf6/U4H/Bh7xl18LLKxjGw/BVUY9BDxW2HfAPOBm4C/+/3Pr2MZOYDsQDSyr+z7EBczNwADu1/2HRtpvuO6pi/3P5yPAyjq172ncOE7h8/gDf92/8t//h4AHgHfVcR+O+N4CX/b34ZPAO+rVRn/5T4CPFa1bl/3YDP801ZGIiDSkZu7iExGRKUwBSkREGpIClIiINCQFKBERaUgKUCIi0pAUoGTaMsb82f8/Zow5p8rP/aVSryUilVOZuUx7xpiTcDNlv3MM24SttflRHu+21s6oRvtEpitlUDJtGWO6/Zsp4PX+tXo+bYwJ+9dIus+fnPSj/vonGXctr1/gThrFGPNbf0LdxwqT6hpjUkCH/3w/D76WP2vEt4wxjxp3va33BZ77VmPML427NtPP/VlHRKatSL0bINIAEgQyKD/QZK21xxhj2oA7jTE3+useCxxp3aUdAM6z1u4wxnQA9xljfmWtTRhjLrTWrijxWu/BTXj6amA/f5vb/ceOAl6Bm0vuTuBE4I7q/7kiU4MyKJF9vQ03R94a3KVQ5uHmTwO4NxCcAD5hjHkIdx2lJYH1RvI64ArrJj59AbgNOCbw3BusmxB1De5CdyLTljIokX0Z4OPW2huGLXRjVT1F998KnGCt3WOMuRVor+C5R9IXuJ1H30+Z5pRBicBuYGbg/g3A3/uXRcEY8zJ/VvdiUWCnH5yOwF22vWCgsH2R24H3+eNc++MuHV7P2bdFGpZ+oYm4GbJzflfdT4CLcN1rD/iFCtsofXn764GPGWMexs2kfXfgsVXAw8aYB6y1fxNY/hvgBNzM1hb4R2vtFj/AiUiAysxFRKQhqYtPREQakgKUiIg0JAUoERFpSApQIiLSkBSgRESkISlAiYhIQ1KAEhGRhvT/ASugLzO5OEV/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "t = range(len(acc_hist))\n",
    "data1 = loss_hist[:-1]\n",
    "data2 = acc_hist\n",
    "\n",
    "fig, ax1 = plt.subplots()\n",
    "\n",
    "color = 'tab:orange'\n",
    "ax1.set_xlabel('iteration')\n",
    "ax1.set_ylabel('loss', color=color)\n",
    "ax1.plot(t, data1, color=color)\n",
    "ax1.tick_params(axis='y', labelcolor=color)\n",
    "\n",
    "ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis\n",
    "\n",
    "color = 'tab:blue'\n",
    "ax2.set_ylabel('accuracy', color=color)  # we already handled the x-label with ax1\n",
    "ax2.plot(t, data2, color=color)\n",
    "ax2.tick_params(axis='y', labelcolor=color)\n",
    "\n",
    "fig.tight_layout()  # otherwise the right y-label is slightly clipped\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}