merge_datasets_SVM_poly.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn import decomposition\n",
    "import os \n",
    "import scipy.stats as sc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.metrics import roc_curve, auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_co = pd.read_csv('Transformed Data/Transformed_Co', index_col = 0)\n",
    "df_pt = pd.read_csv('Transformed Data/Transformed_Pt' , index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#pca1 = decomposition.PCA(n_components=50)\n",
    "#df_co_pca = pca1.fit_transform(df_co)\n",
    "#df_pt_pca = pca1.fit_transform(df_pt)\n",
    "\n",
    "df_co_pca= df_co\n",
    "df_pt_pca= df_pt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_co_len = df_co_pca.shape[0]\n",
    "df_pt_len = df_pt_pca.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_co_pca = pd.DataFrame(df_co_pca)\n",
    "df_pt_pca = pd.DataFrame(df_pt_pca)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = pd.concat([df_co_pca, df_pt_pca])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(92,)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y1 = pd.Series([0]*df_co_len)\n",
    "y1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(214,)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y2 = pd.Series([1]*df_pt_len, index = range(df_co_len-1,(df_co_len + df_pt_len)-1))\n",
    "y2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(306,)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = pd.concat([y1,y2]) \n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\validation.py:547: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 out of 100 done\n",
      "1 out of 100 done\n",
      "2 out of 100 done\n",
      "3 out of 100 done\n",
      "4 out of 100 done\n",
      "5 out of 100 done\n",
      "6 out of 100 done\n",
      "7 out of 100 done\n",
      "8 out of 100 done\n",
      "9 out of 100 done\n",
      "10 out of 100 done\n",
      "11 out of 100 done\n",
      "12 out of 100 done\n",
      "13 out of 100 done\n",
      "14 out of 100 done\n",
      "15 out of 100 done\n",
      "16 out of 100 done\n",
      "17 out of 100 done\n",
      "18 out of 100 done\n",
      "19 out of 100 done\n",
      "20 out of 100 done\n",
      "21 out of 100 done\n",
      "22 out of 100 done\n",
      "23 out of 100 done\n",
      "24 out of 100 done\n",
      "25 out of 100 done\n",
      "26 out of 100 done\n",
      "27 out of 100 done\n",
      "28 out of 100 done\n",
      "29 out of 100 done\n",
      "30 out of 100 done\n",
      "31 out of 100 done\n",
      "32 out of 100 done\n",
      "33 out of 100 done\n",
      "34 out of 100 done\n",
      "35 out of 100 done\n",
      "36 out of 100 done\n",
      "37 out of 100 done\n",
      "38 out of 100 done\n",
      "39 out of 100 done\n",
      "40 out of 100 done\n",
      "41 out of 100 done\n",
      "42 out of 100 done\n",
      "43 out of 100 done\n",
      "44 out of 100 done\n",
      "45 out of 100 done\n",
      "46 out of 100 done\n",
      "47 out of 100 done\n",
      "48 out of 100 done\n",
      "49 out of 100 done\n",
      "50 out of 100 done\n",
      "51 out of 100 done\n",
      "52 out of 100 done\n",
      "53 out of 100 done\n",
      "54 out of 100 done\n",
      "55 out of 100 done\n",
      "56 out of 100 done\n",
      "57 out of 100 done\n",
      "58 out of 100 done\n",
      "59 out of 100 done\n",
      "60 out of 100 done\n",
      "61 out of 100 done\n",
      "62 out of 100 done\n",
      "63 out of 100 done\n",
      "64 out of 100 done\n",
      "65 out of 100 done\n",
      "66 out of 100 done\n",
      "67 out of 100 done\n",
      "68 out of 100 done\n",
      "69 out of 100 done\n",
      "70 out of 100 done\n",
      "71 out of 100 done\n",
      "72 out of 100 done\n",
      "73 out of 100 done\n",
      "74 out of 100 done\n",
      "75 out of 100 done\n",
      "76 out of 100 done\n",
      "77 out of 100 done\n",
      "78 out of 100 done\n",
      "79 out of 100 done\n",
      "80 out of 100 done\n",
      "81 out of 100 done\n",
      "82 out of 100 done\n",
      "83 out of 100 done\n",
      "84 out of 100 done\n",
      "85 out of 100 done\n",
      "86 out of 100 done\n",
      "87 out of 100 done\n",
      "88 out of 100 done\n",
      "89 out of 100 done\n",
      "90 out of 100 done\n",
      "91 out of 100 done\n",
      "92 out of 100 done\n",
      "93 out of 100 done\n",
      "94 out of 100 done\n",
      "95 out of 100 done\n",
      "96 out of 100 done\n",
      "97 out of 100 done\n",
      "98 out of 100 done\n",
      "99 out of 100 done\n",
      "C = 250, gamma = 0.1, Score = 0.935064935064935, Random State = 15\n"
     ]
    }
   ],
   "source": [
    "c_val = 0\n",
    "score_val = 0\n",
    "min_range = 0\n",
    "max_range = 100\n",
    "state = 0\n",
    "gamma = 0\n",
    "for rand_state in range(min_range, max_range):\n",
    "    X_train, X_test, y_train1, y_test1 = train_test_split(X, y, random_state = rand_state)\n",
    "    y_train = pd.DataFrame(y_train1)\n",
    "    y_test = pd.DataFrame(y_test1)\n",
    "\n",
    "    scaler = MinMaxScaler().fit(X_train)\n",
    "    X_train = scaler.transform(X_train)\n",
    "    X_test = scaler.transform(X_test)\n",
    "    \n",
    "    \n",
    "    for this_gamma in ([0.1, 1, 5]):\n",
    "    \n",
    "        for this_C in ([0.1, 1, 15, 250]):\n",
    "\n",
    "            clf = SVC(kernel = 'poly', gamma = this_gamma, C = this_C).fit(X_train, y_train)\n",
    "            score = clf.score(X_test, np.ravel(y_test))\n",
    "\n",
    "        if score > score_val:\n",
    "            gamma = this_gamma\n",
    "            c_val = this_C\n",
    "            score_val = score\n",
    "            state = rand_state\n",
    "\n",
    "    print('{} out of {} done'.format(rand_state - min_range, max_range - min_range))\n",
    "print(\"C = {}, gamma = {}, Score = {}, Random State = {}\".format(c_val, gamma, score_val, state))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gamma = 0.01, C = 0.10, accuracy = 0.6753246753246753\n",
      "gamma = 0.01, C = 1.00, accuracy = 0.6753246753246753\n",
      "gamma = 0.01, C = 15.00, accuracy = 0.6883116883116883\n",
      "gamma = 0.01, C = 250.00, accuracy = 0.8181818181818182\n",
      "gamma = 1.00, C = 0.10, accuracy = 0.8441558441558441\n",
      "gamma = 1.00, C = 1.00, accuracy = 0.8441558441558441\n",
      "gamma = 1.00, C = 15.00, accuracy = 0.8441558441558441\n",
      "gamma = 1.00, C = 250.00, accuracy = 0.8441558441558441\n",
      "gamma = 5.00, C = 0.10, accuracy = 0.8441558441558441\n",
      "gamma = 5.00, C = 1.00, accuracy = 0.8441558441558441\n",
      "gamma = 5.00, C = 15.00, accuracy = 0.8441558441558441\n",
      "gamma = 5.00, C = 250.00, accuracy = 0.8441558441558441\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda\\lib\\site-packages\\sklearn\\utils\\validation.py:547: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    }
   ],
   "source": [
    "for this_gamma in ([0.01, 1, 5]):\n",
    "    \n",
    "    for this_C in ([0.1, 1, 15, 250]):\n",
    "         \n",
    "        clf = SVC(kernel = 'poly', gamma = this_gamma, C = this_C).fit(X_train, y_train)\n",
    "        print('gamma = {:.2f}, C = {:.2f}, accuracy = {}'.format(this_gamma, this_C, clf.score(X_test, np.ravel(y_test))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=250, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma=5, kernel='poly',\n",
       "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "  tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gamma = 5\n",
    "C = 1\n",
    "for d in ([1,2,3,5]):\n",
    "    clf = SVC(kernel = 'poly', gamma = gamma, C = C, degree = d).fit(X_train, y_train)\n",
    "    print('gamma = {:.2f}, C = {:.2f}, accuracy = {}'.format(gamma, C, clf.score(X_test, np.ravel(y_test))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.886462882096\n",
      "{'C': 0.1, 'degree': 3, 'gamma': 1}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, auc, roc_curve\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "best_score = 0\n",
    "for i in range(10):\n",
    "    X_train, X_test, y_train1, y_test1 = train_test_split(X, y)\n",
    "    y_train = pd.DataFrame(y_train1)\n",
    "    y_test = pd.DataFrame(y_test1)\n",
    "\n",
    "    scaler = MinMaxScaler().fit(X_train)\n",
    "    X_train = scaler.transform(X_train)\n",
    "    X_test = scaler.transform(X_test)\n",
    "\n",
    "    clf = SVC(kernel = 'poly', gamma = 1, C = 1).fit(X_train, np.ravel(y_train))\n",
    "    # print('gamma = {:.2f}, C = {:.2f}, accuracy = {}'.format(this_gamma, this_C, clf.score(X_test, np.ravel(y_test))))\n",
    "    grid_vals = {'C': [0.1, 1, 5, 10, 12, 15, 20, 25, 50, 100, 250], 'gamma': [0.01, 1, 2, 3, 5], 'degree': [2, 3, 4]}\n",
    "    grid_clf = GridSearchCV(clf, param_grid=grid_vals, scoring='accuracy')\n",
    "    grid_clf.fit(X_train, y_train1)\n",
    "    decision_fn_scores = grid_clf.decision_function(X_test)\n",
    "    \n",
    "    if best_score < grid_clf.best_score_:\n",
    "        best_score = grid_clf.best_score_\n",
    "        best_params = grid_clf.best_params_\n",
    "        prediction = grid_clf.predict(X_test)\n",
    "        y_score_svm = grid_clf.decision_function(X_test)\n",
    "        fpr_svm, tpr_svm, _ = roc_curve(y_test, y_score_svm)\n",
    "        roc_auc_svm = auc(fpr_svm, tpr_svm)\n",
    "        prec = precision_score(y_test, prediction)\n",
    "        rec = recall_score(y_test, prediction)\n",
    "        f1 = f1_score(y_test, prediction)\n",
    "\n",
    "print(best_score)\n",
    "print(best_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.8793\n",
      "Recall: 0.9444\n",
      "F1: 0.9107\n",
      "AUC: 0.8446\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "print('Precision: {:.4f}'.format(prec))\n",
    "print('Recall: {:.4f}'.format(rec))\n",
    "print('F1: {:.4f}'.format(f1))\n",
    "print('AUC: {:.4f}'.format(roc_auc_svm))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}