Hybrid RS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Muhammad Ahsan Athallah\n",
    "### 1301174068"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "from numba import jit,cuda\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9742, 3)\n",
      "(100836, 4)\n",
      "(3683, 4)\n"
     ]
    }
   ],
   "source": [
    "movies = pd.read_csv('ml-latest-small/movies.csv')\n",
    "ratings = pd.read_csv('ml-latest-small/ratings.csv')\n",
    "tags = pd.read_csv('ml-latest-small/tags.csv')\n",
    "print(movies.shape)\n",
    "print(ratings.shape)\n",
    "print(tags.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>964982703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>964981247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>4.0</td>\n",
       "      <td>964982224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>47</td>\n",
       "      <td>5.0</td>\n",
       "      <td>964983815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>5.0</td>\n",
       "      <td>964982931</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId  movieId  rating  timestamp\n",
       "0       1        1     4.0  964982703\n",
       "1       1        3     4.0  964981247\n",
       "2       1        6     4.0  964982224\n",
       "3       1       47     5.0  964983815\n",
       "4       1       50     5.0  964982931"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "610\n",
      "9742\n"
     ]
    }
   ],
   "source": [
    "users = set(ratings['userId'])\n",
    "movieIds = set(movies['movieId'])\n",
    "print(len(users))\n",
    "print(len(movieIds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mappingRating(colrow,rat):\n",
    "    ratmap = {}\n",
    "    for i in range(len(colrow)):\n",
    "        d = colrow.iloc[i]\n",
    "        if d['userId'] not in ratmap.keys():\n",
    "            ratmap[d['userId']] = {}\n",
    "        if d['movieId'] not in ratmap[d['userId']].keys():\n",
    "            ratmap[d['userId']][d['movieId']] = rat.iloc[i]\n",
    "        else:\n",
    "            print('x')\n",
    "    return ratmap # format : userId1 { movieId1, movieId2}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def isiMatriks(maprat):\n",
    "    matRat = mat.copy()\n",
    "    matProb = mat.copy()\n",
    "    for user in users:\n",
    "        for movie in movies['movieId']:\n",
    "            if user in maprat.keys():\n",
    "                if movie in maprat[user]:\n",
    "                    rating = maprat[user][movie]\n",
    "                    matRat[movie][user] = rating\n",
    "                    matProb[movie][user] = 1\n",
    "    return matRat, matProb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Buat matriks \"cetakan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(610, 9742)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tempmat = [0]*len(movieIds)\n",
    "mat = [tempmat]*len(users)\n",
    "mat = pd.DataFrame(data=mat, columns=movieIds, index=users)\n",
    "mat.shape #matriks rating template"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Buat Matiks MovieLens All Ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100836, 2)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = ratings.drop(['rating','timestamp'],axis=1)\n",
    "y = ratings['rating']\n",
    "map_all = mappingRating(X,y)\n",
    "mat_rat_all, mat_prob_all = isiMatriks(map_all)\n",
    "mat_rat_all.to_csv('rating_all.csv',index=False)\n",
    "mat_prob_all.to_csv('prob_all.csv',index=False)\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Harusnya film yang nanti direkomndasikan telah dirating 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  47,   50,  101,  151,  157,  163,  216,  231,  260,  333,  362,\n",
       "        457,  527,  553,  596,  608,  661,  919,  923,  940,  954, 1023,\n",
       "       1024, 1025, 1029, 1031, 1032, 1049, 1073, 1080, 1089, 1092, 1097,\n",
       "       1136, 1196, 1197, 1198, 1206, 1210, 1213, 1220, 1222, 1224, 1226,\n",
       "       1240, 1256, 1270, 1275, 1278, 1282, 1291, 1298, 1517, 1573, 1587,\n",
       "       1617, 1625, 1732, 1804, 1927, 1954, 2005, 2018, 2033, 2048, 2058,\n",
       "       2078, 2090, 2094, 2115, 2116, 2137, 2139, 2141, 2161, 2291, 2329,\n",
       "       2353, 2387, 2395, 2427, 2459, 2470, 2502, 2529, 2542, 2571, 2580,\n",
       "       2596, 2641, 2654, 2692, 2700, 2716, 2761, 2858, 2872, 2899, 2944,\n",
       "       2947, 2948, 2949, 2959, 2987, 2991, 2993, 3033, 3034, 3052, 3053,\n",
       "       3147, 3253, 3273, 3386, 3441, 3448, 3450, 3578, 3671, 3702, 3703,\n",
       "       3729, 3793, 5060])"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rankTrue = np.array([ np.array([int(j) for j in mat_rat_all.loc[i][mat_rat_all.loc[i] == 5].index.values]) for i in mat_rat_all.index ])\n",
    "rankTrue[0] #film film yang telah di rating 5 oleh user pertama"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start fold 1\n",
      "mapping..\n",
      "filling matrix..\n",
      "start fold 2\n",
      "mapping..\n",
      "filling matrix..\n",
      "start fold 3\n",
      "mapping..\n",
      "filling matrix..\n",
      "start fold 4\n",
      "mapping..\n",
      "filling matrix..\n",
      "start fold 5\n",
      "mapping..\n",
      "filling matrix..\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "kf = KFold(n_splits=5)\n",
    "list_mat_rat_train = []\n",
    "list_mat_prob_train = []\n",
    "test_indexs = []\n",
    "f = 0\n",
    "for train_index, test_index in kf.split(ratings):\n",
    "    f += 1\n",
    "    print('start fold',f)\n",
    "    X_train = ratings[ratings.index.isin(train_index)].drop(['rating','timestamp'],axis=1)\n",
    "    y_train = ratings[ratings.index.isin(train_index)]['rating']\n",
    "    print('mapping..')\n",
    "    map_train = mappingRating(X_train,y_train)\n",
    "    print('filling matrix..')\n",
    "    mat_rat_train, mat_prob_train = isiMatriks(map_train)\n",
    "    list_mat_rat_train.append(mat_rat_train)\n",
    "    list_mat_prob_train.append(mat_prob_train)\n",
    "    test_indexs.append(test_index)\n",
    "with open('list_mat_prob_train.pickle', 'wb') as f:\n",
    "    pickle.dump(list_mat_prob_train, f)\n",
    "with open('list_mat_rat_train.pickle', 'wb') as f:\n",
    "    pickle.dump(list_mat_rat_train, f)    \n",
    "np.save('test_indexs.npy',np.array(test_indexs))\n",
    "print('saved')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocess Movies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Add given tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "      <td>pixar pixar fun</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "      <td>fantasy magic board game robin williams game</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "      <td>moldy old</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>pregnancy remake</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   movieId                               title  \\\n",
       "0        1                    Toy Story (1995)   \n",
       "1        2                      Jumanji (1995)   \n",
       "2        3             Grumpier Old Men (1995)   \n",
       "3        4            Waiting to Exhale (1995)   \n",
       "4        5  Father of the Bride Part II (1995)   \n",
       "\n",
       "                                        genres  \\\n",
       "0  Adventure|Animation|Children|Comedy|Fantasy   \n",
       "1                   Adventure|Children|Fantasy   \n",
       "2                               Comedy|Romance   \n",
       "3                         Comedy|Drama|Romance   \n",
       "4                                       Comedy   \n",
       "\n",
       "                                            tags  \n",
       "0                               pixar pixar fun   \n",
       "1  fantasy magic board game robin williams game   \n",
       "2                                     moldy old   \n",
       "3                                                 \n",
       "4                              pregnancy remake   "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag = [\"\"]*len(movies)\n",
    "for i,v in tags.iterrows():\n",
    "    target = movies.loc[movies['movieId'] == v.movieId]\n",
    "    if len(target) > 0 :\n",
    "        tag[target.index[0]] += v.tag.lower() + ' '\n",
    "movies['tags'] = tag\n",
    "movies.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extract Genres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of genres = 20\n",
      "(9742, 20)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "      <th>tags</th>\n",
       "      <th>Action</th>\n",
       "      <th>Crime</th>\n",
       "      <th>Mystery</th>\n",
       "      <th>Drama</th>\n",
       "      <th>Thriller</th>\n",
       "      <th>Horror</th>\n",
       "      <th>...</th>\n",
       "      <th>War</th>\n",
       "      <th>Animation</th>\n",
       "      <th>Documentary</th>\n",
       "      <th>Adventure</th>\n",
       "      <th>Fantasy</th>\n",
       "      <th>Comedy</th>\n",
       "      <th>Film-Noir</th>\n",
       "      <th>(no genres listed)</th>\n",
       "      <th>Romance</th>\n",
       "      <th>Musical</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "      <td>pixar pixar fun</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "      <td>fantasy magic board game robin williams game</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "      <td>moldy old</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>pregnancy remake</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   movieId                               title  \\\n",
       "0        1                    Toy Story (1995)   \n",
       "1        2                      Jumanji (1995)   \n",
       "2        3             Grumpier Old Men (1995)   \n",
       "3        4            Waiting to Exhale (1995)   \n",
       "4        5  Father of the Bride Part II (1995)   \n",
       "\n",
       "                                        genres  \\\n",
       "0  Adventure|Animation|Children|Comedy|Fantasy   \n",
       "1                   Adventure|Children|Fantasy   \n",
       "2                               Comedy|Romance   \n",
       "3                         Comedy|Drama|Romance   \n",
       "4                                       Comedy   \n",
       "\n",
       "                                            tags  Action  Crime  Mystery  \\\n",
       "0                               pixar pixar fun        0      0        0   \n",
       "1  fantasy magic board game robin williams game        0      0        0   \n",
       "2                                     moldy old        0      0        0   \n",
       "3                                                      0      0        0   \n",
       "4                              pregnancy remake        0      0        0   \n",
       "\n",
       "   Drama  Thriller  Horror  ...  War  Animation  Documentary  Adventure  \\\n",
       "0      0         0       0  ...    0          1            0          1   \n",
       "1      0         0       0  ...    0          0            0          1   \n",
       "2      0         0       0  ...    0          0            0          0   \n",
       "3      1         0       0  ...    0          0            0          0   \n",
       "4      0         0       0  ...    0          0            0          0   \n",
       "\n",
       "   Fantasy  Comedy  Film-Noir  (no genres listed)  Romance  Musical  \n",
       "0        1       1          0                   0        0        0  \n",
       "1        1       0          0                   0        0        0  \n",
       "2        0       1          0                   0        1        0  \n",
       "3        0       1          0                   0        1        0  \n",
       "4        0       1          0                   0        0        0  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#get all genres then store to temporary dataframe\n",
    "genres = []\n",
    "for i in range(len(movies)):\n",
    "    genres +=  movies.iloc[i]['genres'].split('|')\n",
    "    genres = list(set(genres))\n",
    "print('number of genres =',len(genres))\n",
    "trow = [0]*len(genres)\n",
    "tmat = [trow]*len(movies)\n",
    "dftemp = pd.DataFrame(data=tmat,columns=genres)\n",
    "print(dftemp.shape)\n",
    "\n",
    "#concat the temp df with movies\n",
    "for i in range(len(movies)):\n",
    "    genres = movies.iloc[i]['genres'].split('|')\n",
    "    for g in genres:\n",
    "        dftemp.iloc[i][g] = 1\n",
    "movies = pd.concat([movies,dftemp],axis=1)\n",
    "movies.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extract Titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9269\n",
      "(9742, 9269)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "      <th>tags</th>\n",
       "      <th>Action</th>\n",
       "      <th>Crime</th>\n",
       "      <th>Mystery</th>\n",
       "      <th>Drama</th>\n",
       "      <th>Thriller</th>\n",
       "      <th>Horror</th>\n",
       "      <th>...</th>\n",
       "      <th>écume</th>\n",
       "      <th>époque</th>\n",
       "      <th>étage</th>\n",
       "      <th>étrange</th>\n",
       "      <th>été</th>\n",
       "      <th>être</th>\n",
       "      <th>ôdishon</th>\n",
       "      <th>ônibus</th>\n",
       "      <th>ôritsu</th>\n",
       "      <th>über</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "      <td>pixar pixar fun</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "      <td>fantasy magic board game robin williams game</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "      <td>moldy old</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>pregnancy remake</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 9293 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   movieId                               title  \\\n",
       "0        1                    Toy Story (1995)   \n",
       "1        2                      Jumanji (1995)   \n",
       "2        3             Grumpier Old Men (1995)   \n",
       "3        4            Waiting to Exhale (1995)   \n",
       "4        5  Father of the Bride Part II (1995)   \n",
       "\n",
       "                                        genres  \\\n",
       "0  Adventure|Animation|Children|Comedy|Fantasy   \n",
       "1                   Adventure|Children|Fantasy   \n",
       "2                               Comedy|Romance   \n",
       "3                         Comedy|Drama|Romance   \n",
       "4                                       Comedy   \n",
       "\n",
       "                                            tags  Action  Crime  Mystery  \\\n",
       "0                               pixar pixar fun        0      0        0   \n",
       "1  fantasy magic board game robin williams game        0      0        0   \n",
       "2                                     moldy old        0      0        0   \n",
       "3                                                      0      0        0   \n",
       "4                              pregnancy remake        0      0        0   \n",
       "\n",
       "   Drama  Thriller  Horror  ...  écume  époque  étage  étrange  été  être  \\\n",
       "0      0         0       0  ...      0       0      0        0    0     0   \n",
       "1      0         0       0  ...      0       0      0        0    0     0   \n",
       "2      0         0       0  ...      0       0      0        0    0     0   \n",
       "3      1         0       0  ...      0       0      0        0    0     0   \n",
       "4      0         0       0  ...      0       0      0        0    0     0   \n",
       "\n",
       "   ôdishon  ônibus  ôritsu  über  \n",
       "0        0       0       0     0  \n",
       "1        0       0       0     0  \n",
       "2        0       0       0     0  \n",
       "3        0       0       0     0  \n",
       "4        0       0       0     0  \n",
       "\n",
       "[5 rows x 9293 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer()\n",
    "X = vectorizer.fit_transform(movies.title)\n",
    "print(len(vectorizer.get_feature_names()))\n",
    "dftemp = pd.DataFrame(data = X.toarray(), columns=list(vectorizer.get_feature_names()))\n",
    "print(dftemp.shape)\n",
    "movies = pd.concat([movies,dftemp],axis=1)\n",
    "movies.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract Tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1744\n",
      "(9742, 1744)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "      <th>tags</th>\n",
       "      <th>Action</th>\n",
       "      <th>Crime</th>\n",
       "      <th>Mystery</th>\n",
       "      <th>Drama</th>\n",
       "      <th>Thriller</th>\n",
       "      <th>Horror</th>\n",
       "      <th>...</th>\n",
       "      <th>york</th>\n",
       "      <th>you</th>\n",
       "      <th>younger</th>\n",
       "      <th>your</th>\n",
       "      <th>zellweger</th>\n",
       "      <th>zither</th>\n",
       "      <th>zoe</th>\n",
       "      <th>zombie</th>\n",
       "      <th>zombies</th>\n",
       "      <th>zooey</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "      <td>pixar pixar fun</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "      <td>fantasy magic board game robin williams game</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "      <td>moldy old</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>pregnancy remake</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 11037 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   movieId                               title  \\\n",
       "0        1                    Toy Story (1995)   \n",
       "1        2                      Jumanji (1995)   \n",
       "2        3             Grumpier Old Men (1995)   \n",
       "3        4            Waiting to Exhale (1995)   \n",
       "4        5  Father of the Bride Part II (1995)   \n",
       "\n",
       "                                        genres  \\\n",
       "0  Adventure|Animation|Children|Comedy|Fantasy   \n",
       "1                   Adventure|Children|Fantasy   \n",
       "2                               Comedy|Romance   \n",
       "3                         Comedy|Drama|Romance   \n",
       "4                                       Comedy   \n",
       "\n",
       "                                            tags  Action  Crime  Mystery  \\\n",
       "0                               pixar pixar fun        0      0        0   \n",
       "1  fantasy magic board game robin williams game        0      0        0   \n",
       "2                                     moldy old        0      0        0   \n",
       "3                                                      0      0        0   \n",
       "4                              pregnancy remake        0      0        0   \n",
       "\n",
       "   Drama  Thriller  Horror  ...  york  you  younger  your  zellweger  zither  \\\n",
       "0      0         0       0  ...   0.0  0.0      0.0   0.0        0.0     0.0   \n",
       "1      0         0       0  ...   0.0  0.0      0.0   0.0        0.0     0.0   \n",
       "2      0         0       0  ...   0.0  0.0      0.0   0.0        0.0     0.0   \n",
       "3      1         0       0  ...   0.0  0.0      0.0   0.0        0.0     0.0   \n",
       "4      0         0       0  ...   0.0  0.0      0.0   0.0        0.0     0.0   \n",
       "\n",
       "   zoe  zombie  zombies  zooey  \n",
       "0  0.0     0.0      0.0    0.0  \n",
       "1  0.0     0.0      0.0    0.0  \n",
       "2  0.0     0.0      0.0    0.0  \n",
       "3  0.0     0.0      0.0    0.0  \n",
       "4  0.0     0.0      0.0    0.0  \n",
       "\n",
       "[5 rows x 11037 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "vectorizer = TfidfVectorizer()\n",
    "X = vectorizer.fit_transform(movies.tags)\n",
    "print(len(vectorizer.get_feature_names()))\n",
    "dftemp = pd.DataFrame(data = X.toarray(), columns=list(vectorizer.get_feature_names()))\n",
    "print(dftemp.shape)\n",
    "movies = pd.concat([movies,dftemp],axis=1)\n",
    "movies.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate cosine similiarity between movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>...</th>\n",
       "      <th>193565</th>\n",
       "      <th>193567</th>\n",
       "      <th>193571</th>\n",
       "      <th>193573</th>\n",
       "      <th>193579</th>\n",
       "      <th>193581</th>\n",
       "      <th>193583</th>\n",
       "      <th>193585</th>\n",
       "      <th>193587</th>\n",
       "      <th>193609</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>movieId</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.544331</td>\n",
       "      <td>0.251976</td>\n",
       "      <td>0.251976</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0.149071</td>\n",
       "      <td>0.298142</td>\n",
       "      <td>0.408248</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.298142</td>\n",
       "      <td>...</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0.089087</td>\n",
       "      <td>0.149071</td>\n",
       "      <td>0.117851</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.301511</td>\n",
       "      <td>0.301511</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.117851</td>\n",
       "      <td>0.111111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.544331</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.154303</td>\n",
       "      <td>0.154303</td>\n",
       "      <td>0.136083</td>\n",
       "      <td>0.182574</td>\n",
       "      <td>0.182574</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.204124</td>\n",
       "      <td>0.365148</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.123091</td>\n",
       "      <td>0.123091</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.251976</td>\n",
       "      <td>0.154303</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>0.251976</td>\n",
       "      <td>0.169031</td>\n",
       "      <td>0.507093</td>\n",
       "      <td>0.154303</td>\n",
       "      <td>0.188982</td>\n",
       "      <td>0.169031</td>\n",
       "      <td>...</td>\n",
       "      <td>0.125988</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.169031</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.113961</td>\n",
       "      <td>0.113961</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.125988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.251976</td>\n",
       "      <td>0.154303</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.251976</td>\n",
       "      <td>0.169031</td>\n",
       "      <td>0.507093</td>\n",
       "      <td>0.154303</td>\n",
       "      <td>0.188982</td>\n",
       "      <td>0.169031</td>\n",
       "      <td>...</td>\n",
       "      <td>0.125988</td>\n",
       "      <td>0.101015</td>\n",
       "      <td>0.338062</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.113961</td>\n",
       "      <td>0.113961</td>\n",
       "      <td>0.218218</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.125988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.222222</td>\n",
       "      <td>0.136083</td>\n",
       "      <td>0.251976</td>\n",
       "      <td>0.251976</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.149071</td>\n",
       "      <td>0.399929</td>\n",
       "      <td>0.136083</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.149071</td>\n",
       "      <td>...</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0.178174</td>\n",
       "      <td>0.149071</td>\n",
       "      <td>0.117851</td>\n",
       "      <td>0.117851</td>\n",
       "      <td>0.301511</td>\n",
       "      <td>0.100504</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.111111</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 9742 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           1         2         3         4         5         6         7       \\\n",
       "movieId                                                                         \n",
       "1        1.000000  0.544331  0.251976  0.251976  0.222222  0.149071  0.298142   \n",
       "2        0.544331  1.000000  0.154303  0.154303  0.136083  0.182574  0.182574   \n",
       "3        0.251976  0.154303  1.000000  0.428571  0.251976  0.169031  0.507093   \n",
       "4        0.251976  0.154303  0.428571  1.000000  0.251976  0.169031  0.507093   \n",
       "5        0.222222  0.136083  0.251976  0.251976  1.000000  0.149071  0.399929   \n",
       "\n",
       "           8         9         10      ...    193565    193567    193571  \\\n",
       "movieId                                ...                                 \n",
       "1        0.408248  0.166667  0.298142  ...  0.222222  0.089087  0.149071   \n",
       "2        0.500000  0.204124  0.365148  ...  0.000000  0.000000  0.000000   \n",
       "3        0.154303  0.188982  0.169031  ...  0.125988  0.000000  0.169031   \n",
       "4        0.154303  0.188982  0.169031  ...  0.125988  0.101015  0.338062   \n",
       "5        0.136083  0.166667  0.149071  ...  0.222222  0.178174  0.149071   \n",
       "\n",
       "           193573    193579    193581    193583    193585    193587    193609  \n",
       "movieId                                                                        \n",
       "1        0.117851  0.000000  0.301511  0.301511  0.000000  0.117851  0.111111  \n",
       "2        0.000000  0.000000  0.123091  0.123091  0.000000  0.000000  0.000000  \n",
       "3        0.000000  0.000000  0.113961  0.113961  0.000000  0.000000  0.125988  \n",
       "4        0.000000  0.000000  0.113961  0.113961  0.218218  0.000000  0.125988  \n",
       "5        0.117851  0.117851  0.301511  0.100504  0.000000  0.000000  0.111111  \n",
       "\n",
       "[5 rows x 9742 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "M = movies.drop(['movieId','title','genres','tags'],axis=1).values\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "sim = cosine_similarity(M)\n",
    "sim = pd.DataFrame(data=sim,columns=list(movieIds),index=list(movieIds))\n",
    "sim.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Lihat film apa yang mirip dengan Toy Story"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Toy Story (1995) === Adventure|Animation|Children|Comedy|Fantasy === pixar pixar fun  === 0.9999999999999999\n",
      "Toy Story 2 (1999) === Adventure|Animation|Children|Comedy|Fantasy === animation disney funny original pixar sequel tom hanks pixar  === 0.8419185910374267\n",
      "Toy Story 3 (2010) === Adventure|Animation|Children|Comedy|Fantasy|IMAX ===  === 0.7777777777777777\n",
      "Antz (1998) === Adventure|Animation|Children|Comedy|Fantasy ===  === 0.6299407883487119\n",
      "Moana (2016) === Adventure|Animation|Children|Comedy|Fantasy ===  === 0.6299407883487119\n",
      "Turbo (2013) === Adventure|Animation|Children|Comedy|Fantasy ===  === 0.6299407883487119\n"
     ]
    }
   ],
   "source": [
    "for i,v in sim.iloc[0].sort_values(ascending=False)[:6].iteritems():\n",
    "    sel = movies[movies['movieId'] == i ]\n",
    "    print(sel.title.values[0],'===', sel.genres.values[0],'===',sel.tags.values[0],'===', v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Content-Based Filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "def learnUserPref(probTrain,f,cosim):    \n",
    "    print('learning user pref fold',f)\n",
    "    userPref = []\n",
    "    for u in probTrain.index:\n",
    "        pos = list(probTrain.loc[u].loc[probTrain.loc[u] == 1].index)#ambil movieId yang pernah dirating\n",
    "        if len(pos) > 0: #kalau user pernah nge rating sebuah film\n",
    "            posSim = sim.loc[pos] # ambil cosim dari movie yang pernah dirating\n",
    "            mostSim = posSim.max() #untuk setiap movieId, ambil maximum similiarity terhadap movie yang pernah di rating.\n",
    "            userPref.append(mostSim) #simpan prefrensi seorang user u ke  userPref\n",
    "        else: \n",
    "            simD = [1.0]*len(sim.columns) #buat default semua, user pref ga ada, bergantung pada hasil collaborative filtering\n",
    "            simD = pd.Series(simD, index=list(movieIds))\n",
    "            userPref.append(simD)\n",
    "        if u % 10 == 0:\n",
    "            uP = pd.DataFrame(userPref, index=probTrain.index[:u])\n",
    "            uP.to_csv('userPrefFold'+str(f)+'.csv')\n",
    "            print(u,'saved','userPrefFold'+str(f)+'.csv')\n",
    "    uP = pd.DataFrame(userPref, index=probTrain.index[:u])\n",
    "    return uP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "learning user pref fold 1\n",
      "10 saved userPrefFold1.csv\n",
      "20 saved userPrefFold1.csv\n",
      "30 saved userPrefFold1.csv\n",
      "40 saved userPrefFold1.csv\n",
      "50 saved userPrefFold1.csv\n",
      "60 saved userPrefFold1.csv\n",
      "70 saved userPrefFold1.csv\n",
      "80 saved userPrefFold1.csv\n",
      "90 saved userPrefFold1.csv\n",
      "100 saved userPrefFold1.csv\n",
      "110 saved userPrefFold1.csv\n",
      "120 saved userPrefFold1.csv\n",
      "130 saved userPrefFold1.csv\n",
      "140 saved userPrefFold1.csv\n",
      "150 saved userPrefFold1.csv\n",
      "160 saved userPrefFold1.csv\n",
      "170 saved userPrefFold1.csv\n",
      "180 saved userPrefFold1.csv\n",
      "190 saved userPrefFold1.csv\n",
      "200 saved userPrefFold1.csv\n",
      "210 saved userPrefFold1.csv\n",
      "220 saved userPrefFold1.csv\n",
      "230 saved userPrefFold1.csv\n",
      "240 saved userPrefFold1.csv\n",
      "250 saved userPrefFold1.csv\n",
      "260 saved userPrefFold1.csv\n",
      "270 saved userPrefFold1.csv\n",
      "280 saved userPrefFold1.csv\n",
      "290 saved userPrefFold1.csv\n",
      "300 saved userPrefFold1.csv\n",
      "310 saved userPrefFold1.csv\n",
      "320 saved userPrefFold1.csv\n",
      "330 saved userPrefFold1.csv\n",
      "340 saved userPrefFold1.csv\n",
      "350 saved userPrefFold1.csv\n",
      "360 saved userPrefFold1.csv\n",
      "370 saved userPrefFold1.csv\n",
      "380 saved userPrefFold1.csv\n",
      "390 saved userPrefFold1.csv\n",
      "400 saved userPrefFold1.csv\n",
      "410 saved userPrefFold1.csv\n",
      "420 saved userPrefFold1.csv\n",
      "430 saved userPrefFold1.csv\n",
      "440 saved userPrefFold1.csv\n",
      "450 saved userPrefFold1.csv\n",
      "460 saved userPrefFold1.csv\n",
      "470 saved userPrefFold1.csv\n",
      "480 saved userPrefFold1.csv\n",
      "490 saved userPrefFold1.csv\n",
      "500 saved userPrefFold1.csv\n",
      "510 saved userPrefFold1.csv\n",
      "520 saved userPrefFold1.csv\n",
      "530 saved userPrefFold1.csv\n",
      "540 saved userPrefFold1.csv\n",
      "550 saved userPrefFold1.csv\n",
      "560 saved userPrefFold1.csv\n",
      "570 saved userPrefFold1.csv\n",
      "580 saved userPrefFold1.csv\n",
      "590 saved userPrefFold1.csv\n",
      "600 saved userPrefFold1.csv\n",
      "610 saved userPrefFold1.csv\n",
      "learning user pref fold 2\n",
      "10 saved userPrefFold2.csv\n",
      "20 saved userPrefFold2.csv\n",
      "30 saved userPrefFold2.csv\n",
      "40 saved userPrefFold2.csv\n",
      "50 saved userPrefFold2.csv\n",
      "60 saved userPrefFold2.csv\n",
      "70 saved userPrefFold2.csv\n",
      "80 saved userPrefFold2.csv\n",
      "90 saved userPrefFold2.csv\n",
      "100 saved userPrefFold2.csv\n",
      "110 saved userPrefFold2.csv\n",
      "120 saved userPrefFold2.csv\n",
      "130 saved userPrefFold2.csv\n",
      "140 saved userPrefFold2.csv\n",
      "150 saved userPrefFold2.csv\n",
      "160 saved userPrefFold2.csv\n",
      "170 saved userPrefFold2.csv\n",
      "180 saved userPrefFold2.csv\n",
      "190 saved userPrefFold2.csv\n",
      "200 saved userPrefFold2.csv\n",
      "210 saved userPrefFold2.csv\n",
      "220 saved userPrefFold2.csv\n",
      "230 saved userPrefFold2.csv\n",
      "240 saved userPrefFold2.csv\n",
      "250 saved userPrefFold2.csv\n",
      "260 saved userPrefFold2.csv\n",
      "270 saved userPrefFold2.csv\n",
      "280 saved userPrefFold2.csv\n",
      "290 saved userPrefFold2.csv\n",
      "300 saved userPrefFold2.csv\n",
      "310 saved userPrefFold2.csv\n",
      "320 saved userPrefFold2.csv\n",
      "330 saved userPrefFold2.csv\n",
      "340 saved userPrefFold2.csv\n",
      "350 saved userPrefFold2.csv\n",
      "360 saved userPrefFold2.csv\n",
      "370 saved userPrefFold2.csv\n",
      "380 saved userPrefFold2.csv\n",
      "390 saved userPrefFold2.csv\n",
      "400 saved userPrefFold2.csv\n",
      "410 saved userPrefFold2.csv\n",
      "420 saved userPrefFold2.csv\n",
      "430 saved userPrefFold2.csv\n",
      "440 saved userPrefFold2.csv\n",
      "450 saved userPrefFold2.csv\n",
      "460 saved userPrefFold2.csv\n",
      "470 saved userPrefFold2.csv\n",
      "480 saved userPrefFold2.csv\n",
      "490 saved userPrefFold2.csv\n",
      "500 saved userPrefFold2.csv\n",
      "510 saved userPrefFold2.csv\n",
      "520 saved userPrefFold2.csv\n",
      "530 saved userPrefFold2.csv\n",
      "540 saved userPrefFold2.csv\n",
      "550 saved userPrefFold2.csv\n",
      "560 saved userPrefFold2.csv\n",
      "570 saved userPrefFold2.csv\n",
      "580 saved userPrefFold2.csv\n",
      "590 saved userPrefFold2.csv\n",
      "600 saved userPrefFold2.csv\n",
      "610 saved userPrefFold2.csv\n",
      "learning user pref fold 3\n",
      "10 saved userPrefFold3.csv\n",
      "20 saved userPrefFold3.csv\n",
      "30 saved userPrefFold3.csv\n",
      "40 saved userPrefFold3.csv\n",
      "50 saved userPrefFold3.csv\n",
      "60 saved userPrefFold3.csv\n",
      "70 saved userPrefFold3.csv\n",
      "80 saved userPrefFold3.csv\n",
      "90 saved userPrefFold3.csv\n",
      "100 saved userPrefFold3.csv\n",
      "110 saved userPrefFold3.csv\n",
      "120 saved userPrefFold3.csv\n",
      "130 saved userPrefFold3.csv\n",
      "140 saved userPrefFold3.csv\n",
      "150 saved userPrefFold3.csv\n",
      "160 saved userPrefFold3.csv\n",
      "170 saved userPrefFold3.csv\n",
      "180 saved userPrefFold3.csv\n",
      "190 saved userPrefFold3.csv\n",
      "200 saved userPrefFold3.csv\n",
      "210 saved userPrefFold3.csv\n",
      "220 saved userPrefFold3.csv\n",
      "230 saved userPrefFold3.csv\n",
      "240 saved userPrefFold3.csv\n",
      "250 saved userPrefFold3.csv\n",
      "260 saved userPrefFold3.csv\n",
      "270 saved userPrefFold3.csv\n",
      "280 saved userPrefFold3.csv\n",
      "290 saved userPrefFold3.csv\n",
      "300 saved userPrefFold3.csv\n",
      "310 saved userPrefFold3.csv\n",
      "320 saved userPrefFold3.csv\n",
      "330 saved userPrefFold3.csv\n",
      "340 saved userPrefFold3.csv\n",
      "350 saved userPrefFold3.csv\n",
      "360 saved userPrefFold3.csv\n",
      "370 saved userPrefFold3.csv\n",
      "380 saved userPrefFold3.csv\n",
      "390 saved userPrefFold3.csv\n",
      "400 saved userPrefFold3.csv\n",
      "410 saved userPrefFold3.csv\n",
      "420 saved userPrefFold3.csv\n",
      "430 saved userPrefFold3.csv\n",
      "440 saved userPrefFold3.csv\n",
      "450 saved userPrefFold3.csv\n",
      "460 saved userPrefFold3.csv\n",
      "470 saved userPrefFold3.csv\n",
      "480 saved userPrefFold3.csv\n",
      "490 saved userPrefFold3.csv\n",
      "500 saved userPrefFold3.csv\n",
      "510 saved userPrefFold3.csv\n",
      "520 saved userPrefFold3.csv\n",
      "530 saved userPrefFold3.csv\n",
      "540 saved userPrefFold3.csv\n",
      "550 saved userPrefFold3.csv\n",
      "560 saved userPrefFold3.csv\n",
      "570 saved userPrefFold3.csv\n",
      "580 saved userPrefFold3.csv\n",
      "590 saved userPrefFold3.csv\n",
      "600 saved userPrefFold3.csv\n",
      "610 saved userPrefFold3.csv\n",
      "learning user pref fold 4\n",
      "10 saved userPrefFold4.csv\n",
      "20 saved userPrefFold4.csv\n",
      "30 saved userPrefFold4.csv\n",
      "40 saved userPrefFold4.csv\n",
      "50 saved userPrefFold4.csv\n",
      "60 saved userPrefFold4.csv\n",
      "70 saved userPrefFold4.csv\n",
      "80 saved userPrefFold4.csv\n",
      "90 saved userPrefFold4.csv\n",
      "100 saved userPrefFold4.csv\n",
      "110 saved userPrefFold4.csv\n",
      "120 saved userPrefFold4.csv\n",
      "130 saved userPrefFold4.csv\n",
      "140 saved userPrefFold4.csv\n",
      "150 saved userPrefFold4.csv\n",
      "160 saved userPrefFold4.csv\n",
      "170 saved userPrefFold4.csv\n",
      "180 saved userPrefFold4.csv\n",
      "190 saved userPrefFold4.csv\n",
      "200 saved userPrefFold4.csv\n",
      "210 saved userPrefFold4.csv\n",
      "220 saved userPrefFold4.csv\n",
      "230 saved userPrefFold4.csv\n",
      "240 saved userPrefFold4.csv\n",
      "250 saved userPrefFold4.csv\n",
      "260 saved userPrefFold4.csv\n",
      "270 saved userPrefFold4.csv\n",
      "280 saved userPrefFold4.csv\n",
      "290 saved userPrefFold4.csv\n",
      "300 saved userPrefFold4.csv\n",
      "310 saved userPrefFold4.csv\n",
      "320 saved userPrefFold4.csv\n",
      "330 saved userPrefFold4.csv\n",
      "340 saved userPrefFold4.csv\n",
      "350 saved userPrefFold4.csv\n",
      "360 saved userPrefFold4.csv\n",
      "370 saved userPrefFold4.csv\n",
      "380 saved userPrefFold4.csv\n",
      "390 saved userPrefFold4.csv\n",
      "400 saved userPrefFold4.csv\n",
      "410 saved userPrefFold4.csv\n",
      "420 saved userPrefFold4.csv\n",
      "430 saved userPrefFold4.csv\n",
      "440 saved userPrefFold4.csv\n",
      "450 saved userPrefFold4.csv\n",
      "460 saved userPrefFold4.csv\n",
      "470 saved userPrefFold4.csv\n",
      "480 saved userPrefFold4.csv\n",
      "490 saved userPrefFold4.csv\n",
      "500 saved userPrefFold4.csv\n",
      "510 saved userPrefFold4.csv\n",
      "520 saved userPrefFold4.csv\n",
      "530 saved userPrefFold4.csv\n",
      "540 saved userPrefFold4.csv\n",
      "550 saved userPrefFold4.csv\n",
      "560 saved userPrefFold4.csv\n",
      "570 saved userPrefFold4.csv\n",
      "580 saved userPrefFold4.csv\n",
      "590 saved userPrefFold4.csv\n",
      "600 saved userPrefFold4.csv\n",
      "610 saved userPrefFold4.csv\n",
      "learning user pref fold 5\n",
      "10 saved userPrefFold5.csv\n",
      "20 saved userPrefFold5.csv\n",
      "30 saved userPrefFold5.csv\n",
      "40 saved userPrefFold5.csv\n",
      "50 saved userPrefFold5.csv\n",
      "60 saved userPrefFold5.csv\n",
      "70 saved userPrefFold5.csv\n",
      "80 saved userPrefFold5.csv\n",
      "90 saved userPrefFold5.csv\n",
      "100 saved userPrefFold5.csv\n",
      "110 saved userPrefFold5.csv\n",
      "120 saved userPrefFold5.csv\n",
      "130 saved userPrefFold5.csv\n",
      "140 saved userPrefFold5.csv\n",
      "150 saved userPrefFold5.csv\n",
      "160 saved userPrefFold5.csv\n",
      "170 saved userPrefFold5.csv\n",
      "180 saved userPrefFold5.csv\n",
      "190 saved userPrefFold5.csv\n",
      "200 saved userPrefFold5.csv\n",
      "210 saved userPrefFold5.csv\n",
      "220 saved userPrefFold5.csv\n",
      "230 saved userPrefFold5.csv\n",
      "240 saved userPrefFold5.csv\n",
      "250 saved userPrefFold5.csv\n",
      "260 saved userPrefFold5.csv\n",
      "270 saved userPrefFold5.csv\n",
      "280 saved userPrefFold5.csv\n",
      "290 saved userPrefFold5.csv\n",
      "300 saved userPrefFold5.csv\n",
      "310 saved userPrefFold5.csv\n",
      "320 saved userPrefFold5.csv\n",
      "330 saved userPrefFold5.csv\n",
      "340 saved userPrefFold5.csv\n",
      "350 saved userPrefFold5.csv\n",
      "360 saved userPrefFold5.csv\n",
      "370 saved userPrefFold5.csv\n",
      "380 saved userPrefFold5.csv\n",
      "390 saved userPrefFold5.csv\n",
      "400 saved userPrefFold5.csv\n",
      "410 saved userPrefFold5.csv\n",
      "420 saved userPrefFold5.csv\n",
      "430 saved userPrefFold5.csv\n",
      "440 saved userPrefFold5.csv\n",
      "450 saved userPrefFold5.csv\n",
      "460 saved userPrefFold5.csv\n",
      "470 saved userPrefFold5.csv\n",
      "480 saved userPrefFold5.csv\n",
      "490 saved userPrefFold5.csv\n",
      "500 saved userPrefFold5.csv\n",
      "510 saved userPrefFold5.csv\n",
      "520 saved userPrefFold5.csv\n",
      "530 saved userPrefFold5.csv\n",
      "540 saved userPrefFold5.csv\n",
      "550 saved userPrefFold5.csv\n",
      "560 saved userPrefFold5.csv\n",
      "570 saved userPrefFold5.csv\n",
      "580 saved userPrefFold5.csv\n",
      "590 saved userPrefFold5.csv\n",
      "600 saved userPrefFold5.csv\n",
      "610 saved userPrefFold5.csv\n"
     ]
    }
   ],
   "source": [
    "# with open('list_mat_prob_train.pickle') as f:\n",
    "#     list_mat_prob_train = pickle.load(f)\n",
    "userPrefs = []\n",
    "for i in range(len(list_mat_prob_train)):\n",
    "    userPref = learnUserPref(list_mat_prob_train[i],i+1,sim)\n",
    "    userPrefs.append(userPref)\n",
    "    with open('userPrefs.pickle', 'wb') as f:\n",
    "        pickle.dump(userPrefs, f) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collaborative Filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import TruncatedSVD\n",
    "svd = TruncatedSVD(n_components=100, n_iter=100, random_state=42) #LDA approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluateSVD(res,test):\n",
    "    sse = []\n",
    "    for i,v in test.iterrows():\n",
    "        sse.append(pow(v.rating-res.loc[v.userId][v.movieId],2))\n",
    "    mse = sum(sse)/len(sse)\n",
    "    return mse, np.sqrt(mse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('list_mat_rat_train.pickle','rb') as f:\n",
    "    list_mat_rat_train = pickle.load(f)\n",
    "test_indexs = np.load('test_indexs.npy',allow_pickle=True)\n",
    "Rpreds = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold 1\n",
      "training..\n",
      "evaluating..\n",
      "fold 1 MSE = 14.026476260933185 RMSE = 3.745193754792025\n",
      "saved\n",
      "fold 2\n",
      "training..\n",
      "evaluating..\n",
      "fold 2 MSE = 13.541222101353231 RMSE = 3.6798399559428168\n",
      "saved\n",
      "fold 3\n",
      "training..\n",
      "evaluating..\n",
      "fold 3 MSE = 13.053029616509153 RMSE = 3.612897675897998\n",
      "saved\n",
      "fold 4\n",
      "training..\n",
      "evaluating..\n",
      "fold 4 MSE = 13.18615431800806 RMSE = 3.631274475718967\n",
      "saved\n",
      "fold 5\n",
      "training..\n",
      "evaluating..\n",
      "fold 5 MSE = 12.868936806796878 RMSE = 3.5873300387331075\n",
      "saved\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(list_mat_rat_train)):\n",
    "    print('fold',i+1)\n",
    "    print('training..')\n",
    "    result = svd.fit_transform(list_mat_rat_train[i])\n",
    "    result = np.dot(result,svd.components_)\n",
    "    result = pd.DataFrame(result,index=list(users),columns=list(movieIds))\n",
    "    test_fold = ratings.loc[test_indexs[i]]\n",
    "    print('evaluating..')\n",
    "    MSE,RMSE = evaluateSVD(result,test_fold)\n",
    "    print('fold',i+1,'MSE =',MSE, 'RMSE =',RMSE)\n",
    "    Rpreds.append(result)\n",
    "    with open('Rpreds.pickle', 'wb') as f:\n",
    "        pickle.dump(Rpreds, f)   \n",
    "    print('saved')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Recommendations -> Prediction will rate * Predicted Rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('Rpreds.pickle','rb') as f:\n",
    "    Rpreds = pickle.load(f)\n",
    "with open('userPrefs.pickle','rb') as f:\n",
    "    userPrefs = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "recoms_all_fold = []\n",
    "for fold in range(len(Rpreds)):\n",
    "    newPred = Rpreds[fold].values * userPrefs[fold].values #ini bagian paralelization\n",
    "    newPred = pd.DataFrame(newPred, index=range(1,611), columns=movieIds)\n",
    "    recoms = [list(newPred.loc[i].sort_values(ascending=False)[:20].index) for i in newPred.index]\n",
    "    recoms = pd.DataFrame(recoms, index=range(1,611), columns=range(1,21))\n",
    "    recoms_all_fold.append(recoms)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Lihat movie apa aja yang direkomandasiin untuk user 1 di Fold 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Grease (1978)', 'Truman Show, The (1998)', 'Titanic (1997)',\n",
       "       'A.I. Artificial Intelligence (2001)',\n",
       "       \"Harry Potter and the Sorcerer's Stone (a.k.a. Harry Potter and the Philosopher's Stone) (2001)\",\n",
       "       \"Ocean's Eleven (2001)\", 'Lilo & Stitch (2002)',\n",
       "       'Harry Potter and the Chamber of Secrets (2002)',\n",
       "       'Pirates of the Caribbean: The Curse of the Black Pearl (2003)',\n",
       "       \"Monty Python's The Meaning of Life (1983)\",\n",
       "       'Kill Bill: Vol. 1 (2003)', 'The Butterfly Effect (2004)',\n",
       "       '50 First Dates (2004)', 'Kill Bill: Vol. 2 (2004)',\n",
       "       'Mean Girls (2004)', 'Meet the Fockers (2004)',\n",
       "       'X-Men: The Last Stand (2006)', 'Ratatouille (2007)',\n",
       "       'Guardians of the Galaxy (2014)', 'Zootopia (2016)'], dtype=object)"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies[movies['movieId'].isin(recoms_all_fold[0].loc[1])].title.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Lihat movie apa aja yang dirating 5 oleh user 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Mighty Aphrodite (1995)', 'Georgia (1995)',\n",
       "       'Anne Frank Remembered (1995)', 'Mad Love (1995)',\n",
       "       'Nine Months (1995)', 'Showgirls (1995)', 'I.Q. (1994)',\n",
       "       'My Crazy Life (Mi vida loca) (1993)', 'Quiz Show (1994)',\n",
       "       'River Wild, The (1994)', 'Being Human (1993)',\n",
       "       'Romper Stomper (1992)', 'Aristocats, The (1970)',\n",
       "       'Kids in the Hall: Brain Candy (1996)',\n",
       "       'Ghost in the Shell (Kôkaku kidôtai) (1995)', 'Heavy (1995)',\n",
       "       'Bound (1996)', 'Killer, The (Die xue shuang xiong) (1989)',\n",
       "       'Full Metal Jacket (1987)', 'Dead Alive (Braindead) (1992)',\n",
       "       'Bad Taste (1987)', 'Blob, The (1958)', 'Blood Beach (1981)',\n",
       "       'Body Parts (1991)', 'Burnt Offerings (1976)', 'Cape Fear (1991)',\n",
       "       'Cape Fear (1962)', \"Preacher's Wife, The (1996)\",\n",
       "       'Jerry Maguire (1996)', 'Ghosts of Mississippi (1996)',\n",
       "       'Mother (1996)', 'Portrait of a Lady, The (1996)',\n",
       "       'Zeus and Roxanne (1997)', 'Selena (1997)',\n",
       "       'Picture Perfect (1997)', 'In the Company of Men (1997)',\n",
       "       'Career Girls (1997)', 'Money Talks (1997)',\n",
       "       '187 (One Eight Seven) (1997)', 'Kiss Me, Guido (1997)',\n",
       "       'Kiss the Girls (1997)', 'Wishmaster (1997)', 'Game, The (1997)',\n",
       "       'U Turn (1997)', 'Fast, Cheap & Out of Control (1997)',\n",
       "       'Deceiver (1997)', 'I Love You, I Love You Not (1996)',\n",
       "       'Alien: Resurrection (1997)', 'Artemisia (1997)', 'Flubber (1997)',\n",
       "       'Titanic (1997)', 'Big Lebowski, The (1998)',\n",
       "       'Great Mouse Detective, The (1986)',\n",
       "       'Man with Two Brains, The (1983)', 'Snake Eyes (1998)',\n",
       "       'NeverEnding Story, The (1984)', 'Wrongfully Accused (1998)',\n",
       "       'Tales from the Darkside: The Movie (1990)', 'Rambo III (1988)',\n",
       "       'Baby Geniuses (1999)', 'Hideous Kinky (1998)',\n",
       "       'Pork Chop Hill (1959)', 'Ideal Husband, An (1999)',\n",
       "       \"Autumn Tale, An (Conte d'automne) (1998)\", \"Killer's Kiss (1955)\",\n",
       "       'Assassination (1987)', 'Sixth Sense, The (1999)',\n",
       "       'Pit and the Pendulum (1961)',\n",
       "       \"Monty Python's And Now for Something Completely Different (1971)\",\n",
       "       'Perfect Blue (1997)', 'In Too Deep (1999)',\n",
       "       'Minus Man, The (1999)', 'Only Angels Have Wings (1939)',\n",
       "       'Saturn 3 (1980)', 'Tommy (1975)', 'Face in the Crowd, A (1957)',\n",
       "       'Babes in Toyland (1934)', \"'night Mother (1986)\",\n",
       "       'Carnal Knowledge (1971)', 'Next Friday (2000)',\n",
       "       'Where the Buffalo Roam (1980)', 'Bad Lieutenant (1992)',\n",
       "       'Tigger Movie, The (2000)', 'Blood Feast (1963)',\n",
       "       \"Let's Get Harry (1986)\", 'Raise the Titanic (1980)',\n",
       "       'Teenage Mutant Ninja Turtles II: The Secret of the Ooze (1991)',\n",
       "       'Good Mother, The (1988)',\n",
       "       'Close Encounters of the Third Kind (1977)',\n",
       "       'American Psycho (2000)', 'Caddyshack (1980)',\n",
       "       'Regret to Inform (1998)', 'Mission: Impossible II (2000)',\n",
       "       'Fighting Seabees, The (1944)', 'Starman (1984)',\n",
       "       'Autumn in New York (2000)', 'Pumpkinhead (1988)',\n",
       "       'Bring It On (2000)', 'Meet the Parents (2000)',\n",
       "       'Two Family House (2000)', 'Contender, The (2000)',\n",
       "       'Dr. T and the Women (2000)', 'Billy Elliot (2000)',\n",
       "       'Bounty, The (1984)', 'Transformers: The Movie (1986)',\n",
       "       'Born on the Fourth of July (1989)', 'The Man in the Moon (1991)',\n",
       "       'Mystic Pizza (1988)', 'Big Easy, The (1987)',\n",
       "       'Born in East L.A. (1987)', 'Tailor of Panama, The (2001)',\n",
       "       'Cannonball Run, The (1981)', 'Sweet Smell of Success (1957)',\n",
       "       'Innocent Man, An (1989)',\n",
       "       \"Cat o' Nine Tails, The (Gatto a nove code, Il) (1971)\",\n",
       "       'Chocolat (1988)', 'Cage aux Folles, La (1978)', 'Spy Game (2001)',\n",
       "       'Dragon: The Bruce Lee Story (1993)', \"Hart's War (2002)\",\n",
       "       'John Q (2002)', 'Royal Wedding (1951)',\n",
       "       'Rome, Open City (a.k.a. Open City) (Roma, città aperta) (1945)',\n",
       "       'Dark Days (2000)'], dtype=object)"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.loc[mat_rat_all.loc[1][mat_rat_all.loc[1] == 5].index].title.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation: True Positif jika Film yang masuk top 20 rekomendasi pernah dirating 5\n",
    "#### Evaluation method: Recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "recall fold 1 0.22573770491803277\n",
      "recall fold 2 0.21852459016393444\n",
      "recall fold 3 0.22844262295081966\n",
      "recall fold 4 0.2174590163934426\n",
      "recall fold 5 0.2262295081967213\n"
     ]
    }
   ],
   "source": [
    "for fold in range(len(recoms_all_fold)):\n",
    "    recall = []\n",
    "    for i in recoms_all_fold[fold].index:\n",
    "        tp = 0\n",
    "        for ratU in recoms_all_fold[fold].loc[i]:\n",
    "            if ratU in rankTrue[i-1]:\n",
    "                tp += 1\n",
    "        recall.append(tp/len(recoms_all_fold[fold].loc[i])) #hitung recall rekomendasi seorang user\n",
    "    print('recall fold',fold+1,np.mean(recall)) #hitung rata rata recall semua user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}