MF_cosine-similarity_CF.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This notebook explains the mixed approach of Recommender System - MF(Matrix Factorization) with item similarity based Collaborative Filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Sourish\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sys\n",
    "# import tensorflow,keras\n",
    "import os\n",
    "import sys;\n",
    "import numpy,sklearn\n",
    "from pandas import read_csv;\n",
    "# from keras.layers import Input, Embedding, Flatten, Lambda,Concatenate,Dense,Dropout\n",
    "# from keras.models import Model\n",
    "# from keras.initializers import RandomNormal\n",
    "# from keras.regularizers import l2\n",
    "# from keras.callbacks import EarlyStopping\n",
    "# from keras import backend as K\n",
    "import time\n",
    "import scipy.sparse as sparse\n",
    "from scipy.sparse.linalg import spsolve\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "# from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "import datetime\n",
    "# import fasttext\n",
    "import codecs\n",
    "from bs4 import BeautifulSoup\n",
    "import nltk\n",
    "import os\n",
    "import nltk.corpus,nltk.tokenize\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download(\"stopwords\")\n",
    "import re\n",
    "import string\n",
    "import matplotlib.pyplot as plt\n",
    "from pylab import *\n",
    "# import plotly\n",
    "# import plotly.graph_objects as go\n",
    "import sklearn\n",
    "from IPython import display\n",
    "import ipywidgets as widgets\n",
    "import sklearn.metrics as metrics\n",
    "import operator\n",
    "from sklearn.metrics.pairwise import cosine_similarity,pairwise_distances \n",
    "import implicit\n",
    "import seaborn as sns\n",
    "import os\n",
    "import evaluation_func as eval\n",
    "from pygments import highlight\n",
    "from pygments.lexers import get_lexer_by_name, get_lexer_for_filename\n",
    "from pygments.formatters import HtmlFormatter\n",
    "from IPython.display import display, HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv('user_item_action.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### sneak peek of data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   sessionId  userId  itemId action  timeframe eventdate  userID\n",
      "0         48       2   24764  click       8863  4/9/2016       2\n",
      "1         48       2   24764  click     496381  4/9/2016       2\n",
      "2         48       2   24764  click     265216  4/9/2016       2\n",
      "3         48       2   24764  click     519975  4/9/2016       2\n",
      "4         48       2   24764  click     456437  4/9/2016       2\n",
      "unique userId: 9999\n",
      "unique itemId: 16177\n",
      "avg. distinct item count per user: 3.216821682168217\n",
      "avg. distinct user count per item: 1.9883167459974038\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHY9JREFUeJzt3XmUVeWZ7/HvTxCnmABSehGIYEsG\n41oab0VJJ+lri2EwRuy1NE2uaUtDml653nSm7gSj3RiHjnbnxsTVHXNpIeIQlRgTaWPaENQM916H\nchYnSkQoQSkFcTaiz/1jvyWb8pyqc4qqc0re32ets87e7373u5/91qn97OmcrYjAzMzys1OzAzAz\ns+ZwAjAzy5QTgJlZppwAzMwy5QRgZpYpJwAzs0w5AQwxkn4k6R8GqK33SnpR0rA0foukLwxE26m9\nX0lqG6j26ljuOZKekfRUjfVD0gFpuF/927MvB1Nazv6DvZwqy14h6YhmLHt7NbPf3qnk7wE0jqTV\nwD7AFuAN4EHgUmBBRLzZj7a+EBG/qWOeW4DLI+LiepaV5j0TOCAiPlfvvANJ0gTgUWC/iNhQ4zwB\nTI6IjjqWs5o6+7fGdi8BOiPijMGo30iD1Uc1LvsW+vlZtq18BNB4n46IPYH9gPOAbwILB3ohkoYP\ndJtDxH7As7Vu/G1oUsHbn2aLCL8a9AJWA0f1KDsMeBM4KI1fApyThscA1wPPARuB31Mk7cvSPK8A\nLwLfACYCAcwB1gC/K5UNT+3dAnwHuB3YDFwHjE7TjqDY03xbvMAM4I/A62l595ba+0Ia3gk4A3gC\n2EBxZPOeNK07jrYU2zPA6b3003vS/F2pvTNS+0eldX4zxXFJlfn/HlgPrAM+n5Z9wAD1b7kvzwb+\nD/AC8GtgTCmGjwP/N7W9FjgZmJv68I+p3f+oEn8AB1SrD+wL/Cz1z+PA35bmPRP4KXB5iut+4H3A\naenvshaYVstnNLW1JP0tXgBWAK1p2tv6KJVPKa33vcARpbZvAc5NffZKWsdTgIdS+6uAv+kRzyzg\nHuB54DGKz+K5FEfQr6Zl/2u533r7DKVpJwN/AL4LbEp9OLPZ24embJOaHUBOLyokgFS+BvhiGr6E\nrRuo7wA/AnZOr0+w9bTdNm2xdQN1KbAHsBuVN1pPAgelOj+jOIyGXhJAGj6zu25p+i1sTQCfBzqA\n/YF3AdcCl/WI7d9TXAcDrwEfrNJPl1Ikpz3TvI8Cc6rF2WPeGcDTpXX8CdUTQH/6t9yXj1FsXHdL\n4+elae+l2KB9NrW7F3BIz+X3sg4V403jOwF3Av8IjEj9vQqYXvo7vQpMB4anvnwcOD3F8tfA47V8\nRkttHQ0MS/11a7XPMzAOeDbV3wn4ZBpvKfXZGuBDKbadgU8BfwII+G/Ay8Chqf5hFDsqn0ztjQM+\n0POzV6XfevsMnUyRWP86rdcXKXYW1OxtRKNfPgQbGtYBoyuUvw6MpTjf/XpE/D7SJ7gXZ0bESxHx\nSpXpl0XEAxHxEvAPwGcG6MLmicD3ImJVRLxIscc5u8epqG9HxCsRcS/F3uHBPRtJsfwlcFpEvBAR\nq4H/BfxVjXF8BvhxaR3P7KVuf/q37McR8Wjq6yXAIan8ROA3EXFlavfZiLinjnZ78xGKDepZEfHH\niFhFkVhnl+r8PiJujIgtFEcDLRTJ6XXgKmCipJE1Lu8PEXFDRLxBsdf/tr9ZyeeAG1L9NyNiGdBO\nkRC6XRIRKyJiS+qbX0bEY1H4LcWR1CdS3TnAoohYltp7MiIe7ivgGj9DT0TEv6f1WkzxOdinlg7Z\nkTgBDA3jKE5B9PQvFHvVv5a0StK8GtpaW8f0Jyj2wsbUFGXv9k3tldsezrb/VOW7dl6mOFLoaQzF\nnm3PtsbVEUfPdaymP/1bVm19JlAcHQyG/YB9JT3X/QK+xbb9/HRp+BXgmbSh6x6Hyn1fSc913LWX\n60v7ASf0iO3jFBvXbtt8PiXNlHSrpI2p/tFs/Tz2tx9r+Qy9tV4R8XIarLVPdhhOAE0m6SMUH8w/\n9JyW9l6+HhH7A58GviZpavfkKk32tQc7oTT8Xoq94GeAl4DdS3ENo9hzrLXddRQbgHLbW9h2Y1SL\nZ1JMPdt6ssb51/P2dayon/1bi7UUpzUqLrbOtnrWX0txCmdk6bVnRBxdaeZBVim2y3rEtkdEnFdp\nHkm7UJyG/C6wT0SMBG6gOB3U3V5/+nF7P0PZcAJoEknvlnQMxSH55RFxf4U6x0g6QJIoLoK9kV5Q\nbFj7c8/z5yQdKGl34CzgmrR3+CjF3t2nJO1McdFsl9J8T1OcOqj2mbkS+KqkSZLeBfwTcHU6DVGz\nFMsS4FxJe0raD/gaxUXNWiwBTi6t4/xqFQepfwGuAI6S9BlJwyXtJan79FC97fasfzvwvKRvStpN\n0jBJB6UdiUbrGdvlwKclTU9x7SrpCEnjq8w/guIz1gVskTQTmFaavhA4RdJUSTtJGifpA1WW/ZYB\n+Axlwwmg8f5D0gsUezenA9+juBOiksnAbyjudPh/wA8j4pY07TvAGelQ++/qWP5lFBcWnwJ2Bf4W\nICI2A/8DuJhiT+kloLM030/T+7OS7qrQ7qLU9u8oLjq+CnypjrjKvpSWv4riyOgnqf0+RcSvgO8D\nN1Gc3rmpl+qD0b9ExBqKUxlfpzi1dw9bz50vBA5M7f6ihua2qZ82bp+muN7wOMXe7sUUd7002jZ9\nFBFrKe7a+RbFRn0txR1ZFbczEfECxedvCcXdOP8dWFqafjvF/8YFFBeDf8vWvfofAMdL2iTpwgrN\n9/szlBN/EczMLFM+AjAzy5QTgJlZppwAzMwy5QRgZpapIf2DYWPGjImJEyc2Owwzs3eUO++885mI\naOmr3pBOABMnTqS9vb3ZYZiZvaNI6u0b8G/xKSAzs0w5AZiZZcoJwMwsU04AZmaZcgIwM8uUE4CZ\nWaacAMzMMlVTApD0VUkrJD0g6cr0O9+TJN0maaWkqyWNSHV3SeMdafrEUjunpfJHJE0fnFUyM7Na\n9JkAJI2j+M3u1og4iOIhyrOB84ELImIyxW95z0mzzAE2RcQBFL/jfX5q58A034coHtz9wwF6Fq2Z\nmfVDrd8EHg7sJul1iscGrgeOpHiAAxQPVT4TuIjigRBnpvJrgH9NT1yaBVwVEa8Bj0vqAA6jeBDH\noJg475cVy1ef96nBWqSZ2TtGn0cAEfEkxTM711Bs+DcDdwLPlR7318nWBy6PIz34OU3fDOxVLq8w\nz1skzZXULqm9q6urP+tkZmY1qOUU0CiKvfdJwL7AHsDMClW7Hy2mKtOqlW9bELEgIlojorWlpc/f\nMjIzs36q5SLwUcDjEdEVEa8D1wJ/CoyU1H0KaTywLg13AhMA0vT3UDwX9a3yCvOYmVmD1ZIA1gBT\nJO2ezuVPBR4EbgaOT3XagOvS8NI0Tpp+UxQPHl4KzE53CU2ieCD37QOzGmZmVq8+LwJHxG2SrgHu\nArYAdwMLgF8CV0k6J5UtTLMsBC5LF3k3Utz5Q0SskLSEInlsAU6NiDcGeH3MzKxGNd0FFBHzgfk9\nildR3MXTs+6rwAlV2jkXOLfOGM3MbBD4m8BmZplyAjAzy5QTgJlZppwAzMwy5QRgZpYpJwAzs0w5\nAZiZZcoJwMwsU04AZmaZcgIwM8uUE4CZWaacAMzMMuUEYGaWKScAM7NMOQGYmWXKCcDMLFNOAGZm\nmeozAUh6v6R7Sq/nJX1F0mhJyyStTO+jUn1JulBSh6T7JB1aaqst1V8pqa36Us3MbLD1mQAi4pGI\nOCQiDgH+K/Ay8HNgHrA8IiYDy9M4wEyKB75PBuYCFwFIGk3xWMnDKR4lOb87aZiZWePVewpoKvBY\nRDwBzAIWp/LFwHFpeBZwaRRuBUZKGgtMB5ZFxMaI2AQsA2Zs9xqYmVm/1JsAZgNXpuF9ImI9QHrf\nO5WPA9aW5ulMZdXKtyFprqR2Se1dXV11hmdmZrWqOQFIGgEcC/y0r6oVyqKX8m0LIhZERGtEtLa0\ntNQanpmZ1ameI4CZwF0R8XQafzqd2iG9b0jlncCE0nzjgXW9lJuZWRPUkwA+y9bTPwBLge47edqA\n60rlJ6W7gaYAm9MpohuBaZJGpYu/01KZmZk1wfBaKknaHfgk8Del4vOAJZLmAGuAE1L5DcDRQAfF\nHUOnAETERklnA3ekemdFxMbtXgMzM+uXmhJARLwM7NWj7FmKu4J61g3g1CrtLAIW1R+mmZkNNH8T\n2MwsU04AZmaZcgIwM8uUE4CZWaacAMzMMuUEYGaWKScAM7NMOQGYmWXKCcDMLFNOAGZmmXICMDPL\nlBOAmVmmnADMzDLlBGBmliknADOzTDkBmJllqqYEIGmkpGskPSzpIUkflTRa0jJJK9P7qFRXki6U\n1CHpPkmHltppS/VXSmqrvkQzMxtstR4B/AD4z4j4AHAw8BAwD1geEZOB5WkciofHT06vucBFAJJG\nA/OBw4HDgPndScPMzBqvzwQg6d3AnwELASLijxHxHDALWJyqLQaOS8OzgEujcCswUtJYYDqwLCI2\nRsQmYBkwY0DXxszMalbLEcD+QBfwY0l3S7pY0h7APhGxHiC9753qjwPWlubvTGXVyrchaa6kdknt\nXV1dda+QmZnVppYEMBw4FLgoIj4MvMTW0z2VqEJZ9FK+bUHEgohojYjWlpaWGsIzM7P+qCUBdAKd\nEXFbGr+GIiE8nU7tkN43lOpPKM0/HljXS7mZmTVBnwkgIp4C1kp6fyqaCjwILAW67+RpA65Lw0uB\nk9LdQFOAzekU0Y3ANEmj0sXfaanMzMyaYHiN9b4EXCFpBLAKOIUieSyRNAdYA5yQ6t4AHA10AC+n\nukTERklnA3ekemdFxMYBWQszM6tbTQkgIu4BWitMmlqhbgCnVmlnEbCongDNzGxw+JvAZmaZcgIw\nM8uUE4CZWaacAMzMMuUEYGaWKScAM7NMOQGYmWXKCcDMLFNOAGZmmXICMDPLlBOAmVmmnADMzDLl\nBGBmliknADOzTDkBmJllygnAzCxTTgBmZpmqKQFIWi3pfkn3SGpPZaMlLZO0Mr2PSuWSdKGkDkn3\nSTq01E5bqr9SUlu15ZmZ2eCr5wjgzyPikIjofjTkPGB5REwGlqdxgJnA5PSaC1wERcIA5gOHA4cB\n87uThpmZNd72nAKaBSxOw4uB40rll0bhVmCkpLHAdGBZRGyMiE3AMmDGdizfzMy2Q60JIIBfS7pT\n0txUtk9ErAdI73un8nHA2tK8namsWvk2JM2V1C6pvaurq/Y1MTOzugyvsd7HImKdpL2BZZIe7qWu\nKpRFL+XbFkQsABYAtLa2vm26mZkNjJqOACJiXXrfAPyc4hz+0+nUDul9Q6reCUwozT4eWNdLuZmZ\nNUGfCUDSHpL27B4GpgEPAEuB7jt52oDr0vBS4KR0N9AUYHM6RXQjME3SqHTxd1oqMzOzJqjlFNA+\nwM8lddf/SUT8p6Q7gCWS5gBrgBNS/RuAo4EO4GXgFICI2CjpbOCOVO+siNg4YGtiZmZ16TMBRMQq\n4OAK5c8CUyuUB3BqlbYWAYvqD9PMzAaavwlsZpYpJwAzs0w5AZiZZcoJwMwsU04AZmaZcgIwM8uU\nE4CZWaacAMzMMuUEYGaWKScAM7NMOQGYmWXKCcDMLFNOAGZmmXICMDPLlBOAmVmmnADMzDLlBGBm\nlqmaE4CkYZLulnR9Gp8k6TZJKyVdLWlEKt8ljXek6RNLbZyWyh+RNH2gV8bMzGpXzxHAl4GHSuPn\nAxdExGRgEzAnlc8BNkXEAcAFqR6SDgRmAx8CZgA/lDRs+8I3M7P+qikBSBoPfAq4OI0LOBK4JlVZ\nDByXhmelcdL0qan+LOCqiHgtIh6neGj8YQOxEmZmVr9ajwC+D3wDeDON7wU8FxFb0ngnMC4NjwPW\nAqTpm1P9t8orzPMWSXMltUtq7+rqqmNVzMysHn0mAEnHABsi4s5ycYWq0ce03ubZWhCxICJaI6K1\npaWlr/DMzKyfhtdQ52PAsZKOBnYF3k1xRDBS0vC0lz8eWJfqdwITgE5Jw4H3ABtL5d3K85iZWYP1\neQQQEadFxPiImEhxEfemiDgRuBk4PlVrA65Lw0vTOGn6TRERqXx2uktoEjAZuH3A1sTMzOpSyxFA\nNd8ErpJ0DnA3sDCVLwQuk9RBsec/GyAiVkhaAjwIbAFOjYg3tmP5Zma2HepKABFxC3BLGl5Fhbt4\nIuJV4IQq858LnFtvkGZmNvD8TWAzs0w5AZiZZcoJwMwsU04AZmaZcgIwM8uUE4CZWaacAMzMMuUE\nYGaWKScAM7NMOQGYmWXKCcDMLFNOAGZmmXICMDPLlBOAmVmmnADMzDLlBGBmlqlaHgq/q6TbJd0r\naYWkb6fySZJuk7RS0tWSRqTyXdJ4R5o+sdTWaan8EUnTB2ulzMysb7UcAbwGHBkRBwOHADMkTQHO\nBy6IiMnAJmBOqj8H2BQRBwAXpHpIOpDi8ZAfAmYAP5Q0bCBXxszMalfLQ+EjIl5MozunVwBHAtek\n8sXAcWl4VhonTZ8qSan8qoh4LSIeBzqo8EhJMzNrjJquAUgaJukeYAOwDHgMeC4itqQqncC4NDwO\nWAuQpm8G9iqXV5jHzMwarKYEEBFvRMQhwHiKvfYPVqqW3lVlWrXybUiaK6ldUntXV1ct4ZmZWT/U\ndRdQRDwH3AJMAUZKGp4mjQfWpeFOYAJAmv4eYGO5vMI85WUsiIjWiGhtaWmpJzwzM6tDLXcBtUga\nmYZ3A44CHgJuBo5P1dqA69Lw0jROmn5TREQqn53uEpoETAZuH6gVMTOz+gzvuwpjgcXpjp2dgCUR\ncb2kB4GrJJ0D3A0sTPUXApdJ6qDY858NEBErJC0BHgS2AKdGxBsDuzpmZlarPhNARNwHfLhC+Soq\n3MUTEa8CJ1Rp61zg3PrDNDOzgeZvApuZZcoJwMwsU04AZmaZcgIwM8uUE4CZWaacAMzMMuUEYGaW\nKScAM7NMOQGYmWXKCcDMLFNOAGZmmXICMDPLlBOAmVmmnADMzDLlBGBmliknADOzTDkBmJllqpZn\nAk+QdLOkhyStkPTlVD5a0jJJK9P7qFQuSRdK6pB0n6RDS221pforJbVVW6aZmQ2+Wo4AtgBfj4gP\nAlOAUyUdCMwDlkfEZGB5GgeYSfHA98nAXOAiKBIGMB84nOJRkvO7k4aZmTVenwkgItZHxF1p+AXg\nIWAcMAtYnKotBo5Lw7OAS6NwKzBS0lhgOrAsIjZGxCZgGTBjQNfGzMxqVtc1AEkTKR4QfxuwT0Ss\nhyJJAHunauOAtaXZOlNZtfKey5grqV1Se1dXVz3hmZlZHWpOAJLeBfwM+EpEPN9b1Qpl0Uv5tgUR\nCyKiNSJaW1paag3PzMzqVFMCkLQzxcb/ioi4NhU/nU7tkN43pPJOYEJp9vHAul7KzcysCWq5C0jA\nQuChiPheadJSoPtOnjbgulL5SeluoCnA5nSK6EZgmqRR6eLvtFRmZmZNMLyGOh8D/gq4X9I9qexb\nwHnAEklzgDXACWnaDcDRQAfwMnAKQERslHQ2cEeqd1ZEbByQtTAzs7r1mQAi4g9UPn8PMLVC/QBO\nrdLWImBRPQGamdng8DeBzcwy5QRgZpYpJwAzs0w5AZiZZcoJwMwsU04AZmaZcgIwM8uUE4CZWaac\nAMzMMuUEYGaWKScAM7NMOQGYmWXKCcDMLFNOAGZmmXICMDPLlBOAmVmmnADMzDLV5xPBJC0CjgE2\nRMRBqWw0cDUwEVgNfCYiNqXnB/+A4pGQLwMnR8RdaZ424IzU7DkRsXhgV6V2E+f9smL56vM+1eBI\nzMyap5YjgEuAGT3K5gHLI2IysDyNA8wEJqfXXOAieCthzAcOBw4D5qcHw5uZWZP0mQAi4ndAz4e3\nzwK69+AXA8eVyi+Nwq3ASEljgenAsojYGBGbgGW8PamYmVkD9fcawD4RsR4gve+dyscBa0v1OlNZ\ntfK3kTRXUruk9q6urn6GZ2ZmfRnoi8CqUBa9lL+9MGJBRLRGRGtLS8uABmdmZlv1NwE8nU7tkN43\npPJOYEKp3nhgXS/lZmbWJP1NAEuBtjTcBlxXKj9JhSnA5nSK6EZgmqRR6eLvtFRmZmZNUsttoFcC\nRwBjJHVS3M1zHrBE0hxgDXBCqn4DxS2gHRS3gZ4CEBEbJZ0N3JHqnRURPS8sm5lZA/WZACLis1Um\nTa1QN4BTq7SzCFhUV3RmZjZo/E1gM7NMOQGYmWXKCcDMLFNOAGZmmXICMDPLlBOAmVmmnADMzDLl\nBGBmliknADOzTPX5TeCc+ElhZpYTHwGYmWXKCcDMLFNOAGZmmXICMDPLlC8C18AXh81sR+QjADOz\nTDkBmJllquEJQNIMSY9I6pA0r9HLNzOzQkOvAUgaBvwb8EmgE7hD0tKIeLCRcQwUXxsws3eyRl8E\nPgzoiIhVAJKuAmYB78gEUE21xFAvJxIzG0yNTgDjgLWl8U7g8HIFSXOBuWn0RUmP9GM5Y4Bn+hXh\n4KorLp0/iJFsa4forwYbqrE5rvoN1di2J679aqnU6ASgCmWxzUjEAmDBdi1Eao+I1u1pYzA4rvoM\n1bhg6MbmuOo3VGNrRFyNvgjcCUwojY8H1jU4BjMzo/EJ4A5gsqRJkkYAs4GlDY7BzMxo8CmgiNgi\n6X8CNwLDgEURsWIQFrVdp5AGkeOqz1CNC4ZubI6rfkM1tkGPSxHRdy0zM9vh+JvAZmaZcgIwM8vU\nDpUAhsrPTEiaIOlmSQ9JWiHpy6l8tKRlklam91FNim+YpLslXZ/GJ0m6LcV1dbpA34y4Rkq6RtLD\nqe8+OhT6TNJX09/xAUlXStq1WX0maZGkDZIeKJVV7CMVLkz/D/dJOrTBcf1L+lveJ+nnkkaWpp2W\n4npE0vRGxlWa9neSQtKYNN6w/uotNklfSv2yQtI/l8oHvs8iYod4UVxUfgzYHxgB3Asc2KRYxgKH\npuE9gUeBA4F/Bual8nnA+U2K72vAT4Dr0/gSYHYa/hHwxSbFtRj4QhoeAYxsdp9RfHnxcWC3Ul+d\n3Kw+A/4MOBR4oFRWsY+Ao4FfUXz/ZgpwW4PjmgYMT8Pnl+I6MP1/7gJMSv+3wxoVVyqfQHEzyhPA\nmEb3Vy999ufAb4Bd0vjeg9lng/6BbdQL+ChwY2n8NOC0ZseVYrmO4vePHgHGprKxwCNNiGU8sBw4\nErg+fdifKf2jbtOPDYzr3WlDqx7lTe0ztn57fTTFXXPXA9Ob2WfAxB4bjYp9BPxv4LOV6jUirh7T\n/gK4Ig1v87+ZNsQfbWRcwDXAwcDqUgJoaH9V+VsuAY6qUG9Q+mxHOgVU6WcmxjUplrdImgh8GLgN\n2Cci1gOk972bENL3gW8Ab6bxvYDnImJLGm9Wv+0PdAE/TqenLpa0B03us4h4EvgusAZYD2wG7mRo\n9Fm3an00lP4nPk+xdw1NjkvSscCTEXFvj0lDob/eB3winV78raSPDGZsO1IC6PNnJhpN0ruAnwFf\niYjnmxlLiucYYENE3FkurlC1Gf02nOJw+KKI+DDwEsXpjKZK59NnURx27wvsAcysUHUo3k89JP62\nkk4HtgBXdBdVqNaQuCTtDpwO/GOlyRXKGt1fw4FRFKeg/h5YIkkMUmw7UgIYUj8zIWlnio3/FRFx\nbSp+WtLYNH0ssKHBYX0MOFbSauAqitNA3wdGSur+UmCz+q0T6IyI29L4NRQJodl9dhTweER0RcTr\nwLXAnzI0+qxbtT5q+v+EpDbgGODESOcumhzXn1Ak83vT/8F44C5J/6XJcXXrBK6Nwu0UR+pjBiu2\nHSkBDJmfmUgZeyHwUER8rzRpKdCWhtsorg00TEScFhHjI2IiRf/cFBEnAjcDxzcrrhTbU8BaSe9P\nRVMpfia8qX1GcepniqTd09+1O66m91lJtT5aCpyU7m6ZAmzuPlXUCJJmAN8Ejo2Il3vEO1vSLpIm\nAZOB2xsRU0TcHxF7R8TE9H/QSXHDxlM0ub+SX1DsmCHpfRQ3QzzDYPXZYF7gaPSL4ir+oxRXyE9v\nYhwfpzg8uw+4J72OpjjfvhxYmd5HNzHGI9h6F9D+6cPUAfyUdAdCE2I6BGhP/fYLikPhpvcZ8G3g\nYeAB4DKKOzGa0mfAlRTXIl6n2HjNqdZHFKcN/i39P9wPtDY4rg6K89bd/wM/KtU/PcX1CDCzkXH1\nmL6arReBG9ZfvfTZCODy9Fm7CzhyMPvMPwVhZpapHekUkJmZ1cEJwMwsU04AZmaZcgIwM8uUE4CZ\nWaacAMzMMuUEYGaWqf8POIA68qPt9ksAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = pd.read_csv(\"user_item_action.csv\")\n",
    "data['action']='click' #Adding user action(click) column \n",
    "print(data.head(5))\n",
    "print('unique userId:',len(list(data['userId'].unique())))\n",
    "print('unique itemId:',len(list(data['itemId'].unique())))\n",
    "print('avg. distinct item count per user:',data.groupby('userId')['itemId'].nunique().mean())\n",
    "print('avg. distinct user count per item:',data.groupby('itemId')['userId'].nunique().mean())\n",
    "user_item_interaction = data.groupby('userId')['itemId'].nunique()\n",
    "plt.hist(user_item_interaction, bins = 50)\n",
    "plt.title('Distribution of distinct item interaction')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For most of the users distinct item interaction is < 5.So we expect extreme sparsity in user-item matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Preperation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sessionId</th>\n",
       "      <th>userId</th>\n",
       "      <th>itemId</th>\n",
       "      <th>action</th>\n",
       "      <th>eventdate</th>\n",
       "      <th>userID</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>month_day</th>\n",
       "      <th>day_seq</th>\n",
       "      <th>UserID_day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2562</th>\n",
       "      <td>172</td>\n",
       "      <td>10</td>\n",
       "      <td>36602</td>\n",
       "      <td>click</td>\n",
       "      <td>2/21/2016</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27931</th>\n",
       "      <td>172</td>\n",
       "      <td>10</td>\n",
       "      <td>36602</td>\n",
       "      <td>click</td>\n",
       "      <td>2/21/2016</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28730</th>\n",
       "      <td>172</td>\n",
       "      <td>10</td>\n",
       "      <td>10691</td>\n",
       "      <td>click</td>\n",
       "      <td>2/21/2016</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23215</th>\n",
       "      <td>172</td>\n",
       "      <td>10</td>\n",
       "      <td>47373</td>\n",
       "      <td>click</td>\n",
       "      <td>2/21/2016</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>2</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27117</th>\n",
       "      <td>172</td>\n",
       "      <td>10</td>\n",
       "      <td>10858</td>\n",
       "      <td>click</td>\n",
       "      <td>2/21/2016</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       sessionId userId itemId action  eventdate  userID month day month_day  \\\n",
       "2562         172     10  36602  click  2/21/2016      10     1   1        11   \n",
       "27931        172     10  36602  click  2/21/2016      10     1   1        16   \n",
       "28730        172     10  10691  click  2/21/2016      10     1   1        16   \n",
       "23215        172     10  47373  click  2/21/2016      10     1   1        14   \n",
       "27117        172     10  10858  click  2/21/2016      10     1   1        15   \n",
       "\n",
       "      day_seq UserID_day  \n",
       "2562        0        100  \n",
       "27931       1        101  \n",
       "28730       1        101  \n",
       "23215       2        102  \n",
       "27117       5        105  "
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Sorting data based on time stamp\n",
    "\n",
    "data[\"timeframe\"] = pd.to_datetime(pd.to_numeric(data[\"timeframe\"]), unit=\"s\")\n",
    "data_sort = data.sort_values(by=[\"timeframe\"],ascending=True)\n",
    "data_sort['month'] = data_sort[\"timeframe\"].dt.month\n",
    "data_sort['day'] = data_sort[\"timeframe\"].dt.day\n",
    "data_sort['day'] = data_sort['day'].astype('str')\n",
    "data_sort['month'] = data_sort['month'].astype('str')\n",
    "data_sort['month_day'] = data_sort[['month', 'day']].apply(lambda x: ''.join(x), axis=1)\n",
    "data_sort['day'] = data_sort['month'].astype('str')\n",
    "data_sort['userId'] = data_sort['userId'].astype('str')\n",
    "day_sr_no = {}\n",
    "# Mapping external to internal days\n",
    "for i in range(len(data_sort)):\n",
    "    month_day = data_sort['month_day'][i]\n",
    "    if month_day not in day_sr_no :\n",
    "        day_sr_no[month_day] = len(day_sr_no)\n",
    "month_day = list(day_sr_no.keys())\n",
    "sr_no = list(day_sr_no.values())\n",
    "day_sr = pd.DataFrame({'month_day':month_day,'day_seq':sr_no})     \n",
    "data_sort = pd.merge(left=data_sort.reset_index(drop=True),right=day_sr.reset_index(drop=True),how='left',on='month_day')\n",
    "data_sort.sort_values([\"userId\",'day_seq'],ascending=True,inplace=True)\n",
    "data_sort['day_seq'] = data_sort['day_seq'].astype('str')\n",
    "data_sort['UserID_day'] = data_sort[['userId', 'day_seq']].apply(lambda x: ''.join(x), axis=1)\n",
    "data_sort[['userId','itemId']] = data_sort[['userId','itemId']].astype('object')\n",
    "data_sort.drop('timeframe',axis =1).head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Identifying/Filtering users who appeared more than one days for splitting data into train and validation to simulate A/B  test environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\sourish\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>day_seq</th>\n",
       "      <th>user</th>\n",
       "      <th>max_day</th>\n",
       "      <th>user_day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>100</td>\n",
       "      <td>6</td>\n",
       "      <td>1006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1000</td>\n",
       "      <td>7</td>\n",
       "      <td>10007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>10000</td>\n",
       "      <td>7</td>\n",
       "      <td>100007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>10001</td>\n",
       "      <td>7</td>\n",
       "      <td>100017</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   day_seq   user max_day user_day\n",
       "0        5     10       6      106\n",
       "1        2    100       6     1006\n",
       "2        3   1000       7    10007\n",
       "3        2  10000       7   100007\n",
       "4        3  10001       7   100017"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sort_user = data_sort[['userId','day_seq']]\n",
    "data_sort_user.drop_duplicates(subset=None, keep='first', inplace=True)\n",
    "user_id_repeat = pd.DataFrame(data_sort_user.groupby('userId').day_seq.nunique())\n",
    "user_id_repeat['user'] = user_id_repeat.index\n",
    "user_id_repeat.reset_index(drop=True, inplace = True)\n",
    "user_id_repeat = user_id_repeat[(user_id_repeat['day_seq']>1)]\n",
    "data_sort_user = data_sort_user[data_sort_user['userId'].isin(list(user_id_repeat['user']))]\n",
    "\n",
    "#To get the last day for each user\n",
    "max_day = data_sort_user.groupby('userId').day_seq.max()\n",
    "user_id_repeat['max_day'] = max_day.values\n",
    "user_id_repeat['user_day'] = user_id_repeat[['user', 'max_day']].apply(lambda x: ''.join(x), axis=1)\n",
    "user_id_repeat.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Filtering out data including only repeat users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\sourish\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>itemId</th>\n",
       "      <th>action</th>\n",
       "      <th>day_seq</th>\n",
       "      <th>UserID_day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2562</th>\n",
       "      <td>10</td>\n",
       "      <td>36602</td>\n",
       "      <td>click</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27931</th>\n",
       "      <td>10</td>\n",
       "      <td>36602</td>\n",
       "      <td>click</td>\n",
       "      <td>1</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28730</th>\n",
       "      <td>10</td>\n",
       "      <td>10691</td>\n",
       "      <td>click</td>\n",
       "      <td>1</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23215</th>\n",
       "      <td>10</td>\n",
       "      <td>47373</td>\n",
       "      <td>click</td>\n",
       "      <td>2</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27117</th>\n",
       "      <td>10</td>\n",
       "      <td>10858</td>\n",
       "      <td>click</td>\n",
       "      <td>5</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      userId itemId action  day_seq UserID_day\n",
       "2562      10  36602  click        0        100\n",
       "27931     10  36602  click        1        101\n",
       "28730     10  10691  click        1        101\n",
       "23215     10  47373  click        2        102\n",
       "27117     10  10858  click        5        105"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sort_repeat = data_sort[data_sort['userId'].isin(list(user_id_repeat['user']))]\n",
    "data_sort_repeat.sort_values([\"userId\",'day_seq'],ascending=True,inplace=True)\n",
    "data_sort_repeat = data_sort_repeat[[\"userId\",'itemId','action','day_seq','UserID_day']]\n",
    "data_sort_repeat['day_seq'] = data_sort_repeat['day_seq'].astype('int')\n",
    "data_sort_repeat.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train(common users of all but last day) and hold out validation/test samples(common users of last day) split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(24930, 5)\n",
      "(7642, 5)\n"
     ]
    }
   ],
   "source": [
    "#Train-test-1 day test\n",
    "test = data_sort_repeat[data_sort_repeat['UserID_day'].isin(list(user_id_repeat['user_day']))]\n",
    "train = data_sort_repeat[~data_sort_repeat['UserID_day'].isin(list(user_id_repeat['user_day']))]\n",
    "print(train.shape)\n",
    "print(test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MF training data with implicit feedback format(count of clicks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\sourish\\anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:3140: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self[k1] = value[k2]\n",
      "c:\\users\\sourish\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>click</th>\n",
       "      <th>userId</th>\n",
       "      <th>itemId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>10000</td>\n",
       "      <td>20024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>10001</td>\n",
       "      <td>109029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>10001</td>\n",
       "      <td>134792</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   click  userId  itemId\n",
       "0      2   10000   20024\n",
       "1      4   10001  109029\n",
       "2      2   10001  134792"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[['userId', 'itemId']] = train[['userId', 'itemId']].astype(str)\n",
    "train['user_item'] = train[['userId', 'itemId']].apply(lambda x: '_'.join(x), axis=1)\n",
    "training = train[['user_item','action']]\n",
    "training = pd.DataFrame(pd.pivot_table(training, values='action', index=['user_item'],columns=['action'], aggfunc=np.count_nonzero))\n",
    "training['user_item'] = training.index\n",
    "training = pd.DataFrame(np.array(training))\n",
    "training.columns = ['click', 'user_item']\n",
    "training.fillna(0,inplace=True)\n",
    "training['userId'] = training['user_item'].apply(lambda x: str(x).split('_')[0])\n",
    "training['itemId'] = training['user_item'].apply(lambda x: str(x).split('_')[1])\n",
    "training.drop('user_item',axis=1,inplace=True)\n",
    "#training[['UserID', 'itemID']] = training[['UserID', 'itemID']].apply(pd.to_numeric, errors='coerce')\n",
    "training['userId'] = pd.to_numeric(training['userId'])\n",
    "training['itemId'] = training['itemId'].astype(int)\n",
    "training.columns=['click', 'userId', 'itemId']\n",
    "training.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Creating hold-out test sample user-item unique interaction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\sourish\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>itemId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15115</th>\n",
       "      <td>10</td>\n",
       "      <td>10263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14104</th>\n",
       "      <td>100</td>\n",
       "      <td>3717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15547</th>\n",
       "      <td>100</td>\n",
       "      <td>3655</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      userId itemId\n",
       "15115     10  10263\n",
       "14104    100   3717\n",
       "15547    100   3655"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing = test[['userId', 'itemId']]\n",
    "testing.drop_duplicates(subset=None, keep='first', inplace=True)\n",
    "testing.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For evaluation purpose keeping only users who are present both in training and validation(testing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique_user in training: 5352 and unique_user in testing: 5352\n"
     ]
    }
   ],
   "source": [
    "def intersection(lst1, lst2): \n",
    "    lst3 = [value for value in lst1 if value in lst2] \n",
    "    return lst3 \n",
    "final_users = intersection(list(train['userId'].unique()),list(test['userId'].unique()))\n",
    "testing = testing[testing['userId'].isin(list(final_users))]\n",
    "training = training[training['userId'].isin(list(final_users))]\n",
    "print('unique_user in training:',len(list(training['userId'].unique())),'and unique_user in testing:',len(list(testing['userId'].unique())))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Although we loss a significant no of users now the data is suitable for offline performance assessment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "testing[['userId', 'itemId']] = testing[['userId', 'itemId']].apply(pd.to_numeric)\n",
    "training[['userId', 'itemId']] = training[['userId', 'itemId']].apply(pd.to_numeric)\n",
    "testing.sort_values(['userId','itemId'],axis=0,inplace=True)\n",
    "training.sort_values(['userId','itemId'],axis=0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Mapping external to internal user and item Ids \n",
    "user_dict = {}\n",
    "for i,user in enumerate(sorted(training.userId.unique())):\n",
    "    user_dict[user] = i \n",
    "    \n",
    "item_dict = {}\n",
    "for i,item in enumerate(training.itemId.unique()):\n",
    "    item_dict[item] = i    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "training['userId'] = [user_dict[i] for i in training.userId]\n",
    "testing['userId'] = [user_dict[i] for i in testing.userId]\n",
    "training['score'] = training.click.values  #Currently using the simple formula,will be changed based on relative importance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing user-item sparse matrix for matrix factorization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sparsity in data: 0.9996901198420426\n"
     ]
    }
   ],
   "source": [
    "item_user_matrix = pd.DataFrame(training.pivot(index='userId', columns='itemId', values='score'))\n",
    "item_user_matrix.fillna(0,inplace=True)\n",
    "print('sparsity in data:',(1.0 - (np.count_nonzero(item_user_matrix) / float(item_user_matrix.size))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's see how the algorithms works in this super sparse data. First we check the match rate as screening before further assessment   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5352, 12620)"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# item_user_matrix = pd.DataFrame(training.pivot(index='userId', columns='itemId', values='score'))\n",
    "# item_user_matrix.fillna(0.000001,inplace=True) #Instead of  0 Replacing with a very small value to avoid possible NaN error during computation\n",
    "item_dict = {}\n",
    "for i,item in enumerate(item_user_matrix.columns):\n",
    "    item_dict[item] = i\n",
    "item_user_matrix_sparse = sparse.csr_matrix(np.array(item_user_matrix))\n",
    "item_user_matrix_sparse.shape    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([     2,      3,      4, ..., 214354, 231673, 241688], dtype=int64)"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Reversing user-dict to get the userIds back\n",
    "\n",
    "user_rev_dict = {v: k for k, v in user_dict.items()}\n",
    "user_id = pd.Series(list(user_rev_dict.values())).values\n",
    "user_id = user_id[:,np.newaxis]\n",
    "user_id = user_id.reshape((len(user_id)))\n",
    "user_id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Popularity based- non personalized.Will recommend every user fixed 20 items based on popularity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2660, 1664, 5162, 2463, 4957, 5324, 1846, 5350, 2174, 5096, 12585, 154, 5303, 5179, 1674, 5174, 255, 1443, 923, 766]\n"
     ]
    }
   ],
   "source": [
    "pop_20 = list(np.argsort(-np.sum(item_user_matrix_sparse.toarray(),axis=0))[:20])\n",
    "print(pop_20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Match-rate actual vs. predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Actual\n",
    "\n",
    "testing[['userId']] = testing[['userId']].apply(pd.to_numeric, errors='coerce')\n",
    "users = []\n",
    "items = []\n",
    "for u,i in enumerate(np.unique(testing['userId'])):\n",
    "    user = user_rev_dict[u]\n",
    "    users.append(user)\n",
    "    l = list(testing[testing['userId'] == i]['itemId'])\n",
    "    item = [item_dict[x] if x in item_dict.keys() else 9999 for x in l]\n",
    "    items.append(item)\n",
    "test_reco = pd.DataFrame({'user_id':user_id,'items':items})\n",
    "\n",
    "#Predicted pop_20\n",
    "\n",
    "NN_pop = [pop_20 for i in range(len(user_id))]\n",
    "recommended_pop = pd.DataFrame({'user_id':user_id,'items':NN_pop})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "match rate: 0.02466367713004484\n"
     ]
    }
   ],
   "source": [
    "#Calculating actual match rate(atleast one product match) between recommended vs. actual\n",
    "\n",
    "cnt = 0\n",
    "match = []\n",
    "for i in range(recommended_pop.shape[0]):\n",
    "    l1=recommended_pop['items'][i]\n",
    "    l2=test_reco['items'][i]\n",
    "    if len(intersection(l1,l2)) > 0:\n",
    "        cnt+=1\n",
    "        match.append(1)\n",
    "    else:\n",
    "        match.append(0)\n",
    "print('match rate:',cnt/recommended_pop.shape[0])   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Only 2.47% match.Popularity based approach is a poor fit for this data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### item-item CF(Cosine similarity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5352, 20)\n"
     ]
    }
   ],
   "source": [
    "def recommend_items_to_users_cosine(item_user_matrix_sparse,metric='cosine',top_items=20):\n",
    "    item_similarity = pairwise_distances(item_user_matrix.T, metric=metric)\n",
    "    cos_similarity = np.array(item_user_matrix_sparse.dot(item_similarity))/ np.array([np.abs(item_similarity).sum(axis=1)])\n",
    "    nn = np.argsort(-cos_similarity)[:, :top_items]\n",
    "    return nn\n",
    "nn = recommend_items_to_users_cosine(item_user_matrix_sparse, metric='cosine', top_items=20)\n",
    "print(nn.shape)\n",
    "NN_CF = [nn[i] for i in range(len(nn))]\n",
    "recommended_CF = pd.DataFrame({'user_id':user_id,'items':NN_CF})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Match rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "match rate: 0.0476457399103139\n"
     ]
    }
   ],
   "source": [
    "#Calculating actual match rate(atleast one product match) between recommended vs. actual\n",
    "\n",
    "cnt = 0\n",
    "match = []\n",
    "for i in range(recommended_CF.shape[0]):\n",
    "    l1=recommended_CF['items'][i]\n",
    "    l2=test_reco['items'][i]\n",
    "    if len(intersection(l1,l2)) > 0:\n",
    "        cnt+=1\n",
    "        match.append(1)\n",
    "    else:\n",
    "        match.append(0)\n",
    "print('match rate:',cnt/recommended_CF.shape[0]) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Better with 4.76% match rate, almost 100% improvement. However CF alone not suitable to handle this sparsity and massive product universe. Let's try some ML based approach"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pure MF-ALS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████| 100.0/100 [00:00<00:00, 104.82it/s]\n"
     ]
    }
   ],
   "source": [
    "model = implicit.als.AlternatingLeastSquares(factors=32, regularization=0.1, iterations=100)\n",
    "alpha = 15\n",
    "model.fit((item_user_matrix_sparse * alpha_val).astype('double'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recommend_items_to_users_MF(model,top_items=20):\n",
    "    score = np.dot(model.item_factors,model.user_factors.T) \n",
    "    nn = np.argsort(-score)[:, :top_items]\n",
    "    return nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5352, 20)\n",
      "match rate: 0.3678998505231689\n"
     ]
    }
   ],
   "source": [
    "#Calculating actual match rate(atleast one product match) between recommended vs. actual\n",
    "nn_mf = recommend_items_to_users_MF(model,top_items=20)\n",
    "print(nn.shape)\n",
    "NN_MF = [nn_mf[i] for i in range(len(nn_mf))]\n",
    "recommended_MF = pd.DataFrame({'user_id':user_id,'items':NN_MF})\n",
    "cnt = 0\n",
    "match = []\n",
    "for i in range(recommended_MF.shape[0]):\n",
    "    l1=recommended_MF['items'][i]\n",
    "    l2=test_reco['items'][i]\n",
    "    if len(intersection(l1,l2)) > 0:\n",
    "        cnt+=1\n",
    "        match.append(1)\n",
    "    else:\n",
    "        match.append(0)\n",
    "print('match rate:',cnt/recommended_MF.shape[0]) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Drastic improvement with 36.8% match rate, almost 700% improvement over pure CF based recommendation.Keeping in mind only 20 product pick out of massive(12620) product space it's a great start. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MF coupled with cosine similarity.Let's explore wheather we can further improve the result mixing MF and CF."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:implicit:This method is deprecated. Please use the AlternatingLeastSquares class instead\n",
      "100%|█████████████████████████████████████████████████████████████████████████████| 100.0/100 [00:00<00:00, 107.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5352, 32) (12620, 32)\n"
     ]
    }
   ],
   "source": [
    "alpha = 15\n",
    "user_vecs, item_vecs = implicit.alternating_least_squares(\n",
    "    (item_user_matrix_sparse*alpha).astype('double'), \n",
    "     factors=32, \n",
    "     regularization = 0.1, \n",
    "     iterations = 100)\n",
    "\n",
    "print(user_vecs.shape, item_vecs.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Calculate cosine similarity between respective user vector(embeddings numbers- 1 X32) and item-item embeddings(12620 X latent vector dimension).The matrix dot product will pick up similarity values(12620 X 1) for all products.\n",
    "#### Sorting the similarity values to arrive to get top n items(20) for respective user."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recommend_items_to_users_hybrid(indexes, users_vec, items_vec, top_items=20):\n",
    "    cos_similarity = cosine_similarity(        #cosine similarity between product pairs for user vector\n",
    "        users_vec[indexes, :], items_vec)\n",
    "    nn = np.argsort(-cos_similarity)[:, :top_items]\n",
    "    return nn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### product recommendation basis on MF + item similarity based CF approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5352, 20) [[10729  8264   904  9697 11170 11725 10415  9419 11898  8720 10193  9142\n",
      "  10191 11695 11877  6102  6803 10866  1328  8201]\n",
      " [11576  5668   469  6980   904 10651 10654 10574  8894  8880 12204  1328\n",
      "  10704  3918 10713  8888 12185  8990  5740 12124]]\n"
     ]
    }
   ],
   "source": [
    "indexes = list(range(user_vecs.shape[0]))\n",
    "#indexes = list[0]\n",
    "nn_hybrid = recommend_items_to_users_hybrid(indexes, user_vecs, item_vecs)\n",
    "print(nn.shape, nn[:2, :])\n",
    "NN_hybrid = [list(nn_hybrid[i]) for i in range(len(nn_hybrid))]\n",
    "recommended_hybrid = pd.DataFrame({'user_id':user_id,'items':NN_hybrid})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sneak-peak between recommended vs. actual "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   user_id               items\n",
      "0        2              [3669]\n",
      "1        3              [5372]\n",
      "2        4              [2092]\n",
      "3        6              [8826]\n",
      "4        7        [1498, 5875]\n",
      "5        9              [4889]\n",
      "6       10              [1978]\n",
      "7       14              [2061]\n",
      "8       16              [9999]\n",
      "9       20  [1061, 1234, 5324]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>items</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>[3669, 10172, 1086, 5246, 1170, 3246, 6161, 32...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>[9855, 5678, 5311, 12083, 7392, 12071, 12051, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>[1129, 8390, 9481, 2092, 1356, 2067, 1978, 435...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>[1517, 3395, 11111, 6949, 2097, 8989, 7736, 19...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>[3427, 7295, 634, 8249, 7307, 2674, 5198, 990,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9</td>\n",
       "      <td>[8552, 3860, 10566, 2256, 4103, 6563, 5594, 73...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10</td>\n",
       "      <td>[9481, 8390, 2092, 1356, 5491, 8517, 2067, 435...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>14</td>\n",
       "      <td>[1369, 7103, 12560, 5361, 8354, 1583, 6520, 23...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>16</td>\n",
       "      <td>[8686, 7186, 8812, 5911, 11076, 9095, 7557, 10...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>20</td>\n",
       "      <td>[5394, 999, 7962, 1097, 2459, 2572, 2458, 2750...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id                                              items\n",
       "0        2  [3669, 10172, 1086, 5246, 1170, 3246, 6161, 32...\n",
       "1        3  [9855, 5678, 5311, 12083, 7392, 12071, 12051, ...\n",
       "2        4  [1129, 8390, 9481, 2092, 1356, 2067, 1978, 435...\n",
       "3        6  [1517, 3395, 11111, 6949, 2097, 8989, 7736, 19...\n",
       "4        7  [3427, 7295, 634, 8249, 7307, 2674, 5198, 990,...\n",
       "5        9  [8552, 3860, 10566, 2256, 4103, 6563, 5594, 73...\n",
       "6       10  [9481, 8390, 2092, 1356, 5491, 8517, 2067, 435...\n",
       "7       14  [1369, 7103, 12560, 5361, 8354, 1583, 6520, 23...\n",
       "8       16  [8686, 7186, 8812, 5911, 11076, 9095, 7557, 10...\n",
       "9       20  [5394, 999, 7962, 1097, 2459, 2572, 2458, 2750..."
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(test_reco.head(10))\n",
    "recommended_hybrid.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Apparently can see a high match-rate(TP) and calculate the match rate "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.40115844544095663\n"
     ]
    }
   ],
   "source": [
    "#Calculating actual match rate(atleast one product match) between recommended vs. actual\n",
    "\n",
    "cnt = 0\n",
    "match = []\n",
    "for i in range(recommended_hybrid.shape[0]):\n",
    "    l1=recommended_hybrid['items'][i]\n",
    "    l2=test_reco['items'][i]\n",
    "    if len(intersection(l1,l2)) > 0:\n",
    "        cnt+=1\n",
    "        match.append(1)\n",
    "    else:\n",
    "        match.append(0)\n",
    "print(cnt/recommended_hybrid.shape[0])    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### with the hybrid approach we achieve 40.12% match rate, almost 9% improvement over pure MF based recommendation.Although most of the intelligence lies in creating embeddings during MF process,CF(further cosine similarity based filtering of product pairs for each user) adds significant extra lift to the recommendation engine. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparing overall performance with pure MF based and hybrid recommendation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's explore the custom functions to calculate performance metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>\n",
       ".pygments .hll { background-color: #ffffcc }\n",
       ".pygments  { background: #f8f8f8; }\n",
       ".pygments .c { color: #408080; font-style: italic } /* Comment */\n",
       ".pygments .err { border: 1px solid #FF0000 } /* Error */\n",
       ".pygments .k { color: #008000; font-weight: bold } /* Keyword */\n",
       ".pygments .o { color: #666666 } /* Operator */\n",
       ".pygments .ch { color: #408080; font-style: italic } /* Comment.Hashbang */\n",
       ".pygments .cm { color: #408080; font-style: italic } /* Comment.Multiline */\n",
       ".pygments .cp { color: #BC7A00 } /* Comment.Preproc */\n",
       ".pygments .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */\n",
       ".pygments .c1 { color: #408080; font-style: italic } /* Comment.Single */\n",
       ".pygments .cs { color: #408080; font-style: italic } /* Comment.Special */\n",
       ".pygments .gd { color: #A00000 } /* Generic.Deleted */\n",
       ".pygments .ge { font-style: italic } /* Generic.Emph */\n",
       ".pygments .gr { color: #FF0000 } /* Generic.Error */\n",
       ".pygments .gh { color: #000080; font-weight: bold } /* Generic.Heading */\n",
       ".pygments .gi { color: #00A000 } /* Generic.Inserted */\n",
       ".pygments .go { color: #888888 } /* Generic.Output */\n",
       ".pygments .gp { color: #000080; font-weight: bold } /* Generic.Prompt */\n",
       ".pygments .gs { font-weight: bold } /* Generic.Strong */\n",
       ".pygments .gu { color: #800080; font-weight: bold } /* Generic.Subheading */\n",
       ".pygments .gt { color: #0044DD } /* Generic.Traceback */\n",
       ".pygments .kc { color: #008000; font-weight: bold } /* Keyword.Constant */\n",
       ".pygments .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */\n",
       ".pygments .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */\n",
       ".pygments .kp { color: #008000 } /* Keyword.Pseudo */\n",
       ".pygments .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */\n",
       ".pygments .kt { color: #B00040 } /* Keyword.Type */\n",
       ".pygments .m { color: #666666 } /* Literal.Number */\n",
       ".pygments .s { color: #BA2121 } /* Literal.String */\n",
       ".pygments .na { color: #7D9029 } /* Name.Attribute */\n",
       ".pygments .nb { color: #008000 } /* Name.Builtin */\n",
       ".pygments .nc { color: #0000FF; font-weight: bold } /* Name.Class */\n",
       ".pygments .no { color: #880000 } /* Name.Constant */\n",
       ".pygments .nd { color: #AA22FF } /* Name.Decorator */\n",
       ".pygments .ni { color: #999999; font-weight: bold } /* Name.Entity */\n",
       ".pygments .ne { color: #D2413A; font-weight: bold } /* Name.Exception */\n",
       ".pygments .nf { color: #0000FF } /* Name.Function */\n",
       ".pygments .nl { color: #A0A000 } /* Name.Label */\n",
       ".pygments .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */\n",
       ".pygments .nt { color: #008000; font-weight: bold } /* Name.Tag */\n",
       ".pygments .nv { color: #19177C } /* Name.Variable */\n",
       ".pygments .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */\n",
       ".pygments .w { color: #bbbbbb } /* Text.Whitespace */\n",
       ".pygments .mb { color: #666666 } /* Literal.Number.Bin */\n",
       ".pygments .mf { color: #666666 } /* Literal.Number.Float */\n",
       ".pygments .mh { color: #666666 } /* Literal.Number.Hex */\n",
       ".pygments .mi { color: #666666 } /* Literal.Number.Integer */\n",
       ".pygments .mo { color: #666666 } /* Literal.Number.Oct */\n",
       ".pygments .sa { color: #BA2121 } /* Literal.String.Affix */\n",
       ".pygments .sb { color: #BA2121 } /* Literal.String.Backtick */\n",
       ".pygments .sc { color: #BA2121 } /* Literal.String.Char */\n",
       ".pygments .dl { color: #BA2121 } /* Literal.String.Delimiter */\n",
       ".pygments .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */\n",
       ".pygments .s2 { color: #BA2121 } /* Literal.String.Double */\n",
       ".pygments .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */\n",
       ".pygments .sh { color: #BA2121 } /* Literal.String.Heredoc */\n",
       ".pygments .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */\n",
       ".pygments .sx { color: #008000 } /* Literal.String.Other */\n",
       ".pygments .sr { color: #BB6688 } /* Literal.String.Regex */\n",
       ".pygments .s1 { color: #BA2121 } /* Literal.String.Single */\n",
       ".pygments .ss { color: #19177C } /* Literal.String.Symbol */\n",
       ".pygments .bp { color: #008000 } /* Name.Builtin.Pseudo */\n",
       ".pygments .fm { color: #0000FF } /* Name.Function.Magic */\n",
       ".pygments .vc { color: #19177C } /* Name.Variable.Class */\n",
       ".pygments .vg { color: #19177C } /* Name.Variable.Global */\n",
       ".pygments .vi { color: #19177C } /* Name.Variable.Instance */\n",
       ".pygments .vm { color: #19177C } /* Name.Variable.Magic */\n",
       ".pygments .il { color: #666666 } /* Literal.Number.Integer.Long */\n",
       "</style>\n",
       "<div class=\"pygments\"><pre><span></span><span class=\"kn\">import</span> <span class=\"nn\">numpy</span> <span class=\"kn\">as</span> <span class=\"nn\">np</span>\n",
       "<span class=\"kn\">import</span> <span class=\"nn\">sklearn.metrics</span> <span class=\"kn\">as</span> <span class=\"nn\">metrics</span>\n",
       "<span class=\"kn\">import</span> <span class=\"nn\">operator</span>\n",
       "<span class=\"kn\">from</span> <span class=\"nn\">sklearn.metrics.pairwise</span> <span class=\"kn\">import</span> <span class=\"n\">cosine_similarity</span><span class=\"p\">,</span><span class=\"n\">pairwise_distances</span> \n",
       "<span class=\"kn\">import</span> <span class=\"nn\">implicit</span>\n",
       "<span class=\"kn\">import</span> <span class=\"nn\">seaborn</span> <span class=\"kn\">as</span> <span class=\"nn\">sns</span>\n",
       "<span class=\"kn\">import</span> <span class=\"nn\">os</span>\n",
       "\n",
       "<span class=\"k\">def</span> <span class=\"nf\">mean_reciprocal_rank</span><span class=\"p\">(</span><span class=\"n\">rs</span><span class=\"p\">):</span>\n",
       "    <span class=\"sd\">&quot;&quot;&quot;Score is reciprocal of the rank of the first relevant item</span>\n",
       "<span class=\"sd\">    First element is &#39;rank 1&#39;.  Relevance is binary (nonzero is relevant).</span>\n",
       "<span class=\"sd\">    Example from http://en.wikipedia.org/wiki/Mean_reciprocal_rank</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; rs = [[0, 0, 1], [0, 1, 0], [1, 0, 0]]</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; mean_reciprocal_rank(rs)</span>\n",
       "<span class=\"sd\">    0.61111111111111105</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; rs = np.array([[0, 0, 0], [0, 1, 0], [1, 0, 0]])</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; mean_reciprocal_rank(rs)</span>\n",
       "<span class=\"sd\">    0.5</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; rs = [[0, 0, 0, 1], [1, 0, 0], [1, 0, 0]]</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; mean_reciprocal_rank(rs)</span>\n",
       "<span class=\"sd\">    0.75</span>\n",
       "<span class=\"sd\">    Args:</span>\n",
       "<span class=\"sd\">        rs: Iterator of relevance scores (list or numpy) in rank order</span>\n",
       "<span class=\"sd\">            (first element is the first item)</span>\n",
       "<span class=\"sd\">    Returns:</span>\n",
       "<span class=\"sd\">        Mean reciprocal rank</span>\n",
       "<span class=\"sd\">    &quot;&quot;&quot;</span>\n",
       "    <span class=\"n\">rs</span> <span class=\"o\">=</span> <span class=\"p\">(</span><span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">asarray</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">)</span><span class=\"o\">.</span><span class=\"n\">nonzero</span><span class=\"p\">()[</span><span class=\"mi\">0</span><span class=\"p\">]</span> <span class=\"k\">for</span> <span class=\"n\">r</span> <span class=\"ow\">in</span> <span class=\"n\">rs</span><span class=\"p\">)</span>\n",
       "    <span class=\"k\">return</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">mean</span><span class=\"p\">([</span><span class=\"mf\">1.</span> <span class=\"o\">/</span> <span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">[</span><span class=\"mi\">0</span><span class=\"p\">]</span> <span class=\"o\">+</span> <span class=\"mi\">1</span><span class=\"p\">)</span> <span class=\"k\">if</span> <span class=\"n\">r</span><span class=\"o\">.</span><span class=\"n\">size</span> <span class=\"k\">else</span> <span class=\"mf\">0.</span> <span class=\"k\">for</span> <span class=\"n\">r</span> <span class=\"ow\">in</span> <span class=\"n\">rs</span><span class=\"p\">])</span>\n",
       "\n",
       "<span class=\"k\">def</span> <span class=\"nf\">precision_at_k</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">,</span> <span class=\"n\">k</span><span class=\"p\">):</span>\n",
       "    <span class=\"sd\">&quot;&quot;&quot;Score is precision @ k</span>\n",
       "<span class=\"sd\">    Relevance is binary (nonzero is relevant).</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; r = [0, 0, 1]</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; precision_at_k(r, 1)</span>\n",
       "<span class=\"sd\">    0.0</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; precision_at_k(r, 2)</span>\n",
       "<span class=\"sd\">    0.0</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; precision_at_k(r, 3)</span>\n",
       "<span class=\"sd\">    0.33333333333333331</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; precision_at_k(r, 4)</span>\n",
       "<span class=\"sd\">    Traceback (most recent call last):</span>\n",
       "<span class=\"sd\">        File &quot;&lt;stdin&gt;&quot;, line 1, in ?</span>\n",
       "<span class=\"sd\">    ValueError: Relevance score length &lt; k</span>\n",
       "<span class=\"sd\">    Args:</span>\n",
       "<span class=\"sd\">        r: Relevance scores (list or numpy) in rank order</span>\n",
       "<span class=\"sd\">            (first element is the first item)</span>\n",
       "<span class=\"sd\">    Returns:</span>\n",
       "<span class=\"sd\">        Precision @ k</span>\n",
       "<span class=\"sd\">    Raises:</span>\n",
       "<span class=\"sd\">        ValueError: len(r) must be &gt;= k</span>\n",
       "<span class=\"sd\">    &quot;&quot;&quot;</span>\n",
       "    <span class=\"k\">assert</span> <span class=\"n\">k</span> <span class=\"o\">&gt;=</span> <span class=\"mi\">1</span>\n",
       "    <span class=\"n\">r</span> <span class=\"o\">=</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">asarray</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">)[:</span><span class=\"n\">k</span><span class=\"p\">]</span> <span class=\"o\">!=</span> <span class=\"mi\">0</span>\n",
       "    <span class=\"k\">if</span> <span class=\"n\">r</span><span class=\"o\">.</span><span class=\"n\">size</span> <span class=\"o\">!=</span> <span class=\"n\">k</span><span class=\"p\">:</span>\n",
       "        <span class=\"k\">raise</span> <span class=\"ne\">ValueError</span><span class=\"p\">(</span><span class=\"s1\">&#39;Relevance score length &lt; k&#39;</span><span class=\"p\">)</span>\n",
       "    <span class=\"k\">return</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">mean</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">)</span>\n",
       "\n",
       "\n",
       "<span class=\"k\">def</span> <span class=\"nf\">average_precision</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">):</span>\n",
       "    <span class=\"sd\">&quot;&quot;&quot;Score is average precision (area under PR curve)</span>\n",
       "<span class=\"sd\">    Relevance is binary (nonzero is relevant).</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; r = [1, 1, 0, 1, 0, 1, 0, 0, 0, 1]</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; delta_r = 1. / sum(r)</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; sum([sum(r[:x + 1]) / (x + 1.) * delta_r for x, y in enumerate(r) if y])</span>\n",
       "<span class=\"sd\">    0.7833333333333333</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; average_precision(r)</span>\n",
       "<span class=\"sd\">    0.78333333333333333</span>\n",
       "<span class=\"sd\">    Args:</span>\n",
       "<span class=\"sd\">        r: Relevance scores (list or numpy) in rank order</span>\n",
       "<span class=\"sd\">            (first element is the first item)</span>\n",
       "<span class=\"sd\">    Returns:</span>\n",
       "<span class=\"sd\">        Average precision</span>\n",
       "<span class=\"sd\">    &quot;&quot;&quot;</span>\n",
       "    <span class=\"n\">r</span> <span class=\"o\">=</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">asarray</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">)</span> <span class=\"o\">!=</span> <span class=\"mi\">0</span>\n",
       "    <span class=\"n\">out</span> <span class=\"o\">=</span> <span class=\"p\">[</span><span class=\"n\">precision_at_k</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">,</span> <span class=\"n\">k</span> <span class=\"o\">+</span> <span class=\"mi\">1</span><span class=\"p\">)</span> <span class=\"k\">for</span> <span class=\"n\">k</span> <span class=\"ow\">in</span> <span class=\"nb\">range</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"o\">.</span><span class=\"n\">size</span><span class=\"p\">)</span> <span class=\"k\">if</span> <span class=\"n\">r</span><span class=\"p\">[</span><span class=\"n\">k</span><span class=\"p\">]]</span>\n",
       "    <span class=\"k\">if</span> <span class=\"ow\">not</span> <span class=\"n\">out</span><span class=\"p\">:</span>\n",
       "        <span class=\"k\">return</span> <span class=\"mf\">0.</span>\n",
       "    <span class=\"k\">return</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">mean</span><span class=\"p\">(</span><span class=\"n\">out</span><span class=\"p\">)</span>\n",
       "\n",
       "\n",
       "<span class=\"k\">def</span> <span class=\"nf\">mean_average_precision</span><span class=\"p\">(</span><span class=\"n\">rs</span><span class=\"p\">):</span>\n",
       "    <span class=\"sd\">&quot;&quot;&quot;Score is mean average precision</span>\n",
       "<span class=\"sd\">    Relevance is binary (nonzero is relevant).</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; rs = [[1, 1, 0, 1, 0, 1, 0, 0, 0, 1]]</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; mean_average_precision(rs)</span>\n",
       "<span class=\"sd\">    0.78333333333333333</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; rs = [[1, 1, 0, 1, 0, 1, 0, 0, 0, 1], [0]]</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; mean_average_precision(rs)</span>\n",
       "<span class=\"sd\">    0.39166666666666666</span>\n",
       "<span class=\"sd\">    Args:</span>\n",
       "<span class=\"sd\">        rs: Iterator of relevance scores (list or numpy) in rank order</span>\n",
       "<span class=\"sd\">            (first element is the first item)</span>\n",
       "<span class=\"sd\">    Returns:</span>\n",
       "<span class=\"sd\">        Mean average precision</span>\n",
       "<span class=\"sd\">    &quot;&quot;&quot;</span>\n",
       "    <span class=\"k\">return</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">mean</span><span class=\"p\">([</span><span class=\"n\">average_precision</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">)</span> <span class=\"k\">for</span> <span class=\"n\">r</span> <span class=\"ow\">in</span> <span class=\"n\">rs</span><span class=\"p\">])</span>\n",
       "\n",
       "<span class=\"k\">def</span> <span class=\"nf\">precision_at_k</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">,</span> <span class=\"n\">k</span><span class=\"p\">):</span>\n",
       "    <span class=\"sd\">&quot;&quot;&quot;Score is precision @ k</span>\n",
       "<span class=\"sd\">    Relevance is binary (nonzero is relevant).</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; r = [0, 0, 1]</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; precision_at_k(r, 1)</span>\n",
       "<span class=\"sd\">    0.0</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; precision_at_k(r, 2)</span>\n",
       "<span class=\"sd\">    0.0</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; precision_at_k(r, 3)</span>\n",
       "<span class=\"sd\">    0.33333333333333331</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; precision_at_k(r, 4)</span>\n",
       "<span class=\"sd\">    Traceback (most recent call last):</span>\n",
       "<span class=\"sd\">        File &quot;&lt;stdin&gt;&quot;, line 1, in ?</span>\n",
       "<span class=\"sd\">    ValueError: Relevance score length &lt; k</span>\n",
       "<span class=\"sd\">    Args:</span>\n",
       "<span class=\"sd\">        r: Relevance scores (list or numpy) in rank order</span>\n",
       "<span class=\"sd\">            (first element is the first item)</span>\n",
       "<span class=\"sd\">    Returns:</span>\n",
       "<span class=\"sd\">        Precision @ k</span>\n",
       "<span class=\"sd\">    Raises:</span>\n",
       "<span class=\"sd\">        ValueError: len(r) must be &gt;= k</span>\n",
       "<span class=\"sd\">    &quot;&quot;&quot;</span>\n",
       "    <span class=\"k\">assert</span> <span class=\"n\">k</span> <span class=\"o\">&gt;=</span> <span class=\"mi\">1</span>\n",
       "    <span class=\"n\">r</span> <span class=\"o\">=</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">asarray</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">)[:</span><span class=\"n\">k</span><span class=\"p\">]</span> <span class=\"o\">!=</span> <span class=\"mi\">0</span>\n",
       "    <span class=\"k\">if</span> <span class=\"n\">r</span><span class=\"o\">.</span><span class=\"n\">size</span> <span class=\"o\">!=</span> <span class=\"n\">k</span><span class=\"p\">:</span>\n",
       "        <span class=\"k\">raise</span> <span class=\"ne\">ValueError</span><span class=\"p\">(</span><span class=\"s1\">&#39;Relevance score length &lt; k&#39;</span><span class=\"p\">)</span>\n",
       "    <span class=\"k\">return</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">mean</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">)</span>\n",
       "<span class=\"k\">def</span> <span class=\"nf\">check_one_match</span><span class=\"p\">(</span><span class=\"n\">act_prods</span><span class=\"p\">,</span> <span class=\"n\">pred_prods_mf</span><span class=\"p\">,</span><span class=\"n\">pred_prods_hybrid</span><span class=\"p\">,</span> <span class=\"n\">top_n</span><span class=\"o\">=</span><span class=\"mi\">20</span><span class=\"p\">):</span>\n",
       "    <span class=\"n\">is_pred_match_mf</span> <span class=\"o\">=</span> <span class=\"mi\">1</span> <span class=\"k\">if</span> <span class=\"nb\">bool</span><span class=\"p\">(</span><span class=\"nb\">set</span><span class=\"p\">(</span><span class=\"n\">act_prods</span><span class=\"p\">)</span><span class=\"o\">.</span><span class=\"n\">intersection</span><span class=\"p\">(</span><span class=\"nb\">set</span><span class=\"p\">(</span><span class=\"n\">pred_prods_mf</span><span class=\"p\">[:</span><span class=\"n\">top_n</span><span class=\"p\">])))</span> <span class=\"k\">else</span> <span class=\"mi\">0</span>\n",
       "    <span class=\"n\">is_pred_match_hybrid</span> <span class=\"o\">=</span> <span class=\"mi\">1</span> <span class=\"k\">if</span> <span class=\"nb\">bool</span><span class=\"p\">(</span><span class=\"nb\">set</span><span class=\"p\">(</span><span class=\"n\">act_prods</span><span class=\"p\">)</span><span class=\"o\">.</span><span class=\"n\">intersection</span><span class=\"p\">(</span><span class=\"nb\">set</span><span class=\"p\">(</span><span class=\"n\">pred_prods_hybrid</span><span class=\"p\">[:</span><span class=\"n\">top_n</span><span class=\"p\">])))</span> <span class=\"k\">else</span> <span class=\"mi\">0</span>\n",
       "    <span class=\"k\">return</span> <span class=\"p\">(</span><span class=\"n\">is_pred_match_mf</span><span class=\"p\">,</span> <span class=\"n\">is_pred_match_hybrid</span><span class=\"p\">)</span>\n",
       "\n",
       "<span class=\"k\">def</span> <span class=\"nf\">dcg_at_k</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">,</span> <span class=\"n\">k</span><span class=\"p\">,</span> <span class=\"n\">method</span><span class=\"o\">=</span><span class=\"mi\">0</span><span class=\"p\">):</span>\n",
       "    <span class=\"sd\">&quot;&quot;&quot;Score is discounted cumulative gain (dcg)</span>\n",
       "<span class=\"sd\">    Relevance is positive real values.  Can use binary</span>\n",
       "<span class=\"sd\">    as the previous methods.</span>\n",
       "<span class=\"sd\">    Example from</span>\n",
       "<span class=\"sd\">    http://www.stanford.edu/class/cs276/handouts/EvaluationNew-handout-6-per.pdf</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; r = [3, 2, 3, 0, 0, 1, 2, 2, 3, 0]</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; dcg_at_k(r, 1)</span>\n",
       "<span class=\"sd\">    3.0</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; dcg_at_k(r, 1, method=1)</span>\n",
       "<span class=\"sd\">    3.0</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; dcg_at_k(r, 2)</span>\n",
       "<span class=\"sd\">    5.0</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; dcg_at_k(r, 2, method=1)</span>\n",
       "<span class=\"sd\">    4.2618595071429155</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; dcg_at_k(r, 10)</span>\n",
       "<span class=\"sd\">    9.6051177391888114</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; dcg_at_k(r, 11)</span>\n",
       "<span class=\"sd\">    9.6051177391888114</span>\n",
       "<span class=\"sd\">    Args:</span>\n",
       "<span class=\"sd\">        r: Relevance scores (list or numpy) in rank order</span>\n",
       "<span class=\"sd\">            (first element is the first item)</span>\n",
       "<span class=\"sd\">        k: Number of results to consider</span>\n",
       "<span class=\"sd\">        method: If 0 then weights are [1.0, 1.0, 0.6309, 0.5, 0.4307, ...]</span>\n",
       "<span class=\"sd\">                If 1 then weights are [1.0, 0.6309, 0.5, 0.4307, ...]</span>\n",
       "<span class=\"sd\">    Returns:</span>\n",
       "<span class=\"sd\">        Discounted cumulative gain</span>\n",
       "<span class=\"sd\">    &quot;&quot;&quot;</span>\n",
       "    <span class=\"n\">r</span> <span class=\"o\">=</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">asfarray</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">)[:</span><span class=\"n\">k</span><span class=\"p\">]</span>\n",
       "    <span class=\"k\">if</span> <span class=\"n\">r</span><span class=\"o\">.</span><span class=\"n\">size</span><span class=\"p\">:</span>\n",
       "        <span class=\"k\">if</span> <span class=\"n\">method</span> <span class=\"o\">==</span> <span class=\"mi\">0</span><span class=\"p\">:</span>\n",
       "            <span class=\"k\">return</span> <span class=\"n\">r</span><span class=\"p\">[</span><span class=\"mi\">0</span><span class=\"p\">]</span> <span class=\"o\">+</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">sum</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">[</span><span class=\"mi\">1</span><span class=\"p\">:]</span> <span class=\"o\">/</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">log2</span><span class=\"p\">(</span><span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">arange</span><span class=\"p\">(</span><span class=\"mi\">2</span><span class=\"p\">,</span> <span class=\"n\">r</span><span class=\"o\">.</span><span class=\"n\">size</span> <span class=\"o\">+</span> <span class=\"mi\">1</span><span class=\"p\">)))</span>\n",
       "        <span class=\"k\">elif</span> <span class=\"n\">method</span> <span class=\"o\">==</span> <span class=\"mi\">1</span><span class=\"p\">:</span>\n",
       "            <span class=\"k\">return</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">sum</span><span class=\"p\">(</span><span class=\"n\">r</span> <span class=\"o\">/</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">log2</span><span class=\"p\">(</span><span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">arange</span><span class=\"p\">(</span><span class=\"mi\">2</span><span class=\"p\">,</span> <span class=\"n\">r</span><span class=\"o\">.</span><span class=\"n\">size</span> <span class=\"o\">+</span> <span class=\"mi\">2</span><span class=\"p\">)))</span>\n",
       "        <span class=\"k\">else</span><span class=\"p\">:</span>\n",
       "            <span class=\"k\">raise</span> <span class=\"ne\">ValueError</span><span class=\"p\">(</span><span class=\"s1\">&#39;method must be 0 or 1.&#39;</span><span class=\"p\">)</span>\n",
       "    <span class=\"k\">return</span> <span class=\"mf\">0.</span>\n",
       "\n",
       "\n",
       "<span class=\"k\">def</span> <span class=\"nf\">ndcg_at_k</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">,</span> <span class=\"n\">k</span><span class=\"p\">,</span> <span class=\"n\">method</span><span class=\"o\">=</span><span class=\"mi\">0</span><span class=\"p\">):</span>\n",
       "    <span class=\"sd\">&quot;&quot;&quot;Score is normalized discounted cumulative gain (ndcg)</span>\n",
       "<span class=\"sd\">    Relevance is positive real values.  Can use binary</span>\n",
       "<span class=\"sd\">    as the previous methods.</span>\n",
       "<span class=\"sd\">    Example from</span>\n",
       "<span class=\"sd\">    http://www.stanford.edu/class/cs276/handouts/EvaluationNew-handout-6-per.pdf</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; r = [3, 2, 3, 0, 0, 1, 2, 2, 3, 0]</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; ndcg_at_k(r, 1)</span>\n",
       "<span class=\"sd\">    1.0</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; r = [2, 1, 2, 0]</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; ndcg_at_k(r, 4)</span>\n",
       "<span class=\"sd\">    0.9203032077642922</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; ndcg_at_k(r, 4, method=1)</span>\n",
       "<span class=\"sd\">    0.96519546960144276</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; ndcg_at_k([0], 1)</span>\n",
       "<span class=\"sd\">    0.0</span>\n",
       "<span class=\"sd\">    &gt;&gt;&gt; ndcg_at_k([1], 2)</span>\n",
       "<span class=\"sd\">    1.0</span>\n",
       "<span class=\"sd\">    Args:</span>\n",
       "<span class=\"sd\">        r: Relevance scores (list or numpy) in rank order</span>\n",
       "<span class=\"sd\">            (first element is the first item)</span>\n",
       "<span class=\"sd\">        k: Number of results to consider</span>\n",
       "<span class=\"sd\">        method: If 0 then weights are [1.0, 1.0, 0.6309, 0.5, 0.4307, ...]</span>\n",
       "<span class=\"sd\">                If 1 then weights are [1.0, 0.6309, 0.5, 0.4307, ...]</span>\n",
       "<span class=\"sd\">    Returns:</span>\n",
       "<span class=\"sd\">        Normalized discounted cumulative gain</span>\n",
       "<span class=\"sd\">    &quot;&quot;&quot;</span>\n",
       "    <span class=\"n\">dcg_max</span> <span class=\"o\">=</span> <span class=\"n\">dcg_at_k</span><span class=\"p\">(</span><span class=\"nb\">sorted</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">,</span> <span class=\"n\">reverse</span><span class=\"o\">=</span><span class=\"bp\">True</span><span class=\"p\">),</span> <span class=\"n\">k</span><span class=\"p\">,</span> <span class=\"n\">method</span><span class=\"p\">)</span>\n",
       "    <span class=\"k\">if</span> <span class=\"ow\">not</span> <span class=\"n\">dcg_max</span><span class=\"p\">:</span>\n",
       "        <span class=\"k\">return</span> <span class=\"mf\">0.</span>\n",
       "    <span class=\"k\">return</span> <span class=\"n\">dcg_at_k</span><span class=\"p\">(</span><span class=\"n\">r</span><span class=\"p\">,</span> <span class=\"n\">k</span><span class=\"p\">,</span> <span class=\"n\">method</span><span class=\"p\">)</span> <span class=\"o\">/</span> <span class=\"n\">dcg_max</span>\n",
       "\n",
       "<span class=\"k\">def</span> <span class=\"nf\">get_performance_metrics</span><span class=\"p\">(</span><span class=\"n\">user</span><span class=\"p\">,</span><span class=\"n\">user_true</span><span class=\"p\">,</span> <span class=\"n\">user_vecs</span><span class=\"p\">,</span> <span class=\"n\">item_vecs</span><span class=\"p\">,</span> <span class=\"n\">top_nn</span><span class=\"p\">):</span>\n",
       "    <span class=\"n\">score</span> <span class=\"o\">=</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">dot</span><span class=\"p\">(</span><span class=\"n\">user_vecs</span><span class=\"p\">[</span><span class=\"n\">user</span><span class=\"p\">,</span> <span class=\"p\">:],</span><span class=\"n\">item_vecs</span><span class=\"o\">.</span><span class=\"n\">T</span><span class=\"p\">)</span>\n",
       "    <span class=\"n\">item_rec_mf</span> <span class=\"o\">=</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">argsort</span><span class=\"p\">(</span><span class=\"o\">-</span><span class=\"n\">score</span><span class=\"o\">.</span><span class=\"n\">reshape</span><span class=\"p\">(</span><span class=\"o\">-</span><span class=\"mi\">1</span><span class=\"p\">))[:</span><span class=\"n\">top_nn</span><span class=\"p\">]</span>\n",
       "    <span class=\"n\">item_score</span> <span class=\"o\">=</span> <span class=\"n\">cosine_similarity</span><span class=\"p\">(</span><span class=\"n\">user_vecs</span><span class=\"p\">[</span><span class=\"n\">user</span><span class=\"p\">,</span> <span class=\"p\">:]</span><span class=\"o\">.</span><span class=\"n\">reshape</span><span class=\"p\">(</span><span class=\"mi\">1</span><span class=\"p\">,</span><span class=\"o\">-</span><span class=\"mi\">1</span><span class=\"p\">),</span> <span class=\"n\">item_vecs</span><span class=\"p\">)</span>\n",
       "    <span class=\"n\">item_rec_hybrid</span> <span class=\"o\">=</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">argsort</span><span class=\"p\">(</span><span class=\"o\">-</span><span class=\"n\">item_score</span><span class=\"o\">.</span><span class=\"n\">reshape</span><span class=\"p\">(</span><span class=\"o\">-</span><span class=\"mi\">1</span><span class=\"p\">))[:</span><span class=\"n\">top_nn</span><span class=\"p\">]</span>\n",
       "    \n",
       "    <span class=\"c1\">#item_recc_org_id = [i[0] for i in item_scores[:top_n]]</span>\n",
       "    <span class=\"c1\">#item_recc = [items_u.index(i) for i in item_recc_org_id]</span>\n",
       "    \n",
       "        <span class=\"c1\">#item_ind = [items_u.index(p_id) for p_id in user_true]</span>\n",
       "    <span class=\"k\">try</span><span class=\"p\">:</span>\n",
       "        <span class=\"n\">item_act</span> <span class=\"o\">=</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">zeros</span><span class=\"p\">(</span><span class=\"n\">item_vecs</span><span class=\"o\">.</span><span class=\"n\">shape</span><span class=\"p\">[</span><span class=\"mi\">0</span><span class=\"p\">])</span>\n",
       "        <span class=\"k\">for</span> <span class=\"n\">i_ind</span> <span class=\"ow\">in</span> <span class=\"n\">user_true</span><span class=\"p\">:</span>\n",
       "            <span class=\"n\">item_act</span><span class=\"p\">[</span><span class=\"n\">i_ind</span><span class=\"p\">]</span> <span class=\"o\">=</span> <span class=\"mi\">1</span>\n",
       "        <span class=\"n\">mf</span> <span class=\"o\">=</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">zeros</span><span class=\"p\">(</span><span class=\"n\">item_vecs</span><span class=\"o\">.</span><span class=\"n\">shape</span><span class=\"p\">[</span><span class=\"mi\">0</span><span class=\"p\">])</span>\n",
       "        <span class=\"n\">mf_cf</span> <span class=\"o\">=</span> <span class=\"n\">np</span><span class=\"o\">.</span><span class=\"n\">zeros</span><span class=\"p\">(</span><span class=\"n\">item_vecs</span><span class=\"o\">.</span><span class=\"n\">shape</span><span class=\"p\">[</span><span class=\"mi\">0</span><span class=\"p\">])</span>\n",
       "        <span class=\"k\">for</span> <span class=\"n\">r_ind</span> <span class=\"ow\">in</span> <span class=\"n\">item_rec_mf</span><span class=\"p\">:</span>\n",
       "            <span class=\"n\">mf</span><span class=\"p\">[</span><span class=\"n\">r_ind</span><span class=\"p\">]</span> <span class=\"o\">=</span> <span class=\"mi\">1</span>\n",
       "        <span class=\"k\">for</span> <span class=\"n\">r_ind</span> <span class=\"ow\">in</span> <span class=\"n\">item_rec_hybrid</span><span class=\"p\">:</span>\n",
       "            <span class=\"n\">mf_cf</span><span class=\"p\">[</span><span class=\"n\">r_ind</span><span class=\"p\">]</span> <span class=\"o\">=</span> <span class=\"mi\">1</span>\n",
       "        <span class=\"n\">match</span> <span class=\"o\">=</span><span class=\"p\">[</span><span class=\"mi\">1</span> <span class=\"k\">if</span> <span class=\"n\">item_rec_mf</span><span class=\"p\">[</span><span class=\"n\">i</span><span class=\"p\">]</span> <span class=\"ow\">in</span> <span class=\"n\">user_true</span> <span class=\"k\">else</span> <span class=\"mi\">0</span> <span class=\"k\">for</span> <span class=\"n\">i</span> <span class=\"ow\">in</span> <span class=\"nb\">range</span><span class=\"p\">(</span><span class=\"n\">top_nn</span><span class=\"p\">)][:</span><span class=\"n\">top_nn</span><span class=\"p\">]</span> \n",
       "        <span class=\"n\">MAR_mf</span> <span class=\"o\">=</span> <span class=\"n\">mean_reciprocal_rank</span><span class=\"p\">(</span><span class=\"n\">match</span><span class=\"p\">)</span>\n",
       "        <span class=\"n\">MAP_mf</span> <span class=\"o\">=</span> <span class=\"n\">average_precision</span><span class=\"p\">(</span><span class=\"n\">match</span><span class=\"p\">)</span>\n",
       "        <span class=\"n\">ndcg_mf</span> <span class=\"o\">=</span> <span class=\"n\">ndcg_at_k</span><span class=\"p\">(</span><span class=\"n\">match</span><span class=\"p\">,</span><span class=\"n\">top_nn</span><span class=\"p\">)</span>\n",
       "        <span class=\"n\">auc_mf</span> <span class=\"o\">=</span> <span class=\"n\">metrics</span><span class=\"o\">.</span><span class=\"n\">roc_auc_score</span><span class=\"p\">(</span><span class=\"n\">item_act</span><span class=\"p\">,</span> <span class=\"n\">mf</span><span class=\"p\">)</span>\n",
       "        <span class=\"n\">match_hybrid</span> <span class=\"o\">=</span> <span class=\"p\">[</span><span class=\"mi\">1</span> <span class=\"k\">if</span> <span class=\"n\">item_rec_hybrid</span><span class=\"p\">[</span><span class=\"n\">i</span><span class=\"p\">]</span> <span class=\"ow\">in</span> <span class=\"n\">user_true</span> <span class=\"k\">else</span> <span class=\"mi\">0</span> <span class=\"k\">for</span> <span class=\"n\">i</span> <span class=\"ow\">in</span> <span class=\"nb\">range</span><span class=\"p\">(</span><span class=\"n\">top_nn</span><span class=\"p\">)][:</span><span class=\"n\">top_nn</span><span class=\"p\">]</span>\n",
       "        <span class=\"n\">MAR_hybrid</span> <span class=\"o\">=</span> <span class=\"n\">mean_reciprocal_rank</span><span class=\"p\">(</span><span class=\"n\">match_hybrid</span><span class=\"p\">)</span>\n",
       "        <span class=\"n\">MAP_hybrid</span> <span class=\"o\">=</span> <span class=\"n\">average_precision</span><span class=\"p\">(</span><span class=\"n\">match_hybrid</span><span class=\"p\">)</span>\n",
       "        <span class=\"n\">ndcg_hybrid</span> <span class=\"o\">=</span> <span class=\"n\">ndcg_at_k</span><span class=\"p\">(</span><span class=\"n\">match_hybrid</span><span class=\"p\">,</span><span class=\"n\">top_nn</span><span class=\"p\">)</span>\n",
       "        <span class=\"n\">auc_hybrid</span> <span class=\"o\">=</span> <span class=\"n\">metrics</span><span class=\"o\">.</span><span class=\"n\">roc_auc_score</span><span class=\"p\">(</span><span class=\"n\">item_act</span><span class=\"p\">,</span> <span class=\"n\">mf_cf</span><span class=\"p\">)</span>\n",
       "    <span class=\"k\">except</span><span class=\"p\">:</span>\n",
       "        <span class=\"k\">print</span><span class=\"p\">(</span><span class=\"s2\">&quot;failing&quot;</span><span class=\"p\">)</span>\n",
       "        <span class=\"n\">auc_mf</span><span class=\"p\">,</span> <span class=\"n\">auc_hybrid</span><span class=\"p\">,</span><span class=\"n\">MAR_mf</span><span class=\"p\">,</span><span class=\"n\">MAR_hybrid</span><span class=\"p\">,</span><span class=\"n\">MAP_mf</span><span class=\"p\">,</span><span class=\"n\">MAP_hybrid</span><span class=\"p\">,</span><span class=\"n\">ndcg_mf</span><span class=\"p\">,</span><span class=\"n\">ndcg_hybrid</span> <span class=\"o\">=</span> <span class=\"mf\">0.5</span><span class=\"p\">,</span><span class=\"mf\">0.5</span><span class=\"p\">,</span><span class=\"mi\">0</span><span class=\"p\">,</span><span class=\"mi\">0</span><span class=\"p\">,</span><span class=\"mi\">0</span><span class=\"p\">,</span><span class=\"mi\">0</span><span class=\"p\">,</span><span class=\"mi\">0</span><span class=\"p\">,</span><span class=\"mi\">0</span>\n",
       "    <span class=\"k\">return</span> <span class=\"p\">(</span><span class=\"n\">auc_mf</span><span class=\"p\">,</span> <span class=\"n\">auc_hybrid</span><span class=\"p\">,</span><span class=\"n\">MAR_mf</span><span class=\"p\">,</span><span class=\"n\">MAR_hybrid</span><span class=\"p\">,</span><span class=\"n\">MAP_mf</span><span class=\"p\">,</span><span class=\"n\">MAP_hybrid</span><span class=\"p\">,</span><span class=\"n\">ndcg_mf</span><span class=\"p\">,</span><span class=\"n\">ndcg_hybrid</span><span class=\"p\">,</span><span class=\"n\">score</span><span class=\"p\">,</span><span class=\"n\">item_score</span><span class=\"p\">)</span>\n",
       "</pre></div>\n",
       "\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "template = \"\"\"<style>\n",
    "{}\n",
    "</style>\n",
    "{}\n",
    "\"\"\"\n",
    "def show_file(filename):\n",
    "    lexer = get_lexer_for_filename(filename)\n",
    "    formatter = HtmlFormatter(cssclass='pygments')\n",
    "    \n",
    "    with open(filename) as f:\n",
    "        code = f.read()\n",
    "    \n",
    "    html_code = highlight(code, lexer, formatter)\n",
    "    css = formatter.get_style_defs('.pygments')\n",
    "    \n",
    "    html = template.format(css, html_code)\n",
    "    \n",
    "    display(HTML(html))\n",
    "    \n",
    "show_file('evaluation_func.py')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's explore match indicator(0/1) for MF vs. Hybrid approach. First and second element in each tuple in field 'match' is indicator for MF and hybrid resp."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>items_act</th>\n",
       "      <th>item_rec_mf</th>\n",
       "      <th>item_rec_hybrid</th>\n",
       "      <th>match</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>[3669]</td>\n",
       "      <td>[3669, 2701, 5350, 5394, 1086, 2660, 6183, 251...</td>\n",
       "      <td>[3669, 10172, 1086, 5246, 1170, 3246, 6161, 32...</td>\n",
       "      <td>(1, 1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>[5372]</td>\n",
       "      <td>[70, 1781, 1779, 2615, 6534, 4479, 6269, 4226,...</td>\n",
       "      <td>[9855, 5678, 5311, 12083, 7392, 12071, 12051, ...</td>\n",
       "      <td>(0, 1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>[2092]</td>\n",
       "      <td>[2092, 2067, 948, 6573, 5491, 4830, 10782, 197...</td>\n",
       "      <td>[1129, 8390, 9481, 2092, 1356, 2067, 1978, 435...</td>\n",
       "      <td>(1, 1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>[8826]</td>\n",
       "      <td>[4886, 3601, 5025, 10897, 4971, 3124, 8660, 53...</td>\n",
       "      <td>[1517, 3395, 11111, 6949, 2097, 8989, 7736, 19...</td>\n",
       "      <td>(0, 0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>[1498, 5875]</td>\n",
       "      <td>[5179, 4802, 2432, 1019, 5260, 1016, 5198, 516...</td>\n",
       "      <td>[3427, 7295, 634, 8249, 7307, 2674, 5198, 990,...</td>\n",
       "      <td>(0, 0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id     items_act                                        item_rec_mf  \\\n",
       "0        2        [3669]  [3669, 2701, 5350, 5394, 1086, 2660, 6183, 251...   \n",
       "1        3        [5372]  [70, 1781, 1779, 2615, 6534, 4479, 6269, 4226,...   \n",
       "2        4        [2092]  [2092, 2067, 948, 6573, 5491, 4830, 10782, 197...   \n",
       "3        6        [8826]  [4886, 3601, 5025, 10897, 4971, 3124, 8660, 53...   \n",
       "4        7  [1498, 5875]  [5179, 4802, 2432, 1019, 5260, 1016, 5198, 516...   \n",
       "\n",
       "                                     item_rec_hybrid   match  \n",
       "0  [3669, 10172, 1086, 5246, 1170, 3246, 6161, 32...  (1, 1)  \n",
       "1  [9855, 5678, 5311, 12083, 7392, 12071, 12051, ...  (0, 1)  \n",
       "2  [1129, 8390, 9481, 2092, 1356, 2067, 1978, 435...  (1, 1)  \n",
       "3  [1517, 3395, 11111, 6949, 2097, 8989, 7736, 19...  (0, 0)  \n",
       "4  [3427, 7295, 634, 8249, 7307, 2674, 5198, 990,...  (0, 0)  "
      ]
     },
     "execution_count": 299,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reco_df = pd.DataFrame({'user_id':user_id,'items_act':items,'item_rec_mf':list(nn_mf),'item_rec_hybrid':list(nn_hybrid)})\n",
    "reco_df['match'] = reco_df.apply(lambda row: eval.check_one_match(row[1], row[2],row[3],top_n=20), axis=1)  \n",
    "reco_df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### For every metric field(eg. auc) first and second element in each tuple is value for MF and hybrid approach resp for each user.¶"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>items_act</th>\n",
       "      <th>item_rec_mf</th>\n",
       "      <th>item_rec_hybrid</th>\n",
       "      <th>match</th>\n",
       "      <th>u_id</th>\n",
       "      <th>score_mf</th>\n",
       "      <th>score_hybrid</th>\n",
       "      <th>auc</th>\n",
       "      <th>MRR</th>\n",
       "      <th>MAP</th>\n",
       "      <th>NDCG</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>[3669]</td>\n",
       "      <td>[3669, 2701, 5350, 5394, 1086, 2660, 6183, 251...</td>\n",
       "      <td>[3669, 10172, 1086, 5246, 1170, 3246, 6161, 32...</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>0</td>\n",
       "      <td>[-0.052490182, 0.0031282501, -0.020766765, 0.0...</td>\n",
       "      <td>[-0.15278631, 0.028971292, -0.16377506, 0.2410...</td>\n",
       "      <td>(0.9992471669704415, 0.9992471669704415)</td>\n",
       "      <td>(0.05, 0.05)</td>\n",
       "      <td>(0.5, 1.0)</td>\n",
       "      <td>(1.0, 1.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>[5372]</td>\n",
       "      <td>[70, 1781, 1779, 2615, 6534, 4479, 6269, 4226,...</td>\n",
       "      <td>[9855, 5678, 5311, 12083, 7392, 12071, 12051, ...</td>\n",
       "      <td>(0, 1)</td>\n",
       "      <td>1</td>\n",
       "      <td>[-0.11705529, -0.025777327, 0.013785686, 0.005...</td>\n",
       "      <td>[-0.18034473, -0.12636015, 0.05754578, 0.01197...</td>\n",
       "      <td>(0.499207544179412, 0.9992471669704415)</td>\n",
       "      <td>(0.0, 0.05)</td>\n",
       "      <td>(0.0, 0.05555555555555555)</td>\n",
       "      <td>(0.0, 0.23981246656813146)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>[2092]</td>\n",
       "      <td>[2092, 2067, 948, 6573, 5491, 4830, 10782, 197...</td>\n",
       "      <td>[1129, 8390, 9481, 2092, 1356, 2067, 1978, 435...</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>2</td>\n",
       "      <td>[0.1269677, 0.0069081597, -0.004788185, 0.0603...</td>\n",
       "      <td>[0.405068, 0.07012245, -0.04138836, 0.29055303...</td>\n",
       "      <td>(0.9992471669704415, 0.9992471669704415)</td>\n",
       "      <td>(0.05, 0.05)</td>\n",
       "      <td>(1.0, 0.25)</td>\n",
       "      <td>(1.0, 0.5)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>[8826]</td>\n",
       "      <td>[4886, 3601, 5025, 10897, 4971, 3124, 8660, 53...</td>\n",
       "      <td>[1517, 3395, 11111, 6949, 2097, 8989, 7736, 19...</td>\n",
       "      <td>(0, 0)</td>\n",
       "      <td>3</td>\n",
       "      <td>[0.04317532, 0.0029751188, -0.014475862, 0.020...</td>\n",
       "      <td>[0.085892364, 0.018831424, -0.07802541, 0.0604...</td>\n",
       "      <td>(0.499207544179412, 0.499207544179412)</td>\n",
       "      <td>(0.0, 0.0)</td>\n",
       "      <td>(0.0, 0.0)</td>\n",
       "      <td>(0.0, 0.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>[1498, 5875]</td>\n",
       "      <td>[5179, 4802, 2432, 1019, 5260, 1016, 5198, 516...</td>\n",
       "      <td>[3427, 7295, 634, 8249, 7307, 2674, 5198, 990,...</td>\n",
       "      <td>(0, 0)</td>\n",
       "      <td>4</td>\n",
       "      <td>[-0.006367788, 0.017509054, 0.005814514, -0.00...</td>\n",
       "      <td>[-0.03171058, 0.27742028, 0.078451574, -0.0073...</td>\n",
       "      <td>(0.49920748137581233, 0.49920748137581233)</td>\n",
       "      <td>(0.0, 0.0)</td>\n",
       "      <td>(0.0, 0.0)</td>\n",
       "      <td>(0.0, 0.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id     items_act                                        item_rec_mf  \\\n",
       "0        2        [3669]  [3669, 2701, 5350, 5394, 1086, 2660, 6183, 251...   \n",
       "1        3        [5372]  [70, 1781, 1779, 2615, 6534, 4479, 6269, 4226,...   \n",
       "2        4        [2092]  [2092, 2067, 948, 6573, 5491, 4830, 10782, 197...   \n",
       "3        6        [8826]  [4886, 3601, 5025, 10897, 4971, 3124, 8660, 53...   \n",
       "4        7  [1498, 5875]  [5179, 4802, 2432, 1019, 5260, 1016, 5198, 516...   \n",
       "\n",
       "                                     item_rec_hybrid   match  u_id  \\\n",
       "0  [3669, 10172, 1086, 5246, 1170, 3246, 6161, 32...  (1, 1)     0   \n",
       "1  [9855, 5678, 5311, 12083, 7392, 12071, 12051, ...  (0, 1)     1   \n",
       "2  [1129, 8390, 9481, 2092, 1356, 2067, 1978, 435...  (1, 1)     2   \n",
       "3  [1517, 3395, 11111, 6949, 2097, 8989, 7736, 19...  (0, 0)     3   \n",
       "4  [3427, 7295, 634, 8249, 7307, 2674, 5198, 990,...  (0, 0)     4   \n",
       "\n",
       "                                            score_mf  \\\n",
       "0  [-0.052490182, 0.0031282501, -0.020766765, 0.0...   \n",
       "1  [-0.11705529, -0.025777327, 0.013785686, 0.005...   \n",
       "2  [0.1269677, 0.0069081597, -0.004788185, 0.0603...   \n",
       "3  [0.04317532, 0.0029751188, -0.014475862, 0.020...   \n",
       "4  [-0.006367788, 0.017509054, 0.005814514, -0.00...   \n",
       "\n",
       "                                        score_hybrid  \\\n",
       "0  [-0.15278631, 0.028971292, -0.16377506, 0.2410...   \n",
       "1  [-0.18034473, -0.12636015, 0.05754578, 0.01197...   \n",
       "2  [0.405068, 0.07012245, -0.04138836, 0.29055303...   \n",
       "3  [0.085892364, 0.018831424, -0.07802541, 0.0604...   \n",
       "4  [-0.03171058, 0.27742028, 0.078451574, -0.0073...   \n",
       "\n",
       "                                          auc           MRR  \\\n",
       "0    (0.9992471669704415, 0.9992471669704415)  (0.05, 0.05)   \n",
       "1     (0.499207544179412, 0.9992471669704415)   (0.0, 0.05)   \n",
       "2    (0.9992471669704415, 0.9992471669704415)  (0.05, 0.05)   \n",
       "3      (0.499207544179412, 0.499207544179412)    (0.0, 0.0)   \n",
       "4  (0.49920748137581233, 0.49920748137581233)    (0.0, 0.0)   \n",
       "\n",
       "                          MAP                        NDCG  \n",
       "0                  (0.5, 1.0)                  (1.0, 1.0)  \n",
       "1  (0.0, 0.05555555555555555)  (0.0, 0.23981246656813146)  \n",
       "2                 (1.0, 0.25)                  (1.0, 0.5)  \n",
       "3                  (0.0, 0.0)                  (0.0, 0.0)  \n",
       "4                  (0.0, 0.0)                  (0.0, 0.0)  "
      ]
     },
     "execution_count": 311,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Performance metric dataset \n",
    "reco_df['u_id'] = reco_df.index\n",
    "# measures = reco_df.apply(lambda row: get_performance_metrics(row[5],row[1], user_vecs,item_vecs,top_nn=20), axis=1)\n",
    "reco_df['score_mf'] = [i[8].reshape(-1) for i in measures]\n",
    "reco_df['score_hybrid'] = [i[9].reshape(-1) for i in measures]\n",
    "reco_df['auc'] = [(i[0],i[1]) for i in measures]\n",
    "reco_df['MRR'] = [(i[2],i[3]) for i in measures]\n",
    "reco_df['MAP'] = [(i[4],i[5]) for i in measures]\n",
    "reco_df['NDCG'] = [(i[6],i[7]) for i in measures]\n",
    "reco_df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Finally we get the comparative performance metric figure for MF vs. Hybrid by averaging each metric across all users. Here it's comparison between two best solution, however for other approaches metrics can be calculated similarly. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "match rate/TP for MF: 0.3678998505231689\n",
      "match rate/TP for hybrid: 0.40115844544095663\n",
      "AUC for MF: 0.6336362538668119\n",
      "AUC for hybrid: 0.6747993938671657\n",
      "MAP for MF: 0.11373469839364617\n",
      "MAP for hybrid: 0.16712752000049622\n",
      "MRR for MF: 0.01783445440956652\n",
      "MRR for hybrid: 0.023252989536621825\n",
      "NDCG for MF: 0.01783445440956652\n",
      "NDCG for hybrid: 0.023252989536621825\n"
     ]
    }
   ],
   "source": [
    "#Performance\n",
    "\n",
    "match_mf = [i[0] for i in reco_df['match']]\n",
    "match_hybrid = [i[1] for i in reco_df['match']]\n",
    "auc_mf = [i[0] for i in reco_df['auc']]\n",
    "auc_hybrid = [i[1] for i in reco_df['auc']]\n",
    "MAP_mf = [i[0] for i in reco_df['MAP']]\n",
    "MAP_hybrid = [i[1] for i in reco_df['MAP']]\n",
    "MRR_mf = [i[0] for i in reco_df['MRR']]\n",
    "MRR_hybrid = [i[1] for i in reco_df['MRR']]\n",
    "NDCG_mf = [i[0] for i in reco_df['NDCG']]\n",
    "NDCG_hybrid = [i[1] for i in reco_df['NDCG']]\n",
    "print('match rate/TP for MF:', np.mean(match)) \n",
    "print('match rate/TP for hybrid:', np.mean(match_hybrid)) \n",
    "print('AUC for MF:', np.mean(auc_mf)) \n",
    "print('AUC for hybrid:', np.mean(auc_hybrid)) \n",
    "print('MAP for MF:', np.mean(MAP_mf)) \n",
    "print('MAP for hybrid:', np.mean(MAP_hybrid)) \n",
    "print('MRR for MF:', np.mean(MRR_mf)) \n",
    "print('MRR for hybrid:', np.mean(MRR_hybrid))\n",
    "print('NDCG for MF:', np.mean(MRR_mf)) \n",
    "print('NDCG for hybrid:', np.mean(MRR_hybrid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Earlier we already explored match rate comparison for pure MF vs. MF-CF approach. Here we see the direction consistency for other evaluation metric as well. In final hybrid solution we achieve average AUC of 0.675. Contrary to binary classification(with only 2 options), here with massive product space of 12620 options and tiny user space(only 5352 users), it's quite a decent result for the business."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Thank You"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}