3.indexing_final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/katya/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /home/katya/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import os\n",
    "from elasticsearch import Elasticsearch\n",
    "from zipfile import ZipFile\n",
    "import re\n",
    "import pandas as pd\n",
    "import json\n",
    "import requests\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n",
    "import pickle\n",
    "import pytrec_eval\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "from elasticsearch import NotFoundError\n",
    "\n",
    "es = Elasticsearch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dir = 'data'\n",
    "output_dir = 'results'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "for filename in os.listdir(input_dir):\n",
    "    if filename.endswith(\".zip\"):\n",
    "        with ZipFile(filename, 'r') as zip:\n",
    "            zip.extractall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "def strip_punct(s):\n",
    "    s = re.sub('[^A-Za-z0-9]', ' ', s)\n",
    "    s = s.lower()\n",
    "    return \" \".join(s.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "def chunks(lst, n):\n",
    "    \"\"\"Yield successive n-sized chunks from lst.\"\"\"\n",
    "    for i in range(0, len(lst), n):\n",
    "        yield lst[i:i + n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search(search_param, filename, q, ind):\n",
    "    \n",
    "    qid = []\n",
    "    Q0 = []\n",
    "    doc = []\n",
    "    rank = []\n",
    "    score = []\n",
    "    tag = []\n",
    "\n",
    "    num=1\n",
    "\n",
    "    for i in q:\n",
    "        for x in search_param['query']['bool']['should']:\n",
    "            if type(i) == dict:\n",
    "                for key in x['match']:\n",
    "                    x['match'][key] = i['title']\n",
    "            else:\n",
    "                for key in x['match']:\n",
    "                    x['match'][key] = i\n",
    "                    \n",
    "        #print(search_param)\n",
    "        response = es.search(index=ind, body=search_param)\n",
    "        r = 1\n",
    "        for x in response['hits']['hits']:\n",
    "            qid.append(num)\n",
    "            Q0.append('Q0')\n",
    "            doc.append(x['_id'])\n",
    "            rank.append(r)\n",
    "            score.append(x['_score'])\n",
    "            tag.append('uh-t2-thor')\n",
    "            r+=1\n",
    "        num+=1\n",
    "    \n",
    "   \n",
    "    qrels = {'qid': qid, 'Q0': Q0, 'doc': doc, 'rank':rank, 'score':score, 'tag':tag}\n",
    "\n",
    "    df = pd.DataFrame(qrels)\n",
    "    df.to_csv((output_dir+'/'+filename+'.txt'), sep = ' ', index = False, header = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_extended(search_param, filename, q, ind):\n",
    "    qid = []\n",
    "    Q0 = []\n",
    "    doc = []\n",
    "    rank = []\n",
    "    score = []\n",
    "    tag = []\n",
    "\n",
    "    num=1\n",
    "#['bool']['should']\n",
    "    for idx, row in q.iterrows():\n",
    "            \n",
    "            #for key in x['match']:\n",
    "                #x['match'][key] = row['query']\n",
    "        for x in search_param['query']['bool']['should']:\n",
    "            for key in x[\"match\"]:\n",
    "                #print(key)\n",
    "                if \"boost\" in x['match'][key]:\n",
    "                    x['match'][key][\"query\"] = row['query']\n",
    "                else:\n",
    "                    x['match'][key] = row['syn']\n",
    "        #print(search_param)\n",
    "        response = es.search(index=ind, body=search_param)\n",
    "        r = 1\n",
    "        for x in response['hits']['hits']:\n",
    "            qid.append(num)\n",
    "            Q0.append('Q0')\n",
    "            doc.append(x['_id'])\n",
    "            rank.append(r)\n",
    "            score.append(x['_score'])\n",
    "            tag.append('uh-t2-thor')\n",
    "            r+=1\n",
    "        num+=1\n",
    "    \n",
    "   \n",
    "    qrels = {'qid': qid, 'Q0': Q0, 'doc': doc, 'rank':rank, 'score':score, 'tag':tag}\n",
    "\n",
    "    df = pd.DataFrame(qrels)\n",
    "    df.to_csv((output_dir+'/'+filename+'.txt'), sep = ' ', index = False, header = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_index(b, k1, index):\n",
    "    \n",
    "    #create template for index\n",
    "    request_body = {\n",
    "    \"settings\" : {\n",
    "\t        \"number_of_shards\": 1,\n",
    "\t        \"number_of_replicas\": 0,\n",
    "        \"similarity\": {\n",
    "      \"default\": { \n",
    "        \"type\": \"BM25\",\n",
    "        \"b\":b,\n",
    "        \"k1\":k1\n",
    "      }\n",
    "    }\n",
    "\t    }\n",
    "\t}\n",
    "\n",
    "    es.indices.create(index = index, body = request_body)\n",
    "    \n",
    "    \n",
    "    #load data to index\n",
    "    with open('data/bulk_data.json') as f:\n",
    "        bulk_data = json.load(f)\n",
    "        \n",
    "    for x in bulk_data[::2]:\n",
    "        x['index']['_index']=index\n",
    "        \n",
    "    bulks = chunks(bulk_data, 100)\n",
    "    for x in bulks:\n",
    "        res = es.bulk(index = index, body = x)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_run(d, run_name, index, mode):\n",
    "    #d, run_name, index\n",
    "    \n",
    "    if mode=='simple':\n",
    "        f = open(\"data/topics_lemmatized.txt\", 'rb')\n",
    "        queries_lem = pickle.load(f)\n",
    "        f.close()\n",
    "        search_param = {\n",
    "    \n",
    "        'size': 50,\n",
    "        \"query\": {\n",
    "        \"bool\": {\n",
    "        \"should\": d\n",
    "        }\n",
    "        }\n",
    "        }\n",
    "        #print(search_param)\n",
    "\n",
    "        search(search_param, run_name, queries_lem, index)\n",
    "    elif mode=='syn':\n",
    "        df_syn = pd.read_csv('data/q_for_syn.tsv', sep = '\\t')\n",
    "        search_param = {\n",
    "    \n",
    "        'size': 50,\n",
    "        \"query\": {\n",
    "        \"bool\": {\n",
    "        \"should\": d\n",
    "        }\n",
    "        }\n",
    "        }\n",
    "        #print(search_param)\n",
    "\n",
    "        search_extended(search_param, run_name, df_syn, index)\n",
    "    else:\n",
    "        print('Wrong arguments!')\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_run(run_path, qrel_path):\n",
    "    run = pd.read_csv(run_path, sep = ' ', names = ['topic','Q0','id','rank','score','team'])\n",
    "\n",
    "    qrel = pd.read_csv(qrel_path, sep = ' ', names = ['topic','Q0','id','relevance'])\n",
    "\n",
    "    qrels = {}\n",
    "    for i in range(1,51):\n",
    "        qrels[str(i)] = {}\n",
    "    \n",
    "    for idx, row in qrel.iterrows():\n",
    "        qrels[str(row['topic'])][row['id']] = row['relevance']\n",
    "    \n",
    "    runs = {}\n",
    "    for i in range(1,51):\n",
    "        runs[str(i)] = {}\n",
    "    \n",
    "    for idx, row in run.iterrows():\n",
    "        runs[str(row['topic'])][row['id']] = row['score']\n",
    "\n",
    "    evaluator = pytrec_eval.RelevanceEvaluator(\n",
    "        qrels,{'map_cut', 'ndcg_cut', 'recall', 'P'})\n",
    "\n",
    "    res = evaluator.evaluate(runs)\n",
    "\n",
    "    results = {}\n",
    "    recall = {}\n",
    "    pres = {}\n",
    "    for key in res:\n",
    "        results[key] = res[key]['ndcg_cut_10']\n",
    "        recall[key] = res[key]['recall_10']\n",
    "        pres[key] = res[key]['P_10']\n",
    "        #print(res[key])\n",
    "    \n",
    "    filename = re.split('\\.', sys.argv[1])\n",
    "    with open((filename[0]+'_evaluated.json'), 'w') as f:\n",
    "        json.dump(results, f)\n",
    "        \n",
    "    s = sum(results.values())/50\n",
    "    r = sum(recall.values())/50\n",
    "    p = sum(pres.values())/50\n",
    "    if r+p != 0:\n",
    "        f1 = (2*(r*p)/(r+p))\n",
    "    else: \n",
    "        f1 = 0\n",
    "    '''\n",
    "    print(\"Average ndcg_cut_10: \", s)\n",
    "    print(\"Average recall_10: \", r)\n",
    "    print(\"Av. precision: \", p)\n",
    "    print(\"F1-score: \",f1)\n",
    "    '''    \n",
    "    return s, r, p, f1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/katya/anaconda3/lib/python3.7/site-packages/elasticsearch/connection/base.py:190: ElasticsearchDeprecationWarning: [types removal] Specifying types in bulk requests is deprecated.\n",
      "  warnings.warn(message, category=ElasticsearchDeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "index = 'final_0.68'\n",
    "create_index(0.68, 1.2, index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#runs for args\n",
    "d=[]\n",
    "d.append({\n",
    "          \"match\": {\n",
    "            \"title_lem\": \"\"\n",
    "          }})\n",
    "create_run(d, 'run_title', index, 'simple')\n",
    "\n",
    "\n",
    "d=[]\n",
    "d.append({\n",
    "          \"match\": {\n",
    "            \"lem\": \"\"\n",
    "          }\n",
    "        })\n",
    "create_run(d, 'run_doc', index, 'simple')\n",
    "\n",
    "\n",
    "d=[]\n",
    "d.append({\n",
    "          \"match\": {\n",
    "            \"args\": \"\"\n",
    "          }\n",
    "        })      \n",
    "create_run(d, 'run_arg', index, 'simple')\n",
    "\n",
    "\n",
    "d=[]\n",
    "d.append({\n",
    "          \"match\": {\n",
    "            \"title_lem\": \"\"\n",
    "          }})\n",
    "d.append({\n",
    "          \"match\": {\n",
    "            \"args\": \"\"\n",
    "          }\n",
    "        })\n",
    "create_run(d, 'run_title_arg', index, 'simple')\n",
    "\n",
    "\n",
    "d=[]\n",
    "d.append({\n",
    "          \"match\": {\n",
    "            \"title_lem\": \"\"\n",
    "          }})\n",
    "d.append({\n",
    "          \"match\": {\n",
    "            \"lem\": \"\"\n",
    "          }\n",
    "        })\n",
    "create_run(d, 'run_title_doc', index, 'simple')\n",
    "\n",
    "\n",
    "d=[]\n",
    "d.append({\n",
    "          \"match\": {\n",
    "            \"lem\": \"\"\n",
    "          }})\n",
    "d.append({\n",
    "          \"match\": {\n",
    "            \"args\": \"\"\n",
    "          }\n",
    "        })\n",
    "create_run(d, 'run_doc_arg', index, 'simple')\n",
    "\n",
    "\n",
    "d=[]\n",
    "d.append({\n",
    "          \"match\": {\n",
    "            \"title_lem\": \"\"\n",
    "          }})\n",
    "d.append({\n",
    "          \"match\": {\n",
    "            \"lem\": \"\"\n",
    "          }\n",
    "        })\n",
    "d.append({\n",
    "          \"match\": {\n",
    "            \"args\": \"\"\n",
    "          }\n",
    "        })\n",
    "create_run(d, 'run_title_doc_arg', index, 'simple')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#runs for synonyms\n",
    "\n",
    "\n",
    "d=[]\n",
    "d.append({\"match\": {\"title_lem\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"title_lem\": {\"\"}}})\n",
    "create_run(d, 'run_title_syn', index, 'syn')\n",
    "\n",
    "\n",
    "d=[]\n",
    "d.append({\"match\": {\"lem\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"lem\": {\"\"}}})\n",
    "create_run(d, 'run_doc_syn', index, 'syn')\n",
    "\n",
    "\n",
    "d=[]\n",
    "d.append({\"match\": {\"args\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"args\": {\"\"}}})\n",
    "create_run(d, 'run_args_syn', index, 'syn')\n",
    "\n",
    "\n",
    "d=[]\n",
    "d.append({\"match\": {\"title_lem\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"args\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"title_lem\": {\"\"}}})\n",
    "d.append({\"match\": {\"args\": {\"\"}}})\n",
    "create_run(d, 'run_title_arg_syn', index, 'syn')\n",
    "\n",
    "\n",
    "d=[]\n",
    "d.append({\"match\": {\"title_lem\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"lem\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"title_lem\": {\"\"}}})\n",
    "d.append({\"match\": {\"lem\": {\"\"}}})\n",
    "create_run(d, 'run_title_doc_syn', index, 'syn')\n",
    "\n",
    "\n",
    "d=[]\n",
    "d.append({\"match\": {\"lem\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"args\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"lem\": {\"\"}}})\n",
    "d.append({\"match\": {\"args\": {\"\"}}})\n",
    "create_run(d, 'run_doc_arg_syn', index, 'syn')\n",
    "\n",
    "\n",
    "d=[]\n",
    "d.append({\"match\": {\"title_lem\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"lem\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"args\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"title_lem\": {\"\"}}})\n",
    "d.append({\"match\": {\"lem\": {\"\"}}})\n",
    "d.append({\"match\": {\"args\": {\"\"}}})\n",
    "create_run(d, 'run_title_doc_arg_syn', index, 'syn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "#evaluation\n",
    "run = []\n",
    "ndcg = []\n",
    "rec = []\n",
    "prec = []\n",
    "f1s = []\n",
    "\n",
    "for filename in os.listdir(output_dir):\n",
    "    if filename.endswith(\".txt\"):\n",
    "        #print(filename)\n",
    "        s, r, p, f1 = evaluate_run((output_dir+\"/{}\").format(filename), 'touche2020-task2-relevance-withbaseline(1).qrels')\n",
    "        run.append(filename)\n",
    "        ndcg.append(s)\n",
    "        rec.append(r)\n",
    "        prec.append(p)\n",
    "        f1s.append(f1)\n",
    "df = pd.DataFrame({'run':run,'ndcg_cut10':ndcg, 'recall_10':rec, 'precision':prec, 'f1-score':f1s})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          run  ndcg_cut10  recall_10  precision  f1-score\n",
      "10      run_title_doc_syn.txt    0.445049   0.280011      0.428  0.338539\n",
      "3           run_title_doc.txt    0.443412   0.271602      0.422  0.330495\n",
      "12          run_title_arg.txt    0.386312   0.253063      0.376  0.302519\n",
      "1       run_title_arg_syn.txt    0.379192   0.250281      0.370  0.298587\n",
      "0       run_title_doc_arg.txt    0.376844   0.234387      0.352  0.281399\n",
      "11  run_title_doc_arg_syn.txt    0.366942   0.229828      0.348  0.276831\n",
      "13          run_title_syn.txt    0.348180   0.218751      0.340  0.266220\n",
      "2               run_title.txt    0.328667   0.205588      0.312  0.247856\n",
      "4             run_doc_syn.txt    0.323637   0.199024      0.300  0.239296\n",
      "7                 run_doc.txt    0.318181   0.196172      0.292  0.234680\n",
      "6             run_doc_arg.txt    0.285249   0.189760      0.276  0.224896\n",
      "5         run_doc_arg_syn.txt    0.281281   0.186087      0.274  0.221645\n",
      "8            run_args_syn.txt    0.270447   0.180989      0.260  0.213416\n",
      "9                 run_arg.txt    0.264587   0.180140      0.262  0.213492\n"
     ]
    }
   ],
   "source": [
    "print(df.sort_values(by = ['ndcg_cut10'], ascending = False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'acknowledged': True}"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#print(index)\n",
    "es.indices.delete(index=index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}