compute_benchmark_accuracy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "f487addf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "7cdb7514",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CorrectACauseB</th>\n",
       "      <th>CorrectBCauseA</th>\n",
       "      <th>pairid</th>\n",
       "      <th>varA</th>\n",
       "      <th>varB</th>\n",
       "      <th>dataset</th>\n",
       "      <th>groundtruth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Altitude</td>\n",
       "      <td>Temperature</td>\n",
       "      <td>DWD</td>\n",
       "      <td>-&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>Altitude</td>\n",
       "      <td>Precipitation</td>\n",
       "      <td>DWD</td>\n",
       "      <td>-&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>Longitude</td>\n",
       "      <td>Temperature</td>\n",
       "      <td>DWD</td>\n",
       "      <td>-&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>Altitude</td>\n",
       "      <td>Sunshine hours</td>\n",
       "      <td>DWD</td>\n",
       "      <td>-&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5</td>\n",
       "      <td>Age</td>\n",
       "      <td>Length</td>\n",
       "      <td>Abalone</td>\n",
       "      <td>-&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>104</td>\n",
       "      <td>time for passing 1. segment</td>\n",
       "      <td>time for passing 2. segment</td>\n",
       "      <td>D. Janzing</td>\n",
       "      <td>-&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>105</td>\n",
       "      <td>pixel vector of a patch</td>\n",
       "      <td>total brightness at the screen</td>\n",
       "      <td>D. Janzing</td>\n",
       "      <td>-&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>106</td>\n",
       "      <td>time required for one round</td>\n",
       "      <td>voltage</td>\n",
       "      <td>D. Janzing</td>\n",
       "      <td>&lt;-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>107</td>\n",
       "      <td>strength of contrast</td>\n",
       "      <td>answer correct or not</td>\n",
       "      <td>Schuett, edited by D. Janzing</td>\n",
       "      <td>-&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>108</td>\n",
       "      <td>time for 1/6 rotation</td>\n",
       "      <td>temperature</td>\n",
       "      <td>D. Janzing</td>\n",
       "      <td>&lt;-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>108 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     CorrectACauseB  CorrectBCauseA  pairid                          varA  \\\n",
       "0               1.0             1.0       1                      Altitude   \n",
       "1               1.0             1.0       2                      Altitude   \n",
       "2               1.0             1.0       3                     Longitude   \n",
       "3               0.0             1.0       4                      Altitude   \n",
       "4               1.0             1.0       5                           Age   \n",
       "..              ...             ...     ...                           ...   \n",
       "103             0.0             1.0     104   time for passing 1. segment   \n",
       "104             0.0             1.0     105       pixel vector of a patch   \n",
       "105             1.0             1.0     106   time required for one round   \n",
       "106             1.0             1.0     107          strength of contrast   \n",
       "107             1.0             0.0     108         time for 1/6 rotation   \n",
       "\n",
       "                               varB                        dataset groundtruth  \n",
       "0                       Temperature                            DWD          ->  \n",
       "1                     Precipitation                            DWD          ->  \n",
       "2                       Temperature                            DWD          ->  \n",
       "3                    Sunshine hours                            DWD          ->  \n",
       "4                            Length                        Abalone          ->  \n",
       "..                              ...                            ...         ...  \n",
       "103     time for passing 2. segment                     D. Janzing         ->   \n",
       "104  total brightness at the screen                     D. Janzing         ->   \n",
       "105                         voltage                     D. Janzing          <-  \n",
       "106           answer correct or not  Schuett, edited by D. Janzing         ->   \n",
       "107                     temperature                     D. Janzing         <-   \n",
       "\n",
       "[108 rows x 7 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#resfile = \"gpt-3.5-turbo_system_results_singleprompt.csv\"\n",
    "#resfile = \"gpt-4_system_results_singleprompt.csv\"\n",
    "resfile = \"gpt-3.5-turbo_results.csv\"\n",
    "#resfile = \"gpt-3.5-turbo_system_results.csv\"\n",
    "results = pd.read_csv(resfile)\n",
    "results[\"pairid\"] = results[\"pairid\"].str[4:]\n",
    "results[\"pairid\"] = results[\"pairid\"].astype(int)\n",
    "if \"singleprompt\" in resfile:\n",
    "    results[\"CorrectACauseB\"] = results[\"CorrectCause\"]\n",
    "    results[\"CorrectBCauseA\"] = results[\"CorrectCause\"]\n",
    "results[\"CorrectACauseB\"] = results[\"CorrectACauseB\"].astype(float)\n",
    "results[\"CorrectBCauseA\"] = results[\"CorrectBCauseA\"].astype(float)\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "018c187f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "73faaa71",
   "metadata": {},
   "outputs": [],
   "source": [
    "if \"singleprompt\" in resfile:\n",
    "    display(sum(results[\"CorrectCause\"]))\n",
    "    len(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "0dd1f5d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Computing accuracy\n",
    "# The first column denotes whether chatGPT answered correctly for the question, \"Does A cause B?\"\n",
    "# The second column denotes whether chatGPT answered correctly for the question, \"Does B cause A?\".\n",
    "# if the result is 0,1 or 1,0, then we can take a random guess to decide the direction of causation. \n",
    "# Therefore, to compute accuracy, we can take the mean of \"Does A cause B\" and \"Does B cause A\" questions. \n",
    "results[\"accuracy\"] = (results[\"CorrectACauseB\"] + results[\"CorrectBCauseA\"])/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "93772ade",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pairid</th>\n",
       "      <th>v1</th>\n",
       "      <th>v2</th>\n",
       "      <th>v3</th>\n",
       "      <th>v4</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>104</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>105</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>106</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>107</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>108</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>108 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     pairid  v1  v2  v3  v4  weight\n",
       "0         1   1   1   2   2   0.166\n",
       "1         2   1   1   2   2   0.166\n",
       "2         3   1   1   2   2   0.167\n",
       "3         4   1   1   2   2   0.166\n",
       "4         5   1   1   2   2   0.143\n",
       "..      ...  ..  ..  ..  ..     ...\n",
       "103     104   1   1   2   2   0.200\n",
       "104     105   1   9  10  10   0.500\n",
       "105     106   2   2   1   1   1.000\n",
       "106     107   1   1   2   2   1.000\n",
       "107     108   2   2   1   1   1.000\n",
       "\n",
       "[108 rows x 6 columns]"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights = pd.read_table(\"pairmeta.txt\", sep=\" \", header=None, names=[\"pairid\", \"v1\", \"v2\", \"v3\", \"v4\", \"weight\"])\n",
    "weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "95d77f1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pairid</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>v1</th>\n",
       "      <th>v2</th>\n",
       "      <th>v3</th>\n",
       "      <th>v4</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>104</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>105</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>0.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>106</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>107</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>108</td>\n",
       "      <td>0.5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>108 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     pairid  accuracy  v1  v2  v3  v4  weight\n",
       "0         1       1.0   1   1   2   2   0.166\n",
       "1         2       1.0   1   1   2   2   0.166\n",
       "2         3       1.0   1   1   2   2   0.167\n",
       "3         4       0.5   1   1   2   2   0.166\n",
       "4         5       1.0   1   1   2   2   0.143\n",
       "..      ...       ...  ..  ..  ..  ..     ...\n",
       "103     104       0.5   1   1   2   2   0.200\n",
       "104     105       0.5   1   9  10  10   0.500\n",
       "105     106       1.0   2   2   1   1   1.000\n",
       "106     107       1.0   1   1   2   2   1.000\n",
       "107     108       0.5   2   2   1   1   1.000\n",
       "\n",
       "[108 rows x 7 columns]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dat = pd.merge(results[[\"pairid\", \"accuracy\"]], weights, on=\"pairid\")\n",
    "dat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "026af3d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pairid</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>v1</th>\n",
       "      <th>v2</th>\n",
       "      <th>v3</th>\n",
       "      <th>v4</th>\n",
       "      <th>weight</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.166</td>\n",
       "      <td>0.166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.166</td>\n",
       "      <td>0.166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.167</td>\n",
       "      <td>0.167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.166</td>\n",
       "      <td>0.083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.143</td>\n",
       "      <td>0.143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>104</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>105</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>0.500</td>\n",
       "      <td>0.250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>106</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>107</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>108</td>\n",
       "      <td>0.5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>108 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     pairid  accuracy  v1  v2  v3  v4  weight  score\n",
       "0         1       1.0   1   1   2   2   0.166  0.166\n",
       "1         2       1.0   1   1   2   2   0.166  0.166\n",
       "2         3       1.0   1   1   2   2   0.167  0.167\n",
       "3         4       0.5   1   1   2   2   0.166  0.083\n",
       "4         5       1.0   1   1   2   2   0.143  0.143\n",
       "..      ...       ...  ..  ..  ..  ..     ...    ...\n",
       "103     104       0.5   1   1   2   2   0.200  0.100\n",
       "104     105       0.5   1   9  10  10   0.500  0.250\n",
       "105     106       1.0   2   2   1   1   1.000  1.000\n",
       "106     107       1.0   1   1   2   2   1.000  1.000\n",
       "107     108       0.5   2   2   1   1   1.000  0.500\n",
       "\n",
       "[108 rows x 8 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy 0.8148148148148148\n",
      "Weighted accuracy 0.828847194336105\n"
     ]
    }
   ],
   "source": [
    "dat[\"score\"] = dat[\"accuracy\"] * dat[\"weight\"]\n",
    "display(dat)\n",
    "print(\"Accuracy\", sum(dat[\"accuracy\"])/len(dat))\n",
    "print(\"Weighted accuracy\", sum(dat[\"score\"])/sum(dat[\"weight\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "e6c05216",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.min(dat[\"accuracy\"])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}