06.combine-names.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert fore names to simplified (one word) fore names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import string\n",
    "import pandas\n",
    "\n",
    "import pubmedpy.names\n",
    "from pubmedpy.names import simplify_fore_name, simplify_last_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_df_paths = [\n",
    "    'data/pubmed/authors.tsv.xz',\n",
    "    'data/pmc/authors.tsv.xz',\n",
    "    'data/iscb/keynotes.tsv',\n",
    "]\n",
    "path_to_df = {}\n",
    "for path in name_df_paths:\n",
    "    name_df = pandas.read_csv(path, keep_default_na=False, na_values=[''], sep='\\t')\n",
    "    path_to_df[path] = name_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## full name table\n",
    "\n",
    "create a table of individuals with full names for prediction methods that require full names."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fore_name</th>\n",
       "      <th>last_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A H</td>\n",
       "      <td>Samad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>W W</td>\n",
       "      <td>Cai</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  fore_name last_name\n",
       "0       A H     Samad\n",
       "1       W W       Cai"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_name_df = pandas.concat(\n",
    "    [df[[\"fore_name\", \"last_name\"]] for df in path_to_df.values()]\n",
    ").drop_duplicates()\n",
    "full_name_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_full_name(name: str) -> str:\n",
    "    \"\"\"\n",
    "    Lightly clean full names by replacing periods with spaces\n",
    "    and standardizing whitespace.\n",
    "    \"\"\"\n",
    "    name = name.replace(\".\", \" \")\n",
    "    name = \" \".join(name.split())\n",
    "    return name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fore_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>fore_name_simple</th>\n",
       "      <th>last_name_simple</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1042181</th>\n",
       "      <td>Azemi</td>\n",
       "      <td>Barama</td>\n",
       "      <td>azemi</td>\n",
       "      <td>barama</td>\n",
       "      <td>Azemi Barama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1064826</th>\n",
       "      <td>Youyong</td>\n",
       "      <td>Wang</td>\n",
       "      <td>youyong</td>\n",
       "      <td>wang</td>\n",
       "      <td>Youyong Wang</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114476</th>\n",
       "      <td>Helgard I</td>\n",
       "      <td>Nirenberg</td>\n",
       "      <td>helgard</td>\n",
       "      <td>nirenberg</td>\n",
       "      <td>Helgard I Nirenberg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44710</th>\n",
       "      <td>Yong-Hua</td>\n",
       "      <td>Gan</td>\n",
       "      <td>yong-hua</td>\n",
       "      <td>gan</td>\n",
       "      <td>Yong-Hua Gan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>612265</th>\n",
       "      <td>Denise P</td>\n",
       "      <td>Barlow</td>\n",
       "      <td>denise</td>\n",
       "      <td>barlow</td>\n",
       "      <td>Denise P Barlow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1068732</th>\n",
       "      <td>Sylvain</td>\n",
       "      <td>Martineau</td>\n",
       "      <td>sylvain</td>\n",
       "      <td>martineau</td>\n",
       "      <td>Sylvain Martineau</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>531754</th>\n",
       "      <td>Won-Hee</td>\n",
       "      <td>Song</td>\n",
       "      <td>won-hee</td>\n",
       "      <td>song</td>\n",
       "      <td>Won-Hee Song</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>372138</th>\n",
       "      <td>Christiane</td>\n",
       "      <td>Rondeau</td>\n",
       "      <td>christiane</td>\n",
       "      <td>rondeau</td>\n",
       "      <td>Christiane Rondeau</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>421503</th>\n",
       "      <td>Jessica</td>\n",
       "      <td>Tyler</td>\n",
       "      <td>jessica</td>\n",
       "      <td>tyler</td>\n",
       "      <td>Jessica Tyler</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478038</th>\n",
       "      <td>Vytas</td>\n",
       "      <td>Svedas</td>\n",
       "      <td>vytas</td>\n",
       "      <td>svedas</td>\n",
       "      <td>Vytas Svedas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          fore_name  last_name fore_name_simple last_name_simple  \\\n",
       "1042181       Azemi     Barama            azemi           barama   \n",
       "1064826     Youyong       Wang          youyong             wang   \n",
       "114476    Helgard I  Nirenberg          helgard        nirenberg   \n",
       "44710      Yong-Hua        Gan         yong-hua              gan   \n",
       "612265     Denise P     Barlow           denise           barlow   \n",
       "1068732     Sylvain  Martineau          sylvain        martineau   \n",
       "531754      Won-Hee       Song          won-hee             song   \n",
       "372138   Christiane    Rondeau       christiane          rondeau   \n",
       "421503      Jessica      Tyler          jessica            tyler   \n",
       "478038        Vytas     Svedas            vytas           svedas   \n",
       "\n",
       "                   full_name  \n",
       "1042181         Azemi Barama  \n",
       "1064826         Youyong Wang  \n",
       "114476   Helgard I Nirenberg  \n",
       "44710           Yong-Hua Gan  \n",
       "612265       Denise P Barlow  \n",
       "1068732    Sylvain Martineau  \n",
       "531754          Won-Hee Song  \n",
       "372138    Christiane Rondeau  \n",
       "421503         Jessica Tyler  \n",
       "478038          Vytas Svedas  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_name_df['fore_name_simple'] = full_name_df.fore_name.map(\n",
    "    lambda x: simplify_fore_name(x, lower=True))\n",
    "full_name_df['last_name_simple'] = full_name_df.last_name.map(\n",
    "    lambda x: simplify_last_name(x, lower=True))\n",
    "full_name_df = full_name_df.dropna(subset=['fore_name_simple', \"last_name_simple\"])\n",
    "full_name_df[\"full_name\"] = (\n",
    "    full_name_df.fore_name.str.strip() + \" \" +\n",
    "    full_name_df.last_name.str.strip()).map(clean_full_name)\n",
    "full_name_df.sample(n=10, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_name_df.to_csv('data/names/full-names.tsv.xz', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## fore and last name tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simplify_names(dfs, name_column):\n",
    "    names = pandas.concat(df[name_column] for df in dfs)\n",
    "    simple_name_df = names.value_counts(dropna=True).reset_index()\n",
    "    simple_name_df.columns = [name_column, 'n_authors']\n",
    "    assert simple_name_df[name_column].isna().sum() == 0\n",
    "    simplifier = getattr(pubmedpy.names, f'simplify_{name_column}')\n",
    "    simple_name_df[f'{name_column}_simple'] = simple_name_df[name_column].map(lambda x: simplifier(x, lower=True))\n",
    "    simple_name_df = simple_name_df.sort_values(name_column)\n",
    "    return simple_name_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fore_name</th>\n",
       "      <th>n_authors</th>\n",
       "      <th>fore_name_simple</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>136030</th>\n",
       "      <td>(Max) Zong-Ming</td>\n",
       "      <td>1</td>\n",
       "      <td>max</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102701</th>\n",
       "      <td>, Yuxin</td>\n",
       "      <td>1</td>\n",
       "      <td>yuxin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107469</th>\n",
       "      <td>-</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A</td>\n",
       "      <td>3787</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138774</th>\n",
       "      <td>A  Gordon</td>\n",
       "      <td>1</td>\n",
       "      <td>gordon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80979</th>\n",
       "      <td>Živojin</td>\n",
       "      <td>1</td>\n",
       "      <td>živojin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141327</th>\n",
       "      <td>Žygimantė</td>\n",
       "      <td>1</td>\n",
       "      <td>žygimantė</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145797</th>\n",
       "      <td>željka</td>\n",
       "      <td>1</td>\n",
       "      <td>željka</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129082</th>\n",
       "      <td>Νicolaos</td>\n",
       "      <td>1</td>\n",
       "      <td>νicolaos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74887</th>\n",
       "      <td>Аndrey D</td>\n",
       "      <td>1</td>\n",
       "      <td>аndrey</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>151487 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              fore_name  n_authors fore_name_simple\n",
       "136030  (Max) Zong-Ming          1              max\n",
       "102701          , Yuxin          1            yuxin\n",
       "107469                -          1             None\n",
       "3                     A       3787             None\n",
       "138774        A  Gordon          1           gordon\n",
       "...                 ...        ...              ...\n",
       "80979           Živojin          1          živojin\n",
       "141327        Žygimantė          1        žygimantė\n",
       "145797           željka          1           željka\n",
       "129082         Νicolaos          1         νicolaos\n",
       "74887          Аndrey D          1           аndrey\n",
       "\n",
       "[151487 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simple_fore_name_df = simplify_names(path_to_df.values(), 'fore_name')\n",
    "simple_fore_name_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>last_name</th>\n",
       "      <th>n_authors</th>\n",
       "      <th>last_name_simple</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>96300</th>\n",
       "      <td>'t Hart</td>\n",
       "      <td>1</td>\n",
       "      <td>t hart</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5389</th>\n",
       "      <td>'t Hoen</td>\n",
       "      <td>26</td>\n",
       "      <td>t hoen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170911</th>\n",
       "      <td>(Holly) Yang</td>\n",
       "      <td>1</td>\n",
       "      <td>holly) yang</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167472</th>\n",
       "      <td>-L Zhu</td>\n",
       "      <td>1</td>\n",
       "      <td>l zhu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9870</th>\n",
       "      <td>A</td>\n",
       "      <td>16</td>\n",
       "      <td>a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161845</th>\n",
       "      <td>žurauskienė</td>\n",
       "      <td>1</td>\n",
       "      <td>žurauskienė</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137908</th>\n",
       "      <td>ʼt Hoen</td>\n",
       "      <td>1</td>\n",
       "      <td>ʼt hoen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172686</th>\n",
       "      <td>Ӧkmen</td>\n",
       "      <td>1</td>\n",
       "      <td>ӧkmen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68537</th>\n",
       "      <td>‘t Hoen</td>\n",
       "      <td>2</td>\n",
       "      <td>‘t hoen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79394</th>\n",
       "      <td>’t Hoen</td>\n",
       "      <td>2</td>\n",
       "      <td>’t hoen</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>183740 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           last_name  n_authors last_name_simple\n",
       "96300        't Hart          1           t hart\n",
       "5389         't Hoen         26           t hoen\n",
       "170911  (Holly) Yang          1      holly) yang\n",
       "167472        -L Zhu          1            l zhu\n",
       "9870               A         16                a\n",
       "...              ...        ...              ...\n",
       "161845   žurauskienė          1      žurauskienė\n",
       "137908       ʼt Hoen          1          ʼt hoen\n",
       "172686         Ӧkmen          1            ӧkmen\n",
       "68537        ‘t Hoen          2          ‘t hoen\n",
       "79394        ’t Hoen          2          ’t hoen\n",
       "\n",
       "[183740 rows x 3 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simple_last_name_df = simplify_names(path_to_df.values(), 'last_name')\n",
    "simple_last_name_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fore_name</th>\n",
       "      <th>n_authors</th>\n",
       "      <th>fore_name_simple</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14602</th>\n",
       "      <td>Kuo-Bin</td>\n",
       "      <td>11</td>\n",
       "      <td>kuo-bin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15588</th>\n",
       "      <td>Mirosław</td>\n",
       "      <td>10</td>\n",
       "      <td>mirosław</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21273</th>\n",
       "      <td>Collin M</td>\n",
       "      <td>7</td>\n",
       "      <td>collin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28687</th>\n",
       "      <td>Lee-Wei</td>\n",
       "      <td>5</td>\n",
       "      <td>lee-wei</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36287</th>\n",
       "      <td>Ronak H</td>\n",
       "      <td>4</td>\n",
       "      <td>ronak</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33168</th>\n",
       "      <td>Baoqiang</td>\n",
       "      <td>4</td>\n",
       "      <td>baoqiang</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33974</th>\n",
       "      <td>Avichai</td>\n",
       "      <td>4</td>\n",
       "      <td>avichai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45063</th>\n",
       "      <td>Christian Clement</td>\n",
       "      <td>3</td>\n",
       "      <td>christian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42722</th>\n",
       "      <td>Sárka</td>\n",
       "      <td>3</td>\n",
       "      <td>sárka</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41268</th>\n",
       "      <td>Ke-Yi</td>\n",
       "      <td>3</td>\n",
       "      <td>ke-yi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53395</th>\n",
       "      <td>Siegfried G</td>\n",
       "      <td>2</td>\n",
       "      <td>siegfried</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138102</th>\n",
       "      <td>Chitkala</td>\n",
       "      <td>1</td>\n",
       "      <td>chitkala</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102277</th>\n",
       "      <td>Jiaxiuyu</td>\n",
       "      <td>1</td>\n",
       "      <td>jiaxiuyu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133267</th>\n",
       "      <td>Su Jeon</td>\n",
       "      <td>1</td>\n",
       "      <td>su</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141512</th>\n",
       "      <td>Mary L.</td>\n",
       "      <td>1</td>\n",
       "      <td>mary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91438</th>\n",
       "      <td>Wufan</td>\n",
       "      <td>1</td>\n",
       "      <td>wufan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122886</th>\n",
       "      <td>Ze-Xiang</td>\n",
       "      <td>1</td>\n",
       "      <td>ze-xiang</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101529</th>\n",
       "      <td>Yin-Rong</td>\n",
       "      <td>1</td>\n",
       "      <td>yin-rong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151418</th>\n",
       "      <td>Moïra B</td>\n",
       "      <td>1</td>\n",
       "      <td>moïra</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76031</th>\n",
       "      <td>Najah T</td>\n",
       "      <td>1</td>\n",
       "      <td>najah</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                fore_name  n_authors fore_name_simple\n",
       "14602             Kuo-Bin         11          kuo-bin\n",
       "15588            Mirosław         10         mirosław\n",
       "21273            Collin M          7           collin\n",
       "28687             Lee-Wei          5          lee-wei\n",
       "36287             Ronak H          4            ronak\n",
       "33168            Baoqiang          4         baoqiang\n",
       "33974             Avichai          4          avichai\n",
       "45063   Christian Clement          3        christian\n",
       "42722               Sárka          3            sárka\n",
       "41268               Ke-Yi          3            ke-yi\n",
       "53395         Siegfried G          2        siegfried\n",
       "138102           Chitkala          1         chitkala\n",
       "102277           Jiaxiuyu          1         jiaxiuyu\n",
       "133267            Su Jeon          1               su\n",
       "141512            Mary L.          1             mary\n",
       "91438               Wufan          1            wufan\n",
       "122886           Ze-Xiang          1         ze-xiang\n",
       "101529           Yin-Rong          1         yin-rong\n",
       "151418            Moïra B          1            moïra\n",
       "76031             Najah T          1            najah"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simple_fore_name_df.sample(n=20, random_state=0).sort_values('n_authors', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>last_name</th>\n",
       "      <th>n_authors</th>\n",
       "      <th>last_name_simple</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2200</th>\n",
       "      <td>Leite</td>\n",
       "      <td>54</td>\n",
       "      <td>leite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10294</th>\n",
       "      <td>Wieland</td>\n",
       "      <td>15</td>\n",
       "      <td>wieland</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36171</th>\n",
       "      <td>Delmont</td>\n",
       "      <td>5</td>\n",
       "      <td>delmont</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35443</th>\n",
       "      <td>Rönnblom</td>\n",
       "      <td>5</td>\n",
       "      <td>rönnblom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35906</th>\n",
       "      <td>Saia-Cereda</td>\n",
       "      <td>5</td>\n",
       "      <td>saia-cereda</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53155</th>\n",
       "      <td>Goichon</td>\n",
       "      <td>3</td>\n",
       "      <td>goichon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49440</th>\n",
       "      <td>van Peij</td>\n",
       "      <td>3</td>\n",
       "      <td>van peij</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79396</th>\n",
       "      <td>Antonovsky</td>\n",
       "      <td>2</td>\n",
       "      <td>antonovsky</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81386</th>\n",
       "      <td>Tala</td>\n",
       "      <td>2</td>\n",
       "      <td>tala</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79182</th>\n",
       "      <td>Lecube</td>\n",
       "      <td>2</td>\n",
       "      <td>lecube</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70978</th>\n",
       "      <td>Mbogning</td>\n",
       "      <td>2</td>\n",
       "      <td>mbogning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84500</th>\n",
       "      <td>Nevarez</td>\n",
       "      <td>2</td>\n",
       "      <td>nevarez</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181468</th>\n",
       "      <td>Yacoubian</td>\n",
       "      <td>1</td>\n",
       "      <td>yacoubian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111230</th>\n",
       "      <td>Divate</td>\n",
       "      <td>1</td>\n",
       "      <td>divate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128755</th>\n",
       "      <td>Lorphelin</td>\n",
       "      <td>1</td>\n",
       "      <td>lorphelin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96323</th>\n",
       "      <td>Litzenberg</td>\n",
       "      <td>1</td>\n",
       "      <td>litzenberg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109717</th>\n",
       "      <td>von Brunn</td>\n",
       "      <td>1</td>\n",
       "      <td>von brunn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134687</th>\n",
       "      <td>Ezquina</td>\n",
       "      <td>1</td>\n",
       "      <td>ezquina</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147649</th>\n",
       "      <td>Abegglen</td>\n",
       "      <td>1</td>\n",
       "      <td>abegglen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173394</th>\n",
       "      <td>Sciote</td>\n",
       "      <td>1</td>\n",
       "      <td>sciote</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          last_name  n_authors last_name_simple\n",
       "2200          Leite         54            leite\n",
       "10294       Wieland         15          wieland\n",
       "36171       Delmont          5          delmont\n",
       "35443      Rönnblom          5         rönnblom\n",
       "35906   Saia-Cereda          5      saia-cereda\n",
       "53155       Goichon          3          goichon\n",
       "49440      van Peij          3         van peij\n",
       "79396    Antonovsky          2       antonovsky\n",
       "81386          Tala          2             tala\n",
       "79182        Lecube          2           lecube\n",
       "70978      Mbogning          2         mbogning\n",
       "84500       Nevarez          2          nevarez\n",
       "181468    Yacoubian          1        yacoubian\n",
       "111230       Divate          1           divate\n",
       "128755    Lorphelin          1        lorphelin\n",
       "96323    Litzenberg          1       litzenberg\n",
       "109717    von Brunn          1        von brunn\n",
       "134687      Ezquina          1          ezquina\n",
       "147649     Abegglen          1         abegglen\n",
       "173394       Sciote          1           sciote"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simple_last_name_df.sample(n=20, random_state=0).sort_values('n_authors', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>last_name_simple</th>\n",
       "      <th>n_authors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>wang</td>\n",
       "      <td>20137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>zhang</td>\n",
       "      <td>17491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>li</td>\n",
       "      <td>17246</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  last_name_simple  n_authors\n",
       "0             wang      20137\n",
       "1            zhang      17491\n",
       "2               li      17246"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Most common last names\n",
    "(\n",
    "    simple_last_name_df\n",
    "    .groupby('last_name_simple')\n",
    "    .n_authors.sum()\n",
    "    .sort_values(ascending=False)\n",
    "    .reset_index()\n",
    "    .head(3)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "simple_fore_name_df.to_csv('data/names/fore-names.tsv.xz', sep='\\t', index=False)\n",
    "simple_last_name_df.to_csv('data/names/last-names.tsv.xz', sep='\\t', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}