wikipron.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "wikipron.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNtl+NATiK9v7AVPskgXbmd",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/deterministic-algorithms-lab/wikipron-modeling/blob/master/wikipron.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B5jguhWiYWMS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PYwkiinfTZLi",
        "colab_type": "text"
      },
      "source": [
        "# Installing Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oKHzEhAnrfci",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install wikipron"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7XiHiGKQ5I06",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install pynini ngram openfst"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KBMlR2rwB8aT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!git clone https://github.com/pytorch/fairseq\n",
        "%cd fairseq\n",
        "!pip install --editable ./"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bGVL40cfCRDC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "276d9aa5-a289-407e-e94a-146ba50bddae"
      },
      "source": [
        "%cd .."
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A3UGPcCWTd23",
        "colab_type": "text"
      },
      "source": [
        "# Preparing Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tzduyw1Zx6_9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!git clone https://github.com/deterministic-algorithms-lab/wikipron-modeling"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a_1EybgQ1GNr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lg = 'hin'"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5j3XRh-Lx_Mi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%cd wikipron-modeling/\n",
        "!mkdir data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LEHDAyG3t-ry",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%cd data\n",
        "!mkdir {train,test,dev}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xJjGiU8JrhyI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#!wikipron {lg} > {lg}.tsv\n",
        "!shuf {lg}.tsv > shuf_{lg}.tsv\n",
        "!rm {lg}.tsv\n",
        "!mv shuf_{lg}.tsv {lg}.tsv"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VHbO31EpFKVQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Cleans Data & splits it(80:10:10). [Remove short words + NFC normalisation + resolve multiple gold targets for same word]\n",
        "!bash ../scripts/prepare"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IWxxr_2iEHTI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mv {lg}_dev.tsv ./dev/\n",
        "!mv {lg}_train.tsv ./train/\n",
        "!mv {lg}_test.tsv ./test/"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KHEs7ZvZTp4I",
        "colab_type": "text"
      },
      "source": [
        "# Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-IVNcw1f1-2E",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "fddfa69c-fd96-41d5-a5cf-a7a0fc9236ab"
      },
      "source": [
        "%cd ../nn"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/wikipron-modeling/nn\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GMbHAEAV4FT3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!bash ./preprocess"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r4zX1mSjDyed",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!fairseq-preprocess --source-lang {lg}.graphemes --target-lang {lg}.phonemes --trainpref ./data-txt/train \\\n",
        "--testpref ./data-txt/test --validpref ./data-txt/dev --destdir ./data-bin/{lg}/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MeZg9TgP5nvD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!bash ./sweep"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rmbD_MsqTRFM",
        "colab_type": "text"
      },
      "source": [
        "# Evaluating On Samples"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jf_rB8U_Yjpr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%cd wikipron-modeling/nn"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TBkX2dKmJZcZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir evaluate"
      ],
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "12aqmNtcIwbX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "path_to_input_file = './data-txt/test.hin.graphemes'"
      ],
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UOaewcz1JxH8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tar_loc = './evaluate/train.'+lg+'.graphemes'\n",
        "!mv {path_to_input_file} {tar_loc}"
      ],
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KwiLDf2dMqXn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "src_dict = './data-bin/'+lg+'/dict.'+lg+'.graphemes.txt'\n",
        "tgt_dict = './data-bin/'+lg+'/dict.'+lg+'.phonemes.txt'\n",
        "\n",
        "!fairseq-preprocess --source-lang {lg}.graphemes --target-lang {lg}.phonemes  --only-source \\\n",
        "                    --trainpref './evaluate/train' --destdir './evaluate/data-bin/' \\\n",
        "                    --srcdict {src_dict} --tgtdict {tgt_dict}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K8Hvr4X7Zi9d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cp {tgt_dict} './evaluate/data-bin/'"
      ],
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L-A0Sbp8Sw2F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "data_dir = './evaluate/data-bin/'\n",
        "for f in os.listdir(data_dir):\n",
        "    path = os.path.join(data_dir,f)\n",
        "    if os.path.isfile(path) and f.startswith('train'):\n",
        "        new_path = os.path.join(data_dir, 'test'+f[5:])\n",
        "        os.rename(path, new_path)"
      ],
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g0Ym_xXXTQoS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "checkpoint_path = 'checkpoints/'+lg+'-64-128-64-128/'+'checkpoint_last.pt'\n",
        "\n",
        "!fairseq-generate './evaluate/data-bin/' \\\n",
        "    --path {checkpoint_path}  --source-lang {lg}.graphemes \\\n",
        "    --target-lang {lg}.phonemes --batch-size 1 --beam 5"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}