tutorials/1-example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torchaudio\n",
    "import sys\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import IPython.display as ipd\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "DeepDebugger_path = \"/home/xiangling/git_space/DLVisDebugger\"\n",
    "sys.path.append(DeepDebugger_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:1\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchaudio.datasets import SPEECHCOMMANDS\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SubsetSC(SPEECHCOMMANDS):\n",
    "    def __init__(self, subset: str = None):\n",
    "        super().__init__(\"./\", download=True)\n",
    "\n",
    "        def load_list(filename):\n",
    "            filepath = os.path.join(self._path, filename)\n",
    "            with open(filepath) as fileobj:\n",
    "                return [os.path.normpath(os.path.join(self._path, line.strip())) for line in fileobj]\n",
    "\n",
    "        if subset == \"validation\":\n",
    "            self._walker = load_list(\"validation_list.txt\")\n",
    "        elif subset == \"testing\":\n",
    "            self._walker = load_list(\"testing_list.txt\")\n",
    "        elif subset == \"training\":\n",
    "            excludes = load_list(\"validation_list.txt\") + load_list(\"testing_list.txt\")\n",
    "            excludes = set(excludes)\n",
    "            self._walker = [w for w in self._walker if w not in excludes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create training and testing split of the data. We do not use validation in this tutorial.\n",
    "train_set = SubsetSC(\"training\")\n",
    "test_set = SubsetSC(\"testing\")\n",
    "\n",
    "waveform, sample_rate, label, speaker_id, utterance_number = train_set[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Shape of waveform: {}\".format(waveform.size()))\n",
    "print(\"Sample rate of waveform: {}\".format(sample_rate))\n",
    "\n",
    "plt.plot(waveform.t().numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = sorted(list(set(datapoint[2] for datapoint in train_set)))\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "waveform_first, *_ = train_set[0]\n",
    "ipd.Audio(waveform_first.numpy(), rate=sample_rate)\n",
    "\n",
    "waveform_second, *_ = train_set[1]\n",
    "ipd.Audio(waveform_second.numpy(), rate=sample_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "waveform_last, *_ = train_set[-1]\n",
    "ipd.Audio(waveform_last.numpy(), rate=sample_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_sample_rate = 8000\n",
    "transform = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=new_sample_rate)\n",
    "transformed = transform(waveform)\n",
    "\n",
    "ipd.Audio(transformed.numpy(), rate=new_sample_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_to_index(word):\n",
    "    # Return the position of the word in labels\n",
    "    return torch.tensor(labels.index(word))\n",
    "\n",
    "\n",
    "def index_to_label(index):\n",
    "    # Return the word corresponding to the index in labels\n",
    "    # This is the inverse of label_to_index\n",
    "    return labels[index]\n",
    "\n",
    "\n",
    "word_start = \"yes\"\n",
    "index = label_to_index(word_start)\n",
    "word_recovered = index_to_label(index)\n",
    "\n",
    "print(word_start, \"-->\", index, \"-->\", word_recovered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sequence(batch):\n",
    "    # Make all tensor in a batch the same length by padding with zeros\n",
    "    batch = [item.t() for item in batch]\n",
    "    batch = torch.nn.utils.rnn.pad_sequence(batch, batch_first=True, padding_value=0.)\n",
    "    return batch.permute(0, 2, 1)\n",
    "\n",
    "\n",
    "def collate_fn(batch):\n",
    "\n",
    "    # A data tuple has the form:\n",
    "    # waveform, sample_rate, label, speaker_id, utterance_number\n",
    "\n",
    "    tensors, targets = [], []\n",
    "\n",
    "    # Gather in lists, and encode labels as indices\n",
    "    for waveform, _, label, *_ in batch:\n",
    "        tensors += [waveform]\n",
    "        targets += [label_to_index(label)]\n",
    "\n",
    "    # Group the list of tensors into a batched tensor\n",
    "    tensors = pad_sequence(tensors)\n",
    "    targets = torch.stack(targets)\n",
    "\n",
    "    return tensors, targets\n",
    "\n",
    "\n",
    "batch_size = 256\n",
    "\n",
    "if device == \"cuda:1\":\n",
    "    num_workers = 1\n",
    "    pin_memory = True\n",
    "else:\n",
    "    num_workers = 0\n",
    "    pin_memory = False\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    train_set,\n",
    "    batch_size=batch_size,\n",
    "    shuffle=True,\n",
    "    collate_fn=collate_fn,\n",
    "    num_workers=num_workers,\n",
    "    pin_memory=pin_memory,\n",
    ")\n",
    "record_train_loader = torch.utils.data.DataLoader(\n",
    "    train_set,\n",
    "    batch_size=batch_size,\n",
    "    shuffle=False,\n",
    "    collate_fn=collate_fn,\n",
    "    num_workers=num_workers,\n",
    "    pin_memory=pin_memory,\n",
    ")\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    test_set,\n",
    "    batch_size=batch_size,\n",
    "    shuffle=False,\n",
    "    drop_last=False,\n",
    "    collate_fn=collate_fn,\n",
    "    num_workers=num_workers,\n",
    "    pin_memory=pin_memory,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class M5(nn.Module):\n",
    "    def __init__(self, n_input=1, n_output=35, stride=16, n_channel=32):\n",
    "        super().__init__()\n",
    "        self.conv1 = nn.Conv1d(n_input, n_channel, kernel_size=80, stride=stride)\n",
    "        self.bn1 = nn.BatchNorm1d(n_channel)\n",
    "        self.pool1 = nn.MaxPool1d(4)\n",
    "        self.conv2 = nn.Conv1d(n_channel, n_channel, kernel_size=3)\n",
    "        self.bn2 = nn.BatchNorm1d(n_channel)\n",
    "        self.pool2 = nn.MaxPool1d(4)\n",
    "        self.conv3 = nn.Conv1d(n_channel, 2 * n_channel, kernel_size=3)\n",
    "        self.bn3 = nn.BatchNorm1d(2 * n_channel)\n",
    "        self.pool3 = nn.MaxPool1d(4)\n",
    "        self.conv4 = nn.Conv1d(2 * n_channel, 2 * n_channel, kernel_size=3)\n",
    "        self.bn4 = nn.BatchNorm1d(2 * n_channel)\n",
    "        self.pool4 = nn.MaxPool1d(4)\n",
    "        self.fc1 = nn.Linear(2 * n_channel, n_output)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = F.relu(self.bn1(x))\n",
    "        x = self.pool1(x)\n",
    "        x = self.conv2(x)\n",
    "        x = F.relu(self.bn2(x))\n",
    "        x = self.pool2(x)\n",
    "        x = self.conv3(x)\n",
    "        x = F.relu(self.bn3(x))\n",
    "        x = self.pool3(x)\n",
    "        x = self.conv4(x)\n",
    "        x = F.relu(self.bn4(x))\n",
    "        x = self.pool4(x)\n",
    "        x = F.avg_pool1d(x, x.shape[-1])\n",
    "        x = x.permute(0, 2, 1)\n",
    "        x = x.reshape(x.shape[0], -1)\n",
    "        x = self.fc1(x)\n",
    "        return x\n",
    "    \n",
    "    def feature(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = F.relu(self.bn1(x))\n",
    "        x = self.pool1(x)\n",
    "        x = self.conv2(x)\n",
    "        x = F.relu(self.bn2(x))\n",
    "        x = self.pool2(x)\n",
    "        x = self.conv3(x)\n",
    "        x = F.relu(self.bn3(x))\n",
    "        x = self.pool3(x)\n",
    "        x = self.conv4(x)\n",
    "        x = F.relu(self.bn4(x))\n",
    "        x = self.pool4(x)\n",
    "        x = F.avg_pool1d(x, x.shape[-1])\n",
    "        x = x.permute(0, 2, 1)\n",
    "        x = x.reshape(x.shape[0], -1)\n",
    "        return x\n",
    "    \n",
    "    def prediction(self,x):\n",
    "        x = self.fc1(x)\n",
    "        return x\n",
    "\n",
    "model = M5(n_input=transformed.shape[0], n_output=len(labels))\n",
    "model.to(device)\n",
    "print(model)\n",
    "\n",
    "\n",
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n",
    "\n",
    "n = count_parameters(model)\n",
    "print(\"Number of parameters: %s\" % n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0001)\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.1)  # reduce the learning after 20 epochs by a factor of 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, epoch, log_interval):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "\n",
    "        # apply transform and model on whole batch directly on device\n",
    "        data = transform(data)\n",
    "        logits = model(data)\n",
    "        output = F.log_softmax(logits, dim=1)\n",
    "\n",
    "        # negative log-likelihood for a tensor of size (batch x 1 x n_output)\n",
    "        loss = F.nll_loss(output, target)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print training stats\n",
    "        if batch_idx % log_interval == 0:\n",
    "            print(f\"Train Epoch: {epoch} [{batch_idx * len(data)}/{len(train_loader.dataset)} ({100. * batch_idx / len(train_loader):.0f}%)]\\tLoss: {loss.item():.6f}\")\n",
    "\n",
    "        # update progress bar\n",
    "        # pbar.set_description(str(pbar_update))\n",
    "        pbar.update(pbar_update)\n",
    "        # record loss\n",
    "        losses.append(loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def number_of_correct(pred, target):\n",
    "    # count number of correct predictions\n",
    "    return pred.squeeze().eq(target).sum().item()\n",
    "\n",
    "\n",
    "def get_likely_index(tensor):\n",
    "    # find most likely label index for each element in the batch\n",
    "    return tensor.argmax(dim=-1)\n",
    "\n",
    "\n",
    "def test(model, epoch):\n",
    "    model.eval()\n",
    "    correct = 0\n",
    "    for data, target in test_loader:\n",
    "\n",
    "        data = data.to(device)\n",
    "        target = target.to(device)\n",
    "\n",
    "        # apply transform and model on whole batch directly on device\n",
    "        data = transform(data)\n",
    "        output = model(data)\n",
    "\n",
    "        pred = get_likely_index(output)\n",
    "        correct += number_of_correct(pred, target)\n",
    "\n",
    "        # update progress bar\n",
    "        pbar.update(pbar_update)\n",
    "\n",
    "    print(f\"\\nTest Epoch: {epoch}\\tAccuracy: {correct}/{len(test_loader.dataset)} ({100. * correct / len(test_loader.dataset):.0f}%)\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from writer.summary_writer import SummaryWriter\n",
    "log_dir = \"/home/xiangling/data/speech_comment\" # User define\n",
    "writer = SummaryWriter(log_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer.add_training_data(record_train_loader) # use test_transform\n",
    "writer.add_testing_data(test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_interval = 150\n",
    "n_epoch = 20\n",
    "\n",
    "pbar_update = 1 / (len(train_loader) + len(test_loader))\n",
    "losses = []\n",
    "\n",
    "# The transform needs to live on the same device as the model and the data.\n",
    "idxs = list(range(len(train_loader.dataset)))\n",
    "transform = transform.to(device)\n",
    "with tqdm(total=n_epoch) as pbar:\n",
    "    for epoch in range(1, n_epoch + 1):\n",
    "        train(model, epoch, log_interval)\n",
    "        writer.add_checkpoint_data(model.state_dict(), idxs, epoch, epoch-1)\n",
    "        test(model, epoch)\n",
    "        scheduler.step()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.13 ('timevis': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "03c8ca38b42f6175a6e2def9ce469b2c78a69830a666e0f6ae7493f7267c2706"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}