ast_colab_is23.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 165,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LWqer1S1S_zo",
        "outputId": "031519c9-b689-4692-f93d-d95e96a9fa51"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: wget in /usr/local/lib/python3.8/dist-packages (3.2)\n"
          ]
        }
      ],
      "source": [
        "import sys\n",
        "if 'google.colab' in sys.modules:\n",
        "    print('Running in Colab.')\n",
        "    !git clone https://github.com/YuanGongND/ast\n",
        "    sys.path.append('./ast')\n",
        "%cd /content/ast/\n",
        "\n",
        "! pip install timm==0.4.5\n",
        "! pip install wget\n",
        "import os, csv, argparse, wget\n",
        "os.environ['TORCH_HOME'] = '/content/ast/pretrained_models'\n",
        "if os.path.exists('/content/ast/pretrained_models') == False:\n",
        "  os.mkdir('/content/ast/pretrained_models')\n",
        "import torch, torchaudio, timm\n",
        "import numpy as np\n",
        "from torch.cuda.amp import autocast\n",
        "import IPython\n",
        "import os\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torchvision.transforms as transforms #pytorch 做data augmentation的套件\n",
        "from torchvision import datasets #pytorch的dataset\n",
        "import pandas as pd\n",
        "from torch.utils.data import DataLoader, Dataset #pytorch 訓練的data以Dataset和dataloader呈現，通常是將一個dataset照自己的檔案整理、標籤好以後丟到dataloader中，以dataloader的形式進行訓練 \n",
        "import time"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch, gc\n",
        "gc.collect()\n",
        "torch.cuda.empty_cache()"
      ],
      "metadata": {
        "id": "JLMbbhVdp1oz"
      },
      "execution_count": 184,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "from src.models import ASTModel\n",
        "\n",
        "# Create a new class that inherits the original ASTModel class\n",
        "class ASTModelVis(ASTModel):\n",
        "    @autocast()\n",
        "    def forward(self, x):\n",
        "        \"\"\"\n",
        "        :param x: the input spectrogram, expected shape: (batch_size, time_frame_num, frequency_bins), e.g., (12, 1024, 128)\n",
        "        :return: prediction\n",
        "        \"\"\"\n",
        "        # expect input x = (batch_size, time_frame_num, frequency_bins), e.g., (12, 1024, 128)\n",
        "        x = x.unsqueeze(1)\n",
        "        x = x.transpose(2, 3)\n",
        "\n",
        "        B = x.shape[0]\n",
        "        x = self.v.patch_embed(x)\n",
        "        cls_tokens = self.v.cls_token.expand(B, -1, -1)\n",
        "        dist_token = self.v.dist_token.expand(B, -1, -1)\n",
        "        x = torch.cat((cls_tokens, dist_token, x), dim=1)\n",
        "        x = x + self.v.pos_embed\n",
        "        x = self.v.pos_drop(x)\n",
        "        for blk in self.v.blocks:\n",
        "            x = blk(x)\n",
        "        x = self.v.norm(x)\n",
        "        x = (x[:, 0] + x[:, 1]) / 2\n",
        "        feature = x\n",
        "\n",
        "        x = self.mlp_head(x)\n",
        "        return x, feature\n",
        "    \n",
        "\n",
        "    def get_att_map(self, block, x):\n",
        "        qkv = block.attn.qkv\n",
        "        num_heads = block.attn.num_heads\n",
        "        scale = block.attn.scale\n",
        "        B, N, C = x.shape\n",
        "        qkv = qkv(x).reshape(B, N, 3, num_heads, C // num_heads).permute(2, 0, 3, 1, 4)\n",
        "        q, k, v = qkv[0], qkv[1], qkv[2]  # make torchscript happy (cannot use tensor as tuple)\n",
        "        attn = (q @ k.transpose(-2, -1)) * scale\n",
        "        attn = attn.softmax(dim=-1)\n",
        "        return attn\n",
        "\n",
        "    def forward_visualization(self, x):\n",
        "        # expect input x = (batch_size, time_frame_num, frequency_bins), e.g., (12, 1024, 128)\n",
        "        x = x.unsqueeze(1)\n",
        "        x = x.transpose(2, 3)\n",
        "\n",
        "        B = x.shape[0]\n",
        "        x = self.v.patch_embed(x)\n",
        "        cls_tokens = self.v.cls_token.expand(B, -1, -1)\n",
        "        dist_token = self.v.dist_token.expand(B, -1, -1)\n",
        "        x = torch.cat((cls_tokens, dist_token, x), dim=1)\n",
        "        x = x + self.v.pos_embed\n",
        "        x = self.v.pos_drop(x)\n",
        "        # save the attention map of each of 12 Transformer layer\n",
        "        att_list = []\n",
        "        for blk in self.v.blocks:\n",
        "            cur_att = self.get_att_map(blk, x)\n",
        "            att_list.append(cur_att)\n",
        "            x = blk(x)\n",
        "        return att_list\n",
        "    \n",
        "\n",
        "def make_features(wav_name, mel_bins, target_length=1024):\n",
        "    waveform, sr = torchaudio.load(wav_name)\n",
        "    assert sr == 16000, 'input audio sampling rate must be 16kHz'\n",
        "\n",
        "    fbank = torchaudio.compliance.kaldi.fbank(\n",
        "        waveform, htk_compat=True, sample_frequency=sr, use_energy=False,\n",
        "        window_type='hanning', num_mel_bins=mel_bins, dither=0.0, frame_shift=10)\n",
        "\n",
        "    n_frames = fbank.shape[0]\n",
        "\n",
        "    p = target_length - n_frames\n",
        "    if p > 0:\n",
        "        m = torch.nn.ZeroPad2d((0, 0, 0, p))\n",
        "        fbank = m(fbank)\n",
        "    elif p < 0:\n",
        "        fbank = fbank[0:target_length, :]\n",
        "\n",
        "    fbank = (fbank - (-4.2677393)) / (4.5689974 * 2)\n",
        "    return fbank\n",
        "\n",
        "\n",
        "def load_label(label_csv):\n",
        "    with open(label_csv, 'r') as f:\n",
        "        reader = csv.reader(f, delimiter=',')\n",
        "        lines = list(reader)\n",
        "    labels = []\n",
        "    ids = []  # Each label has a unique id such as \"/m/068hy\"\n",
        "    for i1 in range(1, len(lines)):\n",
        "        id = lines[i1][1]\n",
        "        label = lines[i1][2]\n",
        "        ids.append(id)\n",
        "        labels.append(label)\n",
        "    return labels\n",
        "\n",
        "# Create an AST model and download the AudioSet pretrained weights\n",
        "audioset_mdl_url = 'https://www.dropbox.com/s/cv4knew8mvbrnvq/audioset_0.4593.pth?dl=1'\n",
        "if os.path.exists('/content/ast/pretrained_models/audio_mdl.pth') == False:\n",
        "  wget.download(audioset_mdl_url, out='/content/ast/pretrained_models/audio_mdl.pth')\n",
        "\n",
        "audioset_mdl_url = 'https://www.dropbox.com/s/q0tbqpwv44pquwy/speechcommands_10_10_0.9812.pth?dl=1'\n",
        "if os.path.exists('/content/ast/pretrained_models/speechcommands_10_10_0.9812.pth') == False:\n",
        "  wget.download(audioset_mdl_url, out='/content/ast/speechcommands_10_10_0.9812.pth')\n",
        "\n",
        "pretrained_mdl_path = audioset_mdl_url\n",
        "# get the frequency and time stride of the pretrained model from its name\n",
        "fstride, tstride = int(pretrained_mdl_path.split('/')[-1].split('_')[1]), int(pretrained_mdl_path.split('/')[-1].split('_')[2].split('.')[0])\n",
        "\n",
        "# Assume each input spectrogram has 1024 time frames\n",
        "input_tdim = 1024\n",
        "checkpoint_path =  '/content/ast/speechcommands_10_10_0.9812.pth'\n",
        "# now load the visualization model\n",
        "ast_mdl = ASTModelVis(label_dim=35, input_tdim=input_tdim, imagenet_pretrain=False, audioset_pretrain=False, fstride=fstride, tstride=tstride)\n",
        "print(f'[*INFO] load checkpoint: {checkpoint_path}')\n",
        "\n",
        "\n",
        "# audioset input sequence length is 1024\n",
        "pretrained_mdl_path = '/content/ast/speechcommands_10_10_0.9812.pth'\n",
        "# get the frequency and time stride of the pretrained model from its name\n",
        "fstride, tstride = int(pretrained_mdl_path.split('/')[-1].split('_')[1]), int(pretrained_mdl_path.split('/')[-1].split('_')[2].split('.')[0])\n",
        "# The input of audioset pretrained model is 1024 frames.\n",
        "input_tdim = 1024\n",
        "\n",
        "# initialize an AST model\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "sd = torch.load(pretrained_mdl_path, map_location=device)\n",
        "\n",
        "new_ckpt = {}\n",
        "for key in sd:\n",
        "    if \"v.pos_embed\" in key or \"mlp_head.1.\" in key:\n",
        "      continue\n",
        "    new_ckpt[key[7:]] = sd[key]\n",
        "\n",
        "audio_model = ASTModelVis(input_tdim=input_tdim, fstride=fstride, tstride=tstride)\n",
        "audio_model = torch.nn.DataParallel(audio_model)\n",
        "audio_model.load_state_dict(new_ckpt, strict=False)"
      ],
      "metadata": {
        "id": "J8UjmbUxTEMU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_label_path = '/content/drive/MyDrive/LogME-CTC/different_upstream_model/n_y_test_speech.npy'\n",
        "test_feature_path = '/content/drive/MyDrive/LogME-CTC/different_upstream_model/n_x_test_speech.npy'\n",
        "test_features = np.load(test_feature_path)\n",
        "test_labels = np.argmax(np.load(test_label_path), axis = 1)\n",
        "test_features = np.squeeze(test_features, axis=3)\n",
        "\n",
        "train_label_path = '/content/drive/MyDrive/LogME-CTC/different_upstream_model/n_y_train_speech.npy'\n",
        "train_feature_path = '/content/drive/MyDrive/LogME-CTC/different_upstream_model/n_x_train_speech.npy'\n",
        "train_features = np.load(train_feature_path)\n",
        "train_labels = np.argmax(np.load(train_label_path),axis = 1)\n",
        "train_features = np.squeeze(train_features, axis=3)"
      ],
      "metadata": {
        "id": "KXpcqaMWUH4z"
      },
      "execution_count": 171,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "torch.manual_seed(0)\n",
        "import random\n",
        "random.seed(0)\n",
        "np.random.seed(0)"
      ],
      "metadata": {
        "id": "J6DnnUcIh386"
      },
      "execution_count": 172,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "wdmAfgmcoB-u"
      },
      "execution_count": 162,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import DataLoader\n",
        "train_set = torch.utils.data.TensorDataset(torch.Tensor(train_features),torch.Tensor(train_labels))\n",
        "val_num = 1000\n",
        "batch_size = 1\n",
        "train_set, val_set = torch.utils.data.random_split(train_set, [17049-val_num, val_num])\n",
        "train_loader = DataLoader(train_set, batch_size=batch_size, sampler=None,pin_memory=True, num_workers=8,shuffle=True)\n",
        "val_loader = DataLoader(train_set, batch_size=batch_size, sampler=None,pin_memory=True, num_workers=8,shuffle=True)\n",
        "\n",
        "test_set = torch.utils.data.TensorDataset(torch.Tensor(test_features),torch.Tensor(test_labels))\n",
        "test_loader = DataLoader(test_set, batch_size=batch_size, sampler=None,pin_memory=True, num_workers=8,shuffle=False)"
      ],
      "metadata": {
        "id": "bqHd2LgyVYzB"
      },
      "execution_count": 203,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data[1][1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "psHOOyd_o91d",
        "outputId": "2ac789b9-dce7-4204-afdb-b20350e52c70"
      },
      "execution_count": 178,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(3.)"
            ]
          },
          "metadata": {},
          "execution_count": 178
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "model = ASTModelVis(label_dim=10, input_tdim=60, imagenet_pretrain=True, audioset_pretrain=False).cuda()\n",
        "loss = nn.CrossEntropyLoss() # 因為是 classification task，所以 loss 使用 CrossEntropyLoss\n",
        "optimizer = torch.optim.Adagrad(model.parameters(), lr=0.0001) # optimizer 使用Adagrad\n",
        "num_epoch = 5\n",
        "\n",
        "#adam (momemtum)\n",
        "\n",
        "\n",
        "\n",
        "for epoch in range(num_epoch):\n",
        "    epoch_start_time = time.time()\n",
        "    train_acc = 0.0\n",
        "    train_loss = 0.0\n",
        "    val_acc = 0.0\n",
        "    val_loss = 0.0\n",
        "\n",
        "    model.train() # 確保 model 是在 train model (開啟 Dropout 等...)\n",
        "    for i, data in enumerate(train_loader):\n",
        "        optimizer.zero_grad() # 用 optimizer 將 model 參數的 gradient \n",
        "        train_pred,_ = model(data[0].cuda()) # 利用 model 得到預測的機率分佈 這邊實際上就是去呼叫 model 的 forward 函數\n",
        "        batch_loss = loss(train_pred, data[1].type(torch.LongTensor).cuda()) # 計算 loss （注意 prediction 跟 label 必須同時在 CPU 或是 GPU 上）\n",
        "        batch_loss.backward() # 利用 back propagation 算出每個參數的 gradient\n",
        "        optimizer.step() # 以 optimizer 用 gradient 更新參數值\n",
        "        print(\"\\r\",i, \"/\",len(train_set)/batch_size,sep=\"\", end = \"\")\n",
        "\n",
        "        train_acc += np.sum(np.argmax(train_pred.cpu().data.numpy(), axis=1) == data[1].numpy())\n",
        "        train_loss += batch_loss.item()\n",
        "    \n",
        "    model.eval()\n",
        "    with torch.no_grad():\n",
        "        for i, data in enumerate(val_loader):\n",
        "            val_pred,_ = model(data[0].cuda())\n",
        "            batch_loss = loss(val_pred, data[1].cuda())\n",
        "\n",
        "            val_acc += np.sum(np.argmax(val_pred.cpu().data.numpy(), axis=1) == data[1].numpy())\n",
        "            val_loss += batch_loss.item()\n",
        "\n",
        "        #將結果 print 出來\n",
        "        print('[%03d/%03d] %2.2f sec(s) Train Acc: %3.6f Loss: %3.6f | Val Acc: %3.6f loss: %3.6f' % \\\n",
        "            (epoch + 1, num_epoch, time.time()-epoch_start_time, \\\n",
        "             train_acc/train_set.__len__(), train_loss/train_set.__len__(), val_acc/val_set.__len__(), val_loss/val_set.__len__()))"
      ],
      "metadata": {
        "id": "OT7C6KDlnTFa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "extracted_features = torch.zeros([0,768])\n",
        "model.eval()\n",
        "train_acc = 0\n",
        "with torch.no_grad():\n",
        "    for idx, (data, label) in enumerate(test_loader):\n",
        "        train_pred, train_feature = model(data.cuda())\n",
        "        extracted_features = torch.cat((extracted_features, train_feature.cpu()),0)\n",
        "        train_acc += np.sum(np.argmax(train_pred.cpu().data.numpy(), axis=1) == label.cpu().data.numpy())\n",
        "        print(\"\\r\",idx,\"/\", len(test_loader),train_acc, end =\"\")\n",
        "features_output = extracted_features\n",
        "print(features_output.shape, train_acc/test_set.__len__())\n",
        "np.save(\"/content/drive/MyDrive/LogME-CTC/different_upstream_model/tuned_mel_features_output.npy\",features_output.detach().numpy())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A2RnIagDri_S",
        "outputId": "75d2ab90-1e10-4ca7-dd9b-4423cc232929"
      },
      "execution_count": 200,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py:554: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " 4262 / 4263 3630torch.Size([4263, 768]) 0.8515130190007038\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from src.models import ASTModel\n",
        "ast_model = ASTModelVis(label_dim=10, input_tdim=60, imagenet_pretrain=True, audioset_pretrain=False).cuda()\n",
        "extracted_features = torch.zeros([0,768])\n",
        "ast_model.eval()\n",
        "train_acc = 0\n",
        "with torch.no_grad():\n",
        "    for idx, (data, label) in enumerate(test_loader):\n",
        "        train_pred, train_feature = ast_model(data.cuda())\n",
        "        extracted_features = torch.cat((extracted_features, train_feature.cpu()),0)\n",
        "        train_acc += np.sum(np.argmax(train_pred.cpu().data.numpy(), axis=1) == label.cpu().data.numpy())\n",
        "        print(\"\\r\",idx,\"/\", len(test_loader),train_acc, end =\"\")\n",
        "features_output = extracted_features\n",
        "print(features_output.shape, train_acc/test_set.__len__())\n",
        "np.save(\"/content/drive/MyDrive/LogME-CTC/different_upstream_model/image_pretrained_mel_features_output.npy\",features_output.detach().numpy())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tYC7C5ILVh07",
        "outputId": "67cd4ae1-7f98-456e-dfd7-d69a31c3b60c"
      },
      "execution_count": 201,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------AST Model Summary---------------\n",
            "ImageNet pretraining: True, AudioSet pretraining: False\n",
            "frequncey stride=10, time stride=10\n",
            "number of patches=60\n",
            " 4262 / 4263 367torch.Size([4263, 768]) 0.08608960825709594\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from src.models import ASTModel\n",
        "ast_model = ASTModelVis(label_dim=10, input_tdim=60, imagenet_pretrain=True, audioset_pretrain=True).cuda()\n",
        "extracted_features = torch.zeros([0,768])\n",
        "ast_model.eval()\n",
        "train_acc = 0\n",
        "with torch.no_grad():\n",
        "    for idx, (data, label) in enumerate(test_loader):\n",
        "        train_pred, train_feature = ast_model(data.cuda())\n",
        "        extracted_features = torch.cat((extracted_features, train_feature.cpu()),0)\n",
        "        train_acc += np.sum(np.argmax(train_pred.cpu().data.numpy(), axis=1) == label.cpu().data.numpy())\n",
        "        print(\"\\r\",idx,\"/\", len(test_loader),train_acc, end =\"\")\n",
        "features_output = extracted_features\n",
        "print(features_output.shape, train_acc/test_set.__len__())\n",
        "np.save(\"/content/drive/MyDrive/LogME-CTC/different_upstream_model/audio_pretrained_mel_features_output.npy\",features_output.detach().numpy())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RDY6LB4ZVuDw",
        "outputId": "cba21a23-5da4-473b-affc-298c343e8a74"
      },
      "execution_count": 202,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------AST Model Summary---------------\n",
            "ImageNet pretraining: True, AudioSet pretraining: True\n",
            "frequncey stride=10, time stride=10\n",
            "number of patches=60\n",
            " 4262 / 4263 447torch.Size([4263, 768]) 0.10485573539760731\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from src.models import ASTModel\n",
        "ast_model = ASTModelVis(label_dim=10, input_tdim=60, imagenet_pretrain=True, audioset_pretrain=True).cuda()\n",
        "audio_model.load_state_dict(new_ckpt, strict=False)\n",
        "extracted_features = torch.zeros([0,768])\n",
        "ast_model.eval()\n",
        "train_acc = 0\n",
        "with torch.no_grad():\n",
        "    for idx, (data, label) in enumerate(test_loader):\n",
        "        train_pred, train_feature = ast_model(data.cuda())\n",
        "        extracted_features = torch.cat((extracted_features, train_feature.cpu()),0)\n",
        "        train_acc += np.sum(np.argmax(train_pred.cpu().data.numpy(), axis=1) == label.cpu().data.numpy())\n",
        "        print(\"\\r\",idx,\"/\", len(test_loader),train_acc, end =\"\")\n",
        "features_output = extracted_features\n",
        "print(features_output.shape, train_acc/test_set.__len__())\n",
        "np.save(\"/content/drive/MyDrive/LogME-CTC/different_upstream_model/ks_pretrained_mel_features_output.npy\",features_output.detach().numpy())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tqrqOG6QuMQ6",
        "outputId": "bc536428-81ca-4213-d18d-2937ec9f371c"
      },
      "execution_count": 204,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------AST Model Summary---------------\n",
            "ImageNet pretraining: True, AudioSet pretraining: True\n",
            "frequncey stride=10, time stride=10\n",
            "number of patches=60\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py:554: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " 4262 / 4263 256torch.Size([4263, 768]) 0.0600516068496364\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from src.models import ASTModel\n",
        "ast_model = ASTModelVis(label_dim=10, input_tdim=60, imagenet_pretrain=False, audioset_pretrain=False).cuda()\n",
        "extracted_features = torch.zeros([0,768])\n",
        "ast_model.eval()\n",
        "train_acc = 0\n",
        "with torch.no_grad():\n",
        "    for idx, (data, label) in enumerate(train_loader):\n",
        "        train_pred, train_feature = ast_model(data.cuda())\n",
        "        extracted_features = torch.cat((extracted_features, train_feature.cpu()),0)\n",
        "        train_acc += np.sum(np.argmax(train_pred.cpu().data.numpy(), axis=1) == np.argmax(label.cpu().data.numpy(), axis=1))\n",
        "        print(\"\\r\",idx,\"/\", len(train_loader),train_acc, end =\"\")\n",
        "        torch.cuda.empty_cache()\n",
        "features_output = extracted_features\n",
        "print(features_output.shape, train_acc/train_set.__len__())\n",
        "np.save(\"/content/drive/MyDrive/LogME-CTC/different_upstream_model/no_pretrained_mel_features_output.npy\",features_output.detach().numpy())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jcitNDwUeyiX",
        "outputId": "1d00113a-f518-402d-a0f4-6683f589246d"
      },
      "execution_count": 153,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------AST Model Summary---------------\n",
            "ImageNet pretraining: False, AudioSet pretraining: False\n",
            "frequncey stride=10, time stride=10\n",
            "number of patches=60\n",
            " 33 / 34 439torch.Size([4263, 768]) 0.10297912268355618\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from src.models import ASTModel\n",
        "ast_model = ASTModelVis(label_dim=10, input_tdim=60, imagenet_pretrain=True, audioset_pretrain=True).cuda()\n",
        "extracted_features = torch.zeros([0,768])\n",
        "ast_model.eval()\n",
        "train_acc = 0\n",
        "with torch.no_grad():\n",
        "    for idx, (data, label) in enumerate(train_loader):\n",
        "        train_pred, train_feature = ast_model(data.cuda())\n",
        "        extracted_features = torch.cat((extracted_features, train_feature.cpu()),0)\n",
        "        train_acc += np.sum(np.argmax(train_pred.cpu().data.numpy(), axis=1) == np.argmax(label.cpu().data.numpy(), axis=1))\n",
        "        print(\"\\r\",idx,\"/\", len(train_loader),train_acc, end =\"\")\n",
        "        torch.cuda.empty_cache()\n",
        "features_output = extracted_features\n",
        "print(features_output.shape, train_acc/train_set.__len__())\n",
        "np.save(\"/content/drive/MyDrive/LogME-CTC/different_upstream_model/speech_command_pretrained_mel_features_output.npy\",features_output.detach().numpy())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qF-I3H2ee3wU",
        "outputId": "1f99fef9-c691-4159-ae28-523219108555"
      },
      "execution_count": 96,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------AST Model Summary---------------\n",
            "ImageNet pretraining: True, AudioSet pretraining: True\n",
            "frequncey stride=10, time stride=10\n",
            "number of patches=60\n",
            " 166 / 167 2339torch.Size([21312, 768]) 0.10975037537537538\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from src.traintest import train, validate"
      ],
      "metadata": {
        "id": "4H7XY-9_mMLM"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}