speaker-recognition.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "470c5706",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-11-30T18:03:54.453934Z",
     "iopub.status.busy": "2023-11-30T18:03:54.453135Z",
     "iopub.status.idle": "2023-11-30T18:03:59.854934Z",
     "shell.execute_reply": "2023-11-30T18:03:59.853948Z"
    },
    "papermill": {
     "duration": 5.420753,
     "end_time": "2023-11-30T18:03:59.863797",
     "exception": false,
     "start_time": "2023-11-30T18:03:54.443044",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "###\n",
    "# Based on: https://www.kaggle.com/code/mohamedatef20/speaker-recognition\n",
    "# Dataset: https://www.kaggle.com/datasets/kongaevans/speaker-recognition-dataset/\n",
    "###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3912a03f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-30T18:03:59.907575Z",
     "iopub.status.busy": "2023-11-30T18:03:59.907066Z",
     "iopub.status.idle": "2023-11-30T18:04:13.375574Z",
     "shell.execute_reply": "2023-11-30T18:04:13.374735Z"
    },
    "papermill": {
     "duration": 13.492354,
     "end_time": "2023-11-30T18:04:13.377873",
     "exception": false,
     "start_time": "2023-11-30T18:03:59.885519",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from itertools import product\n",
    "from datetime import datetime\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import mlflow\n",
    "\n",
    "from Utils.config import DATA_DIR\n",
    "from Utils.create_model import create_model\n",
    "from Utils.get_wav_paths import get_wav_paths\n",
    "from Utils.generate_training_data import generate_training_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4b8efbce",
   "metadata": {},
   "outputs": [],
   "source": [
    "rng = tf.random.experimental.Generator.from_seed(420420)\n",
    "tf.random.set_seed(420420)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "dc55bd0d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-30T18:04:13.460149Z",
     "iopub.status.busy": "2023-11-30T18:04:13.459861Z",
     "iopub.status.idle": "2023-11-30T18:04:13.468762Z",
     "shell.execute_reply": "2023-11-30T18:04:13.467886Z"
    },
    "papermill": {
     "duration": 0.029792,
     "end_time": "2023-11-30T18:04:13.470696",
     "exception": false,
     "start_time": "2023-11-30T18:04:13.440904",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Alexander_Solzhenitsyn',\n",
       " 'Barrack_Obama',\n",
       " 'Benjamin_Netanyau',\n",
       " 'Hillary_Clinton',\n",
       " 'Jens_Stoltenberg',\n",
       " 'John_F_Kennedy',\n",
       " 'Julia_Gillard',\n",
       " 'Margaret_Tarcher',\n",
       " 'Nelson_Mandela',\n",
       " 'Ronald_Reagan']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_dir = DATA_DIR\n",
    "os.listdir(data_dir)\n",
    "\n",
    "# A.Solzhenitsyn was finally excluded from the dataset, \n",
    "# because of language differences (Russian vs English)\n",
    "# and the fact that he was recorded with simultaneous translation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a0dc7bab",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-30T18:04:13.508853Z",
     "iopub.status.busy": "2023-11-30T18:04:13.508521Z",
     "iopub.status.idle": "2023-11-30T18:04:13.515368Z",
     "shell.execute_reply": "2023-11-30T18:04:13.514578Z"
    },
    "papermill": {
     "duration": 0.028077,
     "end_time": "2023-11-30T18:04:13.517220",
     "exception": false,
     "start_time": "2023-11-30T18:04:13.489143",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['0.wav',\n",
       " '1.wav',\n",
       " '10.wav',\n",
       " '100.wav',\n",
       " '1000.wav',\n",
       " '1001.wav',\n",
       " '1002.wav',\n",
       " '1003.wav',\n",
       " '1004.wav',\n",
       " '1005.wav']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nelson_madela = [item for item in os.listdir(data_dir + \"/Nelson_Mandela\")]\n",
    "nelson_madela[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2473f636",
   "metadata": {
    "papermill": {
     "duration": 0.0222,
     "end_time": "2023-11-30T18:04:26.544893",
     "exception": false,
     "start_time": "2023-11-30T18:04:26.522693",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "##  Process training dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e6bf17f8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-30T18:04:26.643963Z",
     "iopub.status.busy": "2023-11-30T18:04:26.643146Z",
     "iopub.status.idle": "2023-11-30T18:04:26.653009Z",
     "shell.execute_reply": "2023-11-30T18:04:26.652278Z"
    },
    "papermill": {
     "duration": 0.035091,
     "end_time": "2023-11-30T18:04:26.655005",
     "exception": false,
     "start_time": "2023-11-30T18:04:26.619914",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of samples for Margaret Tarcher:  1500\n",
      "['0.wav', '1.wav', '10.wav', '100.wav', '1000.wav', '1001.wav', '1002.wav', '1003.wav', '1004.wav', '1005.wav']\n"
     ]
    }
   ],
   "source": [
    "barack_obama_paths = get_wav_paths(\"Barrack_Obama\")\n",
    "benjamin_netanyau_paths = get_wav_paths(\"Benjamin_Netanyau\")\n",
    "hillary_clinton_paths = get_wav_paths(\"Hillary_Clinton\")\n",
    "jens_stoltenberg_paths = get_wav_paths(\"Jens_Stoltenberg\")\n",
    "john_f_kennedy_paths = get_wav_paths(\"John_F_Kennedy\")\n",
    "julia_gillard_paths = get_wav_paths(\"Julia_Gillard\")\n",
    "margaret_thatcher_paths = get_wav_paths(\"Margaret_Tarcher\")\n",
    "nelson_mandela_paths = get_wav_paths(\"Nelson_Mandela\")\n",
    "ronald_reagan_paths = get_wav_paths(\"Ronald_Reagan\")\n",
    "\n",
    "print(\"Number of samples for Margaret Tarcher: \", len(margaret_thatcher_paths))\n",
    "print(margaret_thatcher_paths[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b47997b9",
   "metadata": {
    "papermill": {
     "duration": 0.022636,
     "end_time": "2023-11-30T18:04:26.755467",
     "exception": false,
     "start_time": "2023-11-30T18:04:26.732831",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Additive White Gaussian Noise (AWGN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2903d146",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1266/1266 [00:01<00:00, 968.62it/s]\n",
      "100%|██████████| 1500/1500 [00:01<00:00, 1052.04it/s]\n",
      "100%|██████████| 418/418 [00:00<00:00, 998.52it/s] \n",
      "100%|██████████| 1500/1500 [00:01<00:00, 1036.81it/s]\n",
      "100%|██████████| 418/418 [00:00<00:00, 973.93it/s]\n",
      "100%|██████████| 1500/1500 [00:01<00:00, 1044.57it/s]\n",
      "100%|██████████| 1500/1500 [00:01<00:00, 1015.45it/s]\n",
      "100%|██████████| 1500/1500 [00:01<00:00, 1015.84it/s]\n",
      "100%|██████████| 1911/1911 [00:01<00:00, 972.49it/s] \n"
     ]
    }
   ],
   "source": [
    "barack_obama_wavs, barack_obama_labels = generate_training_data(\n",
    "    barack_obama_paths, \"Barrack_Obama\", 0\n",
    ")\n",
    "benjamin_netanyau_wavs, benjamin_netanyau_labels = generate_training_data(\n",
    "    benjamin_netanyau_paths, \"Benjamin_Netanyau\", 1\n",
    ")\n",
    "hillary_clinton_wavs, hillary_clinton_labels = generate_training_data(\n",
    "    hillary_clinton_paths, \"Hillary_Clinton\", 2\n",
    ")\n",
    "jens_stoltenberg_wavs, jens_stoltenberg_labels = generate_training_data(\n",
    "    jens_stoltenberg_paths, \"Jens_Stoltenberg\", 3\n",
    ")\n",
    "john_f_kennedy_wavs, john_f_kennedy_labels = generate_training_data(\n",
    "    john_f_kennedy_paths, \"John_F_Kennedy\", 4\n",
    ")\n",
    "julia_gillard_wavs, julia_gillard_labels = generate_training_data(\n",
    "    julia_gillard_paths, \"Julia_Gillard\", 5\n",
    ")\n",
    "margaret_thatcher_wavs, margaret_thatcher_labels = generate_training_data(\n",
    "    margaret_thatcher_paths, \"Margaret_Tarcher\", 6\n",
    ")\n",
    "nelson_mandela_wavs, nelson_mandela_labels = generate_training_data(\n",
    "    nelson_mandela_paths, \"Nelson_Mandela\", 7\n",
    ")\n",
    "ronald_reagan_wavs, ronald_reagan_labels = generate_training_data(\n",
    "    ronald_reagan_paths, \"Ronald_Reagan\", 8\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fc230cb",
   "metadata": {},
   "source": [
    "### Make all lists equal in length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a29d2a90",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-30T18:05:16.175580Z",
     "iopub.status.busy": "2023-11-30T18:05:16.174617Z",
     "iopub.status.idle": "2023-11-30T18:05:16.179940Z",
     "shell.execute_reply": "2023-11-30T18:05:16.178907Z"
    },
    "papermill": {
     "duration": 0.074701,
     "end_time": "2023-11-30T18:05:16.182106",
     "exception": false,
     "start_time": "2023-11-30T18:05:16.107405",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_speakers_labels_and_wavs_list = [\n",
    "    barack_obama_labels,\n",
    "    barack_obama_wavs,\n",
    "    benjamin_netanyau_labels,\n",
    "    benjamin_netanyau_wavs,\n",
    "    hillary_clinton_labels,\n",
    "    hillary_clinton_wavs,\n",
    "    jens_stoltenberg_labels,\n",
    "    jens_stoltenberg_wavs,\n",
    "    john_f_kennedy_labels,\n",
    "    john_f_kennedy_wavs,\n",
    "    julia_gillard_labels,\n",
    "    julia_gillard_wavs,\n",
    "    margaret_thatcher_labels,\n",
    "    margaret_thatcher_wavs,\n",
    "    nelson_mandela_labels,\n",
    "    nelson_mandela_wavs,\n",
    "    ronald_reagan_labels,\n",
    "    ronald_reagan_wavs,\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1c1c422e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minimum length:  417\n"
     ]
    }
   ],
   "source": [
    "min_length = min(len(lst) for lst in all_speakers_labels_and_wavs_list)\n",
    "print(\"Minimum length: \", min_length)\n",
    "trimmed_all_speakers_labels_and_wavs = [\n",
    "    lst[:min_length] for lst in all_speakers_labels_and_wavs_list\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "15f7296a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n",
      "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n"
     ]
    }
   ],
   "source": [
    "trimmed_all_speakers_labels = trimmed_all_speakers_labels_and_wavs[0::2]\n",
    "print(len(trimmed_all_speakers_labels))\n",
    "print(trimmed_all_speakers_labels[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9b5eb26f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n",
      "tf.Tensor([[0.00368235 0.00368235 0.00365183 ... 0.00380442 0.00737498 0.00734446]], shape=(1, 16000), dtype=float32)\n",
      "tf.Tensor(\n",
      "[[ 0.06874497  0.01030381 -0.09937637 ...  0.18623765  0.31123766\n",
      "   0.39696154]], shape=(1, 16000), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "trimmed_all_speakers_wavs = trimmed_all_speakers_labels_and_wavs[1::2]\n",
    "print(len(trimmed_all_speakers_wavs))\n",
    "print(trimmed_all_speakers_wavs[1][0])\n",
    "print(trimmed_all_speakers_wavs[1][5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9bb5180c",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_wavs = []\n",
    "\n",
    "for wavs in trimmed_all_speakers_wavs:\n",
    "    all_wavs.extend(wavs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "11b22094",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_labels = []\n",
    "\n",
    "for labels in trimmed_all_speakers_labels:\n",
    "    all_labels.extend(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9926cbee",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-30T18:05:16.490383Z",
     "iopub.status.busy": "2023-11-30T18:05:16.489982Z",
     "iopub.status.idle": "2023-11-30T18:05:17.427239Z",
     "shell.execute_reply": "2023-11-30T18:05:17.426384Z"
    },
    "papermill": {
     "duration": 1.002014,
     "end_time": "2023-11-30T18:05:17.429594",
     "exception": false,
     "start_time": "2023-11-30T18:05:16.427580",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_wavs, test_wavs, train_labels, test_labels = train_test_split(\n",
    "    all_wavs, all_labels, test_size=0.2\n",
    ")\n",
    "\n",
    "train_x = np.array(train_wavs)\n",
    "train_y = np.array(train_labels)\n",
    "test_x, test_y = np.array(test_wavs), np.array(test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6de8e8a1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-30T18:05:17.550289Z",
     "iopub.status.busy": "2023-11-30T18:05:17.549942Z",
     "iopub.status.idle": "2023-11-30T18:05:17.672646Z",
     "shell.execute_reply": "2023-11-30T18:05:17.671647Z"
    },
    "papermill": {
     "duration": 0.185731,
     "end_time": "2023-11-30T18:05:17.674863",
     "exception": false,
     "start_time": "2023-11-30T18:05:17.489132",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3002, 1, 16000)\n",
      "(3002,)\n",
      "[0 1 2 3 4 5 6 7 8]\n",
      "(3002, 1, 16000)\n",
      "(3002, 9)\n",
      "[0. 1.]\n"
     ]
    }
   ],
   "source": [
    "print(train_x.shape)\n",
    "print(train_y.shape)\n",
    "print(np.unique(train_y))\n",
    "\n",
    "test_y_original = test_y\n",
    "\n",
    "train_y = tf.keras.utils.to_categorical(train_y)\n",
    "test_y = tf.keras.utils.to_categorical(test_y)\n",
    "\n",
    "print(train_x.shape)\n",
    "print(train_y.shape)\n",
    "print(np.unique(train_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c44c960d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From d:\\__repos\\wut_easar\\venv\\lib\\site-packages\\keras\\src\\backend.py:873: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "WARNING:tensorflow:From d:\\__repos\\wut_easar\\venv\\lib\\site-packages\\keras\\src\\layers\\pooling\\max_pooling2d.py:161: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:`lr` is deprecated in Keras optimizer, please use `learning_rate` or use the legacy optimizer, e.g.,tf.keras.optimizers.legacy.Adam.\n"
     ]
    }
   ],
   "source": [
    "mlflow.set_experiment(\"speech-recognition\")\n",
    "\n",
    "model = create_model(\"spectrogram\")\n",
    "timestamp = datetime.now().strftime(\"%Y-%m-%d___%H%M__%S%f\")[:-4]\n",
    "classes = [\n",
    "    \"Barrack_Obama\",\n",
    "    \"Benjamin_Netanyau\",\n",
    "    \"Hillary_Clinton\",\n",
    "    \"Jens_Stoltenberg\",\n",
    "    \"John_F_Kennedy\",\n",
    "    \"Julia_Gillard\",\n",
    "    \"Margaret_Tarcher\",\n",
    "    \"Nelson_Mandela\",\n",
    "    \"Ronald_Reagan\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "491ce5f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# hipermarameters tuning - uncomment if needed\n",
    "\n",
    "# spectrogram_types_list = [\"melspectrogram\", \"spectrogram\"]\n",
    "spectrogram_types_list = [\"spectrogram\"]\n",
    "# num_epochs_list = [10, 20, 30, 50, 70]\n",
    "num_epochs_list = [70]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b03d1fea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spectrogram type:  spectrogram\n",
      "Number of epochs:  70\n"
     ]
    }
   ],
   "source": [
    "spectrogram_type = spectrogram_types_list[0]\n",
    "num_epochs = num_epochs_list[0]\n",
    "\n",
    "print(\"Spectrogram type: \", spectrogram_type)\n",
    "print(\"Number of epochs: \", num_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "6a48ca38",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/70\n",
      "WARNING:tensorflow:From d:\\__repos\\wut_easar\\venv\\lib\\site-packages\\keras\\src\\utils\\tf_utils.py:492: The name tf.ragged.RaggedTensorValue is deprecated. Please use tf.compat.v1.ragged.RaggedTensorValue instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From d:\\__repos\\wut_easar\\venv\\lib\\site-packages\\keras\\src\\utils\\tf_utils.py:492: The name tf.ragged.RaggedTensorValue is deprecated. Please use tf.compat.v1.ragged.RaggedTensorValue instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From d:\\__repos\\wut_easar\\venv\\lib\\site-packages\\keras\\src\\engine\\base_layer_utils.py:384: The name tf.executing_eagerly_outside_functions is deprecated. Please use tf.compat.v1.executing_eagerly_outside_functions instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From d:\\__repos\\wut_easar\\venv\\lib\\site-packages\\keras\\src\\engine\\base_layer_utils.py:384: The name tf.executing_eagerly_outside_functions is deprecated. Please use tf.compat.v1.executing_eagerly_outside_functions instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "94/94 [==============================] - 5s 43ms/step - loss: 47.4523 - accuracy: 0.3161 - val_loss: 2.8770 - val_accuracy: 0.6418\n",
      "Epoch 2/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 1.2964 - accuracy: 0.7498 - val_loss: 1.3937 - val_accuracy: 0.7204\n",
      "Epoch 3/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 0.9834 - accuracy: 0.8158 - val_loss: 1.7680 - val_accuracy: 0.6778\n",
      "Epoch 4/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 0.4772 - accuracy: 0.8947 - val_loss: 1.2207 - val_accuracy: 0.7510\n",
      "Epoch 5/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 0.4414 - accuracy: 0.9141 - val_loss: 1.0023 - val_accuracy: 0.7816\n",
      "Epoch 6/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 0.1518 - accuracy: 0.9577 - val_loss: 0.9751 - val_accuracy: 0.8362\n",
      "Epoch 7/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 0.2972 - accuracy: 0.9224 - val_loss: 1.1309 - val_accuracy: 0.8003\n",
      "Epoch 8/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 0.1606 - accuracy: 0.9540 - val_loss: 0.9143 - val_accuracy: 0.8442\n",
      "Epoch 9/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 0.1409 - accuracy: 0.9657 - val_loss: 0.5945 - val_accuracy: 0.8868\n",
      "Epoch 10/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 0.0452 - accuracy: 0.9873 - val_loss: 0.4286 - val_accuracy: 0.9161\n",
      "Epoch 11/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 0.0174 - accuracy: 0.9953 - val_loss: 0.5087 - val_accuracy: 0.8788\n",
      "Epoch 12/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 0.0064 - accuracy: 0.9983 - val_loss: 0.5339 - val_accuracy: 0.8975\n",
      "Epoch 13/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 5.6424e-04 - accuracy: 1.0000 - val_loss: 0.4352 - val_accuracy: 0.9081\n",
      "Epoch 14/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 9.6467e-05 - accuracy: 1.0000 - val_loss: 0.4372 - val_accuracy: 0.9095\n",
      "Epoch 15/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 7.3279e-05 - accuracy: 1.0000 - val_loss: 0.4398 - val_accuracy: 0.9108\n",
      "Epoch 16/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 6.3349e-05 - accuracy: 1.0000 - val_loss: 0.4379 - val_accuracy: 0.9108\n",
      "Epoch 17/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 5.7530e-05 - accuracy: 1.0000 - val_loss: 0.4363 - val_accuracy: 0.9095\n",
      "Epoch 18/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 5.2652e-05 - accuracy: 1.0000 - val_loss: 0.4358 - val_accuracy: 0.9081\n",
      "Epoch 19/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 4.8634e-05 - accuracy: 1.0000 - val_loss: 0.4352 - val_accuracy: 0.9095\n",
      "Epoch 20/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 4.5276e-05 - accuracy: 1.0000 - val_loss: 0.4344 - val_accuracy: 0.9095\n",
      "Epoch 21/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 4.2270e-05 - accuracy: 1.0000 - val_loss: 0.4341 - val_accuracy: 0.9095\n",
      "Epoch 22/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 3.9738e-05 - accuracy: 1.0000 - val_loss: 0.4339 - val_accuracy: 0.9081\n",
      "Epoch 23/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 3.7315e-05 - accuracy: 1.0000 - val_loss: 0.4337 - val_accuracy: 0.9095\n",
      "Epoch 24/70\n",
      "94/94 [==============================] - 4s 40ms/step - loss: 3.5194e-05 - accuracy: 1.0000 - val_loss: 0.4336 - val_accuracy: 0.9081\n",
      "Epoch 25/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 3.3202e-05 - accuracy: 1.0000 - val_loss: 0.4336 - val_accuracy: 0.9081\n",
      "Epoch 26/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 3.1479e-05 - accuracy: 1.0000 - val_loss: 0.4336 - val_accuracy: 0.9081\n",
      "Epoch 27/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 2.9738e-05 - accuracy: 1.0000 - val_loss: 0.4336 - val_accuracy: 0.9068\n",
      "Epoch 28/70\n",
      "94/94 [==============================] - 4s 43ms/step - loss: 2.8327e-05 - accuracy: 1.0000 - val_loss: 0.4337 - val_accuracy: 0.9055\n",
      "Epoch 29/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 2.6830e-05 - accuracy: 1.0000 - val_loss: 0.4331 - val_accuracy: 0.9041\n",
      "Epoch 30/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 2.5549e-05 - accuracy: 1.0000 - val_loss: 0.4331 - val_accuracy: 0.9068\n",
      "Epoch 31/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 2.4365e-05 - accuracy: 1.0000 - val_loss: 0.4332 - val_accuracy: 0.9055\n",
      "Epoch 32/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 2.3227e-05 - accuracy: 1.0000 - val_loss: 0.4333 - val_accuracy: 0.9055\n",
      "Epoch 33/70\n",
      "94/94 [==============================] - 4s 44ms/step - loss: 2.2166e-05 - accuracy: 1.0000 - val_loss: 0.4329 - val_accuracy: 0.9068\n",
      "Epoch 34/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 2.1167e-05 - accuracy: 1.0000 - val_loss: 0.4328 - val_accuracy: 0.9055\n",
      "Epoch 35/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 2.0203e-05 - accuracy: 1.0000 - val_loss: 0.4327 - val_accuracy: 0.9068\n",
      "Epoch 36/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 1.9322e-05 - accuracy: 1.0000 - val_loss: 0.4331 - val_accuracy: 0.9068\n",
      "Epoch 37/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 1.8478e-05 - accuracy: 1.0000 - val_loss: 0.4332 - val_accuracy: 0.9068\n",
      "Epoch 38/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 1.7671e-05 - accuracy: 1.0000 - val_loss: 0.4334 - val_accuracy: 0.9068\n",
      "Epoch 39/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 1.6974e-05 - accuracy: 1.0000 - val_loss: 0.4331 - val_accuracy: 0.9068\n",
      "Epoch 40/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 1.6213e-05 - accuracy: 1.0000 - val_loss: 0.4330 - val_accuracy: 0.9068\n",
      "Epoch 41/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 1.5501e-05 - accuracy: 1.0000 - val_loss: 0.4331 - val_accuracy: 0.9068\n",
      "Epoch 42/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 1.4868e-05 - accuracy: 1.0000 - val_loss: 0.4335 - val_accuracy: 0.9068\n",
      "Epoch 43/70\n",
      "94/94 [==============================] - 4s 43ms/step - loss: 1.4260e-05 - accuracy: 1.0000 - val_loss: 0.4338 - val_accuracy: 0.9068\n",
      "Epoch 44/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 1.3666e-05 - accuracy: 1.0000 - val_loss: 0.4335 - val_accuracy: 0.9068\n",
      "Epoch 45/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 1.3104e-05 - accuracy: 1.0000 - val_loss: 0.4337 - val_accuracy: 0.9068\n",
      "Epoch 46/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 1.2555e-05 - accuracy: 1.0000 - val_loss: 0.4343 - val_accuracy: 0.9068\n",
      "Epoch 47/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 1.2036e-05 - accuracy: 1.0000 - val_loss: 0.4339 - val_accuracy: 0.9068\n",
      "Epoch 48/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 1.1552e-05 - accuracy: 1.0000 - val_loss: 0.4341 - val_accuracy: 0.9068\n",
      "Epoch 49/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 1.1095e-05 - accuracy: 1.0000 - val_loss: 0.4342 - val_accuracy: 0.9068\n",
      "Epoch 50/70\n",
      "94/94 [==============================] - 4s 43ms/step - loss: 1.0664e-05 - accuracy: 1.0000 - val_loss: 0.4344 - val_accuracy: 0.9068\n",
      "Epoch 51/70\n",
      "94/94 [==============================] - 4s 45ms/step - loss: 1.0216e-05 - accuracy: 1.0000 - val_loss: 0.4347 - val_accuracy: 0.9068\n",
      "Epoch 52/70\n",
      "94/94 [==============================] - 4s 43ms/step - loss: 9.7968e-06 - accuracy: 1.0000 - val_loss: 0.4349 - val_accuracy: 0.9068\n",
      "Epoch 53/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 9.4201e-06 - accuracy: 1.0000 - val_loss: 0.4353 - val_accuracy: 0.9068\n",
      "Epoch 54/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 9.0485e-06 - accuracy: 1.0000 - val_loss: 0.4351 - val_accuracy: 0.9068\n",
      "Epoch 55/70\n",
      "94/94 [==============================] - 4s 43ms/step - loss: 8.7117e-06 - accuracy: 1.0000 - val_loss: 0.4356 - val_accuracy: 0.9068\n",
      "Epoch 56/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 8.3661e-06 - accuracy: 1.0000 - val_loss: 0.4358 - val_accuracy: 0.9068\n",
      "Epoch 57/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 8.0465e-06 - accuracy: 1.0000 - val_loss: 0.4358 - val_accuracy: 0.9055\n",
      "Epoch 58/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 7.7253e-06 - accuracy: 1.0000 - val_loss: 0.4358 - val_accuracy: 0.9068\n",
      "Epoch 59/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 7.4173e-06 - accuracy: 1.0000 - val_loss: 0.4363 - val_accuracy: 0.9068\n",
      "Epoch 60/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 7.1310e-06 - accuracy: 1.0000 - val_loss: 0.4364 - val_accuracy: 0.9068\n",
      "Epoch 61/70\n",
      "94/94 [==============================] - 4s 42ms/step - loss: 6.8607e-06 - accuracy: 1.0000 - val_loss: 0.4366 - val_accuracy: 0.9041\n",
      "Epoch 62/70\n",
      "94/94 [==============================] - 4s 43ms/step - loss: 6.5868e-06 - accuracy: 1.0000 - val_loss: 0.4370 - val_accuracy: 0.9055\n",
      "Epoch 63/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 6.3454e-06 - accuracy: 1.0000 - val_loss: 0.4374 - val_accuracy: 0.9055\n",
      "Epoch 64/70\n",
      "94/94 [==============================] - 4s 40ms/step - loss: 6.0914e-06 - accuracy: 1.0000 - val_loss: 0.4370 - val_accuracy: 0.9041\n",
      "Epoch 65/70\n",
      "94/94 [==============================] - 4s 40ms/step - loss: 5.8702e-06 - accuracy: 1.0000 - val_loss: 0.4380 - val_accuracy: 0.9055\n",
      "Epoch 66/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 5.6358e-06 - accuracy: 1.0000 - val_loss: 0.4382 - val_accuracy: 0.9041\n",
      "Epoch 67/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 5.4118e-06 - accuracy: 1.0000 - val_loss: 0.4378 - val_accuracy: 0.9041\n",
      "Epoch 68/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 5.2080e-06 - accuracy: 1.0000 - val_loss: 0.4384 - val_accuracy: 0.9041\n",
      "Epoch 69/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 5.0070e-06 - accuracy: 1.0000 - val_loss: 0.4389 - val_accuracy: 0.9055\n",
      "Epoch 70/70\n",
      "94/94 [==============================] - 4s 41ms/step - loss: 4.8122e-06 - accuracy: 1.0000 - val_loss: 0.4391 - val_accuracy: 0.9055\n",
      "24/24 [==============================] - 0s 8ms/step - loss: 0.4391 - accuracy: 0.9055\n",
      "24/24 [==============================] - 0s 8ms/step\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " static_stft (Spectrogram)   (None, 257, 63, 1)        263168    \n",
      "                                                                 \n",
      " conv2d (Conv2D)             (None, 256, 62, 16)       80        \n",
      "                                                                 \n",
      " max_pooling2d (MaxPooling2  (None, 85, 20, 16)        0         \n",
      " D)                                                              \n",
      "                                                                 \n",
      " flatten (Flatten)           (None, 27200)             0         \n",
      "                                                                 \n",
      " dense (Dense)               (None, 9)                 244809    \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 508057 (1.94 MB)\n",
      "Trainable params: 508057 (1.94 MB)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "train_loss = []\n",
    "train_accuracy = []\n",
    "val_loss = []\n",
    "val_accuracy = []\n",
    "\n",
    "for spectrogram_type, num_epochs in product(spectrogram_types_list, num_epochs_list):\n",
    "    with mlflow.start_run():\n",
    "        mlflow.log_param(\"model\", model)\n",
    "        mlflow.log_param(\"spectrogram\", spectrogram_type)\n",
    "        mlflow.log_param(\"num_epochs\", num_epochs)\n",
    "\n",
    "        history = model.fit(\n",
    "            x=train_x, y=train_y, epochs=num_epochs, validation_data=(test_x, test_y)\n",
    "        )\n",
    "        test_loss, test_accuracy = model.evaluate(test_x, test_y)\n",
    "\n",
    "        mlflow.log_metric(\"train_loss\", history.history[\"loss\"][-1])\n",
    "        mlflow.log_metric(\"train_accuracy\", history.history[\"accuracy\"][-1])\n",
    "        mlflow.log_metric(\"test_loss\", test_loss)\n",
    "        mlflow.log_metric(\"test_accuracy\", test_accuracy)\n",
    "\n",
    "        train_loss.extend(history.history[\"loss\"])\n",
    "        train_accuracy.extend(history.history[\"accuracy\"])\n",
    "        val_loss.extend(history.history[\"val_loss\"])\n",
    "        val_accuracy.extend(history.history[\"val_accuracy\"])\n",
    "\n",
    "        # Confusion Matrix\n",
    "        y_pred = model.predict(test_x)\n",
    "        y_pred_classes = np.argmax(y_pred, axis=1)\n",
    "        y_true = np.argmax(test_y, axis=1)\n",
    "\n",
    "        cm = confusion_matrix(y_true, y_pred_classes)\n",
    "\n",
    "        # Plot Confusion Matrix\n",
    "        plt.figure(figsize=(8, 6))\n",
    "        sns.heatmap(\n",
    "            cm,\n",
    "            annot=True,\n",
    "            fmt=\"g\",\n",
    "            cmap=\"Blues\",\n",
    "            xticklabels=classes,\n",
    "            yticklabels=classes,\n",
    "        )\n",
    "        plt.xlabel(\"Predicted labels\")\n",
    "        plt.ylabel(\"True labels\")\n",
    "        plt.title(\"Confusion Matrix\")\n",
    "        plt.savefig(\"confusion_matrix.png\")\n",
    "        plt.close()\n",
    "\n",
    "        # Log Confusion Matrix as MLflow artifact\n",
    "        mlflow.log_artifact(\"confusion_matrix.png\", \"confusion_matrix_image\")\n",
    "\n",
    "        model.save_weights(f\"{num_epochs}_epochs_{spectrogram_type}.h5\")\n",
    "        mlflow.log_dict(model.summary(), \"model_summary\")\n",
    "\n",
    "        # Plot Accuracy and Loss\n",
    "        plt.figure(figsize=(8, 6))\n",
    "        plt.plot(train_accuracy, label=\"Train Accuracy\")\n",
    "        plt.plot(val_accuracy, label=\"Validation Accuracy\")\n",
    "        plt.xlabel(\"Epochs\")\n",
    "        plt.ylabel(\"Accuracy\")\n",
    "        plt.title(\"Accuracy - Train vs Validation\")\n",
    "        plt.legend()\n",
    "        plt.savefig(\"accuracy_plot.png\")\n",
    "        plt.close()\n",
    "\n",
    "        plt.figure(figsize=(8, 6))\n",
    "        plt.plot(train_loss, label=\"Train Loss\")\n",
    "        plt.plot(val_loss, label=\"Validation Loss\")\n",
    "        plt.xlabel(\"Epochs\")\n",
    "        plt.ylabel(\"Loss\")\n",
    "        plt.title(\"Loss - Train vs Validation\")\n",
    "        plt.legend()\n",
    "        plt.yscale('log') \n",
    "        plt.savefig(\"loss_plot.png\")\n",
    "        plt.close()\n",
    "\n",
    "        # Log Accuracy and Loss plots as MLflow artifacts\n",
    "        mlflow.log_artifact(\"accuracy_plot.png\", \"accuracy_plot\")\n",
    "        mlflow.log_artifact(\"loss_plot.png\", \"loss_plot\")\n"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "nvidiaTeslaT4",
   "dataSources": [
    {
     "datasetId": 470244,
     "sourceId": 881667,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30588,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.3"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 125.75955,
   "end_time": "2023-11-30T18:05:56.575017",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-11-30T18:03:50.815467",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}