eeg_biometrics_data_split.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Autoencoder",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/BRomans/IdMind/blob/main/autoencoder_data.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8sQFvwJpfLSt"
      },
      "source": [
        "from google.colab import drive\r\n",
        "import numpy as np\r\n",
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gXQT6VF9fRAK"
      },
      "source": [
        "drive.mount(\"/content/drive\")\r\n",
        "filepath = \"/content/drive/MyDrive/ml2-eeg-biometrics/eeg_dataset_right_hand_task_subset_9channels.csv\" "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RoKUCENPWgfV"
      },
      "source": [
        "# Read in the full dataset (i.e. the file with right hand task subset 9 channels)\r\n",
        "df=pd.read_csv(filepath)\r\n",
        "print(df.shape)\r\n",
        "df.dropna(axis=1,how='all',inplace=True)\r\n",
        "print(df.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "stFfQTHJGcI7"
      },
      "source": [
        "# Need to exclude S1 0724 Run2 Trial 10 because of incomplete data.\r\n",
        "df = df[~ ((df['Participant']=='S1') & (df['Date']==20200724) & (df['Run']=='Run2') & (df['Trial']==10))]\r\n",
        "df.reset_index(drop=True, inplace=True) # Reset the index because of the dropped rows.\r\n",
        "\r\n",
        "df.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8FlBa6ByYwbR"
      },
      "source": [
        "# Create a train-validation-test split.\r\n",
        "# For testing we want to take a separate person entirely (let's take S12) - this constitutes 5% of the total dataset.\r\n",
        "# For validation we want to keep it equal across participants and sessions. We'll take 1 run per session for each of the participants. \r\n",
        "# # This is one sixth of the remaining data which equates to 15.8% of the total dataset. Thus, we keep 79.2% for training data.\r\n",
        "\r\n",
        "# Convert Partipant ID to an integer column 'Target' so that the NN can handle it.\r\n",
        "# df['Target'] = df['Participant'].astype('category').cat.codes.values\r\n",
        "\r\n",
        "# test_arr = np.array(df[df['Participant']=='S12'])\r\n",
        "# print(\"test_arr shape:\", test_arr.shape)\r\n",
        "\r\n",
        "# Run5 looks most complete in the data.\r\n",
        "# valid_arr = np.array(df[(df['Run']=='Run5') & (df['Participant']!='S12')])\r\n",
        "# print(\"valid_arr shape:\", valid_arr.shape)\r\n",
        "\r\n",
        "\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n2BTlVnVCe_4"
      },
      "source": [
        "def reshape_data(arr, n_timepoints=2500, filename=None):\r\n",
        "  \"\"\" \r\n",
        "  Function to split the ID, target & feature columns and reshape the data into the \r\n",
        "    required format for the AE.\r\n",
        "  Input:\r\n",
        "      arr           np.ndarray containing the 5 ID columns, the channel measurements and integer target (ID)\r\n",
        "      n_timepoints  (Optional) integer specifying the number of timepoints per sample.\r\n",
        "      filename      (Optional) if specified, then the data will be written to file in train-test-data with filename included in the name of the written file.\r\n",
        "  Output: \r\n",
        "    [All np.ndarrays]\r\n",
        "      x_arr   The feature values in shape [n_samples, timepoints, features]\r\n",
        "      y_arr   The (integer) target values in shape [n_samples, 1, 1]\r\n",
        "      id_arr  The categorical identifiers (Date, Run, etc.) in shape [n_samples, 1, 5]\r\n",
        "  \"\"\"\r\n",
        "  x_arr = arr[:,5:14] # Exclude the first 5 ID columns and the 15th column (Target)\r\n",
        "  y_arr = arr[:,14] # Last column - target (participant ID as integer)\r\n",
        "  id_arr = arr[:,:5] # ['Participant','Date', 'Run', 'Task', 'Trial']\r\n",
        "\r\n",
        "  # Find dimensions for new shape.\r\n",
        "  n_rows = len(x_arr)\r\n",
        "  n_samples = int(n_rows/n_timepoints)\r\n",
        "  \r\n",
        "  # Reshape the three arrays to the required shape.\r\n",
        "  x_arr = x_arr.reshape((n_samples, n_timepoints, x_arr.shape[1]))\r\n",
        "  y_arr = y_arr.reshape((n_samples, n_timepoints, 1))\r\n",
        "  id_arr = id_arr.reshape((n_samples, n_timepoints, id_arr.shape[1]))\r\n",
        "\r\n",
        "  # We do not need the target values / ID values to be replicated n_timepoints for each sample.\r\n",
        "  # Reduce these to just 1 value per sample.\r\n",
        "  y_arr_reduced = np.amin(y_arr, axis=1, keepdims=True) # Take the minimum along the second dimension.\r\n",
        "  # Check if this is the same as the max value, if not then something has gone wrong.\r\n",
        "  if not np.all(y_arr_reduced==np.amax(y_arr, axis=1, keepdims=True)):\r\n",
        "    raise ValueError(\"The target value for each sample does not look to be consistent.\")\r\n",
        "\r\n",
        "  id_arr_reduced = np.amin(id_arr, axis=1, keepdims=True) # Take the minimum along the second dimension.\r\n",
        "\r\n",
        "  print(\"x_arr shape: {} \\ny_arr shape: {} \\nid_arr shape: {}\".format(    \\\r\n",
        "                                x_arr.shape, y_arr_reduced.shape, id_arr_reduced.shape))\r\n",
        "\r\n",
        "  if filename is not None:\r\n",
        "    filepath = \"/content/drive/MyDrive/ml2-eeg-biometrics/train-test-data/\"\r\n",
        "    print(\"Writing to file...\") \r\n",
        "    np.save(filepath + \"x_\" + filename + \".npy\", x_arr)\r\n",
        "    np.save(filepath + \"y_\" + filename + \".npy\", y_arr_reduced)\r\n",
        "    np.save(filepath + \"id_\" + filename + \".npy\", id_arr_reduced)\r\n",
        "    print(\"Writing done.\")\r\n",
        "\r\n",
        "  return x_arr, y_arr_reduced, id_arr_reduced"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qcs8nrnSNZd3"
      },
      "source": [
        "b = np.arange(15*6).reshape(6,15)\r\n",
        "# Ensure we have the same value in the 15th column for all samples from the same target. (n_timepoints=2)\r\n",
        "b[0:3, 14] = 10\r\n",
        "b[3:6, 14] = 20\r\n",
        "print(b)\r\n",
        "reshape_data(b, n_timepoints=3)\r\n",
        "# reshape_data(b, n_timepoints=2) throws an error since the target value is not the same for each sample."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PUYtwkMKThTP"
      },
      "source": [
        "# x_valid, y_valid, id_valid = reshape_data(valid_arr, filename='valid')\r\n",
        "# x_test, y_test, id_test = reshape_data(test_arr, filename='test')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xfLvYdsu0HM-"
      },
      "source": [
        "## Have to do it piece by piece for training data since it was crashing RAM\r\n",
        "\r\n",
        "## # *** Crashed RAM when running everything all at once, so wrote to file restarted and read back in. ***\r\n",
        "## train_df = df[(df['Run']!='Run5') & (df['Participant']!='S12')]\r\n",
        "## train_df.to_csv(\"/content/drive/MyDrive/ml2-eeg-biometrics/train_df_tmp.csv\", index=False)\r\n",
        "## train_arr = np.genfromtxt(\"/content/drive/MyDrive/ml2-eeg-biometrics/train_df_tmp.csv\", delimiter=',', skip_header=1) # Doesn't work (RAM)\r\n",
        "## train_arr = pd.read_csv(\"/content/drive/MyDrive/ml2-eeg-biometrics/train_df_tmp.csv\").to_numpy() # Doesn't work (RAM)\r\n",
        "\r\n",
        "# train_id_df = pd.read_csv(\"/content/drive/MyDrive/ml2-eeg-biometrics/train_df_tmp.csv\", usecols=['Participant','Date', 'Run', 'Task', 'Trial'])\r\n",
        "# train_id_arr = train_id_df.to_numpy()\r\n",
        "\r\n",
        "# train_y_df = pd.read_csv(\"/content/drive/MyDrive/ml2-eeg-biometrics/train_df_tmp.csv\", usecols=['Target'])\r\n",
        "# train_y_arr = train_y_df.to_numpy()\r\n",
        "\r\n",
        "# cols_to_skip=['Participant','Date', 'Run', 'Task', 'Trial','Target']\r\n",
        "# train_x_df = pd.read_csv(\"/content/drive/MyDrive/ml2-eeg-biometrics/train_df_tmp.csv\", usecols=lambda x: x not in cols_to_skip)\r\n",
        "# train_x_arr = train_x_df.to_numpy()\r\n",
        "\r\n",
        "\r\n",
        "filepath = \"/content/drive/MyDrive/ml2-eeg-biometrics/train-test-data/\"\r\n",
        "\r\n",
        "# n_samples = int(len(train_id_arr)/2500)\r\n",
        "# train_id_arr = train_id_arr.reshape((n_samples, 2500, 5))\r\n",
        "# train_id_arr_reduced = np.amin(train_id_arr, axis=1, keepdims=True)\r\n",
        "# np.save(filepath + \"id_train.npy\", train_id_arr_reduced)\r\n",
        "# print(train_id_arr_reduced.shape)\r\n",
        "\r\n",
        "# n_samples = int(len(train_y_arr)/2500)\r\n",
        "# train_y_arr = train_y_arr.reshape((n_samples, 2500, 1))\r\n",
        "# train_y_arr_reduced = np.amin(train_y_arr, axis=1, keepdims=True)\r\n",
        "# np.save(filepath + \"y_train.npy\", train_y_arr_reduced)\r\n",
        "# print(train_y_arr_reduced.shape)\r\n",
        "\r\n",
        "# n_samples = int(len(train_x_arr)/2500)\r\n",
        "# train_x_arr = train_x_arr.reshape((n_samples, 2500, train_x_arr.shape[1]))\r\n",
        "# np.save(filepath + \"x_train.npy\", train_x_arr)\r\n",
        "# print(train_x_arr.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8x1TbMf18dfG"
      },
      "source": [
        "After running the relevant code, we have now created the following files in the train-test-data/ folder:\r\n",
        "*   x_train.npy\r\n",
        "*   x_test.npy\r\n",
        "*   x_valid.npy\r\n",
        "*   y_train.npy\r\n",
        "*   y_test.npy\r\n",
        "*   y_valid.npy\r\n",
        "*   id_train.npy\r\n",
        "*   id_test.npy\r\n",
        "*   id_valid.npy"
      ]
    }
  ]
}