From 9686a3874932fe718b13687902cd75a4895eb68c Mon Sep 17 00:00:00 2001 From: Amrit K Date: Tue, 16 Apr 2024 10:53:15 -0400 Subject: [PATCH 1/9] Merge latest from novelty --- .pre-commit-config.yaml | 2 +- evaluation/AttentionVisualization.ipynb | 45 +- finetune.py | 220 ++- odyssey/data/bigbird_data/DataChecker.ipynb | 899 +++++++---- odyssey/data/dataset.py | 236 ++- odyssey/data/sequence.py | 1 + odyssey/data/tokenizer.py | 121 ++ odyssey/models/baseline/Bi-LSTM.ipynb | 18 +- odyssey/models/baseline/XGBoost.ipynb | 1377 +++++++++++++---- odyssey/models/cehr_big_bird/model.py | 65 +- odyssey/models/cehr_big_bird/playground.ipynb | 33 +- odyssey/models/cehr_big_bird/tokenizer.ipynb | 7 +- odyssey/models/configs/cehr_bigbird.yaml | 16 +- odyssey/models/utils.py | 18 + poetry.lock | 671 ++++---- pretrain.py | 32 +- pyproject.toml | 1 + 17 files changed, 2679 insertions(+), 1083 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 60765af..fe39392 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: - id: check-toml - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.3.1' + rev: 'v0.3.7' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/evaluation/AttentionVisualization.ipynb b/evaluation/AttentionVisualization.ipynb index 3946d50..2395ee6 100644 --- a/evaluation/AttentionVisualization.ipynb +++ b/evaluation/AttentionVisualization.ipynb @@ -45,35 +45,15 @@ "outputs": [], "source": [ "import os\n", - "import sys\n", - "from typing import Any, Dict\n", "\n", "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "from matplotlib.patches import Rectangle\n", - "import plotly.figure_factory as ff\n", "import plotly.graph_objects as go\n", - "import seaborn as sns\n", - "\n", - "import pytorch_lightning as pl\n", "import torch\n", - "from torch.utils.data import Subset\n", - "from lightning.pytorch.loggers import WandbLogger\n", - "from pytorch_lightning.callbacks import (\n", - " EarlyStopping,\n", - " LearningRateMonitor,\n", - " ModelCheckpoint,\n", - ")\n", - "from pytorch_lightning.strategies.ddp import DDPStrategy\n", - "from pytorch_lightning.strategies import DeepSpeedStrategy\n", - "from sklearn.model_selection import train_test_split\n", - "from torch.utils.data import DataLoader\n", - "from torch.utils.data import Subset\n", - "\n", - "from bertviz.transformers_neuron_view import BertModel, BertTokenizer\n", + "from bertviz import head_view, model_view\n", "from bertviz.neuron_view import show\n", - "from transformers import AutoTokenizer, AutoModel, utils\n", - "from bertviz import model_view, head_view\n", + "from torch.utils.data import DataLoader, Subset\n", + "from transformers import utils\n", + "\n", "\n", "utils.logging.set_verbosity_error() # Suppress standard warnings\n", "\n", @@ -82,16 +62,11 @@ "os.chdir(ROOT)\n", "\n", "from lib.data import FinetuneDataset\n", + "from lib.prediction import load_finetuned_model, predict_patient_outcomes\n", "from lib.tokenizer import ConceptTokenizer\n", "from lib.utils import (\n", - " get_run_id,\n", - " load_config,\n", " load_finetune_data,\n", - " seed_everything,\n", - ")\n", - "from lib.prediction import load_finetuned_model, predict_patient_outcomes\n", - "from models.big_bird_cehr.model import BigBirdFinetune, BigBirdPretrain\n", - "from models.cehr_bert.model import BertFinetune, BertPretrain" + ")" ] }, { @@ -996,7 +971,7 @@ "\n", "for i in range(len(attention_matrix)):\n", " truncated_attention_matrix.append(\n", - " attention_matrix[i][:, :, :truncate_at, :truncate_at]\n", + " attention_matrix[i][:, :, :truncate_at, :truncate_at],\n", " )\n", "\n", "truncated_attention_matrix = tuple(truncated_attention_matrix)\n", @@ -3218,7 +3193,7 @@ " textangle=-90,\n", " bgcolor=\"red\",\n", " opacity=0.8,\n", - " )\n", + " ),\n", " )\n", "\n", " # Plot the attention matrix as a heatmap\n", @@ -3231,7 +3206,7 @@ " hoverinfo=\"text\",\n", " text=hover_text,\n", " colorscale=\"YlGnBu\",\n", - " )\n", + " ),\n", " )\n", "\n", " fig.update_layout(\n", @@ -3256,7 +3231,7 @@ " print(\n", " f\"Token {tokenizer.id_to_token(concept_ids[token1])} \"\n", " f\"with Token {tokenizer.id_to_token(concept_ids[token2])}: \"\n", - " f\"Attention Value {attention_value:.3f}\"\n", + " f\"Attention Value {attention_value:.3f}\",\n", " )\n", "\n", " fig.show()\n", diff --git a/finetune.py b/finetune.py index 6ecabb6..d67ba61 100644 --- a/finetune.py +++ b/finetune.py @@ -5,8 +5,17 @@ import sys from typing import Any, Dict +import numpy as np import pytorch_lightning as pl import torch +from lib.data import FinetuneDataset, FinetuneMultiDataset +from lib.tokenizer import ConceptTokenizer +from lib.utils import ( + get_run_id, + load_config, + load_finetune_data, + seed_everything, +) from lightning.pytorch.loggers import WandbLogger from pytorch_lightning.callbacks import ( EarlyStopping, @@ -15,23 +24,15 @@ ) from pytorch_lightning.strategies.ddp import DDPStrategy from sklearn.model_selection import train_test_split +from skmultilearn.model_selection import iterative_train_test_split from torch.utils.data import DataLoader -from odyssey.data.dataset import FinetuneDataset -from odyssey.data.tokenizer import ConceptTokenizer from odyssey.models.cehr_bert.model import BertFinetune, BertPretrain from odyssey.models.cehr_big_bird.model import BigBirdFinetune, BigBirdPretrain -from odyssey.models.utils import ( - get_latest_checkpoint, - get_run_id, - load_config, - load_finetune_data, - seed_everything, -) def main( - args: argparse.Namespace, + args: Dict[str, Any], pre_model_config: Dict[str, Any], fine_model_config: Dict[str, Any], ) -> None: @@ -41,7 +42,6 @@ def main( os.environ["CUDA_LAUNCH_BLOCKING"] = "1" torch.cuda.empty_cache() torch.set_float32_matmul_precision("medium") - # Load data fine_tune, fine_test = load_finetune_data( args.data_dir, @@ -50,41 +50,80 @@ def main( args.valid_scheme, args.num_finetune_patients, ) - - fine_tune.rename(columns={args.label_name: "label"}, inplace=True) - fine_test.rename(columns={args.label_name: "label"}, inplace=True) - - # Split data - fine_train, fine_val = train_test_split( - fine_tune, - test_size=args.val_size, - random_state=args.seed, - stratify=fine_tune["label"], - ) + # Split data based on model type + if not args.is_multi_model: + fine_tune.rename(columns={args.label_name: "label"}, inplace=True) + fine_test.rename(columns={args.label_name: "label"}, inplace=True) + # Split data in a stratified way based on problem type + if args.num_labels == 2: # Binary classification + fine_train, fine_val = train_test_split( + fine_tune, + test_size=args.val_size, + random_state=args.seed, + stratify=fine_tune["label"], + ) + else: # Multi label classfication + fine_train_ids, _, fine_val_ids, _ = iterative_train_test_split( + X=fine_tune["patient_id"].to_numpy().reshape(-1, 1), + y=np.array(fine_tune["label"].values.tolist()), + test_size=args.val_size, + ) + fine_train = fine_tune[ + fine_tune["patient_id"].isin(fine_train_ids.flatten().tolist()) + ] + fine_val = fine_tune[ + fine_tune["patient_id"].isin(fine_val_ids.flatten().tolist()) + ] + else: + fine_train, fine_val = train_test_split( + fine_tune, + test_size=args.val_size, + random_state=args.seed, + ) # Train Tokenizer tokenizer = ConceptTokenizer(data_dir=args.vocab_dir) tokenizer.fit_on_vocab() - # Load datasets - train_dataset = FinetuneDataset( - data=fine_train, - tokenizer=tokenizer, - max_len=args.max_len, - ) - - val_dataset = FinetuneDataset( - data=fine_val, - tokenizer=tokenizer, - max_len=args.max_len, - ) - - test_dataset = FinetuneDataset( - data=fine_test, - tokenizer=tokenizer, - max_len=args.max_len, - ) - + # Load datasets based on model type + if args.is_multi_model: + train_dataset = FinetuneMultiDataset( + data=fine_train, + tokenizer=tokenizer, + tasks=args.tasks, + balance_guide=args.balance_guide, + max_len=args.max_len, + ) + val_dataset = FinetuneMultiDataset( + data=fine_val, + tokenizer=tokenizer, + tasks=args.tasks, + balance_guide=args.balance_guide, + max_len=args.max_len, + ) + test_dataset = FinetuneMultiDataset( + data=fine_test, + tokenizer=tokenizer, + tasks=args.tasks, + balance_guide=None, + max_len=args.max_len, + ) + else: + train_dataset = FinetuneDataset( + data=fine_train, + tokenizer=tokenizer, + max_len=args.max_len, + ) + val_dataset = FinetuneDataset( + data=fine_val, + tokenizer=tokenizer, + max_len=args.max_len, + ) + test_dataset = FinetuneDataset( + data=fine_test, + tokenizer=tokenizer, + max_len=args.max_len, + ) train_loader = DataLoader( train_dataset, batch_size=args.batch_size, @@ -93,21 +132,18 @@ def main( shuffle=True, pin_memory=args.pin_memory, ) - val_loader = DataLoader( val_dataset, batch_size=args.batch_size, num_workers=args.num_workers, pin_memory=args.pin_memory, ) - test_loader = DataLoader( test_dataset, batch_size=args.batch_size, num_workers=args.num_workers, pin_memory=args.pin_memory, ) - callbacks = [ ModelCheckpoint( monitor="val_loss", @@ -136,13 +172,11 @@ def main( **pre_model_config, ) pretrained_model.load_state_dict(torch.load(args.pretrained_path)["state_dict"]) - model = BertFinetune( args=args, pretrained_model=pretrained_model, **fine_model_config, ) - elif args.model_type == "cehr_bigbird": pretrained_model = BigBirdPretrain( vocab_size=tokenizer.get_vocab_size(), @@ -153,13 +187,11 @@ def main( model = BigBirdFinetune( pretrained_model=pretrained_model, + num_labels=args.num_labels, + problem_type=args.problem_type, **fine_model_config, ) - - latest_checkpoint = get_latest_checkpoint(args.checkpoint_dir) - run_id = get_run_id(args.checkpoint_dir) - wandb_logger = WandbLogger( project=args.exp_name, save_dir=args.log_dir, @@ -167,7 +199,6 @@ def main( id=run_id, resume="allow", ) - # Setup PyTorchLightning trainer trainer = pl.Trainer( accelerator="gpu", @@ -186,15 +217,13 @@ def main( accumulate_grad_batches=args.acc, gradient_clip_val=1.0, ) - # Train the model trainer.fit( model=model, train_dataloaders=train_loader, val_dataloaders=val_loader, - ckpt_path=latest_checkpoint, + ckpt_path=args.resume_checkpoint, ) - # Test the model if args.test_last: trainer.test( @@ -207,6 +236,13 @@ def main( ckpt_path="best", ) + # Save test predictions + if args.test_output_dir: + torch.save( + model.test_outputs, + f"{args.test_output_dir}/test_outputs_{run_id}.pt", + ) + if __name__ == "__main__": parser = argparse.ArgumentParser() @@ -232,7 +268,6 @@ def main( parser.add_argument( "--label-name", type=str, - required=True, help="Name of the label column", ) parser.add_argument( @@ -244,33 +279,38 @@ def main( parser.add_argument( "--config-dir", type=str, - default="models/configs", + required=True, help="Path to model config file", ) - + parser.add_argument( + "--is-multi-model", + type=bool, + default=False, + help="Is the model a multimodel like multibird or not", + ) # data-related arguments parser.add_argument( "--data-dir", type=str, - default="data_files", + required=True, help="Path to the data directory", ) parser.add_argument( "--sequence-file", type=str, - default="patient_sequences_2048_labeled.parquet", + required=True, help="Path to the patient sequence file", ) parser.add_argument( "--id-file", type=str, - default="dataset_2048_mortality_1month.pkl", + required=True, help="Path to the patient id file", ) parser.add_argument( "--vocab-dir", type=str, - default="data_files/vocab", + required=True, help="Path to the vocabulary directory of json files", ) parser.add_argument( @@ -288,40 +328,68 @@ def main( parser.add_argument( "--num_finetune_patients", type=str, - default="20000_patients", + required=True, help="Define the number of patients to be fine_tuned on", ) - + parser.add_argument( + "--problem_type", + type=str, + required=True, + help="Define if its single_label_classification or multi_label_classification", + ) + parser.add_argument( + "--num_labels", + type=int, + required=True, + help="Define the number of labels", + ) + parser.add_argument( + "--tasks", + type=str, + default=None, + help="Define the finetune tasks for multi model", + ) + parser.add_argument( + "--balance_guide", + type=str, + default=None, + help="Define the positive label ratios for label balancing", + ) # checkpointing and logging arguments parser.add_argument( "--checkpoint-dir", type=str, - default="checkpoints", + required=True, help="Path to the checkpoint directory", ) parser.add_argument( - "--log-dir", + "--log_dir", type=str, default="logs", help="Path to the log directory", ) parser.add_argument( - "--checkpoint-path", + "--resume_checkpoint", type=str, default=None, help="Checkpoint to resume finetuning from", ) - + parser.add_argument( + "--test_output_dir", + type=str, + default=None, + help="Path to saved test outputs", + ) parser.add_argument( "--log_every_n_steps", type=int, default=10, help="Number of steps to log the training", ) - # other arguments parser.add_argument( "--test-last", + default=False, action="store_true", help="Test the last checkpoint", ) @@ -332,18 +400,20 @@ def main( help="Random seed for reproducibility", ) + # Process arguments args = parser.parse_args() - if args.model_type not in ["cehr_bert", "cehr_bigbird"]: print("Invalid model type. Choose 'cehr_bert' or 'cehr_bigbird'.") sys.exit(1) args.checkpoint_dir = os.path.join(args.checkpoint_dir, args.exp_name) + args.test_output_dir = os.path.join(args.test_output_dir, args.exp_name) + os.makedirs(args.checkpoint_dir, exist_ok=True) os.makedirs(args.log_dir, exist_ok=True) + os.makedirs(args.test_output_dir, exist_ok=True) config = load_config(args.config_dir, args.model_type) - finetune_config = config["finetune"] for key, value in finetune_config.items(): if not hasattr(args, key) or getattr(args, key) is None: @@ -351,7 +421,13 @@ def main( pre_model_config = config["model"] args.max_len = pre_model_config["max_seq_length"] - + # Process the tasks and balance guide arguments + args.tasks = args.tasks.strip().split(" ") + args.balance_guide = { + task: float(ratio) + for task, ratio in ( + pair.strip().split("=") for pair in args.balance_guide.split(",") + ) + } fine_model_config = config["model_finetune"] - main(args, pre_model_config, fine_model_config) diff --git a/odyssey/data/bigbird_data/DataChecker.ipynb b/odyssey/data/bigbird_data/DataChecker.ipynb index 7b63b47..4944995 100644 --- a/odyssey/data/bigbird_data/DataChecker.ipynb +++ b/odyssey/data/bigbird_data/DataChecker.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-03-13T16:14:45.546088300Z", @@ -13,32 +13,34 @@ "outputs": [], "source": [ "import os\n", - "import random\n", "import pickle\n", - "\n", - "from os.path import join\n", - "from typing import Dict, Any, List\n", + "import random\n", + "import sys\n", + "from typing import Any, Dict, List, Optional\n", "\n", "import numpy as np\n", "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "\n", - "from sklearn.model_selection import KFold, StratifiedKFold, train_test_split\n", + "from sklearn.model_selection import train_test_split\n", "from skmultilearn.model_selection import iterative_train_test_split\n", - "from sklearn.preprocessing import MultiLabelBinarizer\n", + "\n", + "\n", + "sys.path.append(\"/h/afallah/odyssey/odyssey/lib\")\n", + "from utils import save_object_to_disk\n", + "\n", "\n", "DATA_ROOT = \"/h/afallah/odyssey/odyssey/data/bigbird_data\"\n", "DATASET = f\"{DATA_ROOT}/patient_sequences/patient_sequences_2048.parquet\"\n", "MAX_LEN = 2048\n", "\n", + "SEED = 23\n", "os.chdir(DATA_ROOT)\n", - "random.seed(23)\n", - "np.random.seed(23)" + "random.seed(SEED)\n", + "np.random.seed(SEED)" ] }, { "cell_type": "code", - "execution_count": 93, + "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2024-03-13T16:15:12.321718600Z", @@ -47,6 +49,18 @@ "collapsed": false }, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current columns: Index(['patient_id', 'num_visits', 'deceased', 'death_after_start',\n", + " 'death_after_end', 'length', 'token_length', 'event_tokens_2048',\n", + " 'type_tokens_2048', 'age_tokens_2048', 'time_tokens_2048',\n", + " 'visit_tokens_2048', 'position_tokens_2048', 'elapsed_tokens_2048',\n", + " 'common_conditions', 'rare_conditions'],\n", + " dtype='object')\n" + ] + }, { "data": { "text/html": [ @@ -258,7 +272,7 @@ "4 [1, 0, 0, 0, 0, 0, 0, 0, 0, 1] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] " ] }, - "execution_count": 93, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -267,49 +281,124 @@ "# Load complete dataset\n", "dataset_2048 = pd.read_parquet(DATASET)\n", "\n", - "# dataset_2048.drop(\n", - "# ['event_tokens', 'type_tokens', 'age_tokens', 'time_tokens', 'visit_tokens', 'position_tokens'],\n", - "# axis=1,\n", - "# inplace=True\n", - "# )\n", - "\n", + "print(f\"Current columns: {dataset_2048.columns}\")\n", "dataset_2048.head()" ] }, { "cell_type": "code", - "execution_count": 94, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ - "def process_condition_dataset(dataset: pd.DataFrame) -> pd.DataFrame:\n", - " \"\"\"Process the condition dataset to extract required features.\n", + "def filter_by_num_visit(dataset: pd.DataFrame, minimum_num_visits: int) -> pd.DataFrame:\n", + " \"\"\"Filter the patients based on num_visits threshold.\n", "\n", " Args:\n", - " dataset (pd.DataFrame): The input condition dataset.\n", + " dataset (pd.DataFrame): The input dataset.\n", + " minimum_num_visits (int): The threshold num_visits\n", "\n", - " Returns:\n", - " pd.DataFrame: The processed condition dataset.\n", + " Returns\n", + " -------\n", + " pd.DataFrame: The filtered dataset.\n", " \"\"\"\n", - " dataset_2048[\"all_conditions\"] = dataset_2048.apply(\n", - " lambda row: np.concatenate(\n", - " [row[\"common_conditions\"], row[\"rare_conditions\"]], dtype=np.float64\n", - " ),\n", - " axis=1,\n", - " )\n", + " filtered_dataset = dataset.loc[dataset[\"num_visits\"] >= minimum_num_visits]\n", + " filtered_dataset.reset_index(drop=True, inplace=True)\n", + " return filtered_dataset\n", + "\n", + "\n", + "def filter_by_length_of_stay(dataset: pd.DataFrame, threshold: int = 1) -> pd.DataFrame:\n", + " \"\"\"Filter the patients based on length of stay threshold.\n", + "\n", + " Args:\n", + " dataset (pd.DataFrame): The input dataset.\n", + " minimum_num_visits (int): The threshold length of stay\n", + "\n", + " Returns\n", + " -------\n", + " pd.DataFrame: The filtered dataset.\n", + " \"\"\"\n", + " filtered_dataset = dataset.loc[dataset[\"length_of_stay\"] >= threshold]\n", + "\n", + " # Only keep the patients that their first event happens within threshold\n", + " # TODO: Check how many patients get removed here?\n", + " filtered_dataset = filtered_dataset[\n", + " filtered_dataset.apply(\n", + " lambda row: row[\"elapsed_tokens_2048\"][row[\"last_VS_index\"] + 1]\n", + " < threshold * 24,\n", + " axis=1,\n", + " )\n", + " ]\n", + "\n", + " filtered_dataset.reset_index(drop=True, inplace=True)\n", + " return filtered_dataset\n", + "\n", "\n", - " dataset[\"event_tokens_2048\"] = dataset[\"event_tokens_2048\"].transform(\n", - " lambda token_list: \" \".join(token_list)\n", + "def get_last_occurence_index(seq: List[str], target: str) -> int:\n", + " \"\"\"Return the index of the last occurrence of target in seq.\n", + "\n", + " Args:\n", + " seq (List[str]): The input sequence.\n", + " target (str): The target string to find.\n", + "\n", + " Returns\n", + " -------\n", + " int: The index of the last occurrence of target in seq.\n", + " \"\"\"\n", + " return len(seq) - (seq[::-1].index(target) + 1)\n", + "\n", + "\n", + "def check_readmission_label(row: pd.Series) -> int:\n", + " \"\"\"Check if the label indicates readmission within one month.\n", + "\n", + " Args:\n", + " row (pd.Series): The input row.\n", + "\n", + " Returns\n", + " -------\n", + " bool: True if readmission label is present, False otherwise.\n", + " \"\"\"\n", + " last_vs_index = row[\"last_VS_index\"]\n", + " return int(\n", + " row[\"event_tokens_2048\"][last_vs_index - 1]\n", + " in (\"[W_0]\", \"[W_1]\", \"[W_2]\", \"[W_3]\", \"[M_1]\"),\n", " )\n", "\n", - " return dataset\n", "\n", + "def get_length_of_stay(row: pd.Series) -> pd.Series:\n", + " \"\"\"Determine the length of a given visit.\n", "\n", - "# Process the dataset for conditions including rare and common\n", - "dataset_2048 = process_condition_dataset(dataset_2048)\n", - "dataset_2048.to_parquet(\n", - " f\"{DATA_ROOT}/patient_sequences/patient_sequences_2048_with_conditions.parquet\"\n", - ")" + " Args:\n", + " row (pd.Series): The input row.\n", + "\n", + " Returns\n", + " -------\n", + " pd.Series: The preprocessed row.\n", + " \"\"\"\n", + " admission_time = row[\"last_VS_index\"] + 1\n", + " discharge_time = row[\"last_VE_index\"] - 1\n", + " return (discharge_time - admission_time) / 24\n", + "\n", + "\n", + "def get_visit_cutoff_at_threshold(row: pd.Series, threshold: int = 24) -> int:\n", + " \"\"\"Get the index of the first event token of last visit that occurs after threshold hours.\n", + "\n", + " Args:\n", + " row (pd.Series): The input row.\n", + " threshold (int): The number of hours to consider.\n", + "\n", + " Returns\n", + " -------\n", + " cutoff_index (int): The corrosponding cutoff index.\n", + " \"\"\"\n", + " last_vs_index = row[\"last_VS_index\"]\n", + " last_ve_index = row[\"last_VE_index\"]\n", + "\n", + " for i in range(last_vs_index + 1, last_ve_index):\n", + " if row[\"elapsed_tokens_2048\"][i] > threshold:\n", + " return i\n", + "\n", + " return len(row[\"event_tokens_2048\"])" ] }, { @@ -318,38 +407,73 @@ "metadata": {}, "outputs": [], "source": [ - "def get_stratified_data_split(\n", - " dataset: pd.DataFrame, target: str, test_size: float = 0.15\n", - "):\n", - " \"\"\"\n", - " Split the given dataset into training and testing sets using iterative stratification on given multi-label target.\n", + "def process_length_of_stay_dataset(\n", + " dataset: pd.DataFrame,\n", + " threshold: int = 7,\n", + ") -> pd.DataFrame:\n", + " \"\"\"Process the length of stay dataset to extract required features.\n", + "\n", + " Args:\n", + " dataset (pd.DataFrame): The input dataset.\n", + " threshold (int): The threshold length of stay.\n", + "\n", + " Returns\n", + " -------\n", + " pd.DataFrame: The processed dataset.\n", " \"\"\"\n", - " # Convert all_conditions into a format suitable for multi-label stratification\n", - " Y = np.array(dataset_2048[target].values.tolist())\n", + " dataset[\"last_VS_index\"] = dataset[\"event_tokens_2048\"].transform(\n", + " lambda seq: get_last_occurence_index(list(seq), \"[VS]\"),\n", + " )\n", + " dataset[\"last_VE_index\"] = dataset[\"event_tokens_2048\"].transform(\n", + " lambda seq: get_last_occurence_index(list(seq), \"[VE]\"),\n", + " )\n", + " dataset[\"length_of_stay\"] = dataset.apply(get_length_of_stay, axis=1)\n", "\n", - " # We will split based on Y but we need to keep the association with patient_id\n", - " X = dataset_2048[\"patient_id\"].to_numpy().reshape(-1, 1)\n", + " dataset = filter_by_length_of_stay(dataset, threshold=1)\n", + " dataset[\"label_los_1week\"] = (dataset[\"length_of_stay\"] >= threshold).astype(int)\n", "\n", - " # Perform iterative stratification\n", - " X_train, y_train, X_test, y_test = iterative_train_test_split(\n", - " X, Y, test_size=test_size\n", + " dataset[\"cutoff_los\"] = dataset.apply(\n", + " lambda row: get_visit_cutoff_at_threshold(row, threshold=24),\n", + " axis=1,\n", " )\n", + " dataset[\"token_length\"] = dataset[\"event_tokens_2048\"].apply(len)\n", "\n", - " return X_train.flatten().tolist(), X_test.flatten().tolist()\n", + " return dataset\n", "\n", "\n", - "pretrain_ids, test_ids = get_stratified_data_split(\n", - " dataset_2048, \"all_conditions\", test_size=0.15\n", - ")\n", + "# Process the dataset for length of stay prediction above a threshold\n", + "dataset_2048_los = process_length_of_stay_dataset(dataset_2048.copy(), threshold=7)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def process_condition_dataset(dataset: pd.DataFrame) -> pd.DataFrame:\n", + " \"\"\"Process the condition dataset to extract required features.\n", "\n", - "patient_ids_dict = {\n", - " \"pretrain\": pretrain_ids,\n", - " \"finetune\": {\"few_shot\": {}, \"kfold\": {}},\n", - " \"test\": test_ids,\n", - "}\n", + " Args:\n", + " dataset (pd.DataFrame): The input condition dataset.\n", + "\n", + " Returns\n", + " -------\n", + " pd.DataFrame: The processed condition dataset.\n", + " \"\"\"\n", + " dataset[\"all_conditions\"] = dataset.apply(\n", + " lambda row: np.concatenate(\n", + " [row[\"common_conditions\"], row[\"rare_conditions\"]],\n", + " dtype=np.int64,\n", + " ),\n", + " axis=1,\n", + " )\n", "\n", - "with open(\"sample_pretrain_test_patient_ids_with_conditions.pkl\", \"wb\") as f:\n", - " pickle.dump(patient_ids_dict, f)" + " return dataset\n", + "\n", + "\n", + "# Process the dataset for conditions including rare and common\n", + "dataset_2048_condition = process_condition_dataset(dataset_2048.copy())" ] }, { @@ -370,18 +494,17 @@ " Args:\n", " dataset (pd.DataFrame): The input mortality dataset.\n", "\n", - " Returns:\n", + " Returns\n", + " -------\n", " pd.DataFrame: The processed mortality dataset.\n", " \"\"\"\n", - " dataset[\"event_tokens_2048\"] = dataset[\"event_tokens_2048\"].transform(\n", - " lambda token_list: \" \".join(token_list)\n", - " )\n", " dataset[\"label_mortality_2weeks\"] = (\n", " (dataset[\"death_after_start\"] >= 0) & (dataset[\"death_after_end\"] <= 15)\n", " ).astype(int)\n", " dataset[\"label_mortality_1month\"] = (\n", " (dataset[\"death_after_start\"] >= 0) & (dataset[\"death_after_end\"] <= 32)\n", " ).astype(int)\n", + "\n", " return dataset\n", "\n", "\n", @@ -407,129 +530,297 @@ " Args:\n", " dataset (pd.DataFrame): The input dataset.\n", "\n", - " Returns:\n", + " Returns\n", + " -------\n", " pd.DataFrame: The processed dataset.\n", " \"\"\"\n", - " dataset.drop(\n", - " [\"deceased\", \"death_after_start\", \"death_after_end\", \"length\"],\n", - " axis=1,\n", - " inplace=True,\n", - " )\n", " dataset[\"last_VS_index\"] = dataset[\"event_tokens_2048\"].transform(\n", - " lambda seq: get_last_index(list(seq), \"[VS]\")\n", + " lambda seq: get_last_occurence_index(list(seq), \"[VS]\"),\n", " )\n", + " dataset[\"cutoff_readmission\"] = dataset[\"last_VS_index\"] - 1\n", " dataset[\"label_readmission_1month\"] = dataset.apply(check_readmission_label, axis=1)\n", - " dataset[\"event_tokens_2048\"] = dataset.apply(remove_last_visit, axis=1)\n", + "\n", " dataset[\"num_visits\"] -= 1\n", " dataset[\"token_length\"] = dataset[\"event_tokens_2048\"].apply(len)\n", - " dataset = dataset.apply(truncate_and_pad, axis=1)\n", - " dataset[\"event_tokens_2048\"] = dataset[\"event_tokens_2048\"].transform(\n", - " lambda token_list: \" \".join(token_list)\n", - " )\n", - " return dataset\n", - "\n", - "\n", - "def filter_by_num_visit(dataset: pd.DataFrame, minimum_num_visits: int) -> pd.DataFrame:\n", - " \"\"\"Filter the patients based on num_visits threshold.\n", - "\n", - " Args:\n", - " dataset (pd.DataFrame): The input dataset.\n", - " minimum_num_visits (int): The threshold num_visits\n", "\n", - " Returns:\n", - " pd.DataFrame: The filtered dataset.\n", - " \"\"\"\n", - " filtered_dataset = dataset.loc[dataset[\"num_visits\"] >= minimum_num_visits].copy()\n", - " filtered_dataset.reset_index(drop=True, inplace=True)\n", - " return filtered_dataset\n", - "\n", - "\n", - "def get_last_index(seq: List[str], target: str) -> int:\n", - " \"\"\"Return the index of the last occurrence of target in seq.\n", - "\n", - " Args:\n", - " seq (List[str]): The input sequence.\n", - " target (str): The target string to find.\n", + " return dataset\n", "\n", - " Returns:\n", - " int: The index of the last occurrence of target in seq.\n", "\n", - " Examples:\n", - " >>> get_last_index(['A', 'B', 'A', 'A', 'C', 'D'], 'A')\n", - " 3\n", + "# Process the dataset for hospital readmission in one month task\n", + "dataset_2048_readmission = filter_by_num_visit(\n", + " dataset_2048.copy(),\n", + " minimum_num_visits=2,\n", + ")\n", + "dataset_2048_readmission = process_readmission_dataset(dataset_2048_readmission)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def process_multi_dataset(datasets: Dict[str, pd.DataFrame]):\n", " \"\"\"\n", - " return len(seq) - (seq[::-1].index(target) + 1)\n", - "\n", - "\n", - "def truncate_and_pad(row: pd.Series) -> Any:\n", - " \"\"\"Return a truncated and padded version of row.\n", + " Process the multi-task dataset by merging the original dataset with the other datasets.\n", "\n", " Args:\n", - " row (pd.Series): The input row.\n", + " datasets (Dict): Dictionary mapping each task to its respective dataframe\n", "\n", - " Returns:\n", - " Any: The truncated and padded row.\n", - "\n", - " Note:\n", - " This function assumes the presence of the following columns in row:\n", - " - 'event_tokens_2048'\n", - " - 'type_tokens_2048'\n", - " - 'age_tokens_2048'\n", - " - 'time_tokens_2048'\n", - " - 'visit_tokens_2048'\n", - " - 'position_tokens_2048'\n", + " Returns\n", + " -------\n", + " pd.DataFrame: The processed multi-task dataset\n", " \"\"\"\n", - " seq_len = len(row[\"event_tokens_2048\"])\n", - " row[\"type_tokens_2048\"] = np.pad(\n", - " row[\"type_tokens_2048\"][:seq_len], (0, MAX_LEN - seq_len), mode=\"constant\"\n", + " # Merging datasets on 'patient_id'\n", + " multi_dataset = datasets[\"original\"].merge(\n", + " datasets[\"condition\"][[\"patient_id\", \"all_conditions\"]],\n", + " on=\"patient_id\",\n", + " how=\"left\",\n", " )\n", - " row[\"age_tokens_2048\"] = np.pad(\n", - " row[\"age_tokens_2048\"][:seq_len], (0, MAX_LEN - seq_len), mode=\"constant\"\n", + " multi_dataset = multi_dataset.merge(\n", + " datasets[\"mortality\"][[\"patient_id\", \"label_mortality_1month\"]],\n", + " on=\"patient_id\",\n", + " how=\"left\",\n", " )\n", - " row[\"time_tokens_2048\"] = np.pad(\n", - " row[\"time_tokens_2048\"][:seq_len], (0, MAX_LEN - seq_len), mode=\"constant\"\n", + " multi_dataset = multi_dataset.merge(\n", + " datasets[\"readmission\"][\n", + " [\"patient_id\", \"cutoff_readmission\", \"label_readmission_1month\"]\n", + " ],\n", + " on=\"patient_id\",\n", + " how=\"left\",\n", " )\n", - " row[\"visit_tokens_2048\"] = np.pad(\n", - " row[\"visit_tokens_2048\"][:seq_len], (0, MAX_LEN - seq_len), mode=\"constant\"\n", + " multi_dataset = multi_dataset.merge(\n", + " datasets[\"los\"][[\"patient_id\", \"cutoff_los\", \"label_los_1week\"]],\n", + " on=\"patient_id\",\n", + " how=\"left\",\n", " )\n", - " row[\"position_tokens_2048\"] = np.pad(\n", - " row[\"position_tokens_2048\"][:seq_len], (0, MAX_LEN - seq_len), mode=\"constant\"\n", + "\n", + " # Selecting the required columns\n", + " multi_dataset = multi_dataset[\n", + " [\n", + " \"patient_id\",\n", + " \"num_visits\",\n", + " \"event_tokens_2048\",\n", + " \"type_tokens_2048\",\n", + " \"age_tokens_2048\",\n", + " \"time_tokens_2048\",\n", + " \"visit_tokens_2048\",\n", + " \"position_tokens_2048\",\n", + " \"elapsed_tokens_2048\",\n", + " \"cutoff_los\",\n", + " \"cutoff_readmission\",\n", + " \"all_conditions\",\n", + " \"label_mortality_1month\",\n", + " \"label_readmission_1month\",\n", + " \"label_los_1week\",\n", + " ]\n", + " ]\n", + "\n", + " # Transform conditions from a vector of numbers to binary classes\n", + " conditions_expanded = multi_dataset[\"all_conditions\"].apply(pd.Series)\n", + " conditions_expanded.columns = [f\"condition{i}\" for i in range(20)]\n", + " multi_dataset = multi_dataset.drop(\"all_conditions\", axis=1)\n", + " multi_dataset = pd.concat([multi_dataset, conditions_expanded], axis=1)\n", + "\n", + " # Standardize important column names\n", + " multi_dataset.rename(\n", + " columns={\n", + " \"cutoff_los\": \"cutoff_los_1week\",\n", + " \"cutoff_readmission\": \"cutoff_readmission_1month\",\n", + " },\n", + " inplace=True,\n", " )\n", - " return row\n", + " condition_columns = {f\"condition{i}\": f\"label_c{i}\" for i in range(20)}\n", + " multi_dataset.rename(columns=condition_columns, inplace=True)\n", + "\n", + " numerical_columns = [\n", + " \"cutoff_los_1week\",\n", + " \"cutoff_readmission_1month\",\n", + " \"label_mortality_1month\",\n", + " \"label_readmission_1month\",\n", + " \"label_los_1week\",\n", + " ] + [f\"label_c{i}\" for i in range(20)]\n", + "\n", + " # Fill NaN values and convert to integers\n", + " for column in numerical_columns:\n", + " multi_dataset[column] = multi_dataset[column].fillna(-1).astype(int)\n", + "\n", + " # Reset dataset index\n", + " multi_dataset.reset_index(drop=True, inplace=True)\n", + "\n", + " return multi_dataset\n", + "\n", + "\n", + "multi_dataset = process_multi_dataset(\n", + " datasets={\n", + " \"original\": dataset_2048,\n", + " \"mortality\": dataset_2048_mortality,\n", + " \"condition\": dataset_2048_condition,\n", + " \"readmission\": dataset_2048_readmission,\n", + " \"los\": dataset_2048_los,\n", + " },\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def stratified_train_test_split(\n", + " dataset: pd.DataFrame,\n", + " target: str,\n", + " test_size: float,\n", + " return_test: Optional[bool] = False,\n", + "):\n", + " \"\"\"\n", + " Split the given dataset into training and testing sets using iterative stratification on given multi-label target.\n", + " \"\"\"\n", + " # Convert all_conditions into a format suitable for multi-label stratification\n", + " Y = np.array(dataset[target].values.tolist())\n", + " X = dataset[\"patient_id\"].to_numpy().reshape(-1, 1)\n", + " is_single_label = type(dataset.iloc[0][target]) == np.int64\n", + "\n", + " # Perform stratified split\n", + " if is_single_label:\n", + " X_train, X_test, y_train, y_test = train_test_split(\n", + " X,\n", + " Y,\n", + " stratify=Y,\n", + " test_size=test_size,\n", + " random_state=SEED,\n", + " )\n", "\n", + " else:\n", + " X_train, y_train, X_test, y_test = iterative_train_test_split(\n", + " X,\n", + " Y,\n", + " test_size=test_size,\n", + " )\n", "\n", - "def check_readmission_label(row: pd.Series) -> int:\n", - " \"\"\"Check if the label indicates readmission within one month.\n", + " X_train = X_train.flatten().tolist()\n", + " X_test = X_test.flatten().tolist()\n", "\n", - " Args:\n", - " row (pd.Series): The input row.\n", + " if return_test:\n", + " return X_test\n", + " else:\n", + " return X_train, X_test\n", "\n", - " Returns:\n", - " bool: True if readmission label is present, False otherwise.\n", + "\n", + "def sample_balanced_subset(dataset: pd.DataFrame, target: str, sample_size: int):\n", " \"\"\"\n", - " last_vs_index = row[\"last_VS_index\"]\n", - " return int(\n", - " row[\"event_tokens_2048\"][last_vs_index - 1]\n", - " in (\"[W_0]\", \"[W_1]\", \"[W_2]\", \"[W_3]\", \"[M_1]\")\n", + " Sample a subset of dataset with balanced target labels.\n", + " \"\"\"\n", + " # Sampling positive and negative patients\n", + " pos_patients = dataset[dataset[target] == True].sample(\n", + " n=sample_size // 2,\n", + " random_state=SEED,\n", + " )\n", + " neg_patients = dataset[dataset[target] == False].sample(\n", + " n=sample_size // 2,\n", + " random_state=SEED,\n", " )\n", "\n", + " # Combining and shuffling patient IDs\n", + " sample_patients = (\n", + " pos_patients[\"patient_id\"].tolist() + neg_patients[\"patient_id\"].tolist()\n", + " )\n", + " random.shuffle(sample_patients)\n", "\n", - "def remove_last_visit(row: pd.Series) -> pd.Series:\n", - " \"\"\"Remove the event tokens of last visit in the row\n", + " return sample_patients\n", "\n", - " Args:\n", - " row (pd.Series): The input row.\n", "\n", - " Returns:\n", - " pd.Series: The preprocessed row.\n", - " \"\"\"\n", - " last_vs_index = row[\"last_VS_index\"]\n", - " return row[\"event_tokens_2048\"][: last_vs_index - 1]\n", + "def get_pretrain_test_split(\n", + " dataset: pd.DataFrame,\n", + " stratify_target: Optional[str] = None,\n", + " test_size: float = 0.15,\n", + "):\n", + " \"\"\"Split dataset into pretrain and test set. Stratify on a given target column if needed.\"\"\"\n", + " if stratify_target:\n", + " pretrain_ids, test_ids = stratified_train_test_split(\n", + " dataset,\n", + " target=stratify_target,\n", + " test_size=test_size,\n", + " )\n", "\n", + " else:\n", + " test_patients = dataset.sample(n=test_size, random_state=SEED)\n", + " test_ids = test_patients[\"patient_id\"].tolist()\n", + " pretrain_ids = dataset[~dataset[\"patient_id\"].isin(test_patients)][\n", + " \"patient_id\"\n", + " ].tolist()\n", "\n", - "# Process the dataset for hospital readmission in one month task\n", - "dataset_2048_readmission = filter_by_num_visit(dataset_2048, minimum_num_visits=2)\n", - "dataset_2048_readmission = process_readmission_dataset(dataset_2048_readmission)" + " random.shuffle(pretrain_ids)\n", + "\n", + " return pretrain_ids, test_ids" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Split data\n", + "patient_ids_dict = {\n", + " \"pretrain\": [],\n", + " \"finetune\": {\"few_shot\": {}, \"kfold\": {}},\n", + " \"test\": [],\n", + "}\n", + "\n", + "# Get train-test split\n", + "# pretrain_ids, test_ids = get_pretrain_test_split(dataset_2048_readmission, stratify_target='label_readmission_1month', test_size=0.2)\n", + "# pretrain_ids, test_ids = get_pretrain_test_split(process_condition_dataset, stratify_target='all_conditions', test_size=0.15)\n", + "# patient_ids_dict['pretrain'] = pretrain_ids\n", + "# patient_ids_dict['test'] = test_ids\n", + "\n", + "# Load pretrain and test patient IDs\n", + "pid = pickle.load(open(\"patient_id_dict/dataset_2048_multi.pkl\", \"rb\"))\n", + "patient_ids_dict[\"pretrain\"] = pid[\"pretrain\"]\n", + "patient_ids_dict[\"test\"] = pid[\"test\"]\n", + "set(pid[\"pretrain\"] + pid[\"test\"]) == set(dataset_2048[\"patient_id\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class config:\n", + " task_splits = {\n", + " \"mortality\": {\n", + " \"dataset\": dataset_2048_mortality,\n", + " \"label_col\": \"label_mortality_1month\",\n", + " \"finetune_size\": [250, 500, 1000, 5000, 20000],\n", + " \"save_path\": \"patient_id_dict/dataset_2048_mortality.pkl\",\n", + " \"split_mode\": \"single_label_balanced\",\n", + " },\n", + " \"readmission\": {\n", + " \"dataset\": dataset_2048_readmission,\n", + " \"label_col\": \"label_readmission_1month\",\n", + " \"finetune_size\": [250, 1000, 5000, 20000, 60000],\n", + " \"save_path\": \"patient_id_dict/dataset_2048_readmission.pkl\",\n", + " \"split_mode\": \"single_label_stratified\",\n", + " },\n", + " \"length_of_stay\": {\n", + " \"dataset\": dataset_2048_los,\n", + " \"label_col\": \"label_los_1week\",\n", + " \"finetune_size\": [250, 1000, 5000, 20000, 50000],\n", + " \"save_path\": \"patient_id_dict/dataset_2048_los.pkl\",\n", + " \"split_mode\": \"single_label_balanced\",\n", + " },\n", + " \"condition\": {\n", + " \"dataset\": dataset_2048_condition,\n", + " \"label_col\": \"all_conditions\",\n", + " \"finetune_size\": [50000],\n", + " \"save_path\": \"patient_id_dict/dataset_2048_condition.pkl\",\n", + " \"split_mode\": \"multi_label_stratified\",\n", + " },\n", + " }\n", + "\n", + " all_tasks = list(task_splits.keys())" ] }, { @@ -544,132 +835,59 @@ }, "outputs": [], "source": [ - "def split_dataset_train_test_finetune_datasets(\n", - " dataset: pd.DataFrame,\n", - " label_col: str,\n", - " cv_size: int,\n", - " test_size: int,\n", - " finetune_size: List[int],\n", - " num_splits: int,\n", - " save_path: str,\n", + "def get_finetune_split(\n", + " config: config,\n", + " patient_ids_dict: Dict[str, Any],\n", ") -> Dict[str, Dict[str, List[str]]]:\n", " \"\"\"\n", " Splits the dataset into training and cross-finetuneation sets using k-fold cross-finetuneation\n", " while ensuring balanced label distribution in each fold. Saves the resulting dictionary to disk.\n", - "\n", - " Args:\n", - " dataset (pd.DataFrame): The input dataset.\n", - " label_col (str): The name of the column containing the labels.\n", - " cv_size (int): The number of patients in each cross-finetuneation split.\n", - " test_size (int): The number of patients in the test set.\n", - " finetune_size (List[int]): The number of patients in each fine-tune set\n", - " num_splits (int): The number of splits to create (k value).\n", - " save_path (str): The path to save the resulting dictionary.\n", - "\n", - " Returns:\n", - " Dict[str, Dict[str, List[str]]]: A dictionary containing patient IDs for each split group.\n", " \"\"\"\n", - "\n", - " # Dictionary to hold patient IDs for different sets\n", - " patient_ids_dict = {\n", - " \"pretrain\": [],\n", - " \"finetune\": {\"few_shot\": {}, \"kfold\": {}},\n", - " \"test\": [],\n", - " }\n", - "\n", - " # Sample test patients and remove them from dataset\n", - " if test_size > 0:\n", - " test_patients = dataset.sample(n=test_size, random_state=23)\n", - " dataset.drop(test_patients.index, inplace=True)\n", - " patient_ids_dict[\"test\"] = test_patients[\"patient_id\"].tolist()\n", - "\n", - " # Any remaining data is used for pretraining\n", - " patient_ids_dict[\"pretrain\"] = dataset[\"patient_id\"].tolist()\n", - " random.shuffle(patient_ids_dict[\"pretrain\"])\n", - "\n", - " # few_shot finetune dataset\n", - " for each_finetune_size in finetune_size:\n", - " # For the time being\n", - " test_patients = dataset.sample(n=each_finetune_size, random_state=23)\n", - " test_patients = test_patients[\"patient_id\"].tolist()\n", - " random.shuffle(test_patients)\n", - " patient_ids_dict[\"finetune\"][\"few_shot\"][f\"{each_finetune_size}\"] = (\n", - " test_patients\n", - " )\n", - "\n", - " subset_size = each_finetune_size // 2\n", - "\n", - " # Sampling positive and negative patients\n", - " pos_patients = dataset[dataset[label_col] == True].sample(\n", - " n=subset_size, random_state=23\n", - " )\n", - " neg_patients = dataset[dataset[label_col] == False].sample(\n", - " n=subset_size, random_state=23\n", - " )\n", - "\n", - " # Extracting patient IDs\n", - " pos_patients_ids = pos_patients[\"patient_id\"].tolist()\n", - " neg_patients_ids = neg_patients[\"patient_id\"].tolist()\n", - "\n", - " # Combining and shuffling patient IDs\n", - " finetune_patients = pos_patients_ids + neg_patients_ids\n", - " random.shuffle(finetune_patients)\n", - " patient_ids_dict[\"finetune\"][\"few_shot\"][f\"{each_finetune_size}\"] = (\n", - " finetune_patients\n", - " )\n", - "\n", - " # Performing stratified k-fold split\n", - " # skf = StratifiedKFold(n_splits=num_splits, shuffle=True, random_state=23)\n", - "\n", - " # for i, (train_index, cv_index) in enumerate(skf.split(dataset, dataset[label_col])):\n", - "\n", - " # dataset_cv = dataset.iloc[cv_index]\n", - " # dataset_finetune = dataset.iloc[train_index]\n", - "\n", - " # # Separate positive and negative labeled patients\n", - " # pos_patients = dataset_cv[dataset_cv[label_col] == True]['patient_id'].tolist()\n", - " # neg_patients = dataset_cv[dataset_cv[label_col] == False]['patient_id'].tolist()\n", - "\n", - " # # Calculate the number of positive and negative patients needed for balanced CV set\n", - " # num_pos_needed = cv_size // 2\n", - " # num_neg_needed = cv_size // 2\n", - "\n", - " # # Select positive and negative patients for CV set ensuring balanced distribution\n", - " # cv_patients = pos_patients[:num_pos_needed] + neg_patients[:num_neg_needed]\n", - " # remaining_finetune_patients = pos_patients[num_pos_needed:] + neg_patients[num_neg_needed:]\n", - "\n", - " # # Extract patient IDs for training set\n", - " # finetune_patients = dataset_finetune['patient_id'].tolist()\n", - " # finetune_patients += remaining_finetune_patients\n", - "\n", - " # # Shuffle each list of patients\n", - " # random.shuffle(cv_patients)\n", - " # random.shuffle(finetune_patients)\n", - "\n", - " # patient_ids_dict['finetune']['kfold'][f'group{i+1}'] = {'finetune': finetune_patients, 'cv': cv_patients}\n", + " # Extract task-specific configuration\n", + " task_config = config.task_splits[task]\n", + " dataset = task_config[\"dataset\"]\n", + " label_col = task_config[\"label_col\"]\n", + " finetune_sizes = task_config[\"finetune_size\"]\n", + " save_path = task_config[\"save_path\"]\n", + " split_mode = task_config[\"split_mode\"]\n", + "\n", + " # Get pretrain dataset\n", + " pretrain_ids = patient_ids_dict[\"pretrain\"]\n", + " dataset = dataset[dataset[\"patient_id\"].isin(pretrain_ids)]\n", + "\n", + " # Few-shot finetune patient ids\n", + " for finetune_num in finetune_sizes:\n", + " if split_mode == \"single_label_balanced\":\n", + " finetune_ids = sample_balanced_subset(\n", + " dataset,\n", + " target=label_col,\n", + " sample_size=finetune_num,\n", + " )\n", + "\n", + " elif (\n", + " split_mode == \"single_label_stratified\"\n", + " or split_mode == \"multi_label_stratified\"\n", + " ):\n", + " finetune_ids = stratified_train_test_split(\n", + " dataset,\n", + " target=label_col,\n", + " test_size=finetune_num / len(dataset),\n", + " return_test=True,\n", + " )\n", + "\n", + " patient_ids_dict[\"finetune\"][\"few_shot\"][f\"{finetune_num}\"] = finetune_ids\n", "\n", " # Save the dictionary to disk\n", - " with open(save_path, \"wb\") as f:\n", - " pickle.dump(patient_ids_dict, f)\n", + " save_object_to_disk(patient_ids_dict, save_path)\n", "\n", " return patient_ids_dict\n", "\n", "\n", - "patient_ids_dict = split_dataset_train_test_finetune_datasets(\n", - " dataset=dataset_2048_condition.copy(),\n", - " label_col=\"all_conditions\",\n", - " cv_size=4000,\n", - " test_size=20000, # for mortality used to be 20000, then 15000, then 20000 | readmission 10000 then, now 8000\n", - " finetune_size=[\n", - " 250,\n", - " 1000,\n", - " 5000,\n", - " 20000,\n", - " 50000,\n", - " ], # [250, 500, 1000, 5000, 20000] [250, 1000, 5000, 20000, 50000]\n", - " num_splits=5,\n", - " save_path=\"patient_id_dict/dataset_2048_condition.pkl\",\n", - ")" + "for task in config.all_tasks:\n", + " patient_ids_dict = get_finetune_split(\n", + " config=config,\n", + " patient_ids_dict=patient_ids_dict,\n", + " )" ] }, { @@ -684,11 +902,124 @@ }, "outputs": [], "source": [ - "# dataset_2048_mortality.to_parquet('patient_sequences/patient_sequences_2048_mortality.parquet')\n", - "# dataset_2048_readmission.to_parquet('patient_sequences/patient_sequences_2048_readmission.parquet')\n", + "dataset_2048_mortality.to_parquet(\n", + " \"patient_sequences/patient_sequences_2048_mortality.parquet\",\n", + ")\n", + "dataset_2048_readmission.to_parquet(\n", + " \"patient_sequences/patient_sequences_2048_readmission.parquet\",\n", + ")\n", + "dataset_2048_los.to_parquet(\"patient_sequences/patient_sequences_2048_los.parquet\")\n", "dataset_2048_condition.to_parquet(\n", - " \"patient_sequences/patient_sequences_2048_condition.parquet\"\n", - ")" + " \"patient_sequences/patient_sequences_2048_condition.parquet\",\n", + ")\n", + "multi_dataset.to_parquet(\"patient_sequences/patient_sequences_2048_multi.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load data\n", + "# multi_dataset = pd.read_parquet('patient_sequences/patient_sequences_2048_multi.parquet')\n", + "# pid = pickle.load(open('patient_id_dict/dataset_2048_multi.pkl', 'rb'))\n", + "# multi_dataset = multi_dataset[multi_dataset['patient_id'].isin(pid['finetune']['few_shot']['all'])]\n", + "\n", + "# # Train Tokenizer\n", + "# tokenizer = ConceptTokenizer(data_dir='/h/afallah/odyssey/odyssey/data/vocab')\n", + "# tokenizer.fit_on_vocab()\n", + "\n", + "# # Load datasets\n", + "# tasks = ['mortality_1month', 'los_1week'] + [f'c{i}' for i in range(5)]\n", + "\n", + "# train_dataset = FinetuneMultiDataset(\n", + "# data=multi_dataset,\n", + "# tokenizer=tokenizer,\n", + "# tasks=tasks,\n", + "# balance_guide={'mortality_1month': 0.5, 'los_1week': 0.5},\n", + "# max_len=2048,\n", + "# )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# dataset_2048_condition = pd.read_parquet('patient_sequences/patient_sequences_2048_condition.parquet')\n", + "# pid = pickle.load(open('patient_id_dict/dataset_2048_condition.pkl', 'rb'))\n", + "# condition_finetune = dataset_2048_condition.loc[dataset_2048_condition['patient_id'].isin(pid['finetune']['few_shot']['50000'])]\n", + "# condition_finetune" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# freq = np.array(condition_finetune['all_conditions'].tolist()).sum(axis=0)\n", + "# weights = np.clip(0, 50, sum(freq) / freq)\n", + "# np.max(np.sqrt(freq)) / np.sqrt(freq)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# sorted(patient_ids_dict['pretrain']) == sorted(pickle.load(open('new_data/patient_id_dict/sample_pretrain_test_patient_ids_with_conditions.pkl', 'rb'))['pretrain'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# merged_df = pd.merge(dataset_2048_mortality, dataset_2048_readmission, how='outer', on='patient_id')\n", + "# final_merged_df = pd.merge(merged_df, dataset_2048_condition, how='outer', on='patient_id')\n", + "# final_merged_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Performing stratified k-fold split\n", + "# skf = StratifiedKFold(n_splits=num_splits, shuffle=True, random_state=SEED)\n", + "\n", + "# for i, (train_index, cv_index) in enumerate(skf.split(dataset, dataset[label_col])):\n", + "\n", + "# dataset_cv = dataset.iloc[cv_index]\n", + "# dataset_finetune = dataset.iloc[train_index]\n", + "\n", + "# # Separate positive and negative labeled patients\n", + "# pos_patients = dataset_cv[dataset_cv[label_col] == True]['patient_id'].tolist()\n", + "# neg_patients = dataset_cv[dataset_cv[label_col] == False]['patient_id'].tolist()\n", + "\n", + "# # Calculate the number of positive and negative patients needed for balanced CV set\n", + "# num_pos_needed = cv_size // 2\n", + "# num_neg_needed = cv_size // 2\n", + "\n", + "# # Select positive and negative patients for CV set ensuring balanced distribution\n", + "# cv_patients = pos_patients[:num_pos_needed] + neg_patients[:num_neg_needed]\n", + "# remaining_finetune_patients = pos_patients[num_pos_needed:] + neg_patients[num_neg_needed:]\n", + "\n", + "# # Extract patient IDs for training set\n", + "# finetune_patients = dataset_finetune['patient_id'].tolist()\n", + "# finetune_patients += remaining_finetune_patients\n", + "\n", + "# # Shuffle each list of patients\n", + "# random.shuffle(cv_patients)\n", + "# random.shuffle(finetune_patients)\n", + "\n", + "# patient_ids_dict['finetune']['kfold'][f'group{i+1}'] = {'finetune': finetune_patients, 'cv': cv_patients}" ] }, { @@ -700,13 +1031,13 @@ "outputs": [], "source": [ "# Assuming dataset.event_tokens is your DataFrame column\n", - "dataset.event_tokens.transform(len).plot(kind=\"hist\", bins=100)\n", - "plt.xlim(1000, 8000) # Limit x-axis to 5000\n", - "plt.ylim(0, 6000)\n", - "plt.xlabel(\"Length of Event Tokens\")\n", - "plt.ylabel(\"Frequency\")\n", - "plt.title(\"Histogram of Event Tokens Length\")\n", - "plt.show()" + "# dataset.event_tokens.transform(len).plot(kind='hist', bins=100)\n", + "# plt.xlim(1000, 8000) # Limit x-axis to 5000\n", + "# plt.ylim(0, 6000)\n", + "# plt.xlabel('Length of Event Tokens')\n", + "# plt.ylabel('Frequency')\n", + "# plt.title('Histogram of Event Tokens Length')\n", + "# plt.show()" ] }, { @@ -761,7 +1092,7 @@ "# dataset_2048_readmission = dataset_2048.loc[dataset_2048['num_visits'] > 1]\n", "# dataset_2048_readmission.reset_index(drop=True, inplace=True)\n", "#\n", - "# dataset_2048_readmission['last_VS_index'] = dataset_2048_readmission['event_tokens_2048'].transform(lambda seq: get_last_index(list(seq), '[VS]'))\n", + "# dataset_2048_readmission['last_VS_index'] = dataset_2048_readmission['event_tokens_2048'].transform(lambda seq: get_last_occurence_index(list(seq), '[VS]'))\n", "#\n", "# dataset_2048_readmission['label_readmission_1month'] = dataset_2048_readmission.apply(\n", "# lambda row: row['event_tokens_2048'][row['last_VS_index'] - 1] in ('[W_0]', '[W_1]', '[W_2]', '[W_3]', '[M_1]'), axis=1\n", diff --git a/odyssey/data/dataset.py b/odyssey/data/dataset.py index d46a90a..59c4d44 100644 --- a/odyssey/data/dataset.py +++ b/odyssey/data/dataset.py @@ -1,12 +1,18 @@ """Data module for pretraining and finetuning the model.""" -from typing import Any, Dict, List, Tuple, Union +import random +from typing import Any, Dict, List, Optional, Tuple, Union import pandas as pd import torch from torch.utils.data import Dataset -from odyssey.data.tokenizer import ConceptTokenizer +from odyssey.data.tokenizer import ConceptTokenizer, truncate_and_pad + + +TASK_INDEX = 1 +LABEL_INDEX = 2 +CUTOFF_INDEX = 3 class PretrainDataset(Dataset): @@ -49,6 +55,12 @@ def __init__( self.tokenizer = tokenizer self.max_len = max_len self.mask_prob = mask_prob + # Find the cutoff column in the data if it exists. + for column in self.data.columns: + if "cutoff" in column: + self.cutoff_col = column + else: + self.cutoff_col = None def __len__(self) -> int: """Return the length of the dataset.""" @@ -129,6 +141,8 @@ def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: """ data = self.data.iloc[idx] + cutoff = data[self.cutoff_col] if self.cutoff_col else None + data = truncate_and_pad(data, cutoff=cutoff, max_len=self.max_len) tokenized_input = self.tokenize_data(data[f"event_tokens_{self.max_len}"]) concept_tokens = tokenized_input["input_ids"].squeeze() attention_mask = tokenized_input["attention_mask"].squeeze() @@ -193,6 +207,12 @@ def __init__( self.data = data self.tokenizer = tokenizer self.max_len = max_len + # Find the cutoff column in the data if it exists. + for column in self.data.columns: + if "cutoff" in column: + self.cutoff_col = column + else: + self.cutoff_col = None def __len__(self) -> int: """Return the length of dataset.""" @@ -230,6 +250,8 @@ def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: """ data = self.data.iloc[idx] + cutoff = data[self.cutoff_col] if self.cutoff_col else None + data = truncate_and_pad(data, cutoff=cutoff, max_len=self.max_len) tokenized_input = self.tokenize_data(data[f"event_tokens_{self.max_len}"]) concept_tokens = tokenized_input["input_ids"].squeeze() attention_mask = tokenized_input["attention_mask"].squeeze() @@ -258,3 +280,213 @@ def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: "labels": labels, "attention_mask": attention_mask, } + + +class FinetuneMultiDataset(Dataset): + """Dataset for finetuning the model on multi dataset. + + Parameters + ---------- + data : pd.DataFrame + The input data containing sequences to be tokenized and masked. + tokenizer : ConceptTokenizer + An instance of the ConceptTokenizer class used for tokenizing sequences. + tasks : List[str] + A list of tasks (labels) that need to be predicted. + balance_guide : Optional[Dict[str, float]], optional + A dictionary containing the desired positive ratios for each task, + by default None. + max_len : int, optional + The maximum length of the tokenized sequences, by default 2048. + nan_indicator : int, optional + Value used to represent missing labels in the dataset, by default -1. + + Attributes + ---------- + data : pd.DataFrame + Stores the input data. + tokenizer : ConceptTokenizer + Tokenizer used for tokenizing sequences. + tasks : List[str] + A list of tasks (labels) that need to be predicted. + balance_guide : Optional[Dict[str, float]] + A dictionary containing the desired positive ratios for each task. + max_len : int + Maximum length of the tokenized sequences. + nan_indicator : int + Value used to represent missing labels in the dataset. + task_to_index : Dict[str, List[Tuple[int, str, int, Optional[int]]]] + A dictionary mapping each task to a list of tuples containing the + index, task, label, and cutoff. + index_mapper : List[Tuple[int, str, int, Optional[int]]] + A list of all datapoints to be used by __getitem__. + + """ + + def __init__( + self, + data: pd.DataFrame, + tokenizer: ConceptTokenizer, + tasks: List[str], + balance_guide: Optional[Dict[str, float]] = None, + max_len: int = 2048, + nan_indicator: int = -1, + ): + """Initiate the class.""" + super(FinetuneMultiDataset, self).__init__() + + self.data = data + self.tokenizer = tokenizer + self.tasks = tasks # List of tasks for which the model is being finetuned. + self.balance_guide = balance_guide + self.max_len = max_len + self.nan_indicator = ( + nan_indicator # Value used to indicate missing data in labels. + ) + + # Precompute indices for quick mapping in __getitem__ that + # exclude missing labels. + # This helps in filtering out entries where the label is missing + # for the specified tasks. + self.task_to_index = {task: [] for task in self.tasks} + self.data.reset_index(drop=True, inplace=True) + + for patient in self.data.itertuples(): + index = patient.Index + + for task in self.tasks: + label_col = f"label_{task}" + # Skip this task for the current patient if the label is missing. + if getattr(patient, label_col) == self.nan_indicator: + continue + + label = getattr(patient, label_col) + # Check for the existence of a task-specific cutoff in the data, + # else use None. + if f"cutoff_{task}" in self.data.columns: + cutoff = getattr(patient, f"cutoff_{task}") + else: + cutoff = None + # Append a tuple containing the necessary information + # for training to index_mapper. + datapoint = (index, task, label, cutoff) + self.task_to_index[task].append(datapoint) + + # Balance labels for specified tasks + if self.balance_guide: + for task in self.balance_guide: + self.balance_labels(task=task, positive_ratio=self.balance_guide[task]) + + # Create a list of all datapoints to be used by __getitem__ + self.index_mapper = [ + datapoints + for task_data in self.task_to_index.values() + for datapoints in task_data + ] + del self.task_to_index + + def __len__(self) -> int: + """Return the length of dataset.""" + return len(self.index_mapper) + + def __getitem__(self, idx: int) -> Dict[str, Any]: + """Get data at corresponding index. + + Parameters + ---------- + idx : int + The index of the data to be retrieved. + + Returns + ------- + Dict[str, Any] + A dictionary containing all different token sequences along with + attention mask and labels. + + """ + index, task, labels, cutoff = self.index_mapper[idx] + data = self.data.iloc[index] + + # Swap the first token with the task token. + data["event_tokens_2048"][0] = self.tokenizer.task_to_token(task) + + # Truncate and pad the data to the specified cutoff. + data = truncate_and_pad(data, cutoff, self.max_len) + + # Prepare model input + tokenized_input = self.tokenize_data(data[f"event_tokens_{self.max_len}"]) + concept_tokens = tokenized_input["input_ids"].squeeze() + attention_mask = tokenized_input["attention_mask"].squeeze() + + type_tokens = data[f"type_tokens_{self.max_len}"] + age_tokens = data[f"age_tokens_{self.max_len}"] + time_tokens = data[f"time_tokens_{self.max_len}"] + visit_tokens = data[f"visit_tokens_{self.max_len}"] + position_tokens = data[f"position_tokens_{self.max_len}"] + + type_tokens = torch.tensor(type_tokens) + age_tokens = torch.tensor(age_tokens) + time_tokens = torch.tensor(time_tokens) + visit_tokens = torch.tensor(visit_tokens) + position_tokens = torch.tensor(position_tokens) + labels = torch.tensor(labels) + + return { + "concept_ids": concept_tokens, + "type_ids": type_tokens, + "ages": age_tokens, + "time_stamps": time_tokens, + "visit_orders": position_tokens, + "visit_segments": visit_tokens, + "labels": labels, + "attention_mask": attention_mask, + "task": task, + } + + def tokenize_data(self, sequence: Union[str, List[str]]) -> Any: + """Tokenize the sequence and return input_ids and attention mask. + + Parameters + ---------- + sequence : Union[str, List[str]] + The sequence to be tokenized. + + Returns + ------- + Any + A dictionary containing input_ids and attention_mask. + + """ + return self.tokenizer(sequence, max_length=self.max_len) + + def balance_labels(self, task: str, positive_ratio: float) -> None: + """Balance the labels for the specified task in the dataset. + + This function modifies the dataset to ensure that the ratio of positive samples + to the total number of samples matches the specified positive_ratio, + while keeping all positive data points. + + Parameters + ---------- + task : str + The task for which the labels need to be balanced. + positive_ratio : float + The desired positive ratio for the task. + + """ + # Separate positive and negative datapoints + datapoints = self.task_to_index[task] + positives = [data for data in datapoints if data[LABEL_INDEX] == 1] + negatives = [data for data in datapoints if data[LABEL_INDEX] == 0] + + # Calculate the total number of samples needed to achieve the + # desired positive ratio + num_positives = len(positives) + total_needed = int(num_positives / positive_ratio) - num_positives + num_negatives_to_keep = min(len(negatives), total_needed) + + # Randomly select the negatives to keep + negatives_kept = random.sample(negatives, num_negatives_to_keep) + + # Combine the kept negatives with all positives + self.task_to_index[task] = positives + negatives_kept diff --git a/odyssey/data/sequence.py b/odyssey/data/sequence.py index e11aa8d..9fbb852 100644 --- a/odyssey/data/sequence.py +++ b/odyssey/data/sequence.py @@ -11,6 +11,7 @@ import numpy as np import pandas as pd from dateutil import parser + from odyssey.utils.log import setup_logging diff --git a/odyssey/data/tokenizer.py b/odyssey/data/tokenizer.py index f5b16ce..dae4c6a 100644 --- a/odyssey/data/tokenizer.py +++ b/odyssey/data/tokenizer.py @@ -6,10 +6,86 @@ from itertools import chain from typing import Any, Dict, List, Optional, Set, Union +import numpy as np +import pandas as pd from tokenizers import Tokenizer, models, pre_tokenizers from transformers import BatchEncoding, PreTrainedTokenizerFast +def truncate_and_pad( + row: pd.Series, + cutoff: Optional[int] = None, + max_len: int = 2048, +) -> pd.Series: + """Truncate and pad the input row to the maximum length. + + This function assumes the presence of the following columns in row: + - 'event_tokens_2048' + - 'type_tokens_2048' + - 'age_tokens_2048' + - 'time_tokens_2048' + - 'visit_tokens_2048' + - 'position_tokens_2048' + - 'elapsed_tokens_2048' + + Parameters + ---------- + row: pd.Series + The input row. + cutoff: Optional[int] + The cutoff length. Will be set to length of 'event_tokens_2048' if None. + max_len: int + The maximum length to pad to. + + Returns + ------- + pd.Series + The truncated and padded row. + + """ + # Ensuring row is a copy to prevent SettingWithCopyWarning + row = row.copy() + + if not cutoff: + cutoff = len(row["event_tokens_2048"]) + + row["event_tokens_2048"] = row["event_tokens_2048"][:cutoff] + row["type_tokens_2048"] = np.pad( + row["type_tokens_2048"][:cutoff], + (0, max_len - cutoff), + mode="constant", + ) + row["age_tokens_2048"] = np.pad( + row["age_tokens_2048"][:cutoff], + (0, max_len - cutoff), + mode="constant", + ) + row["time_tokens_2048"] = np.pad( + row["time_tokens_2048"][:cutoff], + (0, max_len - cutoff), + mode="constant", + ) + row["visit_tokens_2048"] = np.pad( + row["visit_tokens_2048"][:cutoff], + (0, max_len - cutoff), + mode="constant", + ) + row["position_tokens_2048"] = np.pad( + row["position_tokens_2048"][:cutoff], + (0, max_len - cutoff), + mode="constant", + ) + row["elapsed_tokens_2048"] = np.pad( + row["elapsed_tokens_2048"][:cutoff], + (0, max_len - cutoff), + mode="constant", + ) + + row["event_tokens_2048"] = " ".join(row["event_tokens_2048"]) + + return row + + class ConceptTokenizer: """Tokenizer for event concepts using HuggingFace Library. @@ -69,6 +145,13 @@ def __init__( self.mask_token = mask_token self.pad_token = pad_token self.unknown_token = unknown_token + self.task_tokens = ["[MOR_1M]", "[LOS_1W]", "[REA_1M]"] + [ + f"[C{i}]" for i in range(0, 5) + ] + self.tasks = ["mortality_1month", "los_1week", "readmission_1month"] + [ + f"c{i}" for i in range(5) + ] + self.task2token = self.create_task_to_token_dict() self.special_tokens = ( [ pad_token, @@ -107,9 +190,12 @@ def fit_on_vocab(self) -> None: vocab_type = file.split("/")[-1].split(".")[0] self.token_type_vocab[vocab_type] = vocab + self.token_type_vocab["task_tokens"] = self.task_tokens + # Create the tokenizer dictionary tokens = list(chain.from_iterable(list(self.token_type_vocab.values()))) self.tokenizer_vocab = {token: i for i, token in enumerate(tokens)} + self.special_tokens += self.task_tokens # Create the tokenizer object self.tokenizer_object = Tokenizer( @@ -369,3 +455,38 @@ def save_tokenizer_to_disk(self, save_dir: str) -> None: """ self.tokenizer.save(path=save_dir) + + def create_task_to_token_dict(self) -> Dict[str, str]: + """Create a dictionary mapping each task to its respective special token. + + Returns + ------- + Dict[str, str] + Dictionary mapping each task to its respective special token + + """ + task2token = { + "mortality_1month": "[MOR_1M]", + "los_1week": "[LOS_1W]", + "readmission_1month": "[REA_1M]", + } + for i in range(5): + task2token[f"c{i}"] = f"[C{i}]" + + return task2token + + def task_to_token(self, task: str) -> str: + """Return the special token representing task. + + Parameters + ---------- + task: str + Task name + + Returns + ------- + str + Special token representing task + + """ + return self.task2token[task] diff --git a/odyssey/models/baseline/Bi-LSTM.ipynb b/odyssey/models/baseline/Bi-LSTM.ipynb index d8210ab..dabed98 100644 --- a/odyssey/models/baseline/Bi-LSTM.ipynb +++ b/odyssey/models/baseline/Bi-LSTM.ipynb @@ -546,7 +546,7 @@ "train_data = pd.concat((pretrain_data, finetune_data))\n", "train_data.reset_index(inplace=True)\n", "train_data.drop_duplicates(subset=\"index\", keep=\"first\", inplace=True).set_index(\n", - " \"index\"\n", + " \"index\",\n", ")\n", "\n", "del pretrain_data, finetune_data\n", @@ -630,7 +630,7 @@ " self.length_data = length_data\n", "\n", " assert len(tokenized_data) == len(\n", - " length_data\n", + " length_data,\n", " ), \"Datasets have different lengths\"\n", "\n", " self.sorted_indices = sorted(\n", @@ -1122,21 +1122,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" } }, "nbformat": 4, diff --git a/odyssey/models/baseline/XGBoost.ipynb b/odyssey/models/baseline/XGBoost.ipynb index cff6e84..37068ee 100644 --- a/odyssey/models/baseline/XGBoost.ipynb +++ b/odyssey/models/baseline/XGBoost.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:17:53.451568Z", @@ -33,7 +33,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:17:54.970742Z", @@ -45,15 +45,19 @@ "# Import dependencies and define useful constants\n", "import os\n", "import pickle\n", - "\n", - "from typing import Any, List, Dict\n", + "from typing import List\n", "\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", "import xgboost as xgb\n", - "from scipy.sparse import csr_matrix, hstack, load_npz, save_npz, vstack\n", - "\n", + "from scipy.sparse import (\n", + " csr_matrix,\n", + " hstack,\n", + " load_npz,\n", + " save_npz,\n", + " vstack,\n", + ")\n", "from sklearn.metrics import (\n", " auc,\n", " average_precision_score,\n", @@ -66,20 +70,21 @@ " roc_curve,\n", ")\n", "from sklearn.model_selection import StratifiedKFold\n", - "\n", "from tqdm import tqdm\n", "\n", + "\n", "plt.style.use(\"seaborn-v0_8\")\n", "%matplotlib inline\n", "\n", "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", "os.chdir(ROOT)\n", "\n", - "TASK = \"readmission\"\n", + "TASK = \"los\"\n", "NUM_PATIENTS = \"50000\"\n", - "DATA_ROOT = f\"{ROOT}/data/bigbird_data/old_data\"\n", + "DURATION = \"1week\"\n", + "DATA_ROOT = f\"{ROOT}/data/bigbird_data\"\n", "DATA_PATH = f\"{DATA_ROOT}/patient_sequences/patient_sequences_2048_{TASK}.parquet\"\n", - "ID_PATH = f\"{DATA_ROOT}/patient_id_dict/dataset_2048_{TASK}_1month.pkl\"\n", + "ID_PATH = f\"{DATA_ROOT}/patient_id_dict/dataset_2048_{TASK}.pkl\"\n", "FREQ_MATRIX_TRAIN = f\"{DATA_ROOT}/patient_freq_matrix/{TASK}/patient_freq_matrix_finetune_{NUM_PATIENTS}.npz\"\n", "FREQ_MATRIX_TEST = (\n", " f\"{DATA_ROOT}/patient_freq_matrix/{TASK}/patient_freq_matrix_test.npz\"\n", @@ -88,7 +93,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 32, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:18:02.554807Z", @@ -119,6 +124,10 @@ " \n", " patient_id\n", " num_visits\n", + " deceased\n", + " death_after_start\n", + " death_after_end\n", + " length\n", " token_length\n", " event_tokens_2048\n", " type_tokens_2048\n", @@ -127,18 +136,47 @@ " visit_tokens_2048\n", " position_tokens_2048\n", " elapsed_tokens_2048\n", - " label\n", + " common_conditions\n", " rare_conditions\n", " last_VS_index\n", - " label_readmission_1month\n", + " last_VE_index\n", + " length_of_stay\n", + " label\n", " \n", " \n", " \n", " \n", " 0\n", - " f5bba8dd-25c0-5336-8d3d-37424c185026\n", + " 35581927-9c95-5ae9-af76-7d74870a349c\n", " 1\n", - " 67\n", + " 0\n", + " NaN\n", + " NaN\n", + " 50\n", + " 40\n", + " [CLS] [VS] 00006473900 00904516561 51079000220...\n", + " [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, ...\n", + " [0, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85...\n", + " [0, 5902, 5902, 5902, 5902, 5902, 5902, 5902, ...\n", + " [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...\n", + " [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...\n", + " [-2.0, -1.0, 1.97, 2.02, 2.02, 2.02, 2.02, 2.0...\n", + " [1, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", + " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", + " 1\n", + " 52\n", + " 2.041667\n", + " 0\n", + " \n", + " \n", + " 1\n", + " f5bba8dd-25c0-5336-8d3d-37424c185026\n", + " 2\n", + " 0\n", + " NaN\n", + " NaN\n", + " 148\n", + " 81\n", " [CLS] [VS] 52135_2 52075_2 52074_2 52073_3 520...\n", " [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...\n", " [0, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83...\n", @@ -149,13 +187,19 @@ " [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", " 68\n", + " 154\n", + " 3.500000\n", " 0\n", " \n", " \n", - " 1\n", + " 2\n", " f4938f91-cadb-5133-8541-a52fb0916cea\n", - " 1\n", - " 48\n", + " 2\n", + " 0\n", + " NaN\n", + " NaN\n", + " 78\n", + " 86\n", " [CLS] [VS] 0RB30ZZ 0RG10A0 00071101441 0090419...\n", " [1, 2, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n", " [0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44...\n", @@ -166,13 +210,19 @@ " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", " 49\n", - " 1\n", + " 84\n", + " 1.375000\n", + " 0\n", " \n", " \n", - " 2\n", + " 3\n", " 6fe2371b-a6f0-5436-aade-7795005b0c66\n", - " 1\n", - " 54\n", + " 2\n", + " 0\n", + " NaN\n", + " NaN\n", + " 86\n", + " 91\n", " [CLS] [VS] 63739057310 49281041688 00597026010...\n", " [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n", " [0, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72...\n", @@ -183,41 +233,32 @@ " [1, 0, 0, 0, 0, 0, 0, 1, 0, 0]\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", " 55\n", + " 92\n", + " 1.458333\n", " 0\n", " \n", " \n", - " 3\n", - " 7376b52c-8e7e-5e5d-b256-1892a7237694\n", - " 1\n", - " 38\n", - " [CLS] [VS] 00071041813 00121197100 00904198261...\n", - " [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n", - " [0, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79...\n", - " [0, 8451, 8451, 8451, 8451, 8451, 8451, 8451, ...\n", - " [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...\n", - " [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...\n", - " [-2.0, -1.0, 0.04, 0.04, 0.04, 0.04, 0.04, 0.0...\n", - " [0, 0, 0, 0, 0, 0, 0, 0, 1, 0]\n", - " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", - " 39\n", - " 1\n", - " \n", - " \n", " 4\n", - " e0bb4cea-1ae3-5716-baa0-b93d56001be8\n", - " 3\n", - " 143\n", - " [CLS] [VS] 51006_0 50983_0 50971_1 50946_4 509...\n", + " 6f7590ae-f3b9-50e5-9e41-d4bb1000887a\n", + " 1\n", + " 0\n", + " NaN\n", + " NaN\n", + " 72\n", + " 56\n", + " [CLS] [VS] 50813_0 52135_0 52075_3 52074_3 520...\n", " [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...\n", - " [0, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28...\n", - " [0, 8342, 8342, 8342, 8342, 8342, 8342, 8342, ...\n", + " [0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47...\n", + " [0, 6379, 6379, 6379, 6379, 6379, 6379, 6379, ...\n", " [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...\n", " [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...\n", " [-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...\n", - " [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]\n", + " [1, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", - " 144\n", " 1\n", + " 74\n", + " 2.958333\n", + " 0\n", " \n", " \n", " ...\n", @@ -235,12 +276,45 @@ " ...\n", " ...\n", " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", + " ...\n", " \n", " \n", - " 75455\n", + " 143474\n", + " 3f300d4e-4554-5f1f-9dff-f209a4916cbc\n", + " 7\n", + " 0\n", + " NaN\n", + " NaN\n", + " 536\n", + " 564\n", + " [CLS] [VS] 51484_0 51146_3 51200_1 51221_4 512...\n", + " [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...\n", + " [0, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64...\n", + " [0, 6921, 6921, 6921, 6921, 6921, 6921, 6921, ...\n", + " [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...\n", + " [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...\n", + " [-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...\n", + " [1, 1, 1, 1, 0, 0, 0, 0, 1, 0]\n", + " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", + " 520\n", + " 562\n", + " 1.666667\n", + " 0\n", + " \n", + " \n", + " 143475\n", " cf2115d7-937e-511d-b159-dd7eb3d5d420\n", - " 1\n", - " 108\n", + " 2\n", + " 0\n", + " NaN\n", + " NaN\n", + " 166\n", + " 142\n", " [CLS] [VS] 33332001001 00781305714 10019017644...\n", " [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 6, 5, ...\n", " [0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32...\n", @@ -251,13 +325,19 @@ " [1, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", " 109\n", - " 1\n", + " 172\n", + " 2.541667\n", + " 0\n", " \n", " \n", - " 75456\n", + " 143476\n", " 31338a39-28f9-54a5-a810-2d05fbaa5166\n", - " 2\n", - " 191\n", + " 3\n", + " 0\n", + " NaN\n", + " NaN\n", + " 283\n", + " 221\n", " [CLS] [VS] 00338011704 00409128331 63323026201...\n", " [1, 2, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...\n", " [0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50...\n", @@ -268,47 +348,42 @@ " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", " 192\n", + " 293\n", + " 4.125000\n", " 0\n", " \n", " \n", - " 75457\n", + " 143477\n", " 0989415d-394c-5f42-8dac-75dc7306a23c\n", - " 1\n", - " 11\n", + " 2\n", + " 0\n", + " NaN\n", + " NaN\n", + " 470\n", + " 140\n", " [CLS] [VS] 49281041550 51079043620 51079088120...\n", - " [1, 2, 6, 6, 6, 6, 6, 6, 6, 3, 8, 0, 0, 0, 0, ...\n", + " [1, 2, 6, 6, 6, 6, 6, 6, 6, 3, 8, 4, 2, 7, 6, ...\n", " [0, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 0,...\n", " [0, 5309, 5309, 5309, 5309, 5309, 5309, 5309, ...\n", - " [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, ...\n", - " [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, ...\n", + " [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 1, 1, 1, ...\n", + " [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, ...\n", " [-2.0, -1.0, 1.3, 1.78, 1.78, 1.78, 1.84, 1.94...\n", " [0, 0, 0, 0, 1, 0, 0, 0, 1, 0]\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", " 12\n", - " 0\n", - " \n", - " \n", - " 75458\n", - " 26fb8fef-b976-5c55-859d-cc190261f94b\n", + " 476\n", + " 19.250000\n", " 1\n", - " 77\n", - " [CLS] [VS] 0SRD0J9 00904224461 00409128331 009...\n", - " [1, 2, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n", - " [0, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61...\n", - " [0, 5075, 5075, 5075, 5075, 5075, 5075, 5075, ...\n", - " [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...\n", - " [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...\n", - " [-2.0, -1.0, 0.0, 23.94, 23.94, 23.94, 23.94, ...\n", - " [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]\n", - " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", - " 78\n", - " 0\n", " \n", " \n", - " 75459\n", + " 143478\n", " 02cee673-6875-50c9-bad9-e7a7746731eb\n", - " 1\n", - " 38\n", + " 2\n", + " 0\n", + " NaN\n", + " NaN\n", + " 98\n", + " 106\n", " [CLS] [VS] 33332001101 00904224461 00904516561...\n", " [1, 2, 6, 6, 6, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, ...\n", " [0, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52...\n", @@ -319,148 +394,163 @@ " [1, 0, 1, 1, 0, 1, 0, 0, 0, 0]\n", " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n", " 39\n", + " 104\n", + " 2.625000\n", " 0\n", " \n", " \n", "\n", - "

75460 rows × 14 columns

\n", + "

143479 rows × 20 columns

\n", "" ], "text/plain": [ - " patient_id num_visits token_length \\\n", - "0 f5bba8dd-25c0-5336-8d3d-37424c185026 1 67 \n", - "1 f4938f91-cadb-5133-8541-a52fb0916cea 1 48 \n", - "2 6fe2371b-a6f0-5436-aade-7795005b0c66 1 54 \n", - "3 7376b52c-8e7e-5e5d-b256-1892a7237694 1 38 \n", - "4 e0bb4cea-1ae3-5716-baa0-b93d56001be8 3 143 \n", - "... ... ... ... \n", - "75455 cf2115d7-937e-511d-b159-dd7eb3d5d420 1 108 \n", - "75456 31338a39-28f9-54a5-a810-2d05fbaa5166 2 191 \n", - "75457 0989415d-394c-5f42-8dac-75dc7306a23c 1 11 \n", - "75458 26fb8fef-b976-5c55-859d-cc190261f94b 1 77 \n", - "75459 02cee673-6875-50c9-bad9-e7a7746731eb 1 38 \n", - "\n", - " event_tokens_2048 \\\n", - "0 [CLS] [VS] 52135_2 52075_2 52074_2 52073_3 520... \n", - "1 [CLS] [VS] 0RB30ZZ 0RG10A0 00071101441 0090419... \n", - "2 [CLS] [VS] 63739057310 49281041688 00597026010... \n", - "3 [CLS] [VS] 00071041813 00121197100 00904198261... \n", - "4 [CLS] [VS] 51006_0 50983_0 50971_1 50946_4 509... \n", - "... ... \n", - "75455 [CLS] [VS] 33332001001 00781305714 10019017644... \n", - "75456 [CLS] [VS] 00338011704 00409128331 63323026201... \n", - "75457 [CLS] [VS] 49281041550 51079043620 51079088120... \n", - "75458 [CLS] [VS] 0SRD0J9 00904224461 00409128331 009... \n", - "75459 [CLS] [VS] 33332001101 00904224461 00904516561... \n", - "\n", - " type_tokens_2048 \\\n", - "0 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", - "1 [1, 2, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "2 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "3 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "4 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", - "... ... \n", - "75455 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 6, 5, ... \n", - "75456 [1, 2, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", - "75457 [1, 2, 6, 6, 6, 6, 6, 6, 6, 3, 8, 0, 0, 0, 0, ... \n", - "75458 [1, 2, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "75459 [1, 2, 6, 6, 6, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "\n", - " age_tokens_2048 \\\n", - "0 [0, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83... \n", - "1 [0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44... \n", - "2 [0, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72... \n", - "3 [0, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79, 79... \n", - "4 [0, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28... \n", - "... ... \n", - "75455 [0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32... \n", - "75456 [0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50... \n", - "75457 [0, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 0,... \n", - "75458 [0, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61... \n", - "75459 [0, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52... \n", - "\n", - " time_tokens_2048 \\\n", - "0 [0, 6594, 6594, 6594, 6594, 6594, 6594, 6594, ... \n", - "1 [0, 8150, 8150, 8150, 8150, 8150, 8150, 8150, ... \n", - "2 [0, 6093, 6093, 6093, 6093, 6093, 6093, 6093, ... \n", - "3 [0, 8451, 8451, 8451, 8451, 8451, 8451, 8451, ... \n", - "4 [0, 8342, 8342, 8342, 8342, 8342, 8342, 8342, ... \n", - "... ... \n", - "75455 [0, 5481, 5481, 5481, 5481, 5481, 5481, 5481, ... \n", - "75456 [0, 4997, 4997, 4997, 4997, 4997, 4997, 4997, ... \n", - "75457 [0, 5309, 5309, 5309, 5309, 5309, 5309, 5309, ... \n", - "75458 [0, 5075, 5075, 5075, 5075, 5075, 5075, 5075, ... \n", - "75459 [0, 5642, 5642, 5642, 5642, 5642, 5642, 5642, ... \n", - "\n", - " visit_tokens_2048 \\\n", - "0 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "1 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "2 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "3 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "4 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "... ... \n", - "75455 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "75456 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "75457 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, ... \n", - "75458 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "75459 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "\n", - " position_tokens_2048 \\\n", - "0 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "1 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "2 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "3 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "4 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "... ... \n", - "75455 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "75456 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "75457 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, ... \n", - "75458 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "75459 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "\n", - " elapsed_tokens_2048 \\\n", - "0 [-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n", - "1 [-2.0, -1.0, 0.0, 0.0, 1.08, 1.08, 13.89, 13.8... \n", - "2 [-2.0, -1.0, 0.75, 0.75, 0.75, 0.75, 0.75, 0.7... \n", - "3 [-2.0, -1.0, 0.04, 0.04, 0.04, 0.04, 0.04, 0.0... \n", - "4 [-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n", - "... ... \n", - "75455 [-2.0, -1.0, 1.16, 1.25, 1.3, 1.3, 1.31, 1.53,... \n", - "75456 [-2.0, -1.0, 1.48, 1.49, 1.52, 1.52, 1.6, 1.6,... \n", - "75457 [-2.0, -1.0, 1.3, 1.78, 1.78, 1.78, 1.84, 1.94... \n", - "75458 [-2.0, -1.0, 0.0, 23.94, 23.94, 23.94, 23.94, ... \n", - "75459 [-2.0, -1.0, 2.64, 2.71, 2.71, 4.15, 4.15, 4.4... \n", - "\n", - " label rare_conditions \\\n", - "0 [0, 0, 0, 0, 0, 0, 0, 1, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "1 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "2 [1, 0, 0, 0, 0, 0, 0, 1, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "3 [0, 0, 0, 0, 0, 0, 0, 0, 1, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "4 [0, 0, 0, 0, 0, 0, 0, 1, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "... ... ... \n", - "75455 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "75456 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "75457 [0, 0, 0, 0, 1, 0, 0, 0, 1, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "75458 [0, 0, 0, 0, 0, 0, 0, 1, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "75459 [1, 0, 1, 1, 0, 1, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "\n", - " last_VS_index label_readmission_1month \n", - "0 68 0 \n", - "1 49 1 \n", - "2 55 0 \n", - "3 39 1 \n", - "4 144 1 \n", - "... ... ... \n", - "75455 109 1 \n", - "75456 192 0 \n", - "75457 12 0 \n", - "75458 78 0 \n", - "75459 39 0 \n", - "\n", - "[75460 rows x 14 columns]" + " patient_id num_visits deceased \\\n", + "0 35581927-9c95-5ae9-af76-7d74870a349c 1 0 \n", + "1 f5bba8dd-25c0-5336-8d3d-37424c185026 2 0 \n", + "2 f4938f91-cadb-5133-8541-a52fb0916cea 2 0 \n", + "3 6fe2371b-a6f0-5436-aade-7795005b0c66 2 0 \n", + "4 6f7590ae-f3b9-50e5-9e41-d4bb1000887a 1 0 \n", + "... ... ... ... \n", + "143474 3f300d4e-4554-5f1f-9dff-f209a4916cbc 7 0 \n", + "143475 cf2115d7-937e-511d-b159-dd7eb3d5d420 2 0 \n", + "143476 31338a39-28f9-54a5-a810-2d05fbaa5166 3 0 \n", + "143477 0989415d-394c-5f42-8dac-75dc7306a23c 2 0 \n", + "143478 02cee673-6875-50c9-bad9-e7a7746731eb 2 0 \n", + "\n", + " death_after_start death_after_end length token_length \\\n", + "0 NaN NaN 50 40 \n", + "1 NaN NaN 148 81 \n", + "2 NaN NaN 78 86 \n", + "3 NaN NaN 86 91 \n", + "4 NaN NaN 72 56 \n", + "... ... ... ... ... \n", + "143474 NaN NaN 536 564 \n", + "143475 NaN NaN 166 142 \n", + "143476 NaN NaN 283 221 \n", + "143477 NaN NaN 470 140 \n", + "143478 NaN NaN 98 106 \n", + "\n", + " event_tokens_2048 \\\n", + "0 [CLS] [VS] 00006473900 00904516561 51079000220... \n", + "1 [CLS] [VS] 52135_2 52075_2 52074_2 52073_3 520... \n", + "2 [CLS] [VS] 0RB30ZZ 0RG10A0 00071101441 0090419... \n", + "3 [CLS] [VS] 63739057310 49281041688 00597026010... \n", + "4 [CLS] [VS] 50813_0 52135_0 52075_3 52074_3 520... \n", + "... ... \n", + "143474 [CLS] [VS] 51484_0 51146_3 51200_1 51221_4 512... \n", + "143475 [CLS] [VS] 33332001001 00781305714 10019017644... \n", + "143476 [CLS] [VS] 00338011704 00409128331 63323026201... \n", + "143477 [CLS] [VS] 49281041550 51079043620 51079088120... \n", + "143478 [CLS] [VS] 33332001101 00904224461 00904516561... \n", + "\n", + " type_tokens_2048 \\\n", + "0 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, ... \n", + "1 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", + "2 [1, 2, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", + "3 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", + "4 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", + "... ... \n", + "143474 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", + "143475 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 6, 5, ... \n", + "143476 [1, 2, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", + "143477 [1, 2, 6, 6, 6, 6, 6, 6, 6, 3, 8, 4, 2, 7, 6, ... \n", + "143478 [1, 2, 6, 6, 6, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", + "\n", + " age_tokens_2048 \\\n", + "0 [0, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85... \n", + "1 [0, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83... \n", + "2 [0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44... \n", + "3 [0, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72... \n", + "4 [0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47... \n", + "... ... \n", + "143474 [0, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64... \n", + "143475 [0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32... \n", + "143476 [0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50... \n", + "143477 [0, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 0,... \n", + "143478 [0, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52... \n", + "\n", + " time_tokens_2048 \\\n", + "0 [0, 5902, 5902, 5902, 5902, 5902, 5902, 5902, ... \n", + "1 [0, 6594, 6594, 6594, 6594, 6594, 6594, 6594, ... \n", + "2 [0, 8150, 8150, 8150, 8150, 8150, 8150, 8150, ... \n", + "3 [0, 6093, 6093, 6093, 6093, 6093, 6093, 6093, ... \n", + "4 [0, 6379, 6379, 6379, 6379, 6379, 6379, 6379, ... \n", + "... ... \n", + "143474 [0, 6921, 6921, 6921, 6921, 6921, 6921, 6921, ... \n", + "143475 [0, 5481, 5481, 5481, 5481, 5481, 5481, 5481, ... \n", + "143476 [0, 4997, 4997, 4997, 4997, 4997, 4997, 4997, ... \n", + "143477 [0, 5309, 5309, 5309, 5309, 5309, 5309, 5309, ... \n", + "143478 [0, 5642, 5642, 5642, 5642, 5642, 5642, 5642, ... \n", + "\n", + " visit_tokens_2048 \\\n", + "0 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", + "1 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", + "2 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", + "3 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", + "4 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", + "... ... \n", + "143474 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", + "143475 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", + "143476 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", + "143477 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 1, 1, 1, ... \n", + "143478 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", + "\n", + " position_tokens_2048 \\\n", + "0 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", + "1 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", + "2 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", + "3 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", + "4 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", + "... ... \n", + "143474 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", + "143475 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", + "143476 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", + "143477 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, ... \n", + "143478 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", + "\n", + " elapsed_tokens_2048 \\\n", + "0 [-2.0, -1.0, 1.97, 2.02, 2.02, 2.02, 2.02, 2.0... \n", + "1 [-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n", + "2 [-2.0, -1.0, 0.0, 0.0, 1.08, 1.08, 13.89, 13.8... \n", + "3 [-2.0, -1.0, 0.75, 0.75, 0.75, 0.75, 0.75, 0.7... \n", + "4 [-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n", + "... ... \n", + "143474 [-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n", + "143475 [-2.0, -1.0, 1.16, 1.25, 1.3, 1.3, 1.31, 1.53,... \n", + "143476 [-2.0, -1.0, 1.48, 1.49, 1.52, 1.52, 1.6, 1.6,... \n", + "143477 [-2.0, -1.0, 1.3, 1.78, 1.78, 1.78, 1.84, 1.94... \n", + "143478 [-2.0, -1.0, 2.64, 2.71, 2.71, 4.15, 4.15, 4.4... \n", + "\n", + " common_conditions rare_conditions \\\n", + "0 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", + "1 [0, 0, 0, 0, 0, 0, 0, 1, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", + "2 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", + "3 [1, 0, 0, 0, 0, 0, 0, 1, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", + "4 [1, 0, 0, 0, 0, 0, 0, 0, 0, 1] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", + "... ... ... \n", + "143474 [1, 1, 1, 1, 0, 0, 0, 0, 1, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", + "143475 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", + "143476 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", + "143477 [0, 0, 0, 0, 1, 0, 0, 0, 1, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", + "143478 [1, 0, 1, 1, 0, 1, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", + "\n", + " last_VS_index last_VE_index length_of_stay label \n", + "0 1 52 2.041667 0 \n", + "1 68 154 3.500000 0 \n", + "2 49 84 1.375000 0 \n", + "3 55 92 1.458333 0 \n", + "4 1 74 2.958333 0 \n", + "... ... ... ... ... \n", + "143474 520 562 1.666667 0 \n", + "143475 109 172 2.541667 0 \n", + "143476 192 293 4.125000 0 \n", + "143477 12 476 19.250000 1 \n", + "143478 39 104 2.625000 0 \n", + "\n", + "[143479 rows x 20 columns]" ] }, - "execution_count": 12, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } @@ -468,16 +558,18 @@ "source": [ "# Load dataset\n", "dataset = pd.read_parquet(\n", - " f\"{ROOT}/data/bigbird_data/patient_sequences/patient_sequences_2048_{TASK}.parquet\"\n", + " f\"{ROOT}/data/bigbird_data/patient_sequences/patient_sequences_2048_{TASK}.parquet\",\n", ")\n", - "dataset.rename(columns={f\"common_conditions\": \"label\"}, inplace=True)\n", - "# dataset.rename(columns={f'label_{TASK}_1month': 'label'}, inplace=True)\n", + "dataset.rename(columns={\"label_length_of_stay_1week\": \"label\"}, inplace=True)\n", + "dataset[\"label\"] = dataset[\"label\"].astype(int)\n", + "# dataset.rename(columns={f'all_conditions': 'label'}, inplace=True)\n", + "# dataset.rename(columns={f'label_{TASK}_{DURATION}': 'label'}, inplace=True)\n", "dataset" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 33, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:18:02.652563Z", @@ -496,7 +588,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 22, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:18:07.521714Z", @@ -508,7 +600,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "Loading Tokens: 100%|██████████| 75460/75460 [00:05<00:00, 14778.71 Patients/s]" + "Loading Tokens: 100%|██████████| 143479/143479 [00:06<00:00, 20885.94 Patients/s]" ] }, { @@ -516,8 +608,8 @@ "output_type": "stream", "text": [ "Complete list of unique event tokens\n", - "Length: 16458\n", - "Head: ['[W_3]', '[W_2]', '[W_1]', '[W_0]', '[VS]', '[VE]', '[REG]', '[M_9]', '[M_8]', '[M_7]', '[M_6]', '[M_5]', '[M_4]', '[M_3]', '[M_2]', '[M_1]', '[M_12]', '[M_11]', '[M_10]', '[M_0]', '[LT]', '[CLS]', 'XY0VX83', 'XW0DXR5', 'XW0DX82', 'XW043C3', 'XW043B3', 'XW04351', 'XW033B3', 'XW03331']...\n" + "Length: 18534\n", + "Head: ['[W_3]', '[W_2]', '[W_1]', '[W_0]', '[VS]', '[VE]', '[REG]', '[M_9]', '[M_8]', '[M_7]', '[M_6]', '[M_5]', '[M_4]', '[M_3]', '[M_2]', '[M_1]', '[M_12]', '[M_11]', '[M_10]', '[M_0]', '[LT]', '[CLS]', 'XY0VX83', 'XW0DXR5', 'XW0DX82', 'XW043C3', 'XW043B3', 'XW04351', 'XW033H4', 'XW033B3']...\n" ] }, { @@ -550,7 +642,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 23, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:18:07.534354Z", @@ -562,7 +654,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "16437 ['id', 'XY0VX83', 'XW0DXR5', 'XW0DX82', 'XW043C3', 'XW043B3', 'XW04351', 'XW033B3', 'XW03331', 'X2RF332', 'X2C1361', 'X2C0361', 'X2A5312', 'HZ87ZZZ', 'HZ85ZZZ', 'HZ2ZZZZ', 'GZB4ZZZ', 'GZB2ZZZ', 'GZB1ZZZ', 'GZB0ZZZ']\n" + "18513 ['id', 'XY0VX83', 'XW0DXR5', 'XW0DX82', 'XW043C3', 'XW043B3', 'XW04351', 'XW033H4', 'XW033B3', 'XW03372', 'XW03331', 'X2RF332', 'X2RF032', 'X2C1361', 'X2C0361', 'X2A5312', 'HZ99ZZZ', 'HZ87ZZZ', 'HZ85ZZZ', 'HZ81ZZZ']\n" ] } ], @@ -593,7 +685,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 24, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:25:27.580887Z", @@ -612,12 +704,14 @@ " buffer_size: int = 50000,\n", ") -> None:\n", " \"\"\"Calculate and save the patient frequency matrix.\"\"\"\n", - "\n", " patient_freq_matrix = None\n", " matrix_buffer = []\n", "\n", " for idx, patient in tqdm(\n", - " data.iterrows(), desc=\"Loading Tokens\", unit=\" Patients\", total=len(data)\n", + " data.iterrows(),\n", + " desc=\"Loading Tokens\",\n", + " unit=\" Patients\",\n", + " total=len(data),\n", " ):\n", " patient_history = {token: 0 for token in feature_event_tokens}\n", " patient_history[\"id\"] = idx\n", @@ -668,7 +762,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 25, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:41:37.007429Z", @@ -679,11 +773,11 @@ { "data": { "text/plain": [ - "<50000x16343 sparse matrix of type ''\n", - "\twith 6072798 stored elements in Compressed Sparse Row format>" + "<50000x18513 sparse matrix of type ''\n", + "\twith 5122992 stored elements in Compressed Sparse Row format>" ] }, - "execution_count": 17, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -698,7 +792,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 26, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:41:37.481671Z", @@ -720,7 +814,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 27, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:41:40.198998Z", @@ -728,29 +822,14 @@ }, "collapsed": false }, - "outputs": [ - { - "ename": "ValueError", - "evalue": "blocks[0,:] has incompatible row dimensions. Got blocks[0,1].shape[0] == 49879, expected 50000.", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[9], line 18\u001b[0m\n\u001b[1;32m 13\u001b[0m patient_freq_matrix \u001b[38;5;241m=\u001b[39m hstack(\n\u001b[1;32m 14\u001b[0m [patient_freq_matrix, csr_matrix([num_visits, min_age, max_age])\u001b[38;5;241m.\u001b[39mT], \u001b[38;5;28mformat\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcsr\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 15\u001b[0m )\n\u001b[1;32m 16\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m patient_freq_matrix[:, \u001b[38;5;241m1\u001b[39m:] \u001b[38;5;66;03m# Drop id feature\u001b[39;00m\n\u001b[0;32m---> 18\u001b[0m patient_freq_matrix \u001b[38;5;241m=\u001b[39m \u001b[43madd_age_to_freq_matrix\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpatient_freq_matrix\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 19\u001b[0m patient_freq_matrix_test \u001b[38;5;241m=\u001b[39m add_age_to_freq_matrix(test_data, patient_freq_matrix_test)\n", - "Cell \u001b[0;32mIn[9], line 13\u001b[0m, in \u001b[0;36madd_age_to_freq_matrix\u001b[0;34m(data, patient_freq_matrix)\u001b[0m\n\u001b[1;32m 10\u001b[0m max_age \u001b[38;5;241m=\u001b[39m [np\u001b[38;5;241m.\u001b[39mmax(patient_age_tokens) \u001b[38;5;28;01mfor\u001b[39;00m patient_age_tokens \u001b[38;5;129;01min\u001b[39;00m data[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mage_tokens_2048\u001b[39m\u001b[38;5;124m\"\u001b[39m]]\n\u001b[1;32m 12\u001b[0m \u001b[38;5;66;03m# Add extra features to the frequency dataset\u001b[39;00m\n\u001b[0;32m---> 13\u001b[0m patient_freq_matrix \u001b[38;5;241m=\u001b[39m \u001b[43mhstack\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 14\u001b[0m \u001b[43m \u001b[49m\u001b[43m[\u001b[49m\u001b[43mpatient_freq_matrix\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcsr_matrix\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[43mnum_visits\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmin_age\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_age\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mT\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcsr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 15\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m patient_freq_matrix[:, \u001b[38;5;241m1\u001b[39m:]\n", - "File \u001b[0;32m/fs01/home/afallah/light/lib/python3.10/site-packages/scipy/sparse/_construct.py:733\u001b[0m, in \u001b[0;36mhstack\u001b[0;34m(blocks, format, dtype)\u001b[0m\n\u001b[1;32m 731\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _block([blocks], \u001b[38;5;28mformat\u001b[39m, dtype)\n\u001b[1;32m 732\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 733\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_block\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[43mblocks\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreturn_spmatrix\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/fs01/home/afallah/light/lib/python3.10/site-packages/scipy/sparse/_construct.py:948\u001b[0m, in \u001b[0;36m_block\u001b[0;34m(blocks, format, dtype, return_spmatrix)\u001b[0m\n\u001b[1;32m 944\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m brow_lengths[i] \u001b[38;5;241m!=\u001b[39m A\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m]:\n\u001b[1;32m 945\u001b[0m msg \u001b[38;5;241m=\u001b[39m (\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mblocks[\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mi\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m,:] has incompatible row dimensions. \u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 946\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mGot blocks[\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mi\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m,\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mj\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m].shape[0] == \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mA\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 947\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mexpected \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mbrow_lengths[i]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m--> 948\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(msg)\n\u001b[1;32m 950\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m bcol_lengths[j] \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 951\u001b[0m bcol_lengths[j] \u001b[38;5;241m=\u001b[39m A\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m1\u001b[39m]\n", - "\u001b[0;31mValueError\u001b[0m: blocks[0,:] has incompatible row dimensions. Got blocks[0,1].shape[0] == 49879, expected 50000." - ] - } - ], + "outputs": [], "source": [ "# Get extra features such as number of visits and age\n", "def add_age_to_freq_matrix(\n", - " data: pd.DataFrame, patient_freq_matrix: np.ndarray\n", + " data: pd.DataFrame,\n", + " patient_freq_matrix: np.ndarray,\n", ") -> np.ndarray:\n", " \"\"\"Add age feature to patient frequency matrix.\"\"\"\n", - "\n", " num_visits = data[\"num_visits\"].values\n", " min_age = [\n", " find_min_greater_than_zero(patient_age_tokens)\n", @@ -774,14 +853,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:41:40.227252Z", "start_time": "2024-03-18T02:41:40.200678Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "How many features have been reported for at least 1 patients?\n", + "14443 Features\n" + ] + } + ], "source": [ "# Get intuition about the frequency of different features in the training dataset\n", "report_threshold = 1\n", @@ -791,7 +879,7 @@ "\n", "print(\n", " f\"How many features have been reported for at least {report_threshold} patients?\\n\"\n", - " f\"{features_above_threshold} Features\"\n", + " f\"{features_above_threshold} Features\",\n", ")\n", "\n", "# Plot the histogram of feature frequency\n", @@ -804,14 +892,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:41:40.255936Z", "start_time": "2024-03-18T02:41:40.228722Z" } }, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "array([18514, 18512, 18513, ..., 7736, 9232, 14290])" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Pick features to train the model on\n", "NUM_FEATURES = 20000\n", @@ -822,7 +921,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:41:40.261325Z", @@ -842,7 +941,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 50, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:41:42.023770Z", @@ -852,8 +951,8 @@ "outputs": [], "source": [ "# Prepare data for model training\n", - "X = vstack([patient_freq_matrix, patient_freq_matrix_test])\n", - "Y = np.hstack([train_data[\"label\"].values, test_data[\"label\"].values])\n", + "# X = vstack([patient_freq_matrix, patient_freq_matrix_test])\n", + "# Y = hstack([np.array(train_data['label'].tolist()), np.array(test_data['label'].tolist())])\n", "\n", "# Optional, Scale features. Didn't improve performance\n", "# scaler = MaxAbsScaler()\n", @@ -867,7 +966,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:41:42.557970Z", @@ -881,20 +980,693 @@ "X_train = patient_freq_matrix\n", "X_test = patient_freq_matrix_test\n", "\n", - "y_train = train_data[\"label\"].values\n", - "y_test = test_data[\"label\"].values" + "y_train = np.array(train_data[\"label\"].tolist())\n", + "y_test = np.array(test_data[\"label\"].tolist())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "models = {\"unweighted\": [], \"weighted1\": [], \"weighted2\": [], \"weighted3\": []}\n", + "weights = {\n", + " \"unweighted\": np.ones(shape=20),\n", + " \"weighted1\": np.array(\n", + " [\n", + " 5.69327441,\n", + " 7.42882028,\n", + " 10.13231198,\n", + " 11.49395911,\n", + " 16.40469558,\n", + " 11.0384684,\n", + " 15.53901244,\n", + " 10.03937008,\n", + " 12.32068141,\n", + " 12.45844666,\n", + " 50.0,\n", + " 50.0,\n", + " 50.0,\n", + " 50.0,\n", + " 50.0,\n", + " 50.0,\n", + " 50.0,\n", + " 50.0,\n", + " 50.0,\n", + " 50.0,\n", + " ],\n", + " ).tolist(),\n", + " \"weighted2\": np.array(\n", + " [\n", + " 1.0,\n", + " 1.30484142,\n", + " 1.77969851,\n", + " 2.01886617,\n", + " 2.88141663,\n", + " 1.93886112,\n", + " 2.72936299,\n", + " 1.76337365,\n", + " 2.16407651,\n", + " 2.1882744,\n", + " 339.421875,\n", + " 60.34166667,\n", + " 164.56818182,\n", + " 156.28057554,\n", + " 30.50983146,\n", + " 145.79194631,\n", + " 66.84,\n", + " 88.66530612,\n", + " 35.55319149,\n", + " 35.32195122,\n", + " ],\n", + " ).tolist(),\n", + " \"weighted3\": np.array(\n", + " [\n", + " 1.0,\n", + " 1.14229656,\n", + " 1.33405341,\n", + " 1.4208681,\n", + " 1.6974736,\n", + " 1.39242993,\n", + " 1.65207838,\n", + " 1.3279208,\n", + " 1.47108005,\n", + " 1.47928172,\n", + " 18.42340563,\n", + " 7.76798987,\n", + " 12.82841307,\n", + " 12.50122296,\n", + " 5.52357054,\n", + " 12.07443358,\n", + " 8.17557337,\n", + " 9.41622568,\n", + " 5.9626497,\n", + " 5.94322734,\n", + " ],\n", + " ).tolist(),\n", + "}\n", + "\n", + "# Train a separate model for each label\n", + "for weighting_scheme in models:\n", + " for i in range(y_train.shape[1]):\n", + " model = xgb.XGBClassifier(\n", + " objective=\"binary:logistic\",\n", + " random_state=23,\n", + " scale_pos_weight=weights[weighting_scheme][i],\n", + " )\n", + " model.fit(X_train, y_train[:, i])\n", + " models[weighting_scheme].append(model)\n", + " print(f\"Model saved for label: {i}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def generate_predictions(model, X):\n", + " \"\"\"\n", + " Generate both class predictions and probabilities.\n", + " \"\"\"\n", + " pred = model.predict(X)\n", + " prob = model.predict_proba(X)[:, 1]\n", + " return pred, prob\n", + "\n", + "\n", + "def calculate_metrics(y_true, y_pred, y_prob):\n", + " \"\"\"\n", + " Calculate and return performance metrics.\n", + " \"\"\"\n", + " metrics = {\n", + " \"Balanced Accuracy\": balanced_accuracy_score(y_true, y_pred),\n", + " \"F1 Score\": f1_score(y_true, y_pred),\n", + " \"Precision\": precision_score(y_true, y_pred),\n", + " \"Recall\": recall_score(y_true, y_pred),\n", + " \"AUROC\": roc_auc_score(y_true, y_prob),\n", + " \"Average Precision Score\": average_precision_score(y_true, y_pred),\n", + " }\n", + "\n", + " precision, recall, _ = precision_recall_curve(y_true, y_pred)\n", + " metrics[\"AUC-PR\"] = auc(recall, precision)\n", + "\n", + " return metrics\n", + "\n", + "\n", + "def plot_roc_curve(y_true, y_prob, label_prefix):\n", + " \"\"\"\n", + " Plot ROC curve for a dataset.\n", + " \"\"\"\n", + " fpr, tpr, _ = roc_curve(y_true, y_prob)\n", + " plt.plot(\n", + " fpr,\n", + " tpr,\n", + " label=f\"{label_prefix} AUROC={roc_auc_score(y_true, y_prob):.3f}\",\n", + " )\n", + "\n", + "\n", + "def assess_model_performance(\n", + " model,\n", + " X_train,\n", + " y_train,\n", + " X_test,\n", + " y_test,\n", + " X_all,\n", + " Y_all,\n", + " plot=False,\n", + " print_metrics=False,\n", + " return_test_metrics=True,\n", + "):\n", + " \"\"\"\n", + " Assess model performance across training, testing, and all data.\n", + " \"\"\"\n", + " # Generate Predictions\n", + " y_train_pred, y_train_prob = generate_predictions(model, X_train)\n", + " y_test_pred, y_test_prob = generate_predictions(model, X_test)\n", + " all_data_pred, all_data_prob = generate_predictions(model, X_all)\n", + "\n", + " # Calculate Metrics\n", + " train_metrics = calculate_metrics(y_train, y_train_pred, y_train_prob)\n", + " test_metrics = calculate_metrics(y_test, y_test_pred, y_test_prob)\n", + " all_data_metrics = calculate_metrics(Y_all, all_data_pred, all_data_prob)\n", + "\n", + " # Print Metrics\n", + " if print_metrics:\n", + " for metric_name in train_metrics:\n", + " print(\n", + " f\"{metric_name}\\nTrain: {train_metrics[metric_name]:.5f} | \"\n", + " f\"Test: {test_metrics[metric_name]:.5f} | \"\n", + " f\"All Data: {all_data_metrics[metric_name]:.5f}\\n\",\n", + " )\n", + "\n", + " # Plot ROC Curve if requested\n", + " if plot:\n", + " plt.figure(figsize=(10, 7))\n", + " plot_roc_curve(y_train, y_train_prob, \"Train\")\n", + " plot_roc_curve(y_test, y_test_prob, \"Test\")\n", + " # plot_roc_curve(Y_all, all_data_prob, \"All Data\")\n", + " plt.plot([0, 1], [0, 1], linestyle=\"--\", color=\"gray\", label=\"Random\")\n", + " plt.xlabel(\"False Positive Rate\")\n", + " plt.ylabel(\"True Positive Rate\")\n", + " plt.title(\"Receiver Operating Characteristic (ROC) Curve\")\n", + " plt.legend()\n", + " plt.show()\n", + "\n", + " if return_test_metrics:\n", + " return test_metrics" ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "y_test[:, 15].sum()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "i = 15\n", + "\n", + "for weighting_scheme in models:\n", + " print(f\"\\n{weighting_scheme}\")\n", + " print(\n", + " assess_model_performance(\n", + " models[weighting_scheme][i],\n", + " X_train,\n", + " y_train[:, i],\n", + " X_test,\n", + " y_test[:, i],\n", + " X,\n", + " Y.toarray()[:, i],\n", + " ),\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 52, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:41:47.279093Z", "start_time": "2024-03-18T02:41:42.845780Z" } }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
+       "              colsample_bylevel=None, colsample_bynode=None,\n",
+       "              colsample_bytree=None, device=None, early_stopping_rounds=None,\n",
+       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
+       "              gamma=None, grow_policy=None, importance_type=None,\n",
+       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
+       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
+       "              max_delta_step=None, max_depth=None, max_leaves=None,\n",
+       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
+       "              multi_strategy=None, n_estimators=None, n_jobs=None,\n",
+       "              num_parallel_tree=None, random_state=23, ...)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "XGBClassifier(base_score=None, booster=None, callbacks=None,\n", + " colsample_bylevel=None, colsample_bynode=None,\n", + " colsample_bytree=None, device=None, early_stopping_rounds=None,\n", + " enable_categorical=False, eval_metric=None, feature_types=None,\n", + " gamma=None, grow_policy=None, importance_type=None,\n", + " interaction_constraints=None, learning_rate=None, max_bin=None,\n", + " max_cat_threshold=None, max_cat_to_onehot=None,\n", + " max_delta_step=None, max_depth=None, max_leaves=None,\n", + " min_child_weight=None, missing=nan, monotone_constraints=None,\n", + " multi_strategy=None, n_estimators=None, n_jobs=None,\n", + " num_parallel_tree=None, random_state=23, ...)" + ] + }, + "execution_count": 52, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Calculate class weights to scale positive weights\n", "total_negative = len(y_train) - sum(y_train)\n", @@ -905,21 +1677,61 @@ "\n", "# Single XGBoost Classifier\n", "xgb_model = xgb.XGBClassifier(\n", - " objective=\"binary:logistic\", random_state=23, scale_pos_weight=scale_pos_weight\n", + " objective=\"binary:logistic\",\n", + " random_state=23,\n", + " scale_pos_weight=scale_pos_weight,\n", ")\n", "xgb_model.fit(X_train, y_train)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 53, "metadata": { "ExecuteTime": { "end_time": "2024-03-18T02:41:48.274852Z", "start_time": "2024-03-18T02:41:47.281488Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Balanced Accuracy\n", + "Train: 0.86064 | Test: 0.77298 | All Data: 0.48225\n", + "\n", + "F1 Score\n", + "Train: 0.85604 | Test: 0.67183 | All Data: 0.58441\n", + "\n", + "Precision\n", + "Train: 0.88527 | Test: 0.60241 | All Data: 0.42988\n", + "\n", + "Recall\n", + "Train: 0.82868 | Test: 0.75934 | All Data: 0.91237\n", + "\n", + "AUROC\n", + "Train: 0.86064 | Test: 0.77298 | All Data: 0.48225\n", + "\n", + "AUC-PR\n", + "Train: 0.89980 | Test: 0.71681 | All Data: 0.69037\n", + "\n", + "Average Precision Score\n", + "Train: 0.81926 | Test: 0.52930 | All Data: 0.43071\n", + "\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0sAAAJuCAYAAABhUplHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzddXwU19oH8N+6ZnfjbiQkEMODBE0CBVqoO21p6Vtq1Evdvb116t7e3roLFiS4ewgWEuKezWZd5rx/LAwsERJIspHnez+97JyZnX02mezub+fMOQLGGAMhhBBCCCGEEA9CbxdACCGEEEIIIT0RhSVCCCGEEEIIaQGFJUIIIYQQQghpAYUlQgghhBBCCGkBhSVCCCGEEEIIaQGFJUIIIYQQQghpAYUlQgghhBBCCGkBhSVCCCGEEEIIaQGFJUIIIYQQQghpAYUlQshZefjhh5GYmNjsv5EjR2LevHnYvn2712pLTEzEf/7zH689PgDU1NTg5ZdfxvTp05GWloZhw4bhkksuwccffwyLxeLV2trr119/RWJiIgoKCrrtMRsbG/Hmm2/i/PPPx5AhQzBy5EhcdNFF+Oijj2A2m5vVVlpa2m219QSZmZm49957z/r+mzdvRmJiItasWXPOtdTV1WHy5Mn49NNPAZz9a8KyZcswb948jB49GikpKRg/fjzuuece7Nmzp9XH/vfffzF37lykp6cjNTUVmZmZeOSRR3Dw4EF+m7vuugtz586F0+ls93Pas2cP7rnnHowfPx4pKSmYMGECbrnlFqxatard+yCE9DGMEELOwkMPPcTGjBnDqqur+f8qKyvZ1q1b2bx581hycjLbvXu3V2qrrq5mRqPRK4/NGGN5eXlszJgx7LLLLmPLly9nxcXF7OjRo+z7779nEydOZLNmzWJ1dXVeq681b7/9NnvooYf4ZYvFwqqrq5nT6eyWxz927BibMmUKmzFjBvv7779ZcXExy8/PZ1988QUbNWoUmzVrFmtoaGCMMfbLL7+whIQEVlJS0i21nYvJkyezTZs2dcq+6urqmMFgaPf2P//8M5szZw6/bLPZWHV1NbPZbOdUh9PpZFdffTW79dZb+bazeU14/PHHWWpqKnvjjTfYvn37WFlZGdu6dSt78MEH2aBBg9i3337rsT3HcezBBx9kaWlp7J133mH5+fmsuLiY5eTksCuuuIKlpqaylStXMsYYa2pqYtnZ2ezFF19s13P6/vvv2aBBg9jChQvZ1q1bWWlpKduyZQt74IEHWEJCAnvzzTfP6WdGCOmdKCwRQs7KQw89xMaNG9fiOrPZzEaOHMkeeOCBbq7K+2w2G8vKymJXXXVVix9IKyoq2OjRo9mdd97pheraNnfuXI+w1N0uu+wyNnXq1BbDwL59+1hSUhJ7++23GWO9JyxVVlayhISETgtLHfXYY495hKXO8vPPP7OkpCR27Ngxvq2jrwk//vgjS0hI4MPN6f7zn/+wpKQklpeXx7d9++23LCEhga1atarZ9jabjV155ZUsOzubORwOxhhj//zzDxs0aBA7dOhQm88nPz+fJSUlsZdffrnF9S+99BJLSkpiR48ebXM/hJC+h7rhEUI6nUKhQHR0NCorK/k2xhi+/PJLXHjhhRg6dCjGjRuHJ598EgaDweO+ubm5uOyyy5CWlobJkyfjueeeg9Fo5NfX1NRg4cKFyMzMRGpqKs4//3z8/PPPHvs40Q2vuLgYiYmJ+OGHH5rVOGPGDNx2223tru3hhx/GhRdeiO+++w7p6el45ZVXWnzuy5YtQ0lJCR5++GFIpdJm60NCQnDLLbdg+fLlKCkpAQC8++67SElJwYEDB3DFFVcgLS0NEyZMwMcff+xx3/Y+948//hjz589Hamoq3y1pzZo1uPrqqzF06FAMGzYMF198MZYtW8bfLzMzExs2bMBvv/2GxMREbN68uVk3vBM/g82bN+OSSy7BkCFDMHXqVPz2228eNeTk5GDGjBlITU3FBRdcgNzcXMybNw/XXXddiz8zANi2bRv27NmDBQsWwMfHp9n65ORkLFmyBHfddZdHe0NDA+68804MGzYMI0aMwLPPPgu73c6vLywsxIIFC5Ceno6UlBRMnToVH3zwATiO47e57rrrcPvtt+Ott97CsGHD8N///rfd9wWA3377DbNmzUJaWhqys7Px9ttvw+l0YvPmzZg4cSIA4Prrr0dmZiZ/nz/++AOXX345hg8fjvT0dNx7772oqqri17/77rsYOXIkcnJyMH78eP55n94N74cffsCsWbMwdOhQjBo1CjfddBPy8vL45/XTTz9hy5YtSExMxK+//tpiN7zdu3fjuuuuw9ChQzF+/HgsXLgQNTU1rf6uGGN47733cMEFFyAqKqrV7U7V0mvCp59+igkTJmDKlCkt3mfBggVQq9X4+uuv+bbPP/8cGRkZmDx5crPtpVIp3n33Xfz5558Qi8UA3H/n0dHReO+999qs7+uvv4ZCoWh2fJ1w9913Izc3F7GxsQDcP9srrrjCY5vTf7Yt/Q6vueYaXHXVVc32//HHHyM5ORl1dXUA3L+TefPmYdy4cRg6dCiuvfZa7Nixo83nQAjpGhSWCCGdzm63o6SkBGFhYXzbBx98gJdffhnnn38+/vzzT7z88stYt24d7rzzTn6bbdu24dZbb0VGRgZ+++03vPTSS1i2bBkee+wxfr833HADtm/fjqeffhp//fUXLrzwQjz++OP4/fffm9URFRWFIUOGYOnSpR7tBw4cwNGjR3HhhRe2uzbA/cE8JycH33zzDebPn9/ic9+0aRN0Oh2GDBnS6s9n8uTJYIxh8+bNfJvD4cCzzz6Le++9F3/88Qdmz56N119/Hf/++2+Hn/tPP/2EESNGYPHixYiNjUVxcTFuv/12DBgwAL///jv++OMP/rqQ/fv3AwB+/vln+Pn5YcaMGVi3bh2GDRvWYu319fVYtGgR/7hxcXF44oknUFFRAQA4fPgw7r77bkRFReGnn37C448/jtdff/2M1z1t3rwZAoEAkyZNanWbyMjIZm0vv/wyLrjgAvz++++46aab8O233+KPP/4A4P5Qf8stt6CiogJffvklli5dirvvvhvvvfcevv32W4/9HDp0CMeOHcMvv/yCCy+8sN33/euvv/DYY4/h0ksvxV9//YWHH34YX375Jd544w0MGzYMr7/+OgD3B+cTwfaPP/7AwoULMXToUPz66694//33cfToUcydO9cj6LlcLnzzzTf44IMP8PTTTzd77hs3bsTTTz+NG2+8Ef/88w+++eYbaLVa3HTTTbBYLHj33XeRnJyMYcOGYd26dZg5c2azfRQVFWHu3LmIjIzEjz/+iEWLFmH//v38FwktycvLQ1lZmUf4O5PTXxOqqqpQVFTUYug5QSqVIiMjA5s2bQIAVFRUoKSkpM1jJDAwEAqFgl8WCASYMmUKcnNzPX62p9uyZQtGjx7tcd9TKRQKBAQEtPUUW3T673DWrFnYtWuXRzAG3NdgjR8/Hv7+/igsLMQNN9wAl8uFTz75BD/88ANCQkJw0003dev1g4QQNwpLhJBOVVtbiyeffBJGoxHXXHMNAHcQ+Oyzz3DhhRfilltuQVRUFCZOnIhHH30Umzdv5r8x/fTTT5GQkIB7770XcXFxGDt2LB5//HGoVCo4HA7k5OSgoKAAL7zwAiZOnIiYmBjccsstyMzMxAcffNBiPbNmzcLmzZvR0NDAt/3777/QaDTIzMxsd22A+wPeQw89hMTEROh0uhYfr7Ky0iMktiQiIoLf9lRz5szB2LFjERsbiwcffBBhYWH4888/AaBDz93Hxwe33HILIiIiIJVKERwcjD/++AOPPfYYYmJiEBUVhTvvvBMulwsbNmwAAPj5+UEoFEIulyMwMLDFs2IAUF1djSeeeALDhw9HbGws5s2bB4fDwYeuv//+GwDw6quvYtCgQRgzZgxee+01Pky1pqqqCj4+PtBoNG1ud7qZM2di+vTpiI6Oxm233QalUukxMMDnn3+ODz/8EElJSQgPD8cFF1yApKQkrF271mM/lZWVePrppzFgwAD+zFZ77vvxxx9j8uTJmDt3LqKjo5GdnY2FCxfC5XJBKpXyz0er1cLPzw8A8OGHH2LUqFH872PkyJF4+eWXcfToUY9gbzabMXfuXKSmpvL3PdW+ffugUCgwe/ZshIeHY9CgQXjhhRfw8ccfQyQSQafTQSwWQyKRIDAwEHK5vNk+vvnmG8hkMjz77LNISEjA0KFD+Z/DibMcp9u6dSsAYNSoUe36HbX0mnDi2A8NDW3zvuHh4aiqqgLHcXzACA8Pb9fjnjBy5EiYzWb+jFtLqqqqOrzf9jj9dzh9+nSIxWIsWbKE36awsBD5+fn8lzdffvklhEIhH3YTExPx4osvQqVS4csvv+z0GgkhbRN7uwBCSO9VV1fncQaC4zhYrVYkJyfj/fff58+uFBQUwGg0IiMjw+P+Y8aMAQDs378fw4cPx549e5Cdne2xzXnnnYfzzjsPgLtrikQiQXp6usc2Y8eOxYoVK2AymaBSqTzWzZw5Ey+99BJycnJw+eWXAwAWL16M6dOnQyqV4sCBA+2qDQBkMhkSEhLa/JkIBAK4XK42t2GM8dueasSIER7LgwcP5rvqdeS5p6SkeGwjk8lw5MgRPPvssygoKIDJZOLX6fX6Nms9nVKp9PgZnPgQf6LLYnFxMaKioqDVavltEhMTzxggBQIB/3PpiKFDh/K3hUIhdDod//wEAgEMBgPeeOMN7N69G3q9HowxWK1WpKameuwnIiLCo+b23NdqteLQoUO44IILPPZ19dVXt1qv0WjE0aNHMXv2bI/2wYMHQ6fTYf/+/Zg1axbffvrv8lQZGRl47733cOWVV+Kyyy7DmDFjEBsb2+ZZzdPt2bMHycnJfLc1wB0uRo4c2ep9qqurIRaL4evr22xde18TThz7p3dpPB1jDEKhEAKBoNnfS3sFBgYCQJtdC8/2+GuPU3+Hvr6+GD9+PJYuXYobbrgBgPvLG7VazZ+p27NnD4YMGeLRHVUmk2H48OFtBj5CSNegsEQIOWs6nc7jeqB9+/bhvvvuw7x58zy6ypy45ujxxx/HU0891Ww/Jz7EGAyGZmHnVEajEQ6Ho1moODE0cE1NTbP7+/v7Y+zYsViyZAkuv/xy7N27F8XFxXjppZc6VBuAFq+lOV1YWBh27NgBxlirH+5OBKDTv8k+/ayKUqlEU1MTX2d7n/vp+1m+fDnuuusuTJ8+HW+99RYCAgIgEAgwbdq0Mz6f0ymVyhbbT3zQ1Ov1Lf4OW/pgfaqwsDA0NTWhvr6+xbMorTn9bMmpH3orKiowZ84cREdH48knn0RkZCTEYjEeeOCBZvs5/WfWnvueCIhtHbOnO3G8vffee82uSbNYLKiurm6zrlMlJSXhhx9+wOeff4533nkHTz/9NOLj43HfffchKyurXfUYDIYznt05XVNTE9RqdYvHd3tfE06E5xN/C60pKSlBaGgoBAIBX2dRUVGH6j3xMzz9+shThYaG4tixYx3ab0cf/4RZs2bh/vvvR1VVFYKDg7F48WKcd955/LFsNBpx8ODBZl1h7XZ7h/42CCGdg8ISIeSsiUQiREdH88vR0dFYunQpXnjhBWRkZPBd1U58Y//ggw/yF7yf6kQI8ff3R2NjY6uPp9FoIJfLW7w+CWi9S8+sWbPw2GOPQa/X499//0V4eDgfOtpbW3uNGzcO33//PTZt2oSxY8e2uE1ubi5EIlGz9SaTyeOaCZPJxH/QOtvnDgB//vkngoOD8eabb0IodPe+Pv1DeWeRSqWwWq3N2lsLUSec+FksW7asxQvgAWDp0qWIj49HXFxcu2rJycmB2WzGG2+8gQEDBvDtBoPB4yzS2d7X19cXQqGwzWP2dCeOp7lz5/JnOk/VWhhtTWJiIl555RUwxrB371588sknWLBgAf7991/ExMSc8f5n+ptriY+PD4xGY4tfCLT3NSEgIAAJCQnIycnBTTfd1OLj2O12bNq0CdOnTwcABAUFIS4uDsuXL8fNN9/c4n2Ki4uxb98+zJgxg6/tREhqK3iOHTsWv/32GxobG1s8NhwOB3744Qdceuml/N/o6WeiTp0HrC2ZmZlQKBRYtmwZxowZg8OHD+OJJ57g12s0GoSEhOD5559vdt8Tf7+EkO5Df3WEkE716KOPwmKx8GduACA2NhYajQYlJSWIjo7m/4uIiIDT6eS/LU1ISMC2bds89rd8+XJce+21MJlMGDp0KKxWKywWi8d+5HI5NBpNq9fZTJ06FWKxGKtXr8bSpUsxe/Zs/oNUe2trr6ysLMTExODVV19tcfLZqqoqfPbZZ5g9ezaCg4M91m3ZssVjef/+/fwH9bN97oD7g55Wq/X4oHViBLvTP/Cda1ek6OhoFBUVeXwA37dvH8rKytq8X1paGkaNGoVFixY1u/gdcP8sFi5ciO+//77dtTgcDgDw+B3u2LEDRUVFZ3ye7bmvRCJBbGwsfw3PCf/73/9wyy23eLSduI9KpUJCQgIKCws9fo/R0dGw2+3w9/dv9/Pbvn07du/eDcB9Ri0tLQ3PP/88XC4XDh061OyxW5KQkIC9e/d6BNxdu3bh6quvRnFxcYv3CQoKgtPpRH19fbvqbOk1AQA/Ue2J69xO995778FsNuP666/3uM+uXbvwyy+/NNvebrfjsccewyuvvOLR1fTE2eET3fFact1118HlcjWr8YR33nkHL774Io4cOQLAHWhOf/67du1qdf+nUigUyM7ORk5ODpYsWYKwsDCP7rVDhw5FYWEhQkNDPY4PxhiCgoLa9RiEkM5DYYkQ0qlCQkKwYMEC/P7771i/fj0AQCwW4+abb8Z3332Hr7/+GkVFRcjPz8cjjzyCyy+/nP9wPG/ePJSUlOC5555DSUkJNm3ahJdeegl+fn5QqVSYMmUKEhIS8MADD2DDhg0oKytDbm4u5syZ4/HN7OlUKhUyMzPx1VdfoaysjL+QuiO1tZdYLMbbb7+NyspKXHnllViyZAlKSkpQWFiIn376CVdccQUiIiL4Ef5O9dVXX2HdunUoLCzEq6++isrKSlx88cUAcNbPHXB/+Dpy5Aj+/fdflJSU4LPPPsPu3bsRGhqK/fv382eZNBoN9u/fj/z8fNTW1nboeZ8wY8YMfmS/I0eOYMuWLXjqqafadfH8K6+8AplMhiuuuAI///wzjh07hiNHjuCrr77C3LlzMXz4cI9hs8/kxPVMH330EUpLS5GTk4Nnn30WU6ZM4X8nrV0z09773nLLLdi4cSM+/PBDlJWVYeXKlXjrrbf4kHviLMX69euxf/9+MMYwf/58rFixAu+++y4KCgpw5MgRvPLKK7j44ov5gTLaY9WqVbj99tuxbNkylJWV4ejRo/jwww8hl8v566o0Gg2Kioqwd+/eFgfZOBESFi5ciMLCQuzZs4cffr2l0QeBkwM7nB4SW9PSawIAXHTRRbjyyivx8MMP49VXX8W+fftQXl6Obdu24eGHH8ann36K559/3uNM4qWXXopLL70UTzzxBF588UXs27cPpaWlyM3NxXXXXYdDhw7hrbfeglqt5u+zdetWKJVKJCcnt1pjXFwcnnnmGfz555+YP38+Nm7ciLKyMuzcuRMLFy7Ep59+iscff5z/uaalpaG0tBQ//vgjSkpK8OuvvyI3N7ddPw/AfbZ7+/btWLJkCWbNmuVxhu7666+HyWTC/fffj71796KkpAQ//vgjLrroohanQSCEdLHumtCJENK3tDUBpcPhYBdccAHLzMxkZrOZb//mm2/Y9OnTWXJyMhs2bBi75ZZb2P79+z3uu2LFCnbxxRezlJQUNnHiRPbss8+ypqYmfn1tbS17+OGH2ZgxY9jgwYPZhAkT2Msvv+zxOAkJCey1115rtt+EhAR26aWXtljzmWpr6/m2pLa2lr388svsvPPOY6mpqWzo0KHskksuYV988UWzyWrfeecdlpCQwHbu3MmuvPJKlpKSwjIyMtinn37abJ9n89xNJhN78MEH2ciRI9moUaPYwoULWVNTE/v8889ZWloau+666xhjjP32229s5MiRbMiQIezff//lJ349cuRIqz+DI0eOsISEBPbLL7/wbT/++CObMmUKS01NZZdccgnbunUru+SSS9hNN910xp+bwWBgb775Jps5cyZLS0tjo0aNYpdffjn77rvvmN1u57c7vbYTpkyZwu655x5++aOPPmLjx49nQ4cOZddddx07cuQI27p1Kxs9ejRLT09nBoOBzZkzh11++eXNamnPfRlj7KeffmIzZsxgKSkpLDMzk7399tt8rS6Xi912220sOTmZjR07ljmdTsaYe7LUiy66iCUnJ7MhQ4awOXPmsI0bN/KPfeKYsFqtrT4/u93O3njjDZaZmclSUlJYeno6u+GGG9jmzZv57detW8fGjRvHUlJS2Oeff842bdrEEhISWG5uLr/N9u3b2dVXX83S0tLYuHHj2AMPPMBqampa/R1xHMemTJnCFi5c6NF+Nq8JjDG2fPlydtNNN7H09HSWnJzMxo8fz+677z6Wn5/fag3//vsvu+GGG9ioUaNYamoqmzZtGnvuuedYWVlZs1rPO+88dvfdd7e6r1Pt3buX3XPPPSwjI4MlJyezSZMmsQULFrCdO3d6bGcymdj999/PRo0axYYNG8YWLFjAtm7d6vGzbe13eOLnMXbsWJaQkMAOHz7cYh033XQTGzp0KEtOTmbnn38++9///teu50AI6VwCxrpo+BdCCCHt8u6772LRokXYs2cPZDKZt8s5Z/X19fDx8YFEIgHgHoQiIyMDM2fObHEQDdL7/PTTT3jmmWewePHiVs9A9QSLFy/Gfffdhz/++OOMI1kSQkhLqBseIYSQTlNQUIAJEybgySefREFBAQoKCvDss8/CYDDgsssu83Z5pJNccsklSElJwcsvv+ztUlplNBrx5ptvYs6cORSUCCFnjcISIYSQThMXF4cPP/wQhYWFuPzyy3HllVfi0KFD+Oijj9q8ZoT0LiKRCIsWLUJeXh4+++wzb5fTokcffRShoaF46KGHvF0KIaQXo254hBBCCCGEENICOrNECCGEEEIIIS2gsEQIIYQQQgghLaCwRAghhBBCCCEtoLBECCGEEEIIIS0Qe7uA7lRT0+TtEgAAQqEAfn4q1NebwHE0vgY5MzpmSEfRMUM6io4Z0lF0zJCO6InHS2Cgzxm3oTNLXiAUCiAQCCAUCrxdCukl6JghHUXHDOkoOmZIR9ExQzqitx4vFJYIIYQQQgghpAUUlgghhBBCCCGkBRSWCCGEEEIIIaQFFJYIIYQQQgghpAUUlgghhBBCCCGkBRSWCCGEEEIIIaQFFJYIIYQQQgghpAUUlgghhBBCCCGkBRSWCCGEEEIIIaQFFJYIIYQQQgghpAUUlgghhBBCCCGkBRSWCCGEEEIIIaQFFJYIIYQQQgghpAUUlgghhBBCCCGkBRSWCCGEEEIIIaQFFJYIIYQQQgghpAUUlgghhBBCCCGkBRSWCCGEEEIIIaQFXg9La9euxbhx43Dvvfe2uR3HcXjzzTeRlZWFUaNGYd68eSgpKemmKgkhhBBCCCH9jVfD0ieffILnn38e0dHRZ9z222+/xV9//YWPP/4Yq1atQkxMDO644w4wxrqhUkIIIYQQQkh/49WwJJPJ8PPPP7crLP3www+YO3cu4uLioFarce+996KgoAC7d+/uhkoJIYQQQggh/Y3Ymw9+/fXXt2s7q9WKI0eOICkpiW9Tq9WIjo7G3r17MXTo0HbtRygUQCgUnE2pnUokEnr8S8iZ0DFDOoqOGdJRdMyQjuorxwxjDA4nB4eTg4tj4DjG/2t3cbDanOA4Bo6x4//i5PKJtlPu42rhtt5oA2OATCJ0t7OT6wvKDIgIVIHjGJzMAQezw8lc4JgTdphRUm1EkK8cEDK4YINLYAMgBAMDYxwYONildRC5VMd7XDF4/j+D0eKAQiYCBBycilqIbBq4+2ad+H/G34aANV+n0AOcCJxDAggAAXD8/0708GKnLbv3AwGDQOyAwAGMyJchqbAC1vNmIGv25V32++xsXg1L7dXY2AjGGLRarUe7VqtFQ0NDu/fj56eCQOD9sHSCRqPwdgmkl6FjhnQUHTOko+iYIR11pmOGMXcocDo5mKwO2B0cnC53MGk02iAUCPhljmOoN1ghlYj4+7k4Bs7Fob7JBo5jkEpEcDhdcLoYbHYn7A4OdocLeYV1iAnVNAs8+UX1CPVXQSwWwOVicB7fX22j9Ryf+fEwIDcBAs59W8BBoDBCILFDILNAILGBOSXH17kDidC3BszsA6HKAOYUAyEMZQIOAmErl5aogcpzrFTsAzhOWXZKzB3fiZCDUOw483anUZoliCvSwCKVoNyfQXr0AHx9VR1/fC/pFWHphHO9Pqm+3tRjzixpNAoYDBa4XJy3yyG9AB0zpKPomCEdRcdM/3AigFhs7pDBcQxOzh1UTBYHXC6GGr0FQqEAxdVG+Cgk7sDi4nCkzACxSIAmswM+Sgk4xiAQCGF3OKE32lFdb0aIvxIuF4PDxcFic/Jna7pLXSsBqKLOBAhcgNAFgcgJSBwQ+ZvcKwUchKpG9zqpO9wIpFYIVY0QCBmYSwSPMycCHA8+Z0+gMrj/FTvPaT8AAHb8s62AQcK5Q4jg+P+OFw0BBHC6GCQiIQQCAcyCBuhYuHudQHB8C/D/nmw72e6CHRw4SJxayCQiiIRC91oBPLY7cWJCyAHCqiaIaprAHT/56JRJkD1nPhoaTOf+vDtBe0JbrwhLOp0OQqEQer3eo12v18Pf37/d+zlxmrSncLk4OJ30hkTaj44Z0lF0zJCOomOmZ+A4BpvDBavdBbvTBYeDg9HicHcZczFY7U6YrE4YTHbojTYoZWJYHS4UVRjgcDKIhAJIxEJY7S6U1hjhp5Gh3mDr8ror6s7ijEWHuM/mqNWASMJBIgLEEgFEEhdc8nrUm4zQ+NvAhA64hDaImRRM4IJZVOvuIXYW35kLRK4O30cqlEAqkoFjLpidFsRqoiAQCCESCCEQCFFtrkGibzwabQYM9B0AkUAEocDdtU4kEMFProNIIAIA+Mp1EAqEEB6/v0QohVIs59t6Uq+pE+rqapGTsxh1dU0AALHTicFVtTj/xedhFsl71WtMrwhLMpkMAwcORF5eHtLT0wEABoMBxcXFSEtL83J1hBBCCOnvXBznDjYODmarw91FzOFCk9kOxgC90QaHi0Oj0Q67k4NQAFhsLhRXNaG+yQqrzQWNWnq8q1onnG04zbkGJbVCApFIgEajHUE6BQxmOwZF+0IuE8Pl4vgMUlVvQXyEFhKREGKxEAqZ+wyEWCSASCiAXCp2n5U4vmxz2QCJFUanAUeNRyAVScHgQrGxGGXmcgQpAuFiLtRYaiGAACeupnEd/89+Wp0iLXDqOQuPZ92OTKGWqGB0mJCgi4NQIIRUJIVEKEa4OpQPJoLjZ0+EcC87OAeifSIhFoogEoggF8sQqAjokSGmO+Tl7cG6davgcrlDpp+hCUnHShE37/8gC/CHuYecVWqvHhuWqqqqcMMNN+CTTz5BZGQkrr76anz88ceYOHEigoOD8Z///AeDBw9Gamqqt0slhBBCSC9yoiuaO9y4YLQ44OLcF/k3mR1wuFyoa7TCanfx3chcHIPTxeFQiR7hASrsKagDxxjEIncQsNg6fvbhdI3G0z/6n9mJLlMcYwjQyiERC1FRZ0ZcuAZmqxMxIT4Qi4So0VsQH6GD1e6Ev0YOhUwMxhg0KilEQqE7vAgEEIkE0KllUMjEkEtFELcxeINYLISvrwoNDaYWzxQwxmB2WlBv1eNoYxHy6w9BBSUOVh+BRChGtaX2jM+v0lx1cn/oWO+gEGUQmhxGJPrGQyQQocGmR3rwcCglSmikPpCIxNBIfSATySAWiiER9tiPxb2KXK6Ay+WCkDHEl5YjsqYOuimZ0I4Z4+3SzopXj4oTQcfpdH+DkpOTAwDYu3cvHA4HCgsLYbe7Xziuuuoq1NTU4LrrroPJZMLo0aOxaNEi7xROCCGEkG7DGEOTxYFGox1WuxN1Bqv7DI3D5THyWHFVE3w1cvdF/ccaEOyrwIFiPUL9lXz3MIlYCKeT6+DHbk/VDRb+tnsUtY7dX3g83EQEqqGSiyERC2GyOqFWSBAb6gOJWAiDyYHwQBXEIgGEAgG0KimEx8/MnAgxaqUEUnHndMPiGAcX4+DinLA4rTA4G1FvdsLqtKLCVA2RQAiOce7R38DhmKEEWrkGTOTCvsqDCFIEHl/PoaCxEA7u3M6OBcj94OCcaLQbMDF8LMRCMRptBkT4hEEukoHB3dVNI9NAdLw7mkwkRZgqBBKR5Jx/HuTsxcUNRJp/MJRr10BttUEaHoGgK6/xdllnTcD60ayuNTVN3i4BwJm/iSHkdHTMkI6iY4Z0VHcdMw6nC/UGG8w2J4wWB2x2F+oMVlTVm2FzuFDVYEFZjQkhfkocq2qCWOS+MN1bJGIhlHIxlDIxREIBREIhiqubkDbAHw1NNkQEqSEUCBARqIJSLoFMKgLHMejUUkjEIoiEAqgV7naZRAiJWNSt9XOMg5NzwsVcMNiNMNgMMNiNqLc2wGBvgpNzYk3Zxm6tKUYTBYvTArVEDalIgvSQ4VBLVAhWBsFPruu33dd6I6vVgtzcFYiLG4j4+EQAgK2kGMeeexrgOIh8NIh9+TUIZbIe+b4UGOhzxm3ofCMhhBBCzorF5h5goLLeDLuTQ1mNEXKpGAVljfBRStBosmPn4TN3tWrJsSr3F5xtBSWZROQ+8yIUHA8yAtQZbFArJAjxU6K6wYwBYVo0GG1IjNQBcJ8JCvZVQCwWwmJzIshXCQEAjUoKiVgIiUgIiUQIjVIKuVTUaz64Ozgn9NZGHDMU45D+KBqseuyvP9hljyc8fqZJIZZDK9Og0lSNJL9EfhACF3PBzjkxyDceGpkGYapgRKjDIBJ2b1gkXae4uAgrVy6F2WxCaekxhISEQVxXh+Lnnua3CbpmDoQymfeK7AQUlgghhJB+zulyj7RWWmdBaWUjympMOFbZBKeLg0wiQmmtCRzHoUZvdZ8hEQthtjk7/YyP4PgQxDKpO6wkRvmC4xhUCjHUCgmCfJXwVcugkIsR4qeASNi7J0NtD7vLjoLGIpQ2lcPuck9W2mDVo96qh1QkwcGGI+BYx7+lFwqE0Eh9IBVKIBKKUGGqQor/IAwJTD0+YpsAvjIdxEIRZCIZAhX+/AAHQoH7594TzxSQrudwOLBp01rs3buLb4uLS4DYxaHsrTf4Nr9ZF8JnVLoXKuxcFJYIIYSQPszh5GC2OaFvsqG4ugkFZQY0me3YfaQOfhoZmo53hWsvm93V7u0DtHJEBKrRaLJjYIQWDieH8ED3BPEBWjk0SimkEiF8lFIo5WIIe8lZnM5Sa6nDvroDcLgcqLHUweq0QiKSYG/NfoSoglHQWHjOjyEXyTEqZBhiNJEQCUSQiiSI8omAVqbhQw8h7VVdXYmcnMXQ6xsAAAqFElOmTEV0VCzK3nkTrib3/FHKlFQEXHixN0vtNBSWCCGEkF6E4xiMFgfqm6wwmh1oMNpQWWeGXCrC/qIGlNYYoVPLYHO4UNvKBJ0nnGn9CckxvoBAAKPZPeiAi2MI9VNCpZDA10cGrUoKH5UUPsevzelvoactjDE0OYyoNFWh3FiFAw2HsLc2n+/G1prWgpJCLIdcJAcDg97WiMF+Cai11GGgbgCClIEIU4cgyicCPlJ1Vz0l0g9xHIcdO7Zg27ZN4Dj3cRsbG4dJk6ZCqVSibNHbMO/bCwBQDk5C+F33erPcTkVhiRBCCPEiu8MFq8OFmgYLmswOVNabUaO3QCYRwWC2w2Cyw2J3orLO3O75d9raztdHhkCdAjKJCBa7E0nRvlDKJdD5yBDop4JaJoRaLoFCRh8R2mJ3OVBprkKlqRrFTaUAAwz2JlSba2B12eBiHOqtDa3e//SgJBNJ4eCc4BiHaE0kyo0VGBKYgnJjJSZHZCDedwDfFY6Q7tbYqMe2bZvBcRwkEgnGj5+CQYOSIRAI0LR1C0y7dgIARDodwu68G4I+1EWWXgkJIYSQTsLYybl7DGYHGk022OwulNeZoTfaIBYKcaSsEUIBUFBu6PTH1yglcLg4WGwuJMX4IjJI7R6hTSJCoFaOsEAVgn2VLd63v19/UmepR17dQVicFjiZCxzjUGGsBABw4LC3Nh+AO9TYXB2fD6klY0JHItE3Hukhwztlf4R0FV9fP4wZk4GjR48gK2s6tFodAMBlNKLyi0/57UJuvLnXD+hwOgpLhBBCSDs4nBzqDVZU6y2o0VtQUWdGk9kOq92FPQV1EIuEcLo6N2Ro1VJEBKqhkIqglIuhUkhgsbkQHqCCTi1DgFYOP417AtG2Jg/trxhjaLQb0GQ3wsm54GIu1FrqcKihAHpbI8qNlfCRqlFuqmz3PtsKSnKRDMn+g6CSKCESiCAUCgEGiIQiRKjDEKeLgUbqQ2eHSI/nHuGuGAkJg/m2IUNGIC1tuPu4hvvvq/y9d8COz4kaMu//oEpO8Uq9XYnCEiGEkH6LMQani0NtoxVHShtRXGWE1e5EaY0JAGCyOtp9XU9HgpJWLYWPwj00ta+PDMMGBoABCPZVIkArh1ohgVBI1/20hmMc7C47XMcnQbW5bNhTux8mhxkc43C44ShKjeVwcI4z7qvJYWzWJhVKIBVJIRQIYXFa4WIuxGljUG/VI8InDLGaqOMToYoQrYlEgMKPrhEifUZBwWHk5i6HzWaDRqNFSEgYAEAgEHgMpd+YuwqWw4cAAMqkZPiMGeeVersahSVCCCF9BscYGo126I02VNab4XBysDlcKKowwGhxHu8eZ0dFnRkAIBQIwJ3j3OwquRixYRr4+cjgo5RCLBIi1F8JiVgImcQdhjQqKZQyca+Zs6cnKDNWoN7agM0V21HSVAZfuQ6VpuoWw825CFUFQyfTIkgZgGifSAwLSoVUJO3UxyCkN7DbbVi3bjUOHMjj2yoqyviwdCrL4UOo/u5bAIBQrUbobXf22dc3CkuEEEJ6NI4xmCwOVOstaDK5R4GrbbSioKwR9QYbVHIxiqvP7gN0a0HJXyOH3miDr48MiZE6GC0OBOgUSIzUQaOSIkArh4/SPYkpOXsc41BhqkKZsQJV5hqYHRYcbSxCqbG82ba11vp279dXpoPZaYaLcZgYPhax2mj4yrQQCUQQHJ9fSCNV99kPd4R0VHl5KVasWIKm40N/q1RqZGVNR0REVLNtrceKUPbOm4DLBQgECLv1DogUiu4uudtQWCKEEOI1Lo6D0exATaMVFXUmlNWY0NBkg97o/q9Gf+YucHXtGCdBJBTAxbmDUXy4FgaTHUF+CmiVUgwI00AiFiEmxAdhASrq/tYJnJwTJU1lsLpsMDssKGw8hhJjGRwuJ441lUAhVsBxfILVjhigjYa/3B9WlxUhyiCEqoLd3eGEIshEUsTrBkBGZ4UIaTeXy4ktWzZi586tfNvAgYMwYUIm5HJ5s+0tRw6j5OUX+OXQW++ActDgZtv1JRSWCCGEdDnGGBqabCitMeFIWSM25VXCZHXCYmvfUNinEwkFkEqE4DhAJhEiIVIHq8OF+DAtnBxDXJgGaqUEwb5KKGViCkDngDEGJ+eEg3OgwdYIm8uGY4ZSNNoMMDnMkIgkMNgMqDLXtHugBIvT0uq61IAkBMj94GQupIcMg7/cfT0QDYpASOdbvPgvFBe75/SSyWSYODEbAwcmtritvaoKZW+9zi/7XTAbPiNGdkud3kRhiRBCyDlxODk0GG0wWRyoN1hR12jFwRI9bA4X9hc1QCYRwebo2BkEXx8ZrHYXAnVyDIkLgEQsRIBWjlB/FYJ8FZBLRdSFqhNYnVbk1R2E0WGCnbNhbdkmhCiDkFd3EL4yHawuKyzO9g1w0RaFWI5YTTSUEgXEQjEabQZE+oRDKVbAybkQr4tFhE8YFOLm32QTQrpOWtowFBcXIiIiGpmZ06BW+7S4HWMMlZ99DM7qfj0IvfUO+Iwc1Z2leg2FJUIIIa0ymO2oa3RfI1RvsGLHoRrY7C4UVxvh6yNDQ5PtjPtoLSglROoQF6ZBiJ8SCpkYQb4KhPgpIZWIOvtp9GtWpxWFjcVochhR0lQGiVCCvLoDLV4XBAB1FvdEqg02fbsfI1gZBIChylyDALkfBvrGIcInDPHaWMjFcvjJdXRmiJAewGw2QaFQ8l82RUXF4MILL0dYWESbX0DV//MXrEcLAAD+sy/qN0EJoLBECCH9msPJoeH4gAl7CupQZ7DC5nCh0WhHyRkGTWgrKKkVEvgoJRCLhIgKVsPXR45gXwW0KinCA9XQqaV0ZqgTWZwWrCheC45x2FG9+/hw2nYYHaYO7efEhKuRPuEQQIBYbRSUYgVMDgtiNJFwMQ7BykDIxTKEqoIpABHSSzDGcOhQPtauXYmMjMkYPPjkfEjh4ZFt3te4exfqfv8VACBS+8B32vQurbWnobBECCF9nNHiwJ6CWhSUGWB3urD9YA38NXLUGayw2jvWPU4mFSEiQAWni0EpFyM+XAuHk0NYgAoalRRBvgoEaOU0QWonOzG5apmxEg6XHQWNRVhZshY+EjWMDhMYOjb8uVAghFQoRZwuBvG6WIwLTYdO6QNfXxUaGkxwOjt3cl1CiPdYrRbk5uagoOAwAGD9+lzExQ2EVCo7433N+ftRvuht94JAgJBbboWwhYEf+jIKS4QQ0stZbE6U1hhhMDlgMNlQWNmE4qomFFe1fmaorLb1Mw5yqQhD4wOgUUmhkIkRGaRGgFYOnVoGH6WEzgh1MYfLgUJDMcxOCzaUb0Fe3YFWt21pziG5SAalRIlQVTCiNZGwuWyIUIchRhMJnUwLiZB+h4T0F8XFhVi5chnMZvdrvkajRVbWjHYFJVt5OUpffxUAIJBIEH7vA1AmtDz4Q19GYYkQQnowF8fBanehocmGmgYLjlU1odFkR35RAwQCoKqh9VHFWiISChAfroWT4xCkUyLETwGlXILoYB+EBSihlEu66JmQllicVuSWbsAxQwma7E0oNBR3eB/pIcMRrAyCRCjGhPCxkIrod0hIf+dwOLBx4xrs27ebb0tKSkVGxiRIJGceXp85nSh/721+OWTe//XLoARQWCKEkB7B4eRQWmNEea0Jq3eVwWRxorLeDLFIAKerY12spBIhhAIBOAYkRGqRHOOHYD8lUmL9qHucl9ldduhtBtRZ6pFTnIsDDYfbfd8L42YgWBmIKJ8IiIQiyEVyCkaEkGZqa2uwbNnf0Ovdg7UoFEpMmTINMTED2nV/5nSi4IF7wBndZ6590sfAZ2R6l9Xb01FYIoSQbsQxhhq9BaXVRhSUGVDfZMWW/OpWt28rKMWFaeCjlEImFSFtgD/CAlQI9lNALhVDLBbS9SdeVG2uQYG+CH8XLoNYKEatpe6M95GLZAhXh0ElUSLBNw6JvvHQyHygFCtoIAVCSLtJJGIYjwed2Ng4TJ48FQqFsl33ZRyH8vff5YMSAITcfEuX1NlbUFgihJAuYHO4UNtoRVmNETV6C/YdrYfR6kC9wXbGiVhlUhFsdheig30QFazGwAgdREIBAn0VCNTKoVHRSHI9Bcc47Kndj//l/4xAZQCqzTUwtzHhaktmD5iOqdGTKRARQjqFVuuLiRMzwRjDoEHJ7X6/YC4Xan74DqY9J7vuxTz/EgTC/v3aRGGJEELOQb3BioYmG8pqTaiqN2NDXiUajfYO7UOrkoJjDBOHhGHUoCBEBKkhpDDUI3GMQ7mxEturd2NlyVo4uZPB19TG9UaD/RIgF8sRp42BQCBAsCIQgUp/6GRaiIX0VkwIOTuMMeTl7UFTkwFjx07g2wcNSu7wvqq+/ByGjesBALLIKEQ+/BiEsjMPBNHX0Ss0IYScgdXuREWdGaXVRhwo1qO4uglyiQgF5YYO7yssQIWpIyMQHeKDAK0CagVdc9KTmRxmNFj1yC1djw0VW8+4fZAiAMn+g2Bz2ZAcMBhJfgk0+hwhpEuYTEasWrUMxcVFAICIiChERkaf1b4a163hg5IkMAjh99xHQek4CkuEEAL3BKuHSvSorDdj8/4qVNabz3pffhoZ4sO1GJkYhCBfBfw0cihlYgiF9IG5J6s0VaNAX4hVpetQYapq9/2ifMJxc8r18Ff4dmF1hBByUkHBIeTm5sBqtQIAfH39oVAoOrwfxnFoWLoYtb/85G4QCBD56OMQ+2g6s9xejcISIaRfsdicaGiy4UhZI3K2lcLp4jocjAQC9xDcE9LCEBaggr9WjkCtHH4aORQyelnt6WrMdagyV6OgsQibKrbBYG9q933lIhmiNJGYPeA8xGii6IwRIaRb2Ww2rFu3CgcP7ufbhgwZgdGjMyAWd/z9p/LzT9C0aSMAQCAWI+KBhygonYbe1QkhfU6T2Y7CiiaYbQ7sL2yAi+OQf6wB+g5cSxQeoIJYLERsqAYquRhh/iqEB7r/E/Xzi117I7vLgWXHVmFxUU677xOriYafXIdk/0EIVQcjQh1GgzAQQrymrKwEK1cuRVOTuwu4SqVGVtZ0REREndX+6pcu5oOS2M8fobfeAcWA9g0v3p9QWCKE9FqMMRSUGbDrSC0aTTbsOlwLk7XtkeZOJRIK4OIYkmJ84aeRI31wEOLDtZBL6aWxt7K7HKiz1uP7g7+iyW6C1WlBYzvOHIWpQhCjicKI4CGI0URBLqa++oSQnoPjOKxevZwPSgMHDsLEiZmQyeRntT/D5k2o/ekHfjl0/m0UlFpBnwgIIb2GxebE1gPVqKw3Y8nm1kcea4laIcHACC1UCgkGRekQHqBGeKCKJmntxWwuO1aVrMWG8q2os9a3+35qiQrXJ12JEGUQ/BV+XVghIYR0DqFQiMzM87B48Z+YODET8fGJZ70vw+aNqPzkI3457I4FUMTFd0aZfRKFJUJIj2S0OFBYYUBpjRFlNSYUlBtQ3WAGa32OVsSGamCxOREXpsGIQUGICFRBo5RCKhF1X+GkyzhcDhxpLMSXed/B6DC16z5BigAk+SfC4rQiyT8Rw4PSqCsdIaTHY4zhyJGDiI9P5K+NDA0Nx3XX3QyJ5OxHUTXn70flpx8DAARSKSLufRCKgQM7pea+isISIaRHKKs1YcmmY9icXwW5VAyjxXHG+yhkIkwfHY2MlBD4ac6uKwLp2UqayvBP4XIc0R+FxWltc9tBvgMRogpCiCoYIcogxOliKBgRQnqdpiYDVqxYgvLyUpjNZgwZMpxfdy5ByVZagtLXX+WXg6+fS0GpHSgsEUK6ldPFobTGiPJaE/IK63GgWI+GJpvHNqcHpQCtHKH+KoT4KTFsYAASInU0DHcfZHPZ0WhrRJGhBOvKNqGgsajN7eN1scgIG42hgamQimi+KkJI78YYw8GD+Vi3biXsdveARAUFh5CWNuycR960V1Wi+Pln+GXf6TOhGTPunPbZX1BYIoR0KcYYDhxrwIodZThSqofBfOYzRmKREONSQjB0YAASI3U0HHcfU2upw6aK7TisL8ARfSF8ZTo02PTtuu9ViRcjxX8wfOW6Lq2REEK6k8ViQW5uDo4ePQwAEAgEGDEiHSNGjDnnoORsMqD4+WfAnO4BkFTDhiPg0svPueb+gj6BEEI6FWOMP2u0alc5qtqYw+jEaHSZw8OhlIsxJC4AsWEaCGnumj7F4rTg18N/Y3dNHkzO5sdDW0FJJ9PimkGXIdn/7C9mJoSQnqy4uBArVy6D2ey+FlOj0SI7ewZCQsLOed+cw47S114GZ7EAAPzOn4WAiy895/32JxSWCCHnxOZw4XCpHvuO1mN/UT1Ka9q+8F6jlGDqqEiMGhQEf62c5izqoxhjWF++Gd8d/LVd22dFTYRUKIVCLEe4OhShqhBoZT5dXCUhhHjXvn27sWbNCn45KSkVGRmTIJFIO2X/1d98BXt5OQBAMzYD/hdd0in77U8oLBFCOsRqd+Kv9UU4VtWE/KIGtDE4HQAgMkiNgRFaTB8dhQCtoltqJN3PwTmxrXInDjQcxvaq3WCtHBmxmihEayIxyG8gBuriaD4jQki/FhMzAJs3r4NQKMKUKdMQE9N5cx3V/vYLDBvWAwCECgWCb5x3zl36+iMKS4SQNtUbrMg/1oCSaiOOVhhwpLSx1W1TYv0QHeKDAK0c/lo5kmP86IW5D2uyG7G4KAe5pRva3E4iFOOi+PORETYaEiG97RBC+i+XywWO4/hR7dRqH8yYcSF8ff2gUCg75TEYY6j96Qc0LFvCt0U9+gQE1JPjrNC7FiGExxhDXaMVOw7XYtnWYtQbbGe8z8hBQUiK8cXowcE0EEM/UGmqwod7voTe1ggH52x1uwCFP8aGjsTI4GEIoIlfCSEEen0DcnIWw8/PH5mZ5/HtYWERnfo4DcuWeASlyEefhDT03K9/6q/okw0h/RjHMeQfa8DWA1XIK2xAnaHteWwAICMlBAMjdRiTFEyTvfYja0o34IdDv7e5TXbUJEyLngKVpHO+HSWEkL6AMYa8vN3YsGENnE4nqqsrkZAwGBERUZ3+WKa9e1D7y08AAKFShdD5t0ExoPO69vVHFJYI6Wcammz4a0MRDpXoUV7b9mAMof5KxIdrkT44GAmROkjEdAq/P3FxLvxTuBxLj61scf2wwFTEaqORGTmBulsSQkgLTCYjVq1ahuLiIgCAUChEevq4Tj+bBACWw4dRtuhtgOMgkMkR+dCjkIWHd/rj9DcUlgjp4xhjOFSix+LNxdhTUNfmtsMGBiDEX4mxySEI9VfSSHX9UJPdiC2VO7CrZh+OtjIp7PWDr0R6yHAKSIQQ0oaCgkNYvToHNpu714afnz+ys2cgICCo0x/LcvgQSl55kV8O/b/5FJQ6CYUlQvogi82Jfzcdw6ESPaoaLDCY7C1u56OUYProKEwcEgaVXNLNVZKegGMcNpZvxeKiFWecGPbe4bchXhfbPYURQkgvZbfbsXbtShw8uJ9vGzJkBEaPzoBY3PkfvS1Hj3oEpaA510M9dFinP05/RWGJkD6CYwzr9lTgr/WFqGtjYIaIQBX+b1YyIoPU3Vgd6UnMDgv21OZhffmWVs8enSCAAC+OfxwaKc15RAgh7SEUClBdXQXAPdpdZuZ5XXJ9EgDYqypR+tpL/LIueyp0kzO75LH6KwpLhPRyDqcL3yw9hHV7K1pcH+KnxHnpkRgSHwCdmua06a8ONRTgs33/hdHR9nVqKf6DMDxoCNICk6EQy7upOkII6TvEYgmys2dgz54dGD9+MmSyrnktNeXtQ+WnH4E5HAAAvwtmIeCiS7vksfozCkuE9EJGiwPr9lSgtMaIDfsqm62XS0W4+7I0DIzUQUjXlfRb5YZK7CjLx9d5P7a5XawmGvPTboCPlM42EkJIR9XW1mDPnh2YPHkqhMev9Q0MDEJW1vQueTzGGOr//Rt1v/3Ct/lOn0lBqYtQWCKklyitbsI/a4/i7w1FrW4zeWgYrsiMh1xKf9r9kYtzYX35FuyvP4C9tfmtbqeVajA7bjoG+yVAK9N0Y4WEENJ3cByH3bu3Y/PmDeA4FzQaLUaOHNPlj9uwbIlHUFIOTkLgZVd0+eP2V/SJipAezOni8PPqAizbWtLieqFAgMhgNTKHhSMjNRRCIZ1F6m9qzHX4LO+/MNia0Gg3tLqdVCjBvJQ5SAkY3I3VEUJI32QwNGLlyiUoLy8DAIhEIsjliq5/3M0bUfvTD/xyxMJHoExI7PLH7c8oLBHSA5VWG/HH+kJsP1jT4vpBUTpMHBqG9MHB1M2uH3JxLiw7thp/Fy5tc7tBAXEYGTQMY0PSu6kyQgjp2xhjOHgwH2vXroTD4R5pNjAwGNnZM+Dr69elj+00GFD5yUf8ctidd1NQ6gYUlgjpIcxWB5ZsKcH6vRVoaGo+ml16UgguGBeFcH8VzW/Tz5gdFlSZa/Dn0SU41HCk1e3SApIRrg7FlMjx0CrU8PVVoaHBBKeT68ZqCSGkb7JYLMjNXY6jR92vwwKBACNGjMaIEaMhEom69LE5mw1lb77GLwddN5eGB+8mFJYI8SKOMew4WIN1eytanDA2LlyD2FANrpmagMAAH/rg24+YHGasLduIv462ffYIAG5JvR5DAlO6oSpCCOm/Nm5cwwclrVaHrKzpCAkJ65bHrv/nL9hK3F3yNWMzoJ04qVsel1BYIqTbMcaQV1iPLQeqsW5Py8N9j0gMxNVZA+GncQ83KhYJu7NE4iUmhxlf7/8B++paH5wBAOK0MUgJGIz0kOHQybTdVB0hhPRvY8ZMwLFjhRgwIB7jxk2CRNI9k7lXf/8/6HOWAQAkwSEIvnEe9TDpRhSWCOkGLo7D7iN1WLWzDEfKGmGzu5ptEx+hxYzRURgaH0Avgv1IsaEUPx76A4WGY61uEyD3w4zYbESowxCmDoFQQOGZEEK6WlVVBbRaHT9wg1KpxNVX39AtAzmcYNi4gQ9KABB26+0QCOk9oDtRWCKkCxVXNeHzf/JRXG1sdZtZ42IwdVQk1Iru+YaK9AxH9IV4c8cHra5XS1SYHpOFSRHjKBwRQkg3crlc2L59M7Zv34wBA+IxbdoF/JeY3RmUjLt2ovKzj/nloGvmQBYZ1W2PT9woLBHSyVwchx9XFmDHoWrUGZoP1OCvkWHysHBkDo+AQkZ/gv2F2WHBlsodWFmyFnXW+ha3GeQ7EBnho5EakASJkI4NQgjpbg0N9VixYjGqq6sAAMXFx9DUZIBG071dno17dqF80dsAAIFYjKA510M7fmK31kDc6N2YkE5UXNWEZ77cCsY82+PCNIgL1+LSSQMgEXftiDmk53BxLryw5U1Umavb3G7O4CswJmQEdb8khBAvYYwhL283NmxYA6fTCQAIDQ1HVtb0bg9KlqMFKH/nLX45+IYboRmb0a01kJMoLBHSCXYdrsU7v+xp1n5lZjxGJwVDp5Z5oSriLRzj8Pr291FkKG51m6lRkzEkMBmx2uhurIwQQsjpTCYjVq5cipIS97WjQqEQo0dnYMiQERB28/VBtvJylL31Br+snTiZgpKXUVgi5Cy5OA4b9lXii38PNFsXEajGI3OGUze7foYxhtWl6/Hz4T+brYvyCcfwoCGYED4WcjGFZ0II6Qnq6mrw++8/wWazAgD8/PyRnT0TAQGB3V6LpeAISl56nl8Ouu4G6CZN6fY6iCf6JEdIB9kcLuw+UotP/94Pp8uzv51KLsbCa4YjMkjtpepId7M4LcirO4itlTuwr655cFZLVHhw5AIEKLp2ZndCCCEdp9P5QaPRoqbGiiFDRmD06AyIxd3/8dheXY3S117ml/1nX0RBqYegsERIOzWa7Fiy+RiWbilpti4uTIObzh+MUH+VFyoj3vBt/s/YULGlzW3uHnYLEnzju6kiQggh7eFyOSESuT8Ci0QiZGfPgNlsRHi4d0aaM+3bg4pPPgI7fq2UfMAA+M++yCu1kOYoLBHSBrPVge9yDmP9vsoW11+VGY9JQ8Mhk9KgDf0BYww7qvfg87xvW90mUOGPO4f+H51JIoSQHsbpdGLz5vUoKyvGpZdezQcmX18/+Pp65zXbVl6O8kXvuIOSQICAiy+F38wLvFILaRmFJUJacKSsEf9uPIZdR2qbrQvQynHZ5DiMTAyCUEijl/UHhxqO4PN9/0OTo/l8WT5SNa5IuAiJvvFQSZReqI4QQsiZ1NbWICdnMerr3e/r27dvRnq6dwdOYE4nKj//hD+jFHjlNfDNnurVmkhzFJYIOY4xhr83HsPW/GqU1jT/UJwW54/Jw8IxJM6fhnjuBxhjKDIU46M9X7UYkgDg4VH3INInrJsrI4QQ0l4cx2H37u3YvHk9OI4DAERFxSA5eYhX62Ich8N3zAdcLgCAz5ixFJR6KApLpN/jGMPqnWX477JDzdb5a+S4dmoC0uL9IaSA1C/YXQ68teNDHGtqfm0aAET5ROCm5GsRqPTv5soIIYR0hMHQiBUrlqCiogwAIBaLMW7cJCQnp3n1S0/GcSh78z98UBIqlQiZO89r9ZC2UVgi/RZjDHmF9fhxVUGzM0npg4NwZeZA+PrQEM/9QZPdiK/2f49GmwHlppavT7tu8BUYEzqymysjhBByNg4cyMPatavgcNgBAEFBwcjKmuG1a5NOYByHmp9+gDl/P98W9fjTEHhhBD7SPvSbIf1OYYUBWw9UY+nmYpw68LdCJkJGaihmjommSWT7AbPDgipzDb7O/x7V5ubXpgFAiv9gXDDgPOpqRwghvUxpaTEcDjsEAgFGjBiNESNGQyTy7mBMjDEUPvIgnHV17gaRCHFvLYJIofBqXaRtFJZIv1HVYMajH20Ca2Hd1JGRuHTSAEglNKpdf/D1/h+wuXJ7i+ukIiki1WG4Z/itEAq6d+Z2QgghnWPChCkwm00YPToDwcGh3i4HjDFUfvLRyaAEIPqpZyko9QIUlki/cOBYA179bqdHm1AgwJVZ8RibHAK1QuKlykh32l61C5/n/a/FdRlho3FV4sUUkAghpJdxOBzYsmUDhg4dAZXKPSm8TCbH7NmXebkyN8YYKj76AMZtx+fmE4kQ+9JrkPjRFBO9AYUl0qcxxvDFvwewbm+FR/uz89IREaj2UlWku3GMw4JVD7e47pFR9yBYFQSJkF4OCSGkt6mqqkBOzmI0NurR0FCH88+/uEeNWMsYQ93vv54MSgCin3qOglIvQp8OSJ+1akcpvjlthLtxKSG4+YIkL1VEvOGYoQSvbnu3Wfu8lDkYHpTmhYoIIYScK5fLhe3bN2P79s1gzN3BXiwWw+l0QiLpGb1FOKsFZW+/Ccth92cRkdoHUU89C4mvr5crIx1BYYn0KVa7E/9ddggb9jUf0eySiQNwwbiY7i+KdDuOcdhauRNf5//QbN1ViZdgQvgYL1RFCCGkMzQ01CMnZzFqaqoAABKJFBMmZCIxcXCPOatkr6lG6WuvwFl//BolgQARDz5MQakXorBE+oSGJhuWbinG6p1lsDs5j3WZw8Nx0YQBdF1SH8cYw+6afcgpXoNCw7EWt7k4/nwKSoQQ0ksxxrBv325s3LgGTqcTABAWFo7MzOnQaLReru4ke3U1ih5dyC8LpFLEvPAKBaVeisIS6dUYY1j0617sPOw59LNCJkJcuBa3zEqmkNSHuTgX1pZtwu8F/8LBOVrd7prES5ERProbKyOEENLZjhw5iLVrVwIAhEIRRo/OwJAhwyEU9pyBeTi7HaWvv8IvS4JDEP3EUxDKadS73orCEum19EYbFv26F0fLDXxbgFaOKzMHYkRioBcrI13Jxbnw19GlWF68us3tJoaPw8zYbPhIaSAPQgjpC+LiErBv327YbDZkZ89AQEDPeq/nrFZUfv4JPzy4dnImgudc7+WqyLmisER6nSazHb/kFmDN7pMj3AkEwNVZA5E9MtKLlZGuVKAvwhs73m91vUggQoJvHC5PuBDByp71BkoIIaTjbDYrzGYzfH3dI8cJhUKcd94FkEplEIt71kdYR30dyt74D+yV7s8m8gEDEHTNHC9XRTpDzzrSCGkDYwzfLDuE1TvLPNqHxgdg/uxkyKQ0oWxftaxoFf44urjFdbMGnIfJEeMhF8u6uSpCCCFdpbS0GCtWLIFYLMYVV1zHj3CnVKq8XFlzxl07Uf7eO8DxUfkUCYkIu30BBD2oeyA5exSWSK9gsTmx6Ne9yD/WwLdpVVLcOHMw0uL8vVgZ6UpLilbir6NLmrWPDhmBi+PPpy52hBDSxzidTmzevA67d+/g2woKDmHQoGQvVtW6ik8/QtOmjfyybup5CLziqh4zKh85dxSWSI+363At3vllj0fblZnxmDYqkl6M+iDGGPbXH8L7uz9rti5BF4e7h8/3QlWEEEK6Wm1tNXJyFqP++HDbMpkckydnIy4uwcuVNcc57Ch783VYDh0EAAjEYoTcPB8+I0d5uTLS2SgskR7LxXH44Pc87DhUw7f5a2R46JrhCNDRqDJ90f66g3ivhZAkEohw25AbMdiv571hEkIIOTccx2HXrm3YsmUDOM49/UdUVAymTJkGlapn9SBgjMG4bSsqv/gUzG7n2yMWPgrFgAFerIx0FQpLpEeyO1xY+MEGGMwnh4M+f2w0Lpk4gM4m9UFlxgq8uOXNFtfdOfRmCkmEENJHuVxO/PnnL6iocF+PLBaLMW7cJCQnp/W493vOZkPlZx/DuGM73yby8UH0U89CrKM5lPoqCkukR7HandiUV4Vvlx+Ci3NfKCmTiPDMTaMQ5Kv0cnWkszHGcF/u47CfNkeSWCDCw+n3IFQV7KXKCCGEdAeRSAw/P39UVJQhKCgYWVkz+NHvehJnYyPKF70Na+FRvk07cRICr74WQonUi5WRrkZhifQIHGP4+M88bMmv9mhXysR44ZYx0KrohagvMTnM+GTv1zisP9psXXbUJFwcf74XqiKEENIdOI7zmEh23LhJ0Ol8kZIyFCJRzxvZ1mWxoOSl5+GodV8WoB6ZjpC5N9JEs/0EhSXidTa7C6//uAtHShv5NoVMhNkZsTSIQx/0z9Fl+Lcop1n7RXEzMTV6cvcXRAghpNsUFR3F+vWrMWvWpdBotAAAiUSCIUNGeLmylnEOB4pfeIYPSn4zL4D/RZfQsOD9CIUl4jWMMWw9UI2fVh1BncEGAFDIxLhoQiwmDw2HREwvRH1JmbECr257F07O6dE+yHcgrk+6ClqZj5cqI4QQ0tUcDjvWr8/F/v17AQArVy7BhRde0SO/EGUcB5fRCEdNNaq//QaOykoAgGJgAgWlfojCEvEKjjHMf201f10SAMSFa3DP5UOgkku8WBnpCm9sfx8FjUUebQO0Mbhn2HyIhD2vywUhhJDOU1lZjpycxTAY3D1IlEoVhg1L7/agxDgOLoMBTr0ezkb3f67GRjj1DXA2NsKpP75saARcLo/7yqKiEbbgHgpK/RCFJdLtqhrMeOzjzeDYyaB08YRYnD8uBsIe+A0TOXtV5ho8u+m1Zu3XJF6KjPDRXqiIEEJId3G5XNi2bRN27NgCdvw9f8CAgZg0KRsKRedd78OcTjgNjXDqG+Fq1LcQho7fNhiAUz57tJcyJQ1ht98JoZSun+6PKCyRbnXgWANe/W6nR9tj141AXLjWSxWRrvLJ3q+xq2afR1uK/yDMT5sLoYC+mSOEkL6soaEeOTmLUVNTBQCQSqWYMCETCQmD231GiXPYPcKOs7ERLr3ec7lRD1dT01nXKZDJIdZpIdbqINZqIdL5Qqw9vqzTQezrC0lwSI/sLki6B4Ul0m2WbinGDyuP8MsquRhv3Dmerk3qg/4pXN4sKM1PvQFpgcleqogQQkh3Kisr4YNSWFg4srJmwMdHA8A9XxF/tufUbnCNerj0jXA2NsCpbwRnNp314wsVCoh1vhDxwUcLsdYXIj4YudtoRDtyJhSWSLf4dtkhrNhRyi9PHhaO66Yl0Dc1fQxjDHeuesijbYA2BvePuN1LFRFCCPGGhKBQFKh84M8EGFBdD/2H76P2eDjiLJaz3q9QrebPAol1OohOCT58GNJoIZTJOvHZkP7Mq2GprKwMzzzzDHbv3g2lUomZM2fi/vvv9xh7H3CPx79o0SL8/vvvaGhoQEREBG677TbMnDnTS5WT9mKM4dmvtuFY5clT5LdemIz0wTTZaF/i4lz4dN9/sac2z6M9yT8RdwyZ56WqCCGEdJcjRw5CoVAiPDwS5vz9KFv0DgbZrBAAMJ7pzgIBRGqfk+FHd7IbnOjUYKTRQiihQaBI9/JqWFqwYAGSk5ORk5ODuro6zJ8/HwEBAbjxxhs9tvvuu+/w008/4auvvkJ0dDTWrFmDO++8EwMGDMCgQYO8VD1pj3mvrPJYfmTOcAyM0HmnGNLpjjYew7cHfkalqarZuhkxWbhgwHleqIoQQkh3sVqtWLr0Hxw8mA+12gfZSi2MS/4FAAgAiHS6U84EHe8Wd6LtRBjy8YFATJ2dSM/ktSNz7969OHDgAL744gv4+PjAx8cHc+fOxVdffdUsLOXl5WHEiBEYMGAAAGDKlCnQ6XQ4ePAghaUeijGGN3/c7dH27Lx0RASqvVQR6SyMMeypzcPHe79udZv7ht+OOF1M9xVFCCGk25WUFGPFiiUwGAwAAJdej8ot26EGAKEQofNvg8+IUV6tkZBz5bWwlJeXh/DwcGi1J0dBS05ORmFhIYxGI9Tqkx+qJ0+ejKeffhr5+fmIi4vD2rVrYbFYkJ6e3qHHFAoFEAq9f42MSCT0+LevsTtdeOXbHThc2si3PXTtcMSEarxYVe/WU46ZcmMlntnwnxbXqSRKPDf+Yagkym6uirSkpxwzpPegY4a0l9PpxIYNa7Fr13a+LdxixcCDRyDmOABA9EOPQJWY6K0SSQ/UW19jvBaW9Ho9NBrPD88nglNDQ4NHWJo2bRry8/Nx0UUXAQAUCgVeeeUVhIaGdugx/fxUPWpAAY2m743AwhjDFY/+A6vdPZmbn0aGp24eiwE0NHin8OYx8+6mL7D22JZm7U9PuRcJAXEQ0+SyPVJffJ0hXYuOGdKWyspK/Prrr6ipqQEAyCUSDDx4GMF699klia8OQ996A1Idve+TlvW21xivdhBl7ZwY7Pfff8fvv/+On376CYmJidi4cSPuv/9+hIaGIi0trd2PV19v6jFnljQaBQwGC1wuztvldKr3f93rEZSevjEdOqUYDQ1nP/wn8e4x4+CceGLtS2iwNXq0P5vxEIJVgQCApkZrt9ZEzqwvv86QrkHHDDkTk8mIL7/8FC6X+30+VCZH/NYdkDmdAADl4CRE3XMvTEwME73vk9P0xNcYX1/VGbfxWljy8/ODXq/3aNPr9RAIBPDz8/No/+9//4srr7ySD0aTJ0/GmDFj8Oeff3YoLHEcA8d1fObmruJycXA6e8bB0hne/30fth2o5pcfv34k1ApJn3qO3tadx4zJYcYXef9Dfv2hZuvey3wVAOh32wv0tdcZ0vXomCEtYU4nXMdKEC9X4bCxEQml5QivqcOJr6ADr7oGuqyp4AQCcHT8kDb0ttcYr4WllJQUVFRUoL6+ng9He/fuRXx8PFQqz5THcRz/LcYJdru922olZ/b72qMeQemZm9KhU9McB73Vol2fthiSLom/AFlRE71QESGEkO7EGIO9ohyVu3ZAfOQIzAcPgNlsCBcI4C+VQGlzfw6T+OoQeNkVUI8e5+WKCekaXgtLSUlJSE1Nxeuvv45HHnkEVVVV+OKLL3DTTTcBAKZPn47nn38eI0eORGZmJn7++WdkZWUhPj4emzZtwsaNG/ltiXet3FGKP9cX8ctzpiUgMohGveuNjugL8eaOD5q1x2qicfuQG6GkwRsIIaTPcjU1wZSfB/P+PDTk52OfVoU6jQ9GHzwMlc0GABCJRAgYEA9lUgp80lIQmjYY+kZLrzpTQEhHePWapXfeeQdPPPEEMjIyoFarcdVVV+Gaa64BABQWFsJsNgMA5s+fD6fTiTvuuAP19fUIDw/H888/j7Fjx3qzfALgi3/zsXZPBb98xZR4ZA6P8GJF5Gy9sf0DFDQWerTFaKJw/4jbIRT0rpFrCCGEnBnncMB65DBM+/NgztsHW0kxwBhqtD7Ij4qA/fgEsEVxMRgfEQtlUjIUAxMglLl7jojFQgiE9P5A+jYBa+8oC31ATU2Tt0sA4H5x8fVVoaHB1Ku/iflfziHkbCvll5+4YSRiaXjwLtGVxwxjDHeueqhZ+z3DbsVA3wGd+lik+/SV1xnSfeiY6fsYY7CXl8GclwfT/n2wHDoIdsplDU6hEIcjQlEW4M+3JQ0cjIzJ2ZAcD06nomOGdERPPF4CA33OuA1Nl0zOyupdZR5B6d4rhlBQ6oUsTiseWPOkR9u40FG4dvDlXqqIEEJIZ3I2NsKcn3c8IOXB1ahvto1AIoElMRG7lDIYnQ4AgFKpQmbmNERFxXZzxYT0LBSWSIfV6i34ZulBfvmpuaMQHXLmZE56lkMNR/D2zo892v4v5ToMDUr1UkWEEELOFWe3w3L4EMz798G8Pw+2kpIWt5NFRUOZlAxVcgryjY3Ysm0T2PGgNGDAQEyenA25vHfNh0NIV6CwRDrExXF46+c9ONF589JJAygo9TIc4/DY+hdgsHt2S30x4wloZfS7JISQ3oRxHOxlpTDlucOR5fAhMIej2XZiX18oBydDmZwC5eAkiDUne4OoD+4HYwxSqRQTJmQiIWEwBALvz0tJSE9AYYl0yLfLDqG81j3RXFqcP84fG+PdgkiHLD+2Gr8X/Nus/eXxT8JHSiMYEkJIb+DUN8CU5x61zrw/D64mQ7NtBFIplImD3OEoKRnS0DA+AJ1+uXpCwmA0NTUhMXEwfHyoSz0hp6KwRNrtlW934GCJHgAQEajG/81K8m5BpEOKm0qbBaXbh8xDsn+ilyoihBDSHpzNBsuhg/yodfbysuYbCQSQRcdAlZQMZVIy5HHxELYwKIPR2ISVK5chLW0YYmIGHL+rACNHju7qp0FIr0RhibTLlvwqPigB7gEdVPLmL8KkZ/o2/2dsqNji0fbK+KeglqpauQchhBBvsldVwrh9G0z782A9chjM6Wy2jdjPH8rkZKiS3F3rROq2ewgcPnwQa9bkwGazoa6uBldddT0UCpo/j5C2UFgiZ7TvaB0+/COPX746eyB8fWRerIh0xKJdnyK//pBH27tTXqa5kwghpIfS565GzQ//8xjWGwAEMjmUgwe7B2ZISoEkOLhd1xZZrVasXbsShw8f4NsGDUqCVCrt9NoJ6WsoLJE2NTTZ8MaPu/nliybEYurISC9WRNrLxblw1+pHmrW/M/klCkqEENJD6XNXo/qbL/llWUwsVCmpUCWnQB47AAJxxz66lZYWY8WKJTCZjAAAHx8NsrKmIyyMJpAnpD0oLJFWHatswjNfbuWXRyQEYnYGzbfQG7QUlMLVoXg0/V4vVUQIIeRMzPn7UfPD//jloOvnQjdx8lnty+l0YvPmddi9ewffNmhQMsaPnwyplHqHENJeFJZIiyrqTB5BSSYR4baLUrxYEWkvxlizoHTPsPkY6BvnpYoIIYScSdOWzaj4+AN+OfLhx6CIH3jW+2toqMfevbsAAHK5HJMmTUVc3Nnvj5D+isISacbp4vDB7/v45awREbh2aoIXKyLt5XA5cE/uYx5tdw69mYISIYT0YKWvvwZz/slrg0P+79ZzCkoAEBgYhJEjx6CysgKZmdOgVNKAPoScDQpLpJkPft+H0hr3XEohfkoKSr1EkaEYr21b5NH2wIg7EKuN9lJFhBBC2uKor0fN9996BKWga+ZAM3pMh/fV2KhHVVUFEhIG823Dh6dDIBDQBLOEnAMKS8RDWa0JOw/XAgDCA1R47PoRXq6ItMcxQ0mzoPT8uEfhK9d5pyBCCCGtYhyHxtUrUf2//3q0h956B3xGjurYvhhDfv4+rF+/Gi6XC76+/ggMDAIACIU0mA8h54rCEuExxvDpX/v55QlDwiCX0iHS0zXaDHh127seba9PfA5yMV3ASwghPVHZ22/AnLfPoy3klls7HJTMZjNyc5ejsLAAgHty2erqSj4sEULOHX0SJrzXf9iFY1VN/PK0UTREeE/XZDfi0fXPe7S9M/kliIQiL1VECCGkLQ05yzyCUsQDD0GROKjDXeWKigqwatVyWCxmAIBO54usrBkIDg7p1HoJ6e8oLBEAwLo9Fdhf1AAAkIqFePGWjveXJt3rl8N/YWXJWo82mmyWEEJ6LkvBEdT89AO/POD1tyDW6jq0D4fDjvXrc7F//16+LTV1KMaMmQCJRNJZpRJCjqOwRGAw2/H5v/n88h2XpMJPI/diReRM3t35CQ40HPZoo8lmCSGkZ2KMoeb7/0G/YjnfFnrbnR0OSowx/PnnL6iqqgAAKJUqZGaeh6iomE6slhByKgpL/RzHGF7/fhe/nBTji9QB/t4riLRpZ9VefLj7q2btb09+kbreEUJID8RZrSh7501YDh10NwgECJpzA3xGjOzwvgQCAYYNG4UlS/5EXFwCJk3Kglyu6OSKCSGnorDUz32z9CBKqo0AgJRYP9x7xRAvV0Ras754a7OgdPnACzE5MsNLFRFCCGkL57Cj7O03YDl8CAAgkMkRcf9CKAYMaPc+jMYmqNU+/PKAAfG45JKrEBwcSkOCE9INKCz1Y8VVTcjdVc4v3zI7mV54e6h1pZvxzf6fPNpuTZuL1IAkL1VECCHkTMoXvcMHJcWgwQi79Q6I1Op23Zcxhr17d2LjxrXIzDwPAwcO4teFhIR1Sb2EkOYoLPVTjDG8/9vJ0XhunDEIagVdGNoT/XN0Gf4tyvFoey/zVS9VQwgh5EwYY6j95Sd+1DtpeAQi7rkfAnH7PnYZjU1YuXIpSkuLAQDr1+ciNjYe4nbenxDSeeivrp/akl+Nar0FADBsYAAmDKFvqXqij/Z8hT21J2d2j9FE4sGRC7xYESGEkDOp/ORDNG3ZDAAQ+fgg4t4H2h2UDh8+gDVrVsBmswEA/P0DkZ09g4ISIV5Cf3n9EMcYPvvn5Oh3N50/2IvVkJYcbjiKt3Z+6NEmForxyJi74XRyXqqKEELImZR/+D6M27YAAIQqFSIefARine6M97NarVi7dgUOHz7Itw0bNgrp6WMhEtHHNUK8hf76+qF1eyrgdLk/cI9IDIRKTt3vepLfjvyDnOJcj7ZLBp6Pq4ZfgIYGk5eqIoQQcia1v//KByUAiH3hlXZdo1RRUY5ly/6GyeQecMnHR4OsrOkIC4vosloJIe1DYamfsdld+HLxAX75xhl0VqmnYIzhtW2LcKypxKN9fuoNGB6a6qWqCCGEnAljDGVvvc5foyRUKhE6//Z2D+Ygk0lhtbq7xg8alIzx4ydDKpV1Wb2EkPajsNTP/LDy5ESmF0+IhVJOh0BPUG2uxTObPAdtGBE0BDelXOuligghhLRX9Tdf8kEJAgGiHn0C0pDQdt/fzy8AEyZMgVyuwIABA7uoSkLI2aBPyv2Iw8lh9SlDhc8cG+3FasgJLs7VLCilhwzHDUlXeakiQggh7WUrL0fj2jUAAKFCgYgHH24zKHEch507t8HlciI9fRzfnpSU1uW1EkI6jsJSP3Jq97uZY6IhEgq9WA0BgEpTNZ7b/B+PtlcmPAW1ROWligghhLQXZ7ej6svPAMbcZ5QeewrSkJBWt29s1GPFiiWorCyHQCBAZGQ0QkPDu7FiQkhHUVjqJ45VNmFjXiW/fOH4WC9WQwBgS+UOfLX/e4+21yY8A6VE4aWKCCGEtBdzOlH46EK49HoAgO/0ma0GJcYY8vP3Yf361XA4HACAoKAQKJXK7iqXEHKWKCz1E6/8bwd/+5bZSZCI6aySN3245wvsrc33aHtu3CMUlAghpBdw1Neh5KUX+KAkCQmB/+yLWtzWbDZj9erlKCoqAAAIhUKMHDkGw4enQ0g9PAjp8Sgs9QOLNx2D1e7il8cktd5FgHS9OktDs6C0aMorEAgEXqqIEEJIe5n27UHlZ5/A1dTEt8U8/XyLk84WFhZg9eplsFjcI93pdL7Izp6BoCB6Hyakt6Cw1MdxjOGn1QX88tt3jfdiNYRjHJ7c+BK/HCD3wzPjHvZiRYQQQtpLvzIH1f/7L78skMkQ9cjjLQYlp9OBNWtW8EEpNXUoxoyZAImE5jYkpDehsNTHvfTf7fzt1AH+8FFKvVgNeWrjKx7LFJQIIaR3MOfvR82PJ68zDb7+RmgnTmp1e7FYgszM87By5VJMmTINUVEx3VAlIaSzUVjqw2wOFwrKDPzybRcle7Ga/q3BqsfjG170aLtzyM1eqoYQQkh7MY5Dw5J/UfvbLwBjEIjFCL/7PigHJ3ls53K5cPjwASQmJvHdqiMjo3HttTdB3MKZJ0JI70B/vX3YL7knu9+NTgqGXEq/bm9gjDULSjNisjHYP8FLFRFCCGkPR00Nyt59C/byMneDSISwu+5tFpTq62uRk7MEtbXV4DiXx5xJFJQI6d3oL7iP4hhDzrZSfnnujEFerKb/4hiHBas8u9o9MfoBhKiCvFQRIYSQ9rBXVuDYM0+CHR/qGyIRQuffDlXSyV4ajDHs2bMTmzathcvlHkipqOgoBg9OpUF7COkjKCz1Ud8sPcjf9tPIIJOIvFhN/3V6ULo0/gIKSoQQ0kMxpxP2qkpYjhxB9Tdf8u3yuHiE33M/RIqT0zs0NTVh5colKCsrAQCIRCKMGTMeaWnDKSgR0odQWOqDLDYncneV88vP3pTuxWr6r1Ul6zyWH0u/D2FqGi6WEEK8jXEcHNXVsJWVwl5eBltZGezlpbBXVQEul8e2mrEZCJn3fx5thw7lY82albDbbQAAf/9AZGfPgL9/QLc9B0JI96Cw1Af9s/EYfzsjJQRKOQ1T2p04xuHFLW+iwlTFty0Y+n8UlAghpJsxjoOzvu54GCpzh6OyMtgrK052r2uFQCaDesjQZkFp27bN2LJlPb88bNgopKePhUhEH6kI6YvoL7sPOlSi52/fOHOw9wrph8qNlXhhyxvN2gf5DfRCNYQQ0j8wxuBq1LtDUVkZbOWlx/8tB7NZ27yvQCyGNCwc0rAwyMIjIA0PhywsHGI/fwiEwmbbDxyYiJ07t0AuVyA7ewZCQ8O76mkRQnoACkt9jNPF4UhZIwBAo5JCKKR+093F4rQ0C0pToybjoviZXqqIEEL6HldT02nd59z/cmZT23cUiSANDoE0LByy8HBIwyMgCwuHJCioxVB0gtPpPgMlFrt7aWi1Opx//sUICAiEVCrrtOdFCOmZKCz1Mad2wbtofKwXK+lfSprK8PLWtz3abku7ESkBdGaPEELOhstsdgeh8uNni44HJJfB0PYdBQJIgoJOhqKwcPcZo+AQCDo4jHd1dRVychYjMjIKEyZk8u1hYRFn85QIIb0QhaU+ZvXOMv72+LRQL1bSv5welBZNeYVGQyKEkHbgbDbYK8r5QRZOnC1y1tef8b5iP3+Ps0TS8HBIQ0IhlJ3bGR+O47Bjx1Zs27YRHMdBr69HYmISgoLo2lNC+hsKS32I0eJAo8kOABgaHwCxqPVuBaRzNFj1zSacfXn8kxSUCCHkNJzDAUdl5fEzRaX8GSNHbQ3AWJv3FWm1kIWdvJ5IevyM0alDeXeWxkY9cnIWo6qqAoC7+9348ZMQGBjc6Y9FCOn5KCz1ITnbSvjbU0dSF4GuZnFamwWlZ8Y+DB+p2ksVEUKI9zGXC47qqtO6z5XDXlUJcFyb9xWqVO4uc6ddVyRSd/3rKmMM+fl7sW5dLn+dUnBwKLKzp0Or9e3yxyeE9EwUlvqQP9cXAQBkEhEGx/h5t5g+zuFy4IE1T3q0PTzqHgQo6OdOCOkfGMfBUVfrHoq7/OQ1RfaKCjCns837CmTyU64nOiUUabVeOTNvtVqwYsVSHDt2FAAgFAoxcuRYDB8+CsI2Bn8ghPR9FJb6CL3Rxt+OCFR5sZK+7+M9X2F3bZ5H21uTX4RESH9OhJC+zbhnF4zbt/PBiNntbW4vkEggDQ073n3ueDe68OPDcveg7soikQgNDXUAAJ3OD9nZMxAURN3uCCEUlvqMX1YX8LdnZdAoeF3lUMORZkFpfuoNFJQIIX0aYwz1f/2Buj9/b3kDkQjSkFDIwsLcZ4mOnzWSBLY9LHdPIZFIkZ09A4cPH8TYseP5YcIJIYQ+4fUR6/dV8rfT4vy9WEnfxRjD2zs/5pdDlEF4cOQCyMU0zwYhpO9yNupR+ubrsJeevC5WmZIGeUzM8cEWIiANDu7wsNzeVFFRhvz8fZgyZRp/hiskJAwhIWFerowQ0tP0nlc20qry2pMT8SVG6rxXSB/3xo73PZafGPOAlyohhJDuYT6Qj8ovPoWzzt1FDQIBYl58BdLAIO8WdpZcLhe2bt2InTu3gjEGP78ADB06wttlEUJ6MApLfcCm/SfPKl2QEeO9Qvqw7w/+hqONJyf8fSz9Pi9WQwghXcvZqEf1d9/CuG0r36adNAX+s2ZDrOudI8PV19ciJ2cxamtrAABSqQwqFV3jSwhpG4WlPmDFdvdEtAIBkEyj4HUqxhjuXPWQR9vE8LEIU9PEhISQvsleXY1jLzwPV6MeACAQixF4zRzoJk72al1nizGGPXt2YtOmtXC5XACA8PBIZGZOh4+Pj5erI4T0dBSWerlGow0Wm3uI1hmjo71cTd/zwJqnPJanRU/BhXEzvFQNIYR0rbpNm3Hk1deB46FCPXIUgq6eA7FW6+XKzk5TUxNWrlyCsjL39VYikQhjxkxAWtqwHjUaHyGk56Kw1MvtKajjb9PADp3rrR0fwuqy8suzB0zHeTGZXqyIEEK6BuM4VP3wI+oW/8u3BVx6OfxmnO/Fqs7dhg2r+aAUEBCI7OwZ8PML8HJVhJDehMJSL5ezvRQAoJKLMTCid37z1xNxjMNh/VF++fYhNyHZf5AXKyKEkM7nMptQ/+8/MG7bCsfxa3kgECD4hhuhHT/Ru8V1goyMySgrK0VSUipGjRoLkUjk7ZIIIb0MhaVezOniUFJtBABo1TLqUtCJFqx6mL/tI1VTUCKE9DmO+nqUvPwCnPUneyiIfdQIvmYOVKPGeLGys1daWoygoGBIpe4pHdRqH1x77U2QyWiKB0LI2aGw1IvtK6znb5+XHunFSvqWZzf9x2P58dH3e6kSQgjpGsZdO1G+6G1+Wezrh8CLLkLsrOnQG6xwOjkvVtdxDocDmzatxd69uzBoUDIyM8/j11FQIoScCwpLvdiaXeX87fTBwV6spO8oM1agylzNL48PGw21hIaWJYT0HcadO1D+3jv8smbCRARffyMkEhEEvbCbWnV1FXJyFkOvd3+BeOzYUZjNJiiV9NpNCDl3FJZ6KY5j/OAOceEayCS97w2uJ3pxy5v87XGho3D1oEu9WA0hhHQexhgali1B7a8/821+My+A/8WX9spu3BzHYceOLdi2bRM4zn0mLDp6AKZMmUpBiRDSaSgs9VJFlU3gGAMAZA6P8HI1fcOne7/xWL5m0GVeqoQQQjqXo74OlZ99AsvBA3xb0LXXQzeld47w2djYgJycJaiqqgAAiMUSjB8/GYMHp/TK4EcI6bkoLPVSOdtL+NsDw2kUvHN1x8qFHssLRy6gN1xCSI/DGANntYIzm8GZzXBZzCdvm83gLGa4TKZm62ylJcDxL9hEWh3Cbr0dioEJXn42Z6eyshx//vkznE73HIPBwaHIzp4BrVbn3cIIIX0ShaVeau8p8ysF6BRerKR3Y4zhzlUPebRNi56CaA0NmEEI6VkMWzah+r9fgzObz3of2omTEXDJZRCp1Z1YWfcKCAiCRqOFXt+AUaPGYtiwURAKhd4uixDSR1FY6oUYYzBZ3d+o0bVKZ8/msuO+3Mc92i6JvwBZUb1/bhFCSN9hryhHza8/w7Rzxxm3FSoUECqUECqVECmP/6tQQqhSQj10OJSDBndDxZ3P4XBAIpEAAMRiMaZOnQmO4xAYSIMbEUK6FoWlXqisxsTfnjEmyouV9F7Lilbhj6OLPdpmxk6loEQI6VHq/vkLdb/9crJBIIB66HD4jBkDkVIF4amBSKHolaPZtcVut2PdulWor6/DJZdcxZ9B8vcP9HJlhJD+gsJSL5R7ypDhY5NDvFhJ7/Sfbe+h0HDMo+3hUfcg0ifMSxURQkhz1d99C/2K5fyydkomAi6+FKJ+MtJbRUUZVqxYAoOhEQCwe/d2DBs2ystVEUL6GwpLvVCdwcrfDtDKvVhJ79Noa/IISkKBEK9PfBZSkdSLVRFCiKf6xf94BKXePHJdR7lcLmzZsgE7d27l2+LjEzF4cKoXqyKE9FcUlnqhXUdqAbiDEo3Y1jGPrn+Ovx3tE4mFoxZ4sRpCCPHEOA61P/2AhuVL+bbYl16DJLB/dDurq6tFTs5i1NXVAABkMhkmTszCwIGDvFwZIaS/orDUyyzdUszfnjCEuo11xOnDgz848k4vVUIIIS0ruO8ucEYjvxxx/8J+EZQYY9izZwc2bVoHl8sFAIiIiMKUKefBx8fHy9URQvozCku9zLaD1fztLJqMtt1e3vKWx/J1g6+gs3KEkB5Fv2qFR1AKu2MBlIOTvFhR9yovL4PL5YJIJMKYMROQljaMXqcJIV5HYamXOVZ58o1UKadf35m0NI/S/6Vej6GBKV6qiBBCmrMWHkXND98BAARSKQa89iZEqv4xkAMACAQCTJ6cDZfLiXHjJsHPz9/bJRFCCACAZnHrReoarXC6OADAFVPivVxNz+fiXM2C0oyYbApKhJAegzmdaNq2BWWL3gFzOiGQShH1yBN9PihZrRbk5ubAYrHwbQqFEhdccAkFJUJIj0KnJnqRfYV1/O3I4N47+3p34BiHu1Y/4tE2e8B0nBfTP0aTIoT0fKZ9e1D52adwNRn4tuC5N0EWGenFqrpecXERVq1aCpPJBIvFjPPOm0Xd7QghPRaFpV7kn40nh7xOjNR5r5Be4JmNr3osP5Z+H8LUNCcVIaRnMO7cjoqPPgBzOvm2oGvmQJM+xotVdS2Hw4FNm9Zi795dfJtMJgfHcRD1scl0CSF9B4WlXqS20T2/klohgVhEPShbs+zYKtRa6/nlG5OupqBECOkx9KtWovp/3wCMQSCVQjclC7rMLEj8A7xdWpeprq5ETs5i6PUNAACFQoHJk6chNjbOy5URQkjbKCz1EhbbyW8fIwL7dl/2c2FxWvBHwWJ++bKBszEyZJgXKyKEkJPqF/+D2l9+AgCI1D4Iu+seKAb03cDAcRx27NiCbds2gePc19zGxMRh8uSpUCqVXq6OEELOjMJSL3G04mSf9uyRfbs/+7l4YM1THstTIsd7qRJCCPHUuDaXD0pif39E3PsgpCF9+6x3fv4+bNmyAQAgFkswfvxkDB6cQtcoEUJ6DQpLvcSuQ7X87ehgmqCvJY22Jo/ldya/5KVKCCHkJJfFgqovP4Nx+zYAgNjXD5EPPNwvJpsdNCgZ+fl7IRSKkJU1HVqtztslEUJIh1BY6iU27a/kb/tr5V6spOd6dP1z/O04bSxEQrpgmBDiPfaKcjSsXAHDhvVgNivfHjr/tj4blMxmE2w2G3x9/QAAIpEIM2deDLlcDqGQrrUlhPQ+FJZ6CZPVfc1SqD/18W5JtbnGY/ne4bd6qRJCSH/GOA6mPbuhX5kD8/48j3WyqGgEXnUNFPEDvVRd1zp69DBWr14OpVKNyy67BmKx+yMGXZtECOnNziksOZ1O/sWQdB2H08XfjqIueC3KLd3A374h6SrqD08I6VYukwmN69agcdVKOGpP+fJGJILPyFHQZWZDPiCuT7422e02rFu3GgcOuMOh1WpFSckxGumOENIndDjpcByHRYsW4bfffkNdXR327NkDi8WCl19+GY899hikUmlX1Nmv5RU28LeHxNPM5qeyuxy4N/cxj7bhQWleqoYQ0t/YSkugX5kDw6aNYHY73y7S6qCbPAXaiZMg7sPX6ZSXl2LFiiVoOj6xrkqlRlbWdERERHm5MkII6RwdDkvvvvsufv31V9xwww146623AABmsxm7du3C22+/jQcffLCza+z3KuvN/O2ECJ33CulhGGPNglKYKgRiIZ3tJIR0HeZywbhrB/QrcmA5dNBjnTwuHrqsbPgMHwlBH+554XI5sWXLRuzcuZVvGzhwECZMyIRcTtfVEkL6jg6/kv/xxx/44IMPkJSUhLfffhsA4O/vjzfffBPXX389haUuUFR5cthwnY/Mi5X0HA6XA/ecFpRuTZuL1IAkL1VECOnrXE1NaFybC/3qlXDWn5z4WiAWwyd9DHRZ2ZBHx3ivwG5it9vx228/oK7O3d1QJpNh4sQsDBw4yMuVEUJI5+twWKqvr0dSUvMPpNHR0WhsbOyUooin8loTf1vYB/u7d5TNZcd9uY97tL2Y8Ti0Mo2XKiKE9GXWoiLoVy5H05bNYM6TE4SL/fygm5wJzYSJEPv0n9cfqVSKgIBA1NXVICIiCpmZ50GtputpCSF9U4fDUlhYGPLz8zF48GAwxvj2DRs2ILCPDoXqbaU17rA0MpF+vi7O1Swo3T3sFgpKhJBOxZxONG3fBv3KHFgLjnisUyQOgi4zG+qhwyAQ9Y8pCjiO8xj6e/z4KQgJCUNSUmqfHLSCEEJO6HBYmj17Nu644w7MmzcPjDEsW7YM+/btw3fffYcbb7yxK2rs1+oNJ+fmkEr6x5tyWx45ZS4lAHhtwtNQSmhYWkJI53A26tGYuxr63FVwndJbQiCVQjNmHHSZWZBFRHqxwu7FGMOhQ/nYunUjLr74SqhUagDurnfJyTSYDiGk7+twWJo/fz7sdjveeecdOBwO3HXXXQgICMCtt95KYakLbD1Qzd+ekBbqxUq879v8n2FynBzs4vlxj1JQIoScM8YYrEcLoF+Zg6ZtWwHXyekaJAGB0E7JhHb8RIhUKi9W2f2sVgtyc1egoOAQAGDVquW44IKLvVwVIYR0rw6HJafTibvuugsLFixAfX09ZDIZ1Go1nE4nKioqEB4e3u59lZWV4ZlnnsHu3buhVCoxc+ZM3H///S3O8l1QUICnn34ae/bsgU6nw4033oi5c+d2tPxep7rBwt8e2I9Hwqs212JDxRZ++eL48+Er13mvIEJIr8c57DBu3YqGlTmwFRV6rFMmp0CXmQ1VahoELbwn9XXFxUVYuXIpzGZ3N3CNRosRI0Z7uSpCCOl+HQ5LI0eOxO7duyEQCODvf3LOH6vViosvvhhbtmxp496eFixYgOTkZOTk5KCurg7z589HQEBAszNUVqsVN998M6699lp8/PHHOHz4MB599FFMmDABcXF9e9K7VTvLAACRQWoIhf23X/gzm17lb48MHorsqElerIYQ0ps56uvRuHolGtfmwtXUxLcLZHJoMzKgm5IFaWiYFyv0HofDgdWrc7Bnzy6+LSkpFRkZkyCR0DyKhJD+p91haePGjdi4cSOcTifeeOONZuuLi4vhPGWUoDPZu3cvDhw4gC+++AI+Pj7w8fHB3Llz8dVXXzULS4sXL4ZarcbNN98MAEhLS8Pff//d7sfqzSRiIRxODnaH68wb91GLC3M8lm9IuspLlRBCeivGGCyHD0G/YjmMO3cAHMevkwSHQJeZBc248RApFF6s0ruqqirw7bdLUFdXBwBQKJSYMmUqYmL69peShBDSlnaHJalUiqKiIrhcrhaDilKpxAMPPNDuB87Ly0N4eDi0Wi3flpycjMLCQhiNRqjVar59+/btSEhIwCOPPILly5cjICAAt99+O2bPnt3uxwMAoVDQI87OiERCj3/b4nC639AjgtQQi/tfVxC9tRF/Fy7jl/8vbQ6kkr470WNrOnLMEALQMXMCZ7OhcdNG1Ocsh62k5OQKgQDqtCHwy86GKjmlX3a1O11FRRkflGJj45CVNQ1KZf+6Tot0DL3OkI7orcdLuz91jhgxAiNGjMAVV1yBH3/88ZwfWK/XQ6PxHO75RHBqaGjwCEuVlZXYtm0bnnvuOTz55JNYsmQJHnroIcTHx7c451Nr/PxUPWqIU42m7W8wK+tOzq80JCEIvr79703r+SUnz2KG+4Rg6uAML1bjfWc6Zgg5XX89ZqxV1ahcvARVy1fAaTTy7SKVEsHZWQiZMR2K0BAvVtjzTJ48EZWV5Rg0aBCGDh3ao94vSc/WX19nyNnpbcdLh7+iby0oOZ1OTJs2DStXrmz3vk6dp+lM2yUnJ2PWrFkAgIsvvhjff/89lixZ0qGwVF9v6jFnljQaBQwGC1wurtXt9hfU8LdDfeVoaDC1um1f9MGuL1HSWM4vP5J+d7/7GZzQ3mOGkBP64zHDGINpfx7qc5bDuGsXcMp7jCw8HH7ZU6EdlwGhTAYrAGs/fT0B3D+rfft2IyAgCKHHr88SiYS46qqr0NRkhV5vPsMeCOmfrzPk7PXE46U9JyI6HJasVivef/997Nq1C3a7nW+vqamB1Wpt456e/Pz8oNfrPdr0ej0EAgH8/Pw82gMDA5ttGx4ejpqaGnQExzFwXPsCWndwuTg4na0fLMWVJy889vORt7ltX7O5Yjt2Ve/jlx8YcScETNSvfgYtOdMxQ8jp+sMxw1mtMGxcD/3KFbBXnPyCBQIB1EOHQ5eVDUXiIAgEAnAAuD7+8zgTk8mIVauWobi4CFqtDldcMYcfvEEgEPSLY4Z0LjpmSEf0tuOlw2HpxRdfxMqVKzF69GgsWbIE559/Pvbu3YuAgAA89thj7d5PSkoKKioqUF9fz4ejvXv3Ij4+HqrT5rKIi4vDd999B8YY3y2grKwMEyZM6Gj5vUrurpNv+lpV/xmF6GjjMXyd/wO/PDJ4KGK1UV6siBDSE9mrKqFftQKG9evAWU5OsyBUqaCdMAm6KZmQ+Ad4scKep6DgMHJzl/NfbgqFIpjNZmi1/ec9hhBCOqLDV1itWrUK3333HV5//XWIRCK8+uqr+Pvvv5GQkIBjx461ez9JSUlITU3F66+/DqPRiIKCAnzxxRe4+uqrAQDTp0/Htm3bAACzZ89GQ0MDPvzwQ1itVvz999/Iy8vr8AAPvc2pXQZ7QvfB7vL69vf421qpD25MvsaL1RBCehLGcTDt3YPSt95A0WMPQ5+znA9KsqhoBM+dhwGvvYnAy66goHQKm82GFSuWYOnSv/igNGTIcFx++bXQanXeLY4QQnqwDp9ZamxsRGRkJABAKBSC4ziIRCLceeeduP766/nritrjnXfewRNPPIGMjAyo1WpcddVVuOYa9wfjwsJCmM3uPtPBwcH46KOP8MILL+D9999HWFgY3nvvPURF9d2zDRabk5+Qdnp6332ep3tgzVMey89ntP9sJSGk73KZzTBsWAf9qhVwVFWdXCESwWf4COgyp0IeH0+DErSgvLwUK1YsQVOTAQCgUqmRlTUdERH9572FEELOVofDUkhICHbu3Ilhw4bBz88Pu3fvxrBhw6BWq1FdXd3hfX3yySctrjt48KDHcnp6Ov7444+Olttr1TaevP5rQJimjS37jry6g7A4T3aleWrMQggFvWt4SUJI57KVl0G/cgUMG9eD2Wx8u8hHA+2kydBNngKxzteLFfZsjY0N+OOPn/gBlQYOHIQJEzIhl8u9XBkhhPQOHQ5L11xzDebMmYMNGzYgKysLd911F6ZOnYr9+/cjMTGxK2rslworDPztyCB1G1v2DU7Oifd3f8YvT4uegiAldaEhpD9iHAfT7l3Qr8yBOX+/xzr5gAHQZWZDPWIUhBKJlyrsPbRaXyQnp+Hw4QOYODELAwcO8nZJhBDSq3Q4LM2dOxdhYWHQaDR48MEHYTabsXHjRkRHR2PhwoVdUWO/tOOQe6Q/pUyMIN/eNR792Xhrx0ceyxfGzfBSJYQQb3EZjWhctwb61SvhrK3l2wViMdSj0uGbmQ157AAvVtjzMcZQX18H/1Ou1xo7diKGD0+HWu3jxcoIIaR36nBYAoBp06YBAKRSKV544YVOLYi4lVS7J1E025x9vg/+mtKNKDScHBzk9YnPebEaQkh3s5UUo2FFDpo2bwRzOPh2kU4H3eRMaCdOhljTP7ojn4umJgNWrFiCmpoqXHnl9dBo3BO9SyQSSOgsHCGEnJUOhSWj0YhNmzZBJpNhzJgxzV58v/76a1x//fWdWmB/dWI+KB9l336DKzNW4IdDv/HL48PHQC6WebEiQkh3YE4njLt2QL8iB5bDhzzWKQYmQJeVDfXQ4RCIz+o7vX6FMYZDh/Kxdu1Kfv7DnTu3YtKkbC9XRgghvV+734UKCwtx4403orKyEgAQFRWFb775BsHBwairq8NDDz2EzZs3U1jqJI0m9xteSqy/lyvpWi9ueZO/PSwwFVcnXuLFagghXc1pMKBxzWo05q6Cs6GBbxdIJPAZPRa6zCzIo6K9WGHvYrVakJubg4KCwwDck8qOGJGOESPGeLkyQgjpG9odlt5++22kpqbiu+++g91ux7PPPot33nkH06ZNwyOPPAJ/f3/8+OOPXVlrv2G2Or1dQrdYUrTSY/nm1Ou8VAkhpKtZiwqhX5GDpq2bwZwnX+PE/v7QTc6CdsJEiNR9fzCbzlRcXIiVK5fBbDYBADQaLbKzZyAkJMzLlRFCSN/R7rC0a9cufP/99wgJCQEAPPnkkzj//PPxxx9/4MYbb8Rdd91FfaI7SXmdib+dEuvnxUq61j+Fy/jbT42hwUEI6WuY04mmbVugX5kD69GjHusUgwbDNysbqiHDIBDSFAEdtWHDGuzatY1fTkpKRUbGJEgkUi9WRQghfU+7w1JDQwMflAAgOtrdTeK///0vhg4d2umF9Wf7jtbxt/vqsOH5dYfAMY5fpmHCCek7nPoG6HPdXe1chpPTIAhkMmjGZkA3JQuy8HAvVtj7abU6AIBCocSUKdMQE0OjBBJCSFc4pytnRSIRBaUucKyyib8dFqDyYiVdw+5yYNHuT/nlmTF0ETIhvR1jDNaCI9CvzEHT9m2Ay8WvkwQGQZeZBU3GeIiUfe81rTswxjxGRk1KSoXVakVSUgoUCqUXKyOEkL6NhhnqgRqaTs5SLxT2vWHD7819zGP5/AHTvFQJIeRccQ47mrZshn5FDmzFxzzWKVNSocvMhiollbranQO9vgErVizG6NHjERERBeDkQA6EEEK6VrvDEmMMRUVFYIy12RYbG9u5FfZDxcfnWOqLZ5X+LVzusfzM2Ie9VAkh5Fw46urQmLsKjWty4TKePBsulMuhyZgA3ZQsSE/puk06jjGGvLw92LAhF06nEytWLMFVV10PmUzu7dIIIaTfaHdYstvtmDFjhkcbY4xvO9FFID8/v3Mr7Ge4U4JnqH/f61rxzylh6f4RtyNA0XcHsCCkr2GMwXLwAPQrc2DcuQM45fVKGhrm7mo3dhyEcoUXq+wbTCYjVq1ahuLiIgCAUChEaupQGsCBEEK6WbvD0tdff92VdZDj6hut/O2B4VovVtL58uoOeiwP0MZ4pxBCSIdxVgtK33wd1oIjJxsFAqiGDIUuMxvKwUke19SQs1dQcBi5ucthtbrfD/z8/JGdPQMBAUFerowQQvqfdoel9HTqG90dqvUW/naIf9/qhvf+7s/424+l3+fFSgghHVX5xWd8UBIqFNBOnATd5CxIAgO9XFnfYbPZsG7dKhw8uJ9vGzJkBEaPzoBYTJcYE0KIN9Crbw9TUWfmb/tr+06/9IP1RzyWw9R0LQMhvQFjDNX/+y+M291z+ojUPoh95T8QymRerqzvqa+v44OSSqVGVtZ0fkAHQggh3kFhqYcpKG/kbwfp+kZYcnEuvLPrY355RkyWF6shhHSEPmcZGlet4JejnnyGglIXCQ0Nw7Bho2A0NmHixEwayIEQQnoACks9TOUpZ5YkYpEXK+k8b+38yGN5OoUlQnqF+n//Ru2vP/PLUU8+A4kfDcrSWWpra9DQUIeBAwfxbWPGjKdrvwghpAehsNTD9MX3yKONRfztNyY9D7GQDjtCerrq776FfsXJ0StDb18AeVS0FyvqOziOw+7d27F58wYIBIC/fyD8/PwBgIISIYT0MGc1S6Ber8evv/6Kd999l28rKyvrtKL6s7JaEwAgLc7fy5V0jl8P//3/7N13eBRVGwXwsy2990pCLwEivbeEEkA6SBEQGyKiVBUU2yeogBRBRBGlWhBRepEQegcpIXQILY30XrbM98fChJgAWdjsbJLzex4eZ2ZnZ94sY5iz98694nILryawVHDYWyJzl7prZ2FQUijgO3EK7Bs3kbaoCiIjIx2bNq3DkSMHoNNpAQApKUkSV0VERI9i8Ff8Fy5cwMsvvwydTofc3Fy8/fbbuHPnDnr16oWffvoJTZrwH9RnUaDWAQDcKsDgDldSr2P3nf3ieu/qYRJWQ0SlkXH4EBLX/iau+44bD9ug+hJWVDEIgoDLly/iwIEIqNUFAAB3d0907twdzs7s2khEZK4MblmaPXs2+vfvj6NHj0Iu17/d398fEyZMwLx584xeYGWi1ujE5dx8rYSVGMc3Dz2r1MKrCZwsK9a8UUQVTea/p5Cwarl+RaFAwGczYdugobRFVQC5ubnYuXMLIiJ2QK0ugEwmQ9OmLdC//xAGJSIiM2dwy9LZs2exZMkSKBSKIn2rhw0bhgULFhiztkonr0AjLlf3dZCwkmd3NfWGuOxq5YyR9QZLWA0RPUnu9WuI+66wa7XH4KGw9PWVsKKKQafTYcOGtUhNTQEAODo6ITQ0DF5ePhJXRkREpWFwWLK2ti7xAdSsrCw+mPqMsnLV4rKdtUrCSp7dT+fXiMuv1h8uYSVE9DiCICBtdzgSf/9F3ObSqw+cQjpLWFXFIZfL0aRJC4SHb0e9eg3Rpk17qFR8dpOIqLwwOCzVr18fixcvxuTJk8VtmZmZmDlzJho3bmzU4iqb1Mx8cdm+HIelPE0eMtVZ4noVez8JqyGiRxEEAfdWr0T6/r3iNuewHnDr00+6oiqAjIx0ODgUdjuuWbMOnJyc4eHBybiJiMobg8PSlClTMHLkSKxfvx4FBQXo1asX7ty5Azs7Oyxbtqwsaqw0bsRmiMs+7nYSVvJs/r62VVzuW70HWxyJzJCg0+HunK+Qe/UKAECmUsF96Itwat9R2sLKMa1Wi1OnjuHUqWMIC+uFqlVrANAPB86gRERUPhkclmrVqoVt27Zh8+bNiI6OhpWVFapWrYrnn38etra2ZVFjpaHRFg7wYG9TPluWUvPScDD2mLgeWqW9hNUQUUm0Odm4/flnUCfe02+QyRD4+RdQublLW1g5lpqagt27t+PevQQAwKFD+xAQUE0cCImIiMong8PSwoUL0b9/f7z00ktlUU+ldi81V1yWl9PWmOmHvxCX2/u2hlzGGwUicyJoNIj9dqEYlFSeXvB/bxqUjhyt8mkIgoCoqLM4fHg/NBr9ID3e3r4IDQ1jUCIiqgAMDkvr16/HkiVL0LRpU/Tv3x9hYWGwtrYui9oqnfiUHKlLeCb3copOrDioVm+JKiGikmizshC7eKHY9U7l5YXAT2dApjT4nwICkJ2dhT17/sHt2zcB6AdzaN68NZ57rimDEhFRBWHwb/N9+/ZhxYoVCAwMxJdffok2bdpg2rRpOHHiRFnUV6m4OugnorVUKSSu5Okcjz8lLncLCGGrEpEZUScn48b7k8WgZFM3CAEffcag9JRu376J339fJQYlFxdXDBw4DI0bN2dQIiKqQJ7qX8kWLVqgRYsW+Pjjj7Fv3z5s374db775JlxcXPDPP/8Yu8ZK4/KdNACAp3P5bKnbfnO3uNy7epiElRDRA5q0NKQeP46kdWshqPXTE1jXqg2fce9AbmkpcXXll5WVNdTqAgBAcHATtGjRBkoGTyKiCueZfrOrVCrUqVMH0dHRuHPnDi5cuGCsuiolQRAAAM725e8G5r9d8IhIOpr0dGScOYW7p08hI+oCcP93CwDYNWkKnzfHSVhdxeDh4Ym2bTvByckZfn5VpC6HiIjKyFOFpfj4eGzfvh3btm3D+fPnUbduXfTu3RtLliwxdn2VUnkcavuzo7PF5ZfqDZGwEqLKSZORgax/TyLz5AnkXr5UJCABgKW/P6xr1oL74GESVVh+abUaHDt2CJaWVmjSpIW4vX79YAmrIiIiUzA4LA0ZMgTnzp2Dq6srevXqhZkzZ6JWrVplUVulk52nH0mpuq+DxJUY5nbG3SLrzTwbSVQJUeWizclG5onjyDxxvMSAZBMYANvGzWDbuCksvDjPz9NISkpEePh2pKQkQS6Xw98/EB4enlKXRUREJmJwWPLz88O4cePQunVrPsRqRFpd4RxLSkX5+Vx1gg7z/i1sUXwl6MVy2TJGVN4UJMTj5scfAlptke0Wvn6wb9oMTi1awCuoJlJTs6HR6B5xFHoUnU6Hs2dP4dixw9Dp9J+xr68/bGxsJK6MiIhMqVRh6datWwgICAAAvPXWW+K2klStWtVIpVUuCSmFcyzp/vPtsDmbdvBzqHX6h8YdLRzQxJPdUojKWvb5SMR++40YlGQWFnDp3hN2TZrB0scHAKBUlp8vXcxNRkY6IiJ2IDY2BgCgVCrRqlV71K8fzC+DiIgqmVKFpd69e+Ps2bMAgO7du5f4j4UgCJDJZLh48aJxK6wkYpOyxeVq3uWjG15ybgqy1IV1963RQ8JqiCqHuGU/IPPoEXHdpcfzcOs/UMKKKg5BEHD58gUcOLBHHOnO3d0TnTt3h7Ozi8TVERGRFEoVln766SdxedWqVWVWTGWWmpkvLnu52kpYSel9fOQrcTnEvx2aezWWsBqiik0QBNxbs6pIUPIc9Qoc27aXsKqKpaAgH0eO7IdaXQCZTIYmTVqgSZMWUCjK59x3RET07EoVlpo2bSouHz16FO+8806xfbKzszFv3jw0b97ceNVVIgpFYWudrZX5z9Xx7ZllRdb713heokqIKjZBEJBz8QKS/16PvOgbAAC5tTV8xo2HTe06EldXsVhaWqFjx644fHgfQkPD4OXlI3VJREQksVJ3atfpdCgoKMBPP/0EtVqNgoKCIn9u3bqFP/74oyxrrdAefgBbITfvPvG5mlxcTLkiro9uMJL9+InKQM7FC7j6+suImTdHDEoAUOXDjxmUjECtVuPChUhxjjsAqFq1OoYMeYlBiYiIABgwGt7SpUuxYMECyGQyNGzYsMR96tata7TCKpvrsRkA9CPhmXvw2Bq9S1y2V9kh2L2+hNUQVUx5N6MR8808cV1uYwvHDh3h0r0nFByR7ZklJMQhPHw70tPToFQqUatW4b9f7HZHREQPlDosjRkzBp06dcKAAQPw+eefF3vd2toarVu3NmpxlcmDbngarXkP8ZuSl4o9dw6K6x+1nCJhNUQVU87FC7g7/2tApwNkMji0agOP4SMht7CQurRyT6vV4tSpYzh16pjYonT37u0iYYmIiOgBgx6OqV27NhYtWoROnTqVVT2VVnaufkJadycriSt5vAX//iAue9q4w1bFb7iJjCnp7/VI2bpZXPcc9Soc27SVsKKKIzU1Bbt3b8e9ewkAAJXKAu3adULt2vUkroyIiMxVqcLSwoULxUEdTp8+jdOnTz9y30mTJhmnskomLlk/BLc5T/R7NvE8kvNSxPWPWrBVichYBEFA0rq1SP1nh7jNY8RLDEpGIAgCoqLO4vDh/dBo9F9MeXv7IjQ0DA4OjhJXR0RE5qxUYWnbtm1iWNqyZcsj95PJZAxLTykpPQ8A4OtmvsOG/3LxT3G5hVcTs3+2iqg8iV20ANnnzorrVaZ/AqtATvJtDIcP78fZs6cAAHK5Ai1atEZwcBOz/nKKiIjMQ6nC0o4dhd90RkRElFkxlZm9jQqZOWrcvZcldSklWn3xD2RrcsT1kfUGS1gNUcWScfyoGJQsvLzhO2ESVG7uEldVcdStWx/nz5+Bo6MzOnfuDjd+tkREVEpPNaHPwYMH0batvmtIVFQUNmzYgOrVq2PIkCFGLa6y0AkCMnPUAIA2DbwkrqY4tVaNo3EnxfX3mr4tYTVEFUve7VuIX/q9uO795jgGpWeUn58PpVIBhUL/T5yLiyt69RoADw8vKJXmP48dERGZD4P7IPzwww+YOnUqACAlJQWjRo3CpUuXsGzZMnz77bdGL7AyiE8ubLFxd7aWsJKSrb5YOH9WHeeaCHDwl7Aaooqj4N49xC5aIK57vf4GLH19pSuoAoiJuYO1a1fh+PEjRbb7+PgxKBERkcEMDkvr1q3DDz/oR0TbtGkT/P39sXr1aixbtgybNm0yeoGVQUJqYVjydDa/0eVOJ0aKy28996qElRBVHJr0dNye8Rk0qakAAM9Rr8ChRSuJqyq/NBoNDh3ah40b1yErKxOnT59AamrKk99IRET0GAZ/zZacnIygoCAAwOHDhxEWFgYACAwMRGJionGrqyTSMvPFZQcb85pHJS0/HTpBP/eTh7Ub5DI+EE30rNTJyYhZtAC6HP0omE5dusGxbXuJqyq/kpISER6+HSkpSQAAS0srdOjQGc7OLhJXRkRE5Z3BYcne3h4pKSmwsLDAiRMnxFHyHmwjw524dE9cdnGwlLCS4m5l3BWXOwd0kLASoooh/85t3J33NbSZGQAAm6D68Bg8VOKqyiedToezZ0/h2LHD0Om0AAB//wCEhHSDra2dxNUREVFFYHBY6ty5M15++WXI5XIEBASgfv36yM/Px8yZM9GiRYuyqLHCuz+JPACY3XDcF1Iui8u1nWtKWAlR+Ze8ZROSN/wlrrv06gPXXn0krKj8ys7Owq5dWxEbGwMAUCqVaN26PYKCgs3u9ygREZVfBoelqVOnYsWKFcjMzMSLL74IQP/tXmpqKr766iujF1gZxCTpu+JYqMyvi9vN9NvisquVs4SVEJVv2efPFQlKjp1C4dann4QVlW9KpRLp6ekAAA8PT4SGdme3OyIiMjqDw5KFhQVGjx5dZJu1tTV+/vlnoxVV2WTl6ocND/C0l7iSonLUObibFQsAcLCw57e1RE8p4/AhxC9fJq57DB8Jp44hElZU/llaWiE0NAyxsXfRpEkLKBQKqUsiIqIKyOCwpNPp8P333+Pvv/9GTIy++0NAQACGDBmCl156yegFVgYKuQxanYD8Aq3UpRTxzeml4rKXjYeElRCVX5n/nkL8zz8CAGQWFvB5cxxsGzSUuKry5+bNG7hx4yo6deoqfnHj51cFfn5VJK6MiIgqMoPD0qJFi/DLL7+gX79+qFGjBnQ6Ha5cuYKFCxfC0tKSE9M+Ba1O/9BS3UDz6eaWp8kXW5UA4JX6L0pYDVH5o8nIQPKGv5B+YJ+4zXvMWwxKBlKrC3Do0H5cuHAOAODp6Y2gIH6GRERkGgaHpY0bN2LJkiVo0qRJke1dunTBzJkzGZYMpHtodAeNVnjMnqY1ef9H4nK/Gj1hb8GRpYhKQ9BqEb/iJ2QePwZoC1uLPUaOgl3DYAkrK3/i42Oxe/cOpKenAQCsrW1gZ2de3ZWJiKhie6p5lho1alRse/PmzcVueVR62ocCko+brYSVFMoqyC6y3smvrUSVEJUvgiAg7ofvkPXvKXGbTd0geIx4CRYe7MpaWlqtFqdOHcWpU8ch3P9CqVq1GujQoQusra0lro6IiCoTg8OSj48Pzp8/j4YNi3aDiIqKgpubm9EKqyx0usKwpJCbxwAK/947Jy4PrtUXCjkfnCZ6EkEQkPTXn0WCUsCnn8PSz1/Cqsqf1NQUhIdvR2JiAgBApbJAu3YhqF27LgeZISIikzM4LPXu3Rtjx47FyJEjUbOmft6dy5cvY/Xq1ejfv7/RC6zotGYYljZc3yout/NtJWElROVH/M8/IvPIYQCAys0dVT7+DAobG4mrKn8OHtwjBiUfH1+EhITBwcFR4qqIiKiyMjgsjR49GlqtFj///DPS0tIAAPb29hg8eDDGjx9v7PoqvAJN4TMNcjMIS2fuRSJfWwAAcLVy4Te5RE8g6HRIXPubGJQAwH3YiwxKT6lDh874889f0ahRMwQHN4Zcbn7zzxERUeVhcFhSKBQYN24cxo0bh8zMTOTn58PV1ZU31U8p+/4cSwBgZ62SsBK9f27tFZcbe3DEKaLH0ebm4tYnH0KTkgIAkFtZIeCTz6Fyd5e4svIjOvo6/PyqQKXS//5zcHDEiBGvQqWykLgyIiIiA8PS3bt3cfDgQahUKnTq1AkuLi6wt+fIRM8iNStfXLa2MDi7Gt2tzDsAAAu5Cn1r9JC4GiLzlXf7FuKX/SAGJYW9A6p88BGDUinl5+fhwIE9uHLlIurXD0b79qHiawxKRERkLkp9dx4ZGYnhw4dDEATodDrMnTsXq1evRvXq1cuyvgovLjlHXLa3lbZladWFteJy5yodJKyEyLzlXr2CO7O+ENcdWrWBx/CRkFtaSlhV+RETcxu7d+9EVlYmAODWrWgUFOTDwoKfHxERmZdSdwZftGgR+vXrh3///Rf//vsvOnXqhG+++aYsa6sUMnMKu+G5OlhJVodap8Gx+MJRvGo5MwQTlSTvZnSRoOQc1gOer7zGoFQKGo0Ghw7txcaNf4pBqXbtenjhhREMSkREZJZK3bL077//4quvvoJSqX/LhAkTMHDgwDIrrLJ4MIeIlYUCSoV0DzJHJV0ssl6TYYmoGF1eHhJWrxTX3YcMg3PnrhJWVH4kJd1DePh2pKQkAwCsrKzQoUNnVK9eS+LKiIiIHq3UYamgoAAuLi7iuru7uzgaHj09jVYHALCxkvZ5pR/PrxaXZ7T+QMJKiMyTJiMDd+fNQcFd/XN9jp1CGZRK6fbtm9i2bQN0Ov3vuypVAtGpU1fY2tpJXBkREdHjlfoOnaPdlY3oOH1XFKWEw+NGJl0osu5s5SRNIURm6t6va5AWES6uW9epC4+hL0pYUfni7e0Le3sHZGdnoXXrDggKash/U4iIqFyQfvi1Si45PRcAkJOvkeT8giDg+3MrxPX3m74jSR1E5kinViPx91+Rvm+PuE3h6AifMW9Bxvl/HkkQBKjValhY6Ee1U6lU6Nr1eahUKjg5OUtcHRERUekZ1A1vyJAhT9z2+++/G6eySiInXz8pbdZD8y2Z0rmkqCLrVRz8JKmDyNwUJN5D3PffIf/WTQCA0tkFXq+/Aeuatdgq8hi5uTnYuzcc+fm56N17kDiprLu7h8SVERERGa7UYalPnz7FbhCqVq1q9IIqGysLBXLzNajl7yTJ+ZdGrhKXX60/XJIaiMxNxrEjuLdmFXS5+pZfm/oN4P3qaCg4r9xj3bx5A3v2/IPcXP2UCBcuRKJ+/WCJqyIiInp6pQ5LX331VVnWUWkpFfoAamWhkLgSoLFHQ6lLIJKUIAhI2bYFyX+v12+Qy+HWtz+cw3qw291jqNUFOHRoHy5ciBS31a8fjNq160lYFRER0bPjM0sS02j1Q4f7uNqa/Nw7bu4Wl5+v2s3k5ycyJzmXLyHp7/XIu3ZV3OY7fhJsg+pLWJX5i4+PRXj4dmRkpAMAbGxs0alTVwQEsOcBERGVfwxLEkvNzAcAKBSmfQZCJ+iw+cZOcb2eK+c6ocorZftWJK1fJ65bePvA550JsOBzNo+k0+lw4sQR/PvvcXG+uGrVaqJDh86wtraWuDoiIiLjYFiSmEIug1YnID27wKTnXXmh6EAcAQ7+Jj0/kTkQdDokrFqOjIMHAAAyCwu49Hgezp27Qm5lJXF15k0mk+HevXgIggALCwu0axeCWrXqcvALIiKqUBiWJCQIAnQ6/Teyns6m+yZWEAScTDgjrs9s86HJzk1kLnT5+bgz5yvk34wGAChdXOE7fhIsfX0lrqx8kMlkCAnphv37I9C2bUfY2ztIXRIREZHRPfUTy2q1Gnfu3DFmLZWOVidAuL9sY2m63LrtZuHkmnYqWzhZOprs3ETmQNBocPvLGWJQAgD/96YyKD1GVlYmdu/egfz8fHGbra0dunfvzaBEREQVlsFhKS8vD++//z4aNWqE7t27AwAyMjLw2muvISMjw+gFVmQ5eYUT0eYWaE123m3Ru8Tl95txElqqXNQpKbj56XQU3NV/2WNVtRpqfPs9VG7uEldmvq5evYy1a1fh8uULOHhwz5PfQEREVEEYHJbmzJmDixcv4uuvv4ZCUTjctVarxddff23U4io6tUYnLpuqG15yboq47GrlAhcrZ5Ocl0gqOnUB1KmpyL9zB1lnTuPWp9Ohjo8XX/eb/C6fT3qEvLw87Nq1Dbt2bRVblKytbcQBHYiIiCo6g/t+7dy5E2vWrEFgYCDef/99AICDgwO+/PJL9O3bF//73/+MXmRFlZ2nFpflJnooeu/dQ+Jy72ocLpwqLnVyEu7OmwN1QkKJr9sE1YfPuPGQq1Qmrqx8uHv3Nnbv3oHs7CwAgL29A0JCusHXl4PBEBFR5WFwWMrOzkZgYGCx7S4uLsjJyTFGTZXGwy1LNlameWYp4s4Bcbm+GyeMpIpJm5uL2MWLHh2U6jeE7zsTONFsCTQaDY4dO4izZ/8Vt9WuXQ/t2nWChYWlhJURERGZnsF36FWqVMGxY8fQokWLIl0xduzYAR8fH6MWV9GlZOSJy9YmGODhUsrVIutWSt74UMWjzcrC7S8+h/qePihZ16kLp46doLC1g9zWFkoHByid2P30USIjT4tBycrKCh06dEb16pyHjYiIKieD79CHDRuGt99+GwMGDIBOp8Py5ctx/vx57Ny5Ex9+yCGoDZGUXhiW7KzLvitQ+O194vIbDV4q8/MRmZqg0SDuhyViULJr0hTeb4xlC5IBGjZshMuXL8LOzh6dOnWBra2d1CURERFJxuCwNHjwYCiVSqxZswYKhQLff/89qlatiq+//hphYWFlUWOFpVIW3sDZ21iU6bnUWjUuplwR1xu6B5Xp+YhMLftCFO6tWVU0KI15i5OkPkFGRjoEQQdHR31rm0KhRJ8+g2BlZcXPjoiIKr2n6vs1YMAADBgwwNi1VDoabWE3RqWibG9KDsQcEZerOQaU6bmITC1pw19I2bJJXLepFwTv18fwZv8xBEHApUtROHhwD5ydXdCv3xBxhFNra9NNkk1ERGTODA5LGzZseOzrffv2fcpSKp+c+6PhKRXyMr+pW39ti7g8qfHYMj0XkSmlhu8qEpQsvLzhM/ZtyJSmm+i5vMnJycG+fbsQHX0dAJCYeA/x8bEc6Y6IiOg/DL6bmDp1askHUiphZWXFsGSAxLRcAIC8jL/8/u/ADvy2nSqKvJs3kfj7LwAAhYMD/Ca/B0tfP4mrMm83b17Hnj27kJurH73UyckZoaHd4enpJXFlRERE5sfgsHTu3Lki61qtFjdu3MDSpUsxcuRIoxVWGaRk6Cd5LHhoCPGy8Pe1reLyuOdeK9NzEZlK5qkTiF+2VFz3HPESg9JjqNUFOHRoHy5ciBS31a8fjFat2kPFuaaIiIhKZPAQURYWFkX+WFtbIygoCB999JHBE9LGxMRg9OjRaNGiBTp16oQ5c+ZAp3t8cEhISECjRo2waNEiQ0s3O452+kEdlIqyG6nraup13M2KBQA4WzqhrguHAKbyL23/XsQtWQxBrYZMqYTnSy/DrlETqcsyW7m5uVi7drUYlGxsbPH88/3Qvn0ogxIREdFjGK1Tv4ODA27dumXQe95++20EBQUhPDwcycnJeOONN+Dm5oaXX375ke+ZMWOG+BByeafT6Qd4sLcpu5uVBad/EJeH1eGgHFT+pR/Yh3u/rBbXfcaNh239BhJWZP6srKzg7u6BjIx0VK9eEx06dIaVFQdxICIiehKDw9LBgweLbcvLy8O2bdvg5VX6Pu+RkZG4dOkSli9fDnt7e9jb22PUqFFYuXLlI8PSvn37cO3aNXTs2NHQss1S8v1JaT2dy+am5UrqNXE5yLUO6rnWLpPzEJlKyrYtSPrrTwCAzMICniNHMSg9glarFZdlMhk6dOiMatVqokaN2nxukYiIqJQMDkuvvfYaZDIZBEEost3JyQlfffVVqY8TFRUFX19fODo6ituCgoIQHR2NrKws2NkVnQgxLy8P//vf/zBz5swnjsj3KHK5DPKyHk2hFBT3u92lZRUA0NelVBq/K97Je2fE5f61epTJOcg0HlwzijLssmnu0o8cFoOSwtYWVSZPgXW16hJXZX4EQcC5c6dx+vQpjB79unjN2NnZom7dehJXR+aMv2fIULxmyBDl9XoxOCzt3r272DYrKyu4uLgY9G1lWloaHBwcimx7EJxSU1OLhaXFixfjueeeQ8uWLZ86LLm42JrVN6ruTtZITs9DUnoenJ1tjX78yKQL4nJ9/xpm9bPT03FwqJxdp3Lu3EXcyhXieoMvZ8A2oIp0BZmpjIwMbNy4ETdu3AAAbN++HQMHDpS4KipvKuvvGXp6vGbIEOXtejE4LK1YsQIffvihUU7+39apR7l27RrWrVuHzZs3P9P5UlKyzaZlycHBGpdupQIAavo5ITU126jnuJeThIz8LABAe79WSEvLMerxybQeXDMZGbnQast29ERzk3fnDm58dP93jkKBqh9OR4GDKwqM/P9MeXflyiXs2bML+fn6UTbd3NzRvn37SnnN0NOpzL9n6OnwmiFDmOP1UprGCoPD0vbt2zFu3Lgi3eeehouLC9LS0opsS0tLg0wmg4uLi7hNEAR8+umnePvtt+Hu7v5M59TpBHFQBXPgbG+J1Mx8pGXlQ2Pk4cOXnF4hLtdyqmH045M0tFpdpfq7zL1xA3e+KBxl033AIKiqVK1Un8GT5OXl4cCBCFy9eknc1qhRU7Rq1Rbu7o5ITc3m50UGqWy/Z+jZ8ZohQ5S368XgsPTee+9h2rRpGDBgAPz9/YsNO1u1atVSHad+/fqIi4tDSkqKGI4iIyNRo0YN2NoWprzY2FicOHECV69excKFCwHoZ5+Xy+WIiIjA33//beiPYDZSM/XfAHu72hj1uIIgIDY7Xlx/zr2+UY9PZArq1FTEfrdQXLdr2hxOXbpJWJH5uXv3Nnbv3oHsbH0rsr29A0JDw+Dj48dnFImIiIzgqcISAERERBR5BkYQBMhkMly8eLFUx6lXrx4aNGiAuXPnYtq0aUhISMDy5cvxyiuvAADCwsIwY8YMNGrUCPv27Svy3i+//BJeXl547bXyO8Gq/vMCBAEoUGuf/AYDhN8u/Lyec6/PZ5Wo3NFmZyNm3hxo77c+q9w94DNmrLRFmaG4uBgxKNWpE4S2bTvCwsJS4qqIiIgqDoPD0qpVq4x28oULF+Kjjz5CmzZtYGdnhyFDhmDYsGEAgOjoaOTk5EChUBQbktza2hp2dnbP3C1PShqtgAePbFXxtDfqsQ/EHBWXB9fuZ9RjE5lC3PeLURB3fzLlLt3gPnioxBWZpyZNWuDevQTUqROE6tVrSl0OERFRhVPqsBQcHIyzZ8+iefPmRju5l5cXfvzxxxJfu3z58iPfZ8gQ5eYqO1ctLmu1xn2OKjkvBQDgZu0KBwvjBjGispb45x/IuagfydG2UWO4vTBE4orMg06nw+nTJxEQEAg3Nw8AgFwuR8+efaUtjIiIqAIrdaf20o5cR6WTkZ0vLrs4GK/bTIG2MIT52JZ+kmAic5B15jRSd2wDAMgsreD18mvsRgogPT0NGzb8gWPHDiI8fDs0Go3UJREREVUKpW5Z4g2LceUVFD6npDLig9hrrxQOeBHkWttoxyUqa9lR5xH77Tfiut+ESVDYGHfwk/JGEARcvHgehw7thVqt/yJEpbJAfn4elEq7J7ybiIiInlWpw5JWq8Uff/zx2BYmmUyGF154wSiFVXQPd72zt7Ew2nGvpl4Xl1t4NTHacYnKUsaxo4j/8Xtx3fOll2Fds5aEFUkvJycHe/fuws2b+v+n5XI5mjZticaNm0Mu50h3REREplDqsKTRaPDxxx8/dh+GpdLT6ArHl1cYaaJcnaBDcp5+oltXK2eoFKonvINIeml7duPeL6vFdZcez8OxXQcJK5JedPR17N37D3JzcwEATk7O6Ny5Ozw82LWWiIjIlEodliwtLXH27NmyrKVSScssfGZJoTDOt8TnEqPE5eZsVaJyIOPY0SJBybVvf7g+31vCiqSXmHgP27dvFNcbNHgOLVu2KzanHREREZU9g4cOJ+NQPhSQlEZqWVp98Q9xuY2P8UYtJCoLqf/sROIfv4nrni+/Bsc2bSWsyDy4u3ugTp0g3L59EyEh3VClSqDUJREREVVapQ5LHA3PuHQPfZ7GGOChQFuAPG1ha5WzldMzH5OorKQf2FckKHm9/gYcWrSSsCLpaLVapKYmi8OBA0Dbth2h0+lgZWUtYWVERERU6rDUp0+fsqyj0tHpCsOS3AgtS3ez4sTlwbX6PvPxiMpK1pnTSFi9Ulz3nfQubOsFSViRdFJSkhEevh2ZmekYPHgk7Oz086JZWBhvOgEiIiJ6eqUOS59//nlZ1lHpFAlLRhiW/ddLf4rLNZyqPfPxiIxNm5WF5C0bkb53D6DTQaZUwmfcO5UyKAmCgMjI0zhy5AC0Wv00ApGRZ9CqVTuJKyMiIqKH8ZkliTzcDe9ZW5ay1NmIy04Q1z1t3J/peETGlh8bg9sz/wchv7CrqO/4SbCpW0/CqqSRlZWJiIiduHv3NgBAoVCgZcu2aNiwscSVERER0X8xLEmkaMvSsx3rYMxRcdnXzhsKueLZDkhkRJnHjyF+5XIxKCmdneE+eFilDEpXr17C/v27kX//s3B1dUfnzt3h6uomcWVERERUEoYliWi0hfMsyZ6xG15U8mVx+f2m7zzTsYiMKePYEcT/+IO47tS5KzyGDJOwIulEROzEpUuFw/s3atQMzZu3gkLBX8NERETmiv9KS+Reaq64/KyT0t5IvwkAsFRYsFWJzEbmyRNFgpJzl25wHzxUwoqk9aD1yN7eAaGhYfDx8ZO4IiIiInoShiWJ2FkXTjBpoXr6gJOrySs8psr2mWoiMpb0QweQsPwncd3rtdFwaNlawopMTxCEIq3GDRs2hk6nQ1BQQ452R0REVE48+wQ/9FSu300HAFhaPFtL0KHYY+Jy3xo9n+lYRM9KEAQk/LJaDEoyS0v4vDOh0gWle/cSsHbtKsTFxYrbZDIZGjVqxqBERERUjjAsScTZQX/DlF+gfabj/H1tq7hck0OGk4R06gIkrPwZ6Xt2AwDk1tbwmzAZdg2fk7YwE9LpdDh58hj++us3pKQkY/fu7VCr1VKXRURERE+J3fAkEpeUDQCwtnz6vwK1tuhNmL2F3TPVRFQagk6HnKjzKIiPgyYtTf8nPQ25ly6K+8itrRHw6edQVaJR3tLT0xAevh0JCfoJopVKJRo1agalkr9miYiIyiv+Ky6RvAINAMDO+un/Ck4mnBGXB9bs/awlET2RLj8ft2f+DwWxMY/cR2Fvj4BPPofSycl0hUlIEARcvHgeBw/uhUaj/wLD09MbnTuHwdHRWeLqiIiI6FkwLEnE3sYCAJCYlveEPR/tUupVcbmtT4tnronocXKvX8Odr2YCD02orHBwgNLJGUonJyidnKFy94BT5y6Qq1SPOVLFkZOTjb17d+HmzRsAALlcjqZNW6Jx4+aQy9nLmYiIqLxjWJKI9v6ktLX8HJ/6GNHptwAA9VxqQ6WoHDenJA1tZiZiF30jBiWrqtXgO34SFHaVu+tncnKSGJScnJzRuXN3eHh4SVwVERERGQvDkkR098OS/BnmWErPzwAAVHHgfC1UdnIuXUTs4oXQ5ernBrNv0Qper41+5smUKwJ//wA0aNAIgICWLdtBVUla1IiIiCoLhiWJaJ8xLOVp8qAR9CPpcWAHKisZRw4hfsXPgFZ/rTm0agOvV1+XuCrpxMXFIDs7CzVq1Ba3tW3bkcGRiIiogmJYkohGowPw9GHpfFLhyGOuVnyInIxLEAQkrV+H1B3bAAByKyt4jnwZdk2bSVyZNLRaLU6cOILTp09AoVDC3d0Tjo5OAMCgREREVIExLEkk9/5oeMqnfAj8WvpNcTnQoYoxSiICABQkJCBm0Xyo4+MBAHIbW/i/OxWW/v4SVyaNlJQkhIdvR1JSIgD9IA7p6WliWCIiIqKKi2FJImq1vmXJ2lLxVO8/EHNEXGY3PDKW7AtRiF28CEK+fpRGhaMj/Kd+CAt3D4krMz1BEHDu3GkcPXoA2vvdEH19/RESEgZ7e3uJqyMiIiJTYFiSiFqrD0sqpeFhKbMgy9jlECE78hxiFy+EoNG3elpVqw7vN8dB5Vz5unlmZmYiImIHYmLuAAAUCgVatmyHhg0bsdsdERFRJcKwJJE7CZkAAKXC8BuvC8mXxeVhdQYYrSaqvNRJiYj5Zp647jFiFJw6dJSuIAlptRr89ddvyM7Wfynh5uaOzp27w8XFTeLKiIiIyNQ4a6JErCz0LUo5+RqD33v9oeeVmno2MlZJVEnl3bqJ2zP/J667DxlWaYMSACgUSjRr1hIA0KhRMwwYMIxBiYiIqJJiy5JEVEo58gq0sHiKbniHYo+Jy5YKC2OWRZVMQUIC7s6bA112NgDAbcAgOHfuKnFVppeWlgonp8LuhnXrNoCnpzdcXd0lrIqIiIikxpYliWi0+nmWXBwsDXrf6XuR4nILryZGrYkql4KEBNydO1sMSu4vDIVL954SV2VaGo0aBw7swa+/LsedO7fE7TKZjEGJiIiI2LIkldz73e8UBs6ztPNWhLjcNaCTUWuiyiM/JgZ3v/4K2kz9s3PuQ16Ec+cuEldlWvfuJSA8fDvS0lIAAEeO7Ief33AO4EBEREQihiUJaHU6cTlfrXvMnsXdyYwRl71sK99wzvTs1CnJiPlmnhiUXPv2r1RBSafT4d9/T+DkySPQ3f9/MSCgGjp16sKgREREREUwLEmg4KGAZG+jKvX78jT54nIr72ZGrYkqh7zbtxCzcD60aWkAAI9hw+EU0lnaokwoPT0N4eHbkZAQBwBQKlVo27YD6tZtwKBERERExTAsSSA7r3AEPAtl6R8bOxJ3QlwOcPAzak1U8WUeP4a4pUvEdde+/StVULp27TIiIv6BRqMGAHh6eqNz5zA4Ola+eaSIiIiodBiWJKDVFrYs2VqVvmXpz6ubxOX6rnWNWhNVbIl//oHUHdvEddc+/eDSs5eEFZmera0dtFoN5HI5mjZthcaNm0Eu5xg3RERE9GgMSxIQHlq2UJXuZq1AW1Bk3dnKyXgFUYUlCAKS/vgdqbt2itu83xgL+2bNJazKdARBELvXeXv7om3bjvD09IaHh5fElREREVF5wLAkAZ2uMC6V9jmJu1lx4vLzVSvfPDhkOEGrRdLf68WgJLe2hu+EybCuXkPiyspeQUEBDh7cA1dXNwQHFw6x36ABJ3EmIiKi0mNYkoAgFIYleSnD0txTi8XlOi41jV4TVSyCTofbX85A/s1ocZv/e9Ng6V9FwqpMIy4uBrt370BGRjoUCgX8/ALg6uomdVlERERUDjEsSeChrITSZCWdUHR48aqOAUauiCoSnboA8T/+IAYlmVKJKh//D5Y+PhJXVra0Wi2OHz+M06cLB0KpWrUGbG1tJayKiIiIyjOGJQnoDGxZOhb/r7jc0a9NmdREFYMmIwN3Zn0BdUI8AEDl5YWA6Z9CbmUlcWVlKyUlCeHh25GUlAgAsLCwRIcOoahZs47ElREREVF5xrAkgYfDUmlalqLTb4rLvap1K4OKqCLQ5eUVCUpW1WvAb+LkCh2UBEHAuXP/4ujRg9BqtQAAX19/hISEwd7eXuLqiIiIqLxjWJKA7qFedXL5k9PS9bSb4rKVsuLe+NLTEwQBd+Z8JQYlh9Zt4DnqVcgq+NDYubk5OHnyKLRaLRQKBVq2bIeGDRtxglkiIiIyCoYlCeTkqcXlh0fGe5T4nHsAACsFgxIVlx8Tg5gFc6FJTQEAWFWrBs+XX6sUgcHGxhbt24fi9OkT6Ny5O1xcOJADERERGQ/DkgQefk7pSZPSxjw0ZLg1W5XoPzJPHEfcD98V2ebz1vgKG5Ty8nJx/fpVBAU1FLfVrFkH1avX4gSzREREZHQMSxLQPNSapFA8/qb2QvJlcXlU0NAyq4nKn6QNfyFly6Yi2wJnzoLS0VGiisrWnTu3EBGxA9nZ2bC2tka1aoVD6DMoERERUVlgWJKAWqMVlxWKx9/kHY07KS5X45DhBP1kszGLvkHO+XPiNp9x42H3XMWccFWtVuPo0QOIjDwjbouPjysSloiIiIjKAsOSBFIz88Vl5RMGeHjwvBIAyGX89ryy02RlIua775BzIQoAILO0QpUPpsPS10/iysrGvXsJCA/fjrQ0/fNY1tbW6NixK6pWrS5xZURERFQZMCxJwEKpEJetLB7/VyCXyaETdKjuWLWsyyIzlx19Ezdnz0VBbCwAQGFnD+833qyQQUmn0+Hff4/j5Mmj0N0fPjIwsBo6duwCGxtOMktERESmwbAkgczcAnFZpXx0a9GtjDvQCfobxepOgWVdFpmx9COHEfPD9+K6XZOm8H59DGTKivm/8N69u3Dpkr71TKlUoW3bjqhbt36FHbiCiIiIzFPFvNMyc1k5hUOHKx8zwMOm6zvE5VrO7HZUGamTEpG2dw9Sd+0UtzmH9YDbgEEVOjg0aNAIV65chLu7Jzp37g5HRyepSyIiIqJKiGFJAjZWhR/74254b6TfBACo5CrUdalV1mWRGREEAQkrf0bGwQNFtvu9/Q5sghtLVFXZycnJgZWVlTiqnbu7B/r0eQGenl4c6Y6IiIgkw7AkAeH+yOGWKsUj99HoNCjQ6VugqjsGmqAqMhea9HTcnTcHBTF3xW0Wnp6o+tJwKOoFQ6PRSVid8d24cRV79+5Cw4aN0bRpS3G7t7ePhFURERERMSxJQnc/LT3uC/Mb6bfE5drONcq6JDITeTdv4vaMTws3KBTwn/I+7OvWgbOzLVJTsyWrzdgKCvJx8OBe8dmkkyePok6dINjZ2UtcGREREZEew5IEHrQsPa4L3pnE8+Lycx4NyrokMgOp4f8gaf06cd2+WXN4vvI65CqVhFWVjdjYu9i9ewcyMzMAALa2dggJ6cagRERERGaFYUkCwv209LiwdDnlqrjsYeNW5jWRdARBQMrmjUjetEHc5twtDG4DB1e4QRy0Wi2OHz+M06dPiNtq1qyNdu1CYWVlJWFlRERERMUxLElAd79l6XHz0SbkJAIA7FScU6YiEwQB8cuWIvPYEQCAwt4BvuMnwSowUNrCykB6eip27NiC5GT9tW1paYn27UNRs2YdiSsjIiIiKhnDkgSe1LKk1mkgQL8PB3eouNRJiYj/6UfkXr0CALDw8obv5PegcnaWuLKyYWFhiZwc/TNXfn5V2O2OiIiIzB7DkgQKn1kq+fVbGXfE5XqutU1QEZlSzuVLSFz7G/JvFw7iofLygt9706B0cJCwsrJlbW2DkJCuSE9PQ4MGjSpcF0MiIiKqeBiWJPCgZUn+iJvFE/H/iss1naqZpCYyjfSDB5Cw4qci26xr1oL3m+MqVFASBAFXrlxEbOxddOzYRQxGAQG8nomIiKj8YFiSwI1Y/Qhgj/pe/XLqNXHZ09bDBBWRKaiTEpGw8mdx3bV3Xzi0bgOVm7uEVRlfXl4u9u3bjevX9d0LfXz8ULt2PYmrIiIiIjIcw5IEvN1scfFWKlIy80t8PTE3Wb+fracpy6IyoisoQMrWzUjduV3sg+n+whA4dw2TuDLju337JiIidorPJjk4OMLBwUnaooiIiIieEsOSBIT7w+F5u9oUey1Xkysue9qwVam8Uycl4s7sr6BJSRa3ufR4Hk5duklYlfGp1WocPXoAkZFnxG316jVA69YdYGFhIV1hRERERM+AYUkC2vthSamQF3stsyBLXK7vVtdkNZHx5Vy6iNjvF0OXpf87tfDzh+eIl2BdvYbElRnXvXvxCA/fjrS0VAD6gRw6deqCwMDqEldGRERE9GwYliTwICzJS5ho6VzSBXHZzcrFZDWRcWnS03H361niulNIKNyHvAiZvHhALs8EQcCBA3vEoBQYWB0dO3aBjU3xVlMiIiKi8oZhSQJ5BRoAJY+Gl5B9T1z2t/c1WU1kPAUJ8YhdvEhc9xg+Ek4dQySsqOzIZDKEhHTDhg1/oGXLtqhTJ4hDghMREVGFwbAkgTsJ+m5ZJd1TyuUKcdlKaWmqkshIcq5cRsyCuRAKCgAA9i1aVaigJAgCrl69hOrVa0Kh0P/6cHZ2wYgRr0KpVElcHREREZFxMSxJwNvNBvfScnErPrPYa1qdFgDgY+tl6rLoGWWdO4O4H5aIQcmlVx+49u4rbVFGlJOTjYiIf3D7djSSkhLRunV78TUGJSIiIqqIGJYkoNXqn1mqX8212Gux2fEAAKWcfzXlhaDVImHlz8g4fAgAIFMq4TX6Tdg3biJxZcZz/fpV7Nu3C3l5eQCA27ej0bx5ayiVvE6JiIio4uKdjgRu39N3w1MqivfDu5VxR/8aw1K5oElLRdzS75F75bK4zev1NypMUCooyMfBg3tx6VKUuK1hw8Zo2bINgxIRERFVeLzbkYDi/ih4ao2u2GtOlo5Iy0/H7cy7pi6LDJRz5TLivvsW2ix9d0rLgED4vPUOVC4VYxTD2Ni72L17BzIzMwAAtrZ2CA0Ng59fFYkrIyIiIjINhiUJ2FmrkJqZj6xcdbHX0vLTAQAh/u1MXRYZIP3gfiSsXglo9c+YOYWEwm3AC5BbVoxBOa5du4J//tkirtesWQft2oXAyspKwqqIiIiITIthSQK6+/MsBXjaF9mu0WnE5XxtvklrotLR5eUidsli5ESdF7d5jnwZju07SFiV8fn7B8DOzh5qdQHatw9FzZp1pC6JiIiIyOQYliTwqElpk/NSxeUAe3+T1kRPplOriwUl7zFjYd+0uYRVGYcgCFCrC2BhoW8Zs7S0RFhYb9jY2MDOzv4J7yYiIiKqmBiWJJCSoR9RTPGfsHQusfAhei9bD5PWRI+ny8/HndlfIv/WTXFb4MxZsPD0lK4oI8nMzMDu3TugVCrRs2c/cVJZD4/y/7MRERERPQuGJQkUPBjY4T+D4Z1NLGyx8LXzNmFF9Di5168hfvkyqOPjxW0B//ui3AclQRBw5cpFHDgQgYL7c0NdvXoJtWrVlbgyIiIiIvPAsCQBGyslcvI0SM0o+lxSdMZtcZlDh0tP0GqRsGoFMg4dELfZ1G8AnzFjIbeylrCyZ5eXl4t9+8Jx/fpVAIBMJkPjxs1RvXotiSsjIiIiMh+8I5fAgwEeqnjaidtS89LE5RpOVU1dEv2HJjMDt//3CTSphc+RuQ14Ac5h3cVuauXV7dvRiIj4Bzk52QAABwdHhIZ2h7e3j8SVEREREZkXhiUJPJhfSaGQi9tS7w8ZDgCtvcv/gAHlWXbkOdz7dY0YlJTOLvAcOQq2DRpKXNmz0Wo1OHRoH86fPytuq1evAdq06QCVykLCyoiIiIjME8OSiQmCII6G9/AAD0m5yeKyj52XyesiQJOejpTtW5AWvkvcZteoCbzHjIVMoZCwMuOQyeRISUkCAFhb26BTpy4IDKwucVVERERE5othycQ0WkFcTssqfGYpMSdJXHaxcjZpTQTk37mNW599XLhBoYD7oMFwCu1S7rvdPSCXyxEa2h1Hjx5EmzYdYWNjI3VJRERERGaNYcnEBKEwLHm5FN6sxmYniMu2Kt7EmlLezWjcmf2luK50doHP2+NhVSVAwqqeXVpaKk6cOIyOHbtCpVIBAOztHdClSw+JKyMiIiIqHxiWTEwoslbYYqHRaUxdCgHIOHYU8T//CGi1AADXfgPg2rOXxFU9G0EQEBV1DocP74NGo4GFhSU6dOgsdVlERERE5Q7Dkqk9lJYe7t11PvkiAKChW5CJC6q8Mo4fRfyP34vrzt26l/uglJ2dhT17/sHt2zcB6Lve2ds7QBCECtOdkIiIiMhUGJZMTHgoLZV066oTdKYrppJSJyUiedNGZBw+KG7zGTceds81krCqZ3f9+lXs27cLeXl5AABnZ1d07twd7u4eEldGREREVD4xLJmY8HA/vPtpKSWvcC6fuq6cFLQs5d2+hZj5X0ObmanfIJPBZ+y4ch2U8vPzcfDgHly+fEHcFhzcGC1atIVSyf/FiYiIiJ4W76QkJLuflg7EHBW3OVs6SlVOhZd56iQSli+D7n7Li1W1avAY/lK5H8jhzJmTYlCytbVDaGgY/PyqSFwVERERUfnHsGRiQgnPLJ25Fylu4zNLZSNl2xYk/fWnuO42aDBcunWXsCLjady4Oa5fvwJ3d0+0bx8CS0srqUsiIiIiqhDkUp48JiYGo0ePRosWLdCpUyfMmTMHOl3Jz+z89ttv6NatGxo1aoQ+ffogPDzcxNUay0PPLN0PSxYKCwCAUq7kQ/hGJggCMo4fRdKGv8Rt3m+MLddBKTk5ERkZ6eK6SqVC//5D0aVLDwYlIiIiIiOStGXp7bffRlBQEMLDw5GcnIw33ngDbm5uePnll4vst3PnTsydOxc//PADGjZsiA0bNmDChAnYvn07/P39Jar+6egeblm63w3vblasRNVUbLr8fMR9vxjZkecAADKlEr4TJsOmTl2JK3s6giDg339P4PDhg/D09EKfPoMgl+u/77CyYkgiIiIiMjbJWpYiIyNx6dIlTJkyBfb29ggMDMSoUaOwdu3aYvvm5eVh0qRJaNKkCVQqFQYNGgRbW1ucOXPG9IUbkwwo0BaIq009n5OulgpGm5ODO3O+EoOS3Noa3mPHlduglJGRjlWrVuHgwX3Q6bS4dy8eSUmJUpdFREREVKFJ1rIUFRUFX19fODoWDmgQFBSE6OhoZGVlwc7OTtzep0+fIu/NyMhAdnY2PD09DTqnXC6DXC5tNzeFovD8CrkcahSGJW87DyiVkvaMrBDy4+Nwd9FC5MfEAACsqlRBwLQPobC2lrgywwmCgEuXLmDfvt0oKNBfK+7unujWrQdcXFwlro7MmUIhL/JfoifhNUOG4jVDhiiv14tkYSktLQ0ODg5Ftj0ITqmpqUXC0sMEQcD06dMRHByM5s2bG3ROFxdbyZ8JkqsKP3JrGwvIrQqf0arpWQXOzrZSlFVhxG3fiRs//CiOpOHesT2qvfE6lDY2EldmuJycHGzdug0XLuhHupPJZGjbti06dOgAhUIhcXVUXjg4lL8vCUhavGbIULxmyBDl7XqR9JklocikQ0+mVqsxdepUXLt2DatWrTL4fCkp2ZK3LGVkF7Yk5eUW4NjNy+K6Nk+G1NRsKcqqEFL3RCBu5Qpx3WPgILj2fB6Z+QKQX74+18zMDKxd+wtycvR1Ozo6YcCA/nB0dENGRp7E1VF5oFDI4eBgjYyMXGi1nOyanozXDBmK1wwZwhyvl9I0UkgWllxcXJCWllZkW1paGmQyGVxcXIrtn5eXh7FjxyI3Nxe//PILnJ2dDT6nTidApzMsoBmbWlN4cQiCgH13DovrgfYB0GjM4+Ipb9IP7kfCqpUA9AM5eI95C3bPNYJWK+DhEQjLCysrW7i5eeD27WjUq9cA7dt3gqenM1JTs3mNkEG0Wh2vGTIIrxkyFK8ZMkR5u14kC0v169dHXFwcUlJSxHAUGRmJGjVqwNa2aMoTBAETJ06EUqnEihUrYGlpKUXJxiE8PHS4DGl5hUNAy2Xlqw+nuUj9ZycS//gNACCztILf5HdhXa26xFUZTqNRQ6lUAdBfGyEhXXHvXgICA6vxWTYiIiIiCUh2B1avXj00aNAAc+fORVZWFq5fv47ly5dj6NChAICwsDCcPHkSALB582Zcu3YN33zzTfkOSvjv0OFAgU4NAPCwcZOmoHIubd/ewqBkYQHf8RPLXVDSarU4ceIIfv99FfLzC7vY2djYIjCwmoSVEREREVVukj6ztHDhQnz00Udo06YN7OzsMGTIEAwbNgwAEB0djZycHADA+vXrERMTU2xAhz59+mDGjBkmr9tYNNCIyw3c6klYSfmUtmc37v2yGgAgt7WF/7tTYelXvubdSktLRXj4dty7Fw8AOHx4Pzp16ipxVUREREQESByWvLy88OOPP5b42uXLhQMfrFy50lQllbmHB7XI0qSJy9YKTipqiMx/T+Her2vEdZ+33ilXQUkQBERFncPhw/ug0ehDs7e3Lxo3NmyERyIiIiIqO5KGpcro4QEe7mTfEpdru9SQopxyKTUiHIlFgtLbsKlVW8KKDJOdnYU9e/7B7ds3AQByuRzNm7fGc881hVzOZ5OIiIiIzAXDkonpHmpZUqoKhzH3sfWWopxyJ+mvP5GybYu47jNuPOyeayRhRYa5fv0K9u4NF59NcnFxRefO3eHm5iFxZURERET0XwxLJvbwAA9Z2jRx2UpZvgeuMIWUbVsKg5JCAb8Jk2FTt3w963XvXrwYlIKDm6BFizZQKvm/IREREZE54l2aiQkPpSWdoJWwkvJD0OkQu2gBsiPPidv8Jk6BTZ26Elb1dJo3b42UlBQEBzeGn18VqcshIiIiosfgAxIm9nA3vNi8uxJWUj4IOh1iv/2mSFCq8vFn5SIoabUaHDlyACkpyeI2hUKJnj37MigRERERlQNsWTIx7UMtSw4qB6QUJEEl519DSXR5uYhd/C1yLkYBAJQurvCf+iFU9ycxNmdJSYkID9+OlJQk3L17C/37D4VCoZC6LCIiIiIyAO/STezhlqWb2TcAADWcOPHofwmCgJhv5iP36hUAgGWVAPiOnwilo5O0hT2BTqfD2bOncOzYYeh0+m6WVlbWUKsLoFBYS1wdERERERmCYcnEsnMLJ6J1sXBDSkESUvJSJazI/AiCgHurV4pBSeHoCP/3pkJuZd5hIyMjHREROxAbGwMAUCqVaNWqPerXD4ZMJnvCu4mIiIjI3DAsmZhK+dBjYjJ9K1MtZ86x9IA2KwvxK35C9pnTAACVhyf83jXvoCQIAi5fvogDByKgVhcAANzdPdG5c3c4O5t/l0EiIiIiKhnDkoQEQT9BrZytDgCArNOnEL/8J+hycgAAKi8v+E95H0onZ4kre7zY2LuIiNgBAJDJZGjSpAWaNGnBZ5SIiIiIyjmGJQnpHoSlSj4ooSAIiP9pKTKPHhG32dRvCO833oTC2nxblB7w8fFD9eq1kJR0D6GhYfDy8pG6JCIiIiIyAoYlU3togAcd9MuV/XmWhBU/FwlKzt26w23gC2b7uajVaqSnp8LNzQOA/u+vY8fOkMvlUKksJK6OiIiIiIyFYUlChd3wKm/LUubJE8g4dACAfiAHv8nvwdLHV+KqHi0hIQ7h4dtRUFCAwYNHwsbGBgBgaWklcWVEREREZGwMSxLK0mQBqLxhKWX7ViStXwcAkFtZwf+9abDw9JK4qpJptVqcOnUMp04dg3C/dfDy5Sg0atRM4sqIiIiIqKwwLEmmsDtetjpbwjpMT9BocG/tr0jfEyFu835znNkGpdTUFOzevR337iUAAFQqC7Rr1wm1a9eTuDIiIiIiKksMSyYmRiRF4XxLPrbektQiBZ1ajbtzZyPv2lX9BoUCPmPfhm1QfWkLK4EgCIiKOovDh/dDo9H/fXl7+yI0NAwODo4SV0dEREREZY1hSSIyq8LWJGtl5XjeRRAEJK37XQxKVtWqwfedSVDY2UlcWXGCIGD79o24efMGAEAul6NFizYIDm4CubxydpskIiIiqmwYliQie6hlycvWQ8JKTEMQBCSs+FkczMHS3x/+738ImZnORSSTyeDh4YWbN2/AxcUVnTv3gJubu9RlEREREZEJMSyZ2INueHKbTHGbncr8WlaM7d6va8SgpHL3gM9b75hdUNLpdEVajRo3bg6VygJBQQ2hVPJ/FSIiIqLKhv2JJCIIhXMIOVjaS1hJ2RJ0OiSsXon0PbsBAEpXV/hP/QAqM2uliYm5g99+W4HExHviNrlcjuDgxgxKRERERJUUw5JEZHKtuKySV8ybcV1+PmK+mYf0fXsAAHJra/i/OxVKRydpC3uIRqPBoUP7sHHjOqSnpyE8fBu0Wu2T30hEREREFV7FvEsvB+R2aQAAG6W1tIWUEUGnw73f1iAn6jwA/TNKPm9PgMrFVeLKCiUlJSI8fDtSUpIA6CeWbdasNRRm1j2QiIiIiKTBsGRq9x9aklnmAgByNLkSFlN2Elb8jIzDBwEAKg9P+E/7CHILC4mr0tPpdDh79hSOHTsMnU7fiuTvH4CQkG6wta34z48RERERUekwLElEUFsCyJK6jDKRvGlDYVDy9IT/u9PMJihlZKQjImIHYmNjAABKpRKtW7dHUFAwZDLZE95NRERERJUJw5JEHsyzVNOpmsSVGFdaRDiSN20AACjs7eH/3gdQOprPBK7JyYliUPLw8ERoaHc4O7tIXBURERERmSOGJYnILPIBAA4WFWckvJSd25G0bi0AQGZhAf/3PzSroAQAVavWQL16DWBjY4smTVrw+SQiIiIieiSGJRMT7j+0JJPp/5utzpGyHKMQBAH31qwqHPXOygq+E6fAwstL4sqAW7duQKPRonr1muK2Dh06s8sdERERET0Rw5IUZDpx0c26fHcBK0i8h9hFC1AQGwsAkNvYwP+9abD085e0LrVajcOH9yEq6hwsLCzh4eEFe3t9Kx6DEhERUcU3a9YMFBQU4KOP/id1KVSOMSxJ4aGw5G0nfevL08q/ewe3Pv1IXFc6u8D/vWlQuUs74WxCQhzCw7cjPT0NAKBQKJCVlSmGJSIiIjIvEye+hbNnTwMAtFotdDodVCqV+Pqvv66Hl5e3Qcd8//3pz1zXmDGv4OLFKPz111a4urqJ2+PiYjFoUG/88sufCAgILPKeJUsWISoqEt9+uxQA0LZtUyiVSsjlD6Y3lcHDwwNhYT0xfPgoKJWFt+MnTx7Hr7+uwoULUVCrC+Dt7YMuXcIwdOgIWPxnsKxdu3Zg/fo/cP36NcjlMlSpEoC+fQeiZ8/epf75du/+B6tW/YzY2Fj4+1fBmDHj0Lx5yxL3jY2NwTfffI3IyHOQyYD69RvinXcmw9fXD4B+7srvvvsGO3Zsg0ajQfPmLfDeex/CwcHx/utqzJo1C8uXL8ecOd+gZcvWpa5TSgxLErOUm8cocYbKvXYVd+d/La7LLCzgP+1DSedR0mq1OHXqGE6dOgZB0HdzrFq1Bjp27AJr64o5nxUREVFFMH/+YnH5p59+wLFjR7B06QrpCgIQHX0D0dHX0bx5S2zfvgXDh4966mN99dU8MRzodDpcuBCFDz98FwqFAiNGvAwA2LZtM+bPn4N33pmEGTNmwcrKGleuXMK8ebNx6tQJzJv3rRisfvxxCTZt+hvvvvsBWrVqA41Gg8OHD2DOnC8QHx+HV19944k1Xb16GTNnfoqZM2ejceNm2LcvAh98MAW//roeHh6exfb/5JNpCAioij//3AwAmDdvFj755AMsW7YKAPDDD4tx6dJFrFz5G1QqC8yfPwubNv2N4cNHITc3F+PHv4k6dWqJ92jlhfzJu5BRCQBkhRdJeewSVnDvHu7OmwMhPx+QyeA2aDBqfrdU0qCUmpqCv/76HSdPHoUgCFCpLBAS0g1hYb0YlIiIiCqAtm2bYu3aX9CnTzesXr0CAPDPP9sxfPggdOnSDoMG9cbff/8p7j9z5qf45JNpAPRB5KWXhmL79i0YOLAXunRpj08+mQaNRvPI823ZshFt2rRHly5h2LZts9F+Drlcjvr1G6Bfv4HYv1//vHd2dhYWLPgaY8e+g169+sLGxhZyuRx16tTD7NkLcOnSRWzZshEAcPv2Laxa9TOmTfsY7dt3hEqlgrW1NUJDu2L69P+JYWTixLcQEtK6xD8AsHnzBrRs2QatWrWFpaUlunbtjmrVamDnzu3FahYEAVeuXEZoaFfY2NjAxsYGnTt3w9WrlyEIAvLz8/D33+swfvwUuLt7wMnJCZ999qUYMHNzc9CrV298+eWXRvscTYUtS5IoDEtyWfnKq/l3buPO3NkQCgoAAO5DhsE5tIukNanVBfjrr9+Rn58HAPD29kVoaJjY7EtERFTZ5eRpEJeSbdRjKhVy2GfkIzMzDxqtrtjr3i62sLEy7q3mgQP7sHz5r3B2dkFsbAxmzPgEc+cuQtOmzXHq1AlMmjQODRoEo0aNmsXeGx8fi8uXL2L16j8QFxeL114bgX379iC0hPsYtVqNnTu3Yfr0zxAc3Ahz5nyJs2fPIDj4OaP9LA8HtePHj0Kn06JXr77F9nNyckLXrmHYu3c3+vYdgAMH9sLLywetW7cttm/btu3Rtm17AEVb60py+fIltGrVpsi2WrXq4NKlqGL7ymQytGrVBtu2bUZQUAPIZDLs2rUDrVu3hUwmw+XLl6DRaHDjxjVMn/4+cnNz0K5dB7zzzmRYW1vDxcUV/foNfPKHYoYYliQgU2jF5fLUFJl/5w5uzfgM0Orrd+nxvORBCQBUKgs0bdoSR47sR4sWbRAc3OShfsFERESVW06eBu8tOYyc/Ee3opQFG0slZr/Z2qiBqVOnznC535PF29sHW7aEw8HBAQDQtGlzODu74PLliyWGpZycHIwePRbW1taoVq06qlevgVu3oks8z8GD+6BQyNGsmX6akQ4dOmHr1o1GCUsajQZRUZHYuPEvsQteTMxdeHl5F3l+6WFVqgTg2LGjAPTPDvn7V3nmOtLT02Fv71Bkm4ODA27evFHi/h988AkmTBiLHj1CAADVq9fE/PnfAgASE+8BAI4dO4KfflqFlJQUTJ06CUuXfofx4yc/c61SYliSgqLwl5VKoXrMjuZDm52NmIXzxaDk2rc/XJ8v/QOExpacnFTkQcuGDRshIKAqnJycJauJiIiIytbDgzzIZDJs2PAntmzZiKSkJAACCgoKoFYXlPheR0cn2NjYiuuWllbIz88vcd/Nmzeic+du4nyM3br1wAcfTMGECe/CxsbG4LqnTp0kfpGr1Wrh5OSEESNG4YUXhor7aDTaR70dgiDgwZMbMpkMOt2j9zVM6b+0//jjaQgMrIZ58xZDJtM/N/XuuxPw448rIQgCNBoNXn/9TTg4OMLBwRFDhgzH8uU/MiyRYQQActt0cd3Z0vy7iunUBYhZOB+a1BQA+hYlqYJSfn4eDhzYgytXLqJfv8Hw9vYFoP/FwaBERERUnI2VvoWnTLrh2VuZtBvew5PJb9myAWvWrMRXX81FcHAjKBQK9O/f85HvLW2vk/j4eJw8eQxnzpzCpk1/i9tzc3MREfEPnn++rzhSX15eXrH3Z2dnwdLSqsi2hwd42LjxLyxb9j3CwgprrVIlEAkJccjPzyv2XkD/nNKDUff8/Pxx6NCB+wHq0c++PzzC4H9FRByGs7MT0tPTi2zPyEgv8X4qOvoGTp48jg0btsPJyQkAMGbM2wgL64irV6+IrX0Pjzzs7e2DtLTUJ9Zp7hiWJOZpI+0w208i6HSI/XYh8q5fAwDYPtcIrv0GSFJLTMxt7N69E1lZmQCAY8cOoW/fFySphYiIqDyxsVKiuo9xv6BVKuVwdrZFamo2NJriYamsXbgQheDg59C4cVMA+l4nSUmJz3zcbds2ISAgEF9+ObfI9t9/X4OtWzfh+ef7wtnZBfb2Drhy5RJq164j7iMIAi5cOI+mTZs/8vi9e/fDjh1bsHDhPEyf/hkAoFmzFrC0tMKGDesxePCLRfbPzMxEePhOTJjwLgCgXbuOWLJkEXbt2omuXcOK7Hv8+FEsW/Y9vvtu2ROfWapdux4uX75YZNvFixcQGtq12L46nf7vV6stbNF6uAUvMLAqZDIZrl69giZNmgHQD6/u4eFZroMSwNHwpPFQNzxLhaWEhTyeoNMhbsli5ESdBwBY164D7zfGmvyi12g0OHRoHzZu/FMMSrVq1UX37n1MWgcRERGZD29vH9y6dRMZGRmIj4/DggVfw9PTG4mJTx+YdDodtm3bjJ49e8PPz7/In/79X0Bk5DncunUTCoUCw4aNxLJlS3Dy5HFoNBqkpqZg4cJ5SE5OxrBhLz3yHDKZDO+++wHCw3fi2LEjAABra2tMmDAFP/ywGL/9tgbZ2VnQ6XS4eDEK48a9joYNn0O3bj0AAL6+fnjppVcxe/YMbN68Afn5ecjLy8OuXTvw8cfTEBbW85HPPj2sd+++OHHiOA4fPoj8/Hxs2bIRd+7cRrdu3QEA+/btwdixrwEAAgIC4edXBcuWfY+srCzk5GTjp59+gJ+fP6pVqw4XF1e0a9cR33//LZKTkxAbG4O1a39Bjx69nvrvwlywZcnEBAGQ26WJ6+Y6Gl5BQgLif1qKvBvXAQAW3j7wfWci5CrTPmOVlHQP4eHbkZKSDEDfv7hDh86oUaOWSesgIiIi89K370CcPn0K/fv3gJeXD6ZMmYpLly7gxx+XiN3CDHXy5HEkJSWiW7fi3fmqVauOunWDsHXrRowdOx4jRoyCu7s7vvtuIe7evQMbGxvUq1cfCxd+L3ZVe5Rq1Wpg8OAXMWfOF1i1ai1sbGwQFtYTrq6uWLNmJVauXAa1Wg1vb1+EhfXAkCHDi3QjfOWV0fD19cOff67FwoVzoVSqUL16DXz88ecljpL3qBo++eRzLFw4DwkJcQgMrIrZs+eLz4RnZ2fh7t07AAClUomvv/4GixbNw5AhfSEIAurWDcLs2fPFLonTpn2Mr7/+EkOHDoBcLsfzz/cRB7DYsWMrZs+eKZ77wTNc3br1MMrkwWVJJpSn4dieUWJiptQl4Nz1ZCw+9juUXrcAAItDZktcUXFZ584ifukS6O73w7UMCITfhMlQPNQP1RSios7hwIEIsenX3z8AISHdYGtrZ9I6zIHUXR2o/OE1Q4biNUOG4jVDhjDH68Xd/cn3tmxZksL9SWldLF0kLqS4vJs3EbfkWwhqNaBQwLlrGFx794FcZWHyWhwdnaDT6aBUKtG6dXsEBQWX+36vRERERFR+MCxJQh+WzK0LniY9DbGLF4pByW/CZNjUrWey8z9o5HwQiPz8qqBt207w9w+As7P5BUsiIiIiqtjM6269UhDEliVzCkvarCzcnfe1ODy4W78BJg1Kubk52LFjMy5ciCyyvWHDRgxKRERERCQJtixJQGaln+dAbiZZVRAE3Prfx9Ck6IOSY6cQuIT1MNn5b968gT17/kFubg7u3LkJPz9/ODpyziQiIiIikhbDkgRk8vtj1QvGmn352SQs/0kMSkoXV3gMHW6S86rVBTh0aD8uXDgnbqtTJwg2NpVvAAciIiIiMj8MSxIQNPohFjPV0o/Ol7ZvLzIOHwQAKOztUeXDjyEr5QzXzyI+Pha7d+9AenoaAMDGxhadOnVFQEDVMj83EREREVFpMCyZmCBAfGapil0VSWtJ278X99asFNerfPgxlI7Gnd37v7RaLU6dOopTp46LAzpUq1YTHTp0hrW1dZmem4iIiIjIEAxLEpA76CdYVcql+/jTDx3AvVUr9PXY2MD/3WlQubmX+XlzcrJx9uy/EAQBFhYWaNcuBLVq1eWQ4ERERERkdhiWJCDkW0NmlYtsdbYk58+/cxsJy3/Sr8jl8H17Iiz9/U1ybnt7B7Rt2wmXL0chNLQ77O0dTHJeIiIiIiJDMSxJQdC3olgrTd/tTJefj5hvvxHXfSdMhnXNmmV2vuzsLERHX0f9+sHitjp1glCnThBbk4iIiIjIrDEsmZgAiM8sOVk4mfTcurxc3F0wD5pkfTdA5+49YVsvqMzOd+3aFezbtwv5+flwcHBAlSr6wRsYkoiIiOhhEye+hbNnTwPQP9+s0+mgUqnE13/9dT28vLwNPq5Wq8W6db9hyJDHj/T76acfIjx8J378cSXq1i16b9S2bVN8/fVCtGzZusj2DRv+xJo1K/Hnn5sBAAMH9kJi4j0oFApxHxcXV3To0AmvvjoGNjY24vZLly5i1aqfce7caeTm5sLV1Q0dO4ZgxIhXYG9vX+Q8x48fxS+/rMKlS1HQ6QT4+Pige/fn8cILwyAv5aBcp06dwPffL8KtW7fg4eGJkSNfRteu3UvcNySkdbFtBQUFWLToB8hkMkyaNK7Ia4IgQK1W4+DBkwCAiIhwrFy5DDExd+Ho6ITQ0K4YPXoslEqLUtVqbhiWpHA/LClMPCntzY8/FIcItwmqD7d+A8rkPPn5eThwYA+uXLkobktKShTDEhEREdHD5s9fLC7/9NMPOHbsCJYuXfHMx7169TJ+/XX1Y8NSRkYGDhzYi5CQLti6dVOxsGSIiRPfRd++AwHoQ0R09A188sk05Obm4r33PgQAnDhxFNOmTcGoUa9h6tTpsLd3wK1bN7F48QK8+earWLp0OWxsbAEAmzdvwMKF8zBx4ruYNWseZDIZzpz5F7NmzcC1a1cxffpnT6wpKSkJU6dOwvjxU9ClSxjOnTuDqVMnoUqVANSpU6/Y/hERh4usnzt3Bp9//jHq1QuCpaVVsddXrfoZ165dBaAPgTNnfoIZM2ajRYtWuHnzBsaPHws3N3cMG/ai4R+oGWBYkoDcMk//XxOFJZ1ajej3J0ObkSFu8xn3TpkMER4Tcxu7d+9EVpZ+WHQ7O3uEhobB19c0z0QRERFRcbmaXMRnJxr1mEqFDEk6a2Rm5EKjFYq97mXrbrRHDuLj4zB//mycP38OWq0Obdq0w6RJ78HW1g55eXn4+usvcfToYeTn56N69RqYMOFd6HRavPnmq9BqtQgJaY2vv16Ixo2bFjv2P/9sQ61adTBw4GC8994EvP32RFhaWj1zzTKZDNWqVcfw4aOwaNF8vPfeh9DpdJgz50v07/8Chg8fJe4bGFgVX3zxNYYO7Y9Vq5ZjzJhxyMzMxMKF8zBmzDj06NFL3LdFi1aYMWMWdu7choKCAsyfPxs7d24rsYZff12PiIhw+PtXwfPP9wEANGvWAm3btsfmzRtKDEsP02q1mDdvNt58850SP5P4+HisXfsLfvrpFwCAlZUVPvlkJlq1agMAqFatBho0CEZ09HWDPjtzwrAkoVxtbpmfQ9DpELd0SZGgVHXW15CrjNsUqtFocOzYIZw9e0rcVrt2PbRt2wmWlpZGPRcRERGVXq4mFx8d/gq5mrK/73iYtdIan7ee+syBSRAETJ06GQ0aBGP9+i+Rm5uDTz/9EN9++w3ef/9D/PHHr0hJScEff2yASmWBX35ZidmzZ+Dnn3/B++9Px/fff4tNm3Y+8vhbtmxC37790aBBMOztHbF3bwS6devxTDU/TK1Wi8tXrlxCbGwMBg0aUmw/lUqFvn0HYMuWjRgzZhyOHz8KrVaD3r37Fdu3Xr36qFevPgDg/fen4/33pz/y/JcvX0StWnWKbKtVqw527971xNp37NgKlUqFkJDOJb6+bNkS9OzZG15eXgD0oS8wUN+TSKvV4vTpUzh37jSmT//fE89lrkzbD4yg1WnFZVdL1zI/3+0vPkf26X8BAEo3N1Sd9TVUrm5GP8+uXVvFoGRlZYVu3Z5HaGgYgxIRERE9k0uXLiA6+jrGjn0HVlZWcHZ2wSuvjMY//2yDIAjIysqEUqmEhYUlVCoVXnrpVfz005pSH/vWrWiEhHSBTCZDWFgPbN26ySh163Q6XL16Gb/8shJdu4YBAGJi7sLKygru7h4lvqdKlUDExcVCEATExsbAy8u7yLNbTyMjI73Y6MMODo5IT097Yv1r1qzAiBEvl/h6XFws9u3bg8GDi3ev27FjK0JCWmPatCl4/fWxxZ73Kk/YsmRi+brCb3VkZdwNL+/2LeTfjAYAWHh5w3/adChsbcvkXI0aNcPNmzfg7x+ATp26wtbWrkzOQ0RERIZ50MJTFt3w7B3KvhteTMxdaLVa9OwZWmS7VqtFWloa+vUbhEmTxqF//55o0aIV2rXriHbtOpTq2Js3b0CrVm3h4OAIAOjWrQdWrNAPTuDr62dwrfPnz8HChfPE+qytrTFw4BCMGvWauI9Op4MgCCUOeCUIhZ+jTKbf1xgePm5pHTlyCBqN5pGf5fr1f6BDh05wLeFL+LCwnujcuRuios7js88+hCAIGDhwkME1mAOGJRPTCoUtS04WjmV3ntxc3P7fJ+K6x8hRRg1KmZmZsLOzE/9H9/LyQf/+Q+Dh4cXR7oiIiMyMtdIaVR2rGPWYSqUczs62SJVnQ6Mxzk19SSwtLWFtbYNdu/Y/Yg9nrFmzDv/+exKHDh3AnDlf4J9/tmHGjNmPPW5eXh7Cw3eioKAAXbq0E7cLgoBt2zbj9dffBAAolUrk5+cVe39WVlaxHjQPD/Bw/PhRTJs2Gd269YBSqb/lrlIlAAUFBYiNjSkxjN2+fQv+/gGQyWTw96+CuLhY5Obmwtr60aFz1qwZj31mycnJGRkZ6UW2p6enwdnZ5ZHHBIA9e8LRunXbR97X7d27G+PGTXjk+5VKJYKDn0O/fgOxfv3achuW2A3PxHQo/GWilJdNVtVkZhQJSk6du8CmVm2jHFsQBFy8eB6//76iyPNJAODp6c2gREREREbl6+uH3NwcxMbGiNtycrLFbmQ5OTnQ6XRo1qwFJkyYgh9/XIm9eyOe2M1sz55wKBRKrFz5O5Yv/1X88/rrb2L79i1iq05AQCAuX75U7P1RUZGoVq3GI4/fvHlLtG3bAbNmzRBbdmrUqAV//ypYt+73YvtrNBps3vw3QkO7ANAPxGBlZYV1634rtu+NG9cwfPggZGZm4v33pyMi4nCJf7y8vFGnTl1cvnyxyPsvXbqAeo+ZPkYQBBw6dADNmrUs8fWrVy8jPj4OzZq1KLJ99erl+N//PiqyTSaTiWGxPGJYMrGHm0HLYjQ8dXIybs/4DOrEewAA2wYN4f7CUKMcOycnBzt2bMKePf9ArVbj+PEjyMsz7cOiREREVLnoR1RriG+++RppaWnIzMzE7Nlf4PPPPwYATJ/+Hr79dj5ycrKh0+kQGXkWjo6OsLd3gKWlJbKyspCUlFisdWjz5g3o2jUMVaoEwM/PX/zTr99ApKWl4vjxowCA4cNHYd2637B3727k5+cjMzMTa9aswMmTx8XWp0cZP34yrl27io0b/wKgDw5TpkzDpk1/Y8mSRUhLS4MgCLh16yYmTBgLOzs7DB06AgBgY2OLd96ZjJ9++gGrVy9HdnYWCgoKcOTIIUyZMh6tW7crNidTSbp27Y64uDhs3rwB+fn5OHLkII4cOYTevfsDAC5cOI9hwwYUGYgiLi4WmZkZ8PHxKfGYV65chqOjY7HHLp57rjEiInZhz55waDQa3LhxHRs2rEebNu2fWKe5Kr8xr5x6uGWpLJ5ZuvXpdOhy9QHGoV17eI582SitPTdv3sCePf8gNzcHAODk5IzQ0O6wsjLOkKBEREREj/LJJzMxb94sDBrUCxYWFmjSpDk+/PBTAMB7703HnDlfoG/fHpDJgGrVquOLL+ZCLpejSZPm8PHxwQsv9MGHH36K0NCuAIDbt2/i3LkzmDjx3WLncnBwRNu2HbB16ya0bNkaXbqEwc7ODqtWLcdXX82ASqVCzZq1MH/+d6hSJeCxdbu4uGLMmLewZMlCtGnTDu7uHmjSpBm+++5HLF/+I4YNG4C8vDy4u7ujU6fOGDnyZVhZFQ7R3bNnb7i6umHNmhVYvXoFZDLA3z8Ao0ePRVhYz1J9ds7OLpg9ez4WLPga8+bNgpeXNz7++HPUqFETgL474u3bt4o8H5WSkny//pIHBUtJSYaLS/GByho0CMZnn32BpUu/w+effwxnZxd07twNI0e+UqpazZFMeJonvsqpxMRMqUvA7qgL+CthBQDg5dqj0NT38ePbGyLj8CHE//wjAMDCxxcBn8145qCkVhfg0KF9uHAhUtxWv34wWrVq/8yjs1Dpif3CU8u2XzhVHLxmyFC8ZshQvGbIEOZ4vbi7P7llji1LJlZ0lBPjtSxlHj8mBiUA8Jv83jMHpaSkROzYsUl8KNDGxhYhIV1RpUrVZzouEREREVF5wLBkYnm6HHFZEIyTqgvi4xC3dIm47vfeNCgdn32kPSsrK7F/b7VqNdGxY2d2uyMiIiKiSoNhycRkD42pYaO0eebjaXNyEPPtN+K6S68+Rhv5zs7OHh07doVGo0atWnU50h0RERERVSoMSxKSGyF8JP31J9Tx8QAA57AecOvT76mOIwgCIiPPIDU1GR06dBa3V69e85lrJCIiIiIqjxiWTMyYo2nk372D9H17AAA2QfXh1n/gUx0nKysTERH/4O7dWwAAP78qqF69ltHqJCIiIiIqjxiWTO6huPQMLUu6/HzEfDMPEARAoYDHsOGQyQ0fMOLq1cvYvz8c+fn5AABXVzc4OTk/dV1ERERERBUFw5KEZHi6sKROSsSdOV9Bk5oKAHAJ6wELTy+DjpGXl4cDByJw9WrhjNSNGjVF8+atoVDwsiAiIiIi4l2xiQnP2BFPnZqKW59+BF2efpQ62+cawdXA55Tu3r2N3bt3IDs7CwBgb++A0NAw+Pj4PVNtREREREQVCcOSiT08BbChLUva3FzELJgrBiX75i3gOfJlg7rf6XQ6HDq0VwxKdeoEoW3bjrCwsDSoFiIiIiKiis54s6JSmRJ0OsT9sAQFMXcBAE6du8B79JuQW1kZdBy5XI7Q0O6wtbVFt269EBLSjUGJiIiIyoWBA3thw4Y/AQDjxo3GkiWLJK6IKjq2LJlcYdOSIeM7JKz4GTnnzwEALAOrwn3QkFK9T6fT4dKlKNSuXQ8KhQIA4ObmjuHDX+WzSURERGR2jh8/ikmTxqFfv0GYPPn9pzrGTz/9gBUrlkGlUkEQBFhaWqFWrdp4/vk+6Nq1e6mPc+rUCdja2qJOnXpPVQcAbNz4F+bM+QJjx76DYcNGFnlt3LjRCApqgDfffLvI9lu3buLFFwdi3bpN8Pb2wcyZn2Lnzm1QKgvv3Wxt7RAc/BzefPMd+PoWPkqRlJSElSt/wuHDB5CamgpbW1s0bdocL7/8OqpUCShyntu3b2L58mU4efI4cnKy4eLiijZt2uOVV16Hg4NjqX6++Pg4zJ37FaKizsPa2hqhoV0xZsw4yJ/Q8ykx8R6GDRuIIUNexKuvvgEASEtLw6JFc3H8+DGo1WrUqlUbb701AbVr1xHP9c03c3H27GkoFAq0bNka77wzGfb29qWq9WmwZcnEij6xVLq0lLThL2QcPggAsPD1g9+kdyG7H3weJz09DRs2/IG9e3fh1KljRV5jUCIiIiJztHnzBoSGdkV4+E5xtN6nUbduECIiDmPPniNYs+YP9OrVF4sXL8CsWTNLfYy1a3/BpUsXnroGANiyRf/zbNu2+ZmO06lTKCIiDot/Vq78DXK5Au+/PxFarRYAkJSUiNdfH4mcnCwsXPg9IiIOYcWKX+Hh4YnXXx+JK1cKB/a6evUyXnvtJbi7e2Dlyt/wzz/78cUXc3Dt2hW8+earyM/PK1VdH3zwLtzcPPDHHxuxYMF32L9/L/7449cnvm/BgjlQKIpGkblzv0JKSgrWrPkDmzbtRFBQA7z77njx53v//Ymwt7fHn39uxk8/rUZ09A0sXryglJ/g0+Eds8k91LJUir0zT51EypZNAACFoyP8Jr8HhY3N488gCLh48TwOHdoLtVoNQD+oQ9OmLZ+Y8omIiKji0ebkoCA+zqjHVCjkUNpbISczD1qtrtjrFl7eT7xn+a/09DQcOrQfK1f+jsuXL2H//j3o0iXsmWt1c3NHly5hqF69Bl5++UV06dINjRs3RX5+PhYsmIPDhw8gNzcPNWrUxJQpU1GtWg28//5EHD58EMeOHcGePbvxzTdLcOnSBSxcOA83blyDSmWBDh06YcKEd4u0+Dzs+vVruHHjOmbPXoDBg/vh/PlzqF+/4TP/PADg4uKKceMmoH//nrh9+xaqVq2GH35YDDc3d3z00efifq6ubnjzzbeRkBCPuXNn4YcflgMA5s2bjebNW2Ls2HfEfWvWrI3Zs+djwYKvkZSUhMjIs5g9u+Rw+d57HyIwsCquX7+KBQu+g52dHezs7DBkyDD88cdvGDJk+CNrP3LkIG7ejEbr1u2KbL98+SKGDh0BR0cnAEBYWE+sXr0cyclJsLa2Qe3adTFmzDjY2NjAxsYG3bv3xLp1a5/2IywVhiUTE4p0w3t8XNJmZSH+52Xiut+kd6F0cHjse3JycrBv3y5ER18Xz9GsWSs0btycQYmIiKgS0ubkIHrqFOhyckx6XrmNDap+9bVBgWnHjq2oUaMW/P2roGvXMGzZsskoYemBatVqoFmzFoiICEfjxk3x66+rcOHCeaxe/QesrW0wb94szJjxKX7+eQ1mzZqPgQN7Yfjwl9C370AAwMcfT0O3bj2waNEPSExMxJtvvoLAwKoYOLDkxyM2b96Atm3bw9nZBR07hmDLlo1GC0sAxC/FAf2jF/v378HkyVNL3HfQoKF4441RSEy8B6VSicjIs1i06Idi+9nY2OKDDz4BAPj6+iEsrOcjz79hw3p4eXnD4aH701q16uD27VvIycmGjY1tsffk5eVh3rw5mDbtI2zfvqXIa61bt0N4+E60a9cRtra22L59C2rWrAV3dw/IZDKxrgcSEhLg7u7+yPqMgXfPEnpcVBJ0OtyZ8xWE+02gHiNGwdL38UN737x5HWvXrhKDkpOTM/r3H8oWJSIiIioXtmzZiG7degAAunXrgdOnTyIuLtao56hSJUA85ogRL2PJkp/g4OAIlUqFTp064/r1q9BoNCW+d8WKXzFy5CtQKBTw8vJCcHAjXLp0scR9CwoK8M8/29Gtmz5shIX1REREOHJzc43ycyQm3sO3385HrVq1ERhYFWlpqcjOzoafn3+J+z94Xikm5i5iYmKKbHtaGRnpsLcv+kX+g2ed0tLSSnzPzz//iPr1G6Bx46bFXnvrrfFQqVTo2zcMXbrog9Onn35RYgPDpUsXsH79Wowc+coz/QxPwpYlM3Xnq5niyHfWderCqUPHx+4fFXUO+/aFi+v16wejVav2UKlUZVkmERERmTnF/RaesuiG52BvhQwjdcM7fz4Sd+7cRmhoVwD6Vo2goAbYtm2zOACAMWi1WvFZmdTUFCxYMAdnzvyLnJwcCIIArVYLrVZbYte6kydPYMWKH3Hnzm1otVpoNBp06hRa4nn2798DuVyO5s1bAgAaNWoCOzs7RETsQs+evQ2ue8+e3ThwoDUA/SMXarUa3bs/jylTphUJEw+e7/kv4f78NTKZTNxfpyv+92YoQSj9HKLXrl3Dpk1/Y+XK30t8fe7crwAA69dvgZ2dHdat+x0TJ76FNWvWweaha+ncuTN4//1JGDPmbTRr1uLZfoAnYFgysSIX1CO64WVHnkPejeviuvfrY5543GrVauLEiSMAgJCQbqhSJfCZ6iQiIqKKQ2FjA+tq1Y16TKVSDntnW2hSs6HRPPtN95YtG6DVajFoUC9xm1qtRmLiPbz88utG6yVz5cplBAU1AAB88skHsLCwwPLl+kEQTp48jgkTxpb4vlu3buKjj97HuHET0bt3X1haWuHzzz96ZCvU5s0bkZmZge7dO4nbCgoKsHXrJjEsqVSqEgdSyMrSz4dpaVk4vUunTqH47LMvAQApKckYNmwgmjZtDldXNwCAs7ML7O0dcOtWNBo0CC52zNu3bwHQtyY9+Cyjo2/A3d3jkZ/Vjh1bH/vMkpOTMzIy0otsT09Ph0wmg5OTc5HtgiDg008/xWuvvSHW/LDc3Fxs3boJ3323DJ6eXgCAl156FWvX/ooTJ46iQ4cQAMDBg/vx+ecfYcKEd9G9+/OPrN1YGJYkVFJUErRaxC5eKK77f/AxlI7Fh2588G3Gg/+JrK2t0aNHHzg4OMLKyrqsSiYiIiIyupycHOzevQtTpkxDkybNxO25ubkYPfolnDp1HM2atXzm85w4cQyRkWcxfvwUAMDFi1H46KPP4eHhCQBFRov7rytXLsHCwgKD7k/fIggCrly5jGolhNDY2Bj8++8JzJo1v0hXt7i4WEyc+BZu376FKlUCEBAQiMuXi58zKuocXFxc4ezsUmItLi6uGDNmHBYunIvmzVvB2dkZMpkMHTuGYP36P9CzZ59iXdf+/nsdGjduKh6zUaMm+P33NWLL1wN5eXkYO/Y1TJgwBWFhPR/7zNKVK5eQkBCPtLQ0ODk5AQAuXYpCYGDVIi1BgH7Y7xMnTuDKlav48cfvAej/fmUyGQ4d2o9Fi36AIAhFWrsEQSgSRiMjz2LGjE/w+eezitVdVvggi4k93FApKyEupe0Oh3D/onAfPBTW1aoV2yclJRl//fUbIiJ2Fmmp8vDwYlAiIiKicici4h9YWlqgR49e8PPzF//UrFkLbdq0x5YtG5/p+Pn5+YiICMdnn03H0KEjUKdOXQCAl5c3Llw4D41Gg6NHD+P48aMA9ENwA/qWnZiYGGRlZcHb2wf5+fm4evUyMjIysGTJQqhUFkhKSirWFW3r1k2oVq0GWrduW+TnadasBerUqYetW/UjHQ8YMBjXrl3FTz/9gMzMTOTn5+PAgb1YvnwZxo2b+NjBwPr06Q9//wAsWDBH3Pbaa2OQnp6Od98dj7t37wDQ3zfqR/w7iEmTCuetGj9+CqKiIvHJJx/g3r0E6HQ6XL16GZMnvw0rKyvUq1f/iZ9rrVp1UKdOPXz//SJkZ2fh1q2b+P33X8UBMQBg2LABOHv2DDw8PLFv3z6sXv0bli//FcuX/4o2bdqjb98BmDPnG9ja2qFRoyZYufInpKQkIz8/D6tXL4dSqcRzzzWGRqPBrFkz8Oabb5ssKAEMS2Yl/84dJP7xGwBAZmkJp//0gRUEAefO/Yt169YgMfEeoqOv4dataClKJSIiIjIa/ah33Ut81rpnz944cGBfse5eT3LxYhRCQlojJKQ1evYMxW+/rca4cRPw1lvjxX0mTXoP+/btQffuIdiyZQM+++wL1KtXH6++OhwpKcno1asv/vrrD4wbNxr16zdE//4vYNy40Rgx4gV4eflg/PgpuH79Gj755APxmDqdDtu3b0HPnr1KKgs9e/bGjh1bodVq4e9fBUuXrsD169fw4osD0adPN6xa9TPGj5+Mrl0fPwqgTCbDu+9+gH37InDo0AEA+mHCly5dCQ8PT7zzzhiEhLTBqFHDkJmZiWXLViEwsKr4/ho1amLp0pUQBAGvvDIcXbq0w8cfT0Pjxk0xf/63jxwO/b9mzJiFpKRE9O7dDW+//QbCwnqgf/9B4uu3b99Cbm6OOCiGh4en+MfKygo2NrZit7zPPvsCTk7OePnlYejTpztOnDiGr79eCEdHJ0RFReLmzWh8883X4t/rgz/xRn4e72EywZCnssq5xMRMqUvAX2cOY3fKBgDAe40mI8BZ3+yrUxfg1icfQX0vAQDg8/YE2AU/J74vKysTERE7cffubQCAXK5Ay5ZtERzc+IlDkFP5p1TK4exsi1Qj9Qunio/XDBmK1wwZitcMGcIcrxd3d/sn7sNnlkysaDQtDDkJy38Sg5JD67ZFgtLVq5exf3+4OIu1q6s7OnfuXuLDcUREREREZBwMSyb38KS0+v/GfrcIWf+eAgBY164Dz5dfBQCo1QXYuzccV68WPvjXqFEzNG/eCgoF/+qIiIiIiMoS77hN7L8DPBQkxItBSWFnD+/Rb4rd6uRyBdLTUwEA9vYOCA0Ng4/P4yemJSIiIiIi42BYkpJGg9jFi8RV1379iwwTrlAo0LlzD5w5cxKtW7eHhYVlSUchIiIiIqIywNHwTK6wbUlz5AgKYmMBAM7dwqCuF4SdOzcXGU/eyckZHTt2YVAiIiIiIjIxtiyZ2IOopFLroN62HQCg9PTCrapVcXL9r9DpdLCzc0CbNh2kK5KIiIiIiKRtWYqJicHo0aPRokULdOrUCXPmzCkya+/DVq1ahW7duqFx48YYOnQozp8/b+JqjePBSO2hxzMBtRo5FhY4VbsGjh8/BJ1OB6VSCScnZ4mrJCIiIiIiSVuW3n77bQQFBSE8PBzJycl444034ObmhpdffrnIfhEREVi0aBGWLVuG2rVrY9WqVRgzZgz++ecf2NjYSFT90xJgn61F1bv5iHF1xhV/X2izMgAAnp5eCA3tzrBERERERGQGJGtZioyMxKVLlzBlyhTY29sjMDAQo0aNwtq1a4vtu3btWvTv3x/BwcGwsrLCa6+9BgDYs2ePqcs2iuf3ZeJCQCAuBvhDK5dDLpejefPW6NdvCIMSEREREZGZkKxlKSoqCr6+vnB8aPS3oKAgREdHIysrC3Z2dkX27dGjh7gul8tRt25dREZGomfPnqU+p1wug1wue/KOZcgqLQ2pdm5IcnIAoB/AoVu3nvD09JK0LjJvCoW8yH+JnoTXDBmK1wwZqiJcMz16dMHYsW/j+ed7S11KhVderxfJwlJaWhocHByKbHsQnFJTU4uEpbS0tCKh6sG+qampBp3TxcVWnMNIKlWdXaCIjUeSoz1qNmqE5/v2hUqlkrQmKj8cHKylLoHKGV4zZCheM2QoY10zISEhSEhIgFxeeDPt7u6OLl264J133oGtra1RzvMwuVwGW1tLODsb/9hUsvL2O0bSZ5YeDHZg7H0fJSUlW/KWpeAmbbF3QDJaurviuVYdkZVVAKBA0prI/CkUcjg4WCMjIxdabcmDoBA9jNcMGYrXDBnK2NeMVqvDpEnvoX//gQD0937R0TcwffpUpKVlYOrU6c98jv/S6QRkZ+cjNTXb6Memoszxd0xpQrJkYcnFxQVpaWlFtqWlpUEmk8HFxaXIdmdn5xL3rVmzpkHn1OkE6HTPHrqeVUjPfnB2tkVqajY0GvO4WKh80Gp1vGbIILxmyFC8ZiquhIS4J+7j6ektLmu1GiQlJT5yX4VCjuxsW1hZOYjXTEFBAVJTk4sdq7R0uqLXX5UqVfHiiy9h0aL5mDLlA1y6dAELF87DjRvXoFJZoEOHTpgw4V0olUr8++9JTJs2GZ999iUWLpyLe/cS0LBhI3z66Uw4ODhAo9Fg0aJ52LVrJywsLPDqq2/cP6cAjUaH/Px8LFmyCPv370F6ehrq1KmHCROmoGbN2gCAtm2b4tNPZ+K339YgOvoGmjRpinff/QBffTUDkZFn4O8fgJkzZ8Pb28fgn7syKW+/YyQLS/Xr10dcXBxSUlLEcBQZGYkaNWoUa2atX78+oqKi0K9fPwCAVqvFhQsXMHDgQJPXTURERFQerV//2xP3GTt2krick5PzxPc4OjripZdeF9eTkxPx999rix3rWajVanH544+noVu3Hli06AckJibizTdfQWBgVQwcOAQAkJeXh/Dwnfj+++XIy8vF66+/hM2b/8aLL76ErVs3Yc+e3fjuu2Xw9PTC4sULkJmZIR576dLvcObMv1i8+Ee4uLjg++8X4733JuKPPzaKj0xs2LAes2bNR25uDl56aQimTHkH06d/Bl9fP7z11mj8/vsaTJz4nlF+bjIPkj1hVa9ePTRo0ABz585FVlYWrl+/juXLl2Po0KEAgLCwMJw8eRIAMHToUGzYsAFnzpxBbm4ulixZAgsLC3Ts2FGq8omIiIioDOl0Oly9ehm//LISXbuGAQBWrPgVI0e+AoVCAS8vLwQHN8KlSxfF92i1WgwbNhIODg7w8PBEw4bP4datmwCA/fv3okuXMAQGVoW1tTVee+1NaDQa8b1bt27EiBGj4O3tA0tLK7z++ptITk5CZORZcZ/OnbvBzc0N/v5VUKVKIOrWDUKtWnVga2uHRo2a4M6dO6b5cMhkJH1maeHChfjoo4/Qpk0b2NnZYciQIRg2bBgAIDo6Gjk5OQCA9u3bY9KkSZgwYQKSk5PRoEEDLF26FFZWVlKWT0RERFRuDBgw1KD9bWxsHvsehUIOJ6eivYFcXd0NPs/D5s+fg4UL5wHQBx9ra2sMHDgEo0bpp405efIEVqz4EXfu3IZWq4VGo0GnTqFFjuHj4ysuW1lZIT8/DwCQmJiA1q3biK85OTnB3l4/2FhGRgaysrIQEFBVfN3GxgbOzi6Ii4sVt3l6eorLFhYWcHf3KLJeUJD/1D87mSdJw5KXlxd+/PHHEl+7fPlykfVhw4aJQYqIiIiIDGPoM0QKhfKx71Eq5eIz2A9YWFg81bNKD0yc+C769tU/ZnH8+FFMmzYZ3br1gFKpxK1bN/HRR+9j3LiJ6N27LywtrfD55x8VaR0C8MiRjwsK1NBqtUW26XT6Z2fU6kcPtvXw8WQy+SNfo4qpfA10TkRERESVQvPmLdG2bQfMmjUDgiDgypVLsLCwwKBBQ2BpaXV/2+UnH+g+Nzc3JCQkiOtJSUnIysoEADg7u8DGxlbssgfoW5tSU1PgR9nElQAAE6FJREFU6+tntJ+Jyh+GJSIiIiIyS+PHT8a1a1exceNf8Pb2QX5+Pq5evYyMjAwsWbIQKpUFkpKSSjXFTMuWrREevhN37txGTk42li5dDAsLSwCAXC5Hly7dsGbNcty7l3D/GflF8PHxRf36Dcv6xyQzxrBERERERGbJxcUVY8a8hSVLFsLT0wv9+7+AceNGY8SIF+Dl5YPx46fg+vVr+OSTD554rMGDX0Tr1u0wevQoDB06APXrN4SHR+EzR+PGTUTNmrUxevQoDBjwPJKTk7BgwXdQKBRl+SOSmZMJxpjttZxITMyUugQARfv4lqdx5kk6vGbIULxmyFC8ZshQvGbIEOZ4vbi72z9xH7YsERERERERlYBhiYiIiIiIqAQMS0RERERERCVgWCIiIiIiIioBwxIREREREVEJGJaIiIiIiIhKwLBERERERERUAoYlIiIiIiKiEjAsERERERERlYBhiYiIiIiIqAQMS0RERERERCVgWCIiIiIiIioBwxIREREREVEJGJaIiIiIiIhKIBMEQZC6CCIiIiIiInPDliUiIiIiIqISMCwRERERERGVgGGJiIiIiIioBAxLREREREREJWBYIiIiIiIiKgHDEhERERERUQkYloiIiIiIiErAsERERERERFQChiUiIiIiIqISMCwRERERERGVgGGpjMTExGD06NFo0aIFOnXqhDlz5kCn05W476pVq9CtWzc0btwYQ4cOxfnz501cLZkDQ66Z3377Dd26dUOjRo3Qp08fhIeHm7haMgeGXDMPJCQkoFGjRli0aJGJqiRzYcj1cv36dYwYMQLBwcHo0KEDVqxYYdpiySyU9prR6XRYuHAhQkJC0KhRI/Tq1Qvbtm2ToGIyBwcOHEDr1q0xceLEx+6n0+kwf/58hIaGolmzZnj11Vdx584dE1VZegxLZeTtt9+Gp6cnwsPDsXz5coSHh2PlypXF9ouIiMCiRYswe/ZsHD58GJ06dcKYMWOQk5MjQdUkpdJeMzt37sTcuXPxxRdf4Pjx4xg+fDgmTJhglr9gqGyV9pp52IwZM6BQKExUIZmT0l4veXl5eO2119ChQwccPXoUixYtwp9//onr169LUDVJqbTXzG+//YZ169Zh2bJlOHnyJCZNmoR3330Xly5dkqBqktKPP/6IGTNmICAg4In7/vLLL9i8eTOWLl2KPXv2IDAwEG+99RYEQTBBpaXHsFQGIiMjcenSJUyZMgX29vYIDAzEqFGjsHbt2mL7rl27Fv3790dwcDCsrKzw2muvAQD27Nlj6rJJQoZcM3l5eZg0aRKaNGkClUqFQYMGwdbWFmfOnDF94SQZQ66ZB/bt24dr166hY8eOpiuUzIIh18v27dthZ2eH1157DdbW1mjYsCG2bNmC6tWrS1A5ScWQayYqKgpNmjRBtWrVoFAo0KlTJzg5OeHy5csSVE5SsrS0xJ9//lmqsLR27VqMGjUK1atXh52dHSZOnIjr16/j7NmzJqi09BiWykBUVBR8fX3h6OgobgsKCkJ0dDSysrKK7VuvXj1xXS6Xo27duoiMjDRZvSQ9Q66ZPn36YNiwYeJ6RkYGsrOz4enpabJ6SXqGXDOAPmT/73//wyeffAKlUmnKUskMGHK9nDp1CrVq1cK0adPQtGlThIWFYdOmTaYumSRmyDXTsWNHHD9+HBcvXkRBQQF2796N3NxcNG/e3NRlk8RGjhwJe3v7J+6Xl5eHa9euFbkHtrOzQ0BAgNndAzMslYG0tDQ4ODgU2fbgl01qamqxfR/+RfRg3//uRxWbIdfMwwRBwPTp0xEcHMx/lCoZQ6+ZxYsX47nnnkPLli1NUh+ZF0Oul/j4eOzevRutW7fGgQMH8MYbb+D999/HhQsXTFYvSc+Qa6Zr164YPHgw+vbtiwYNGmDy5Mn48ssv4e3tbbJ6qXxJT0+HIAjl4h6YXy+WEUP6W5pb30yShqHXgVqtxtSpU3Ht2jWsWrWqjKoic1baa+batWtYt24dNm/eXMYVkTkr7fUiCAKCgoLQq1cvAEC/fv3w+++/Y8eOHUW+BaaKr7TXzIYNG7BhwwasW7cOtWvXxpEjRzB58mR4e3ujYcOGZVwllWfl4R6YLUtlwMXFBWlpaUW2paWlQSaTwcXFpch2Z2fnEvf9735UsRlyzQD65us33ngDsbGx+OWXX+Dm5maiSslclPaaEQQBn376Kd5++224u7ubuEoyF4b8jnF3dy/WjcbX1xeJiYllXSaZEUOumTVr1mDw4MFo2LAhLC0t0bFjR7Rs2ZLdN+mRnJycIJfLS7zGXF1dpSnqERiWykD9+vURFxeHlJQUcVtkZCRq1KgBW1vbYvtGRUWJ61qtFhcuXEBwcLDJ6iXpGXLNCIKAiRMnQqlUYsWKFXB2djZ1uWQGSnvNxMbG4sSJE1i4cCFatGiBFi1aYOvWrVi2bBn69esnRekkAUN+x1SvXh1Xrlwp8o1vTEwMfH19TVYvSc+Qa0an00Gr1RbZVlBQYJI6qXyytLREzZo1i9wDZ2Rk4Pbt22bXGsmwVAbq1auHBg0aYO7cucjKysL169exfPlyDB06FAAQFhaGkydPAgCGDh2KDRs24MyZM8jNzcWSJUtgYWHB0aoqGUOumc2bN+PatWv45ptvYGlpKWXZJKHSXjNeXl7Yt28fNm7cKP4JCQnBkCFDsHTpUol/CjIVQ37H9O7dG6mpqfj++++Rl5eHLVu2ICoqCr1795byRyATM+SaCQkJwZ9//olLly5Bo9Hg4MGDOHLkCEJDQ6X8EcjMJCQkICwsTJzqZOjQoVi1ahWuX7+OrKwsfP3116hbty4aNGggcaVF8ZmlMrJw4UJ89NFHaNOmDezs7DBkyBBxBLPo6GhxHqX27dtj0qRJmDBhApKTk9GgQQMsXboUVlZWUpZPEijtNbN+/XrExMQUG9ChT58+mDFjhsnrJumU5ppRKBTw8vIq8j5ra2vY2dmxW14lU9rfMZ6envjhhx8wc+ZMfPfdd/Dx8cHixYtRpUoVKcsnCZT2mnnjjTeg0Wjw1ltvISUlBb6+vpgxYwZatWolZfkkgQdBR6PRAADCw8MB6Fsl1Wo1oqOjxVbHIUOGIDExESNGjEB2djZatGiBb7/9VprCH0MmlIcnq4iIiIiIiEyM3fCIiIiIiIhKwLBERERERERUAoYlIiIiIiKiEjAsERERERERlYBhiYiIiIiIqAQMS0RERERERCVgWCIiIiIiIioBwxIREREREVEJGJaIiMhgx44dQ+3atXH9+nWpS3km06dPx/Dhwx/5eoMGDbBu3ToTVkREROZEKXUBRERkWiNGjMDJkyehVBb/JyAkJATffPONBFUV9ddff2HatGmwsLAosr1mzZp49dVX0bNnT6OcZ8aMGUXWN27ciOeeew4BAQEAgMjISKOc51Fq164NpVIJubzwu0sbGxvUrFkTb7zxBtq1a1fqY8XHx+PAgQMYNGhQWZRKRFQpMSwREVVCYWFhmD9/vtRlPNHJkydhaWkJAMjNzcXmzZsxZcoU6HQ69OrVy6jnEgQBX375JWbPni2GJVOYPn06hg4dKq5nZmZi1apVGDNmDH799VcEBweX6ji7du3C5s2bGZaIiIyI3fCIiKiYnJwcfPrpp2jVqhUaNmyIzp07Y8WKFY/c/8iRI3jhhRfQpEkTNG3aFC+//DKuXbsmvn7ixAmMGDECzZs3R5MmTfDmm2/izp07BtVkbW2NF154Ac2bN8fu3bvF7bt27UL//v3RuHFjtGjRAlOmTEFKSor4+i+//IJu3brhueeeQ6tWrTBt2jRkZWUBAKZOnYoXXngBmZmZaNCgAVJTUzF27Fixa17t2rXx22+/4Y8//kCDBg3E9z0wefJkDBs2DACQnp6ODz/8EB07dkRwcDB69eqFrVu3GvQzAoC9vT3eeusteHl5YfPmzeL2/fv3Y9CgQQgODkbz5s3x+uuv49atWwCAWbNm4YsvvsC5c+fQoEEDHDp0SPxsBg0aJH427777bpHPhoiIHo9hiYiIipk7dy4OHjyIv//+G2fPnsX06dPx5Zdf4sCBA8X2VavVeOutt9C/f38cP34ce/fuRdWqVTF9+nQAwPXr1/Hqq68iNDQUBw/+v737jan57+M4/jwrLakdGmomCuuGEcMUSoSMmo60UW12GmPIZg3ZiLGQZNwRy/zbsjJjNUxTjKUQh/ybMBERDSXNnzqnc90w53KuDj9n143r2rwet9r3+/l8vu/v996rz59zhYqKCnx8fEhLS6Ojo8Pt2mw2m2MJYU1NDStXrmThwoVcu3aNkydPUl9fz6pVqwCora0lJyeH3bt3U1tbS0lJCc+ePaOgoMBpTD8/P8rKygDIz8+nsLDQ6f6sWbMwGAxUVFQ4rn3+/JmLFy8yd+5cAFasWEFTUxPFxcXcvHmTZcuWsXbtWqqrq91+R7vdjs1mc8yqNTc3s3z5cmJjY7l16xbl5eVYrVbWrFkDQGZmJgkJCYSFhXHv3j0mTZrE1atXycjIwGw2U1NTQ2lpKc3NzaSnp7tdj4jI30phSUREusnMzOTUqVMEBgZiMBiYMmUK/fr1o7a2tlvbjo4Ovn79ire3Nx4eHvj6+pKVlUVxcTEAx48fZ9iwYZjNZry8vOjTpw/r16/n5cuXWCyWP66pvb2doqIiLBYLiYmJABQWFjJhwgRMJhNeXl4MHDiQ5cuXc/36dV6/fk1bWxvwfVYKICAggKKiIjIyMtz6Hn5+fsTExHDu3DnHtYqKCux2O7NmzaKuro4bN26QmZlJYGAgPXr0YPbs2URGRlJSUuLWsz58+EBubi7v3r0jISEBgP79+3PlyhXS0tLw8PDAaDQyc+ZM7t+/j9VqdTlOYWEhU6ZMIS4uDk9PTwIDA1m9ejUWi8XtWT0Rkb+V9iyJiPyFysrKnGZJfli6dCnp6em8ffuWnTt3cvPmTT59+gR8D0Xfvn3r1qdXr15kZGSQlZXF/v37mTBhAjNmzGDixIkA1NfX8/DhQ0aOHOnUz9PTk8bGxt/WOW7cOMffPj4+DB06lPz8fMfYDQ0NREREOPUZNmwYAC9evCAiIoLp06cTFxdHWFgYERERxMfHO9q4w2QykZ6ezsePHzEajZw+fZrY2Fh8fX2pr68HICkpyamP3W5n9OjRvx03Ozubbdu2Odp3dnYSGRnJkSNHCA0NdbQ7e/YsxcXFNDY2YrVa6erqwmazOc20/ay+vp6GhoZu393Dw4PGxkaCgoLc/gYiIn8bhSURkb/Q7w546OrqYvHixfTt25eioiIGDRqEwWAgOjr6l+MtXryYpKQkqqqqqKysZMWKFcTExLBr1y68vb2JiorqtvTtT/x8wIMrrsJbV1cXAAaDAS8vL3bv3s3Lly+prKzk0qVLFBQUsHHjRhYsWOBWLZGRkRiNRsrLy4mJiaG6upqDBw8COGq8fPky/v7+bo378wEPHz9+JD4+ntDQUKegWFpaSnZ2NtnZ2cyePZuePXty4sQJx1JHV7y9vZk/fz6bNm1yqx4REfk3LcMTEREn79+/5/nz56SmpjJ48GAMBgNNTU28ffv2l30+fPhA7969iYuLIycnh/z8fM6cOUNrayshISHU1dVhs9kc7W022z/OKv2J4OBgHj165HTtyZMnjntWq5W2tjaCgoJISUmhoKCAJUuWcOzYMbef5enpSVxcHGVlZZw/f56AgADCw8MBCAkJAeD+/ftOfV69euX03v/EaDSydetWjhw54rTXyWKxEBISwrx58xxLCu/cufPbsUJCQnjw4IHTtS9fvtDc3PzH9YiI/O0UlkRExIm/vz9+fn7cunULq9XKo0eP2Lx5M0FBQTQ1NXVrb7FYHIc32Gw2Ojo6qK2tpW/fvhiNRpKTk2ltbSU3N5e2tjba29vJy8sjKSmp2+ly7kpOTubatWuUlJTQ2dlJQ0MDe/fuZerUqQQEBHDgwAFSUlIcP57b2trK48ePGTJkSLexfHx8gO/L137sdfpPJpOJ69evU1payty5czEYDAAMGTKE6OhocnNzefr0KTabjaqqKubMmeO0z+lPTJ48maSkJDIzM2lpaQFg8ODBvHnzhoaGBtrb2yksLHS80+vXr4Hv+7Kam5tpaWnhy5cvmM1m7t69y6FDh/j8+TMtLS1s2LABs9nsmH0TEZHfU1gSEREnHh4e5OTkcOnSJcaNG0dWVhbp6emYzWYuXLjgOIHth7Fjx7Ju3Tq2bt3KmDFjiIqKoqamhv3792MwGBgwYAAFBQXcuXOHqKgooqKiePz4MUePHsXX1/e/qjU6Oprt27dz+PBhxo8fT1paGuHh4ezatQuARYsWMXHiRMxmM2FhYcTHx+Pr6+tyaZq/vz9z5swhLy+P1NRUl88bPnw4wcHB3L59G5PJ5HRvx44djBgxgpSUFEaNGsWWLVtYu3Yt8fHxbr/XunXr8PLyciyzS05OJjIyEpPJRGxsLG/evGHfvn2EhoaSmJhIXV0dCQkJWK1WoqOjqaioICwsjD179lBaWkp4eDjTpk2js7OTAwcOOP0IroiI/JrBbrfb/9dFiIiIiIiI/L/Rv5ZERERERERcUFgSERERERFxQWFJRERERETEBYUlERERERERFxSWREREREREXFBYEhERERERcUFhSURERERExAWFJRERERERERcUlkRERERERFxQWBIREREREXFBYUlERERERMSFfwE8d43vndg8GwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "### ASSESS MODEL PERFORMANCE ###\n", "\n", @@ -1002,9 +1814,9 @@ "\n", "# Plot Information\n", "plt.figure(figsize=(10, 7))\n", - "plt.plot(fpr_train, tpr_train, label=f\"Train AUROC={y_train_auroc:.2f}\")\n", - "plt.plot(fpr_test, tpr_test, label=f\"Test AUROC={y_test_auroc:.2f}\")\n", - "plt.plot(fpr_all_data, tpr_all_data, label=f\"All Data AUROC={all_data_auroc:.2f}\")\n", + "plt.plot(fpr_train, tpr_train, label=f\"Train AUROC={y_train_auroc:.3f}\")\n", + "plt.plot(fpr_test, tpr_test, label=f\"Test AUROC={y_test_auroc:.3f}\")\n", + "plt.plot(fpr_all_data, tpr_all_data, label=f\"All Data AUROC={all_data_auroc:.3f}\")\n", "plt.plot([0, 1], [0, 1], linestyle=\"--\", color=\"gray\", label=\"Random\")\n", "plt.xlabel(\"False Positive Rate\")\n", "plt.ylabel(\"True Positive Rate\")\n", @@ -1049,7 +1861,9 @@ "\n", "# Create a list of tuples (feature, importance) and sort it by importance in descending order\n", "sorted_importances = sorted(\n", - " zip(selected_features, feature_importances), key=lambda x: x[1], reverse=True\n", + " zip(selected_features, feature_importances),\n", + " key=lambda x: x[1],\n", + " reverse=True,\n", ")\n", "\n", "# Display the top 10 most important features\n", @@ -1067,10 +1881,11 @@ "outputs": [], "source": [ "# Save current predictions, labels, and probabilities to disk\n", - "np.save(f\"{ROOT}/xgboost_y_test_pred_two_weeks.npy\", y_test_pred)\n", - "np.save(f\"{ROOT}/xgboost_y_test_pred_two_weeks_labels.npy\", y_test)\n", + "np.save(f\"{DATA_ROOT}/xgboost_y_test_pred_{TASK}.npy\", y_test_pred)\n", + "np.save(f\"{DATA_ROOT}/xgboost_y_test_pred_{TASK}_labels.npy\", y_test)\n", "np.save(\n", - " f\"{ROOT}/xgboost_y_test_pred_two_weeks_prob.npy\", xgb_model.predict_proba(X_test)\n", + " f\"{DATA_ROOT}/xgboost_y_test_pred_{TASK}_prob.npy\",\n", + " xgb_model.predict_proba(X_test),\n", ")" ] }, @@ -1109,7 +1924,9 @@ "\n", " # Create a new XGBoost model for each fold\n", " xgb_model = xgb.XGBClassifier(\n", - " objective=\"binary:logistic\", random_state=23, scale_pos_weight=scale_pos_weight\n", + " objective=\"binary:logistic\",\n", + " random_state=23,\n", + " scale_pos_weight=scale_pos_weight,\n", " )\n", "\n", " # Train the model on the training fold\n", @@ -1172,6 +1989,18 @@ "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" } }, "nbformat": 4, diff --git a/odyssey/models/cehr_big_bird/model.py b/odyssey/models/cehr_big_bird/model.py index 2c5cf67..55fbb3a 100644 --- a/odyssey/models/cehr_big_bird/model.py +++ b/odyssey/models/cehr_big_bird/model.py @@ -2,6 +2,7 @@ from typing import Any, Dict, Optional, Tuple, Union +import numpy as np import pytorch_lightning as pl import torch from sklearn.metrics import ( @@ -12,6 +13,7 @@ roc_auc_score, ) from torch import nn, optim +from torch.cuda.amp import autocast from torch.optim import AdamW from torch.optim.lr_scheduler import LinearLR, SequentialLR from transformers import BigBirdConfig @@ -156,12 +158,14 @@ def training_step(self, batch: Dict[str, Any], batch_idx: int) -> Any: # This is not necessary but makes sure we use the right attention self.model.bert.set_attention_type("block_sparse") - loss = self( - inputs, - attention_mask=attention_mask, - labels=labels, - return_dict=True, - ).loss + # Ensure use of mixed precision + with autocast(): + loss = self( + inputs, + attention_mask=attention_mask, + labels=labels, + return_dict=True, + ).loss (current_lr,) = self.lr_schedulers().get_last_lr() self.log_dict( @@ -204,12 +208,16 @@ def validation_step(self, batch: Dict[str, Any], batch_idx: int) -> Any: def configure_optimizers( self, - ) -> Tuple[list[AdamW], list[dict[str, SequentialLR | str]]]: + ) -> Tuple[list[Any], list[dict[str, SequentialLR | str]]]: """Configure optimizers and learning rate scheduler.""" optimizer = optim.AdamW( self.parameters(), lr=self.learning_rate, ) + # Change optimizer if DeepSpeed strategy is used + # optimizer = DeepSpeedCPUAdam( + # self.parameters(), lr=self.learning_rate, adamw_mode=True + # ) n_steps = self.trainer.estimated_stepping_batches n_warmup_steps = int(0.1 * n_steps) @@ -335,12 +343,13 @@ def training_step(self, batch: Dict[str, Any], batch_idx: int) -> Any: # This is not necessary but makes sure we use the right attention self.model.bert.set_attention_type("block_sparse") + # Ensure use of mixed precision loss = self( inputs, attention_mask=attention_mask, labels=labels, return_dict=True, - )[0] + ).loss (current_lr,) = self.lr_schedulers().get_last_lr() self.log_dict( @@ -365,12 +374,13 @@ def validation_step(self, batch: Dict[str, Any], batch_idx: int) -> Any: # This is not necessary but makes sure we use the right attention self.model.bert.set_attention_type("block_sparse") + # Ensure use of mixed precision loss = self( inputs, attention_mask=attention_mask, labels=labels, return_dict=True, - )[0] + ).loss (current_lr,) = self.lr_schedulers().get_last_lr() self.log_dict( @@ -406,7 +416,7 @@ def test_step(self, batch: Dict[str, Any], batch_idx: int) -> Any: loss = outputs[0] logits = outputs[1] preds = torch.argmax(logits, dim=1) - log = {"loss": loss, "preds": preds, "labels": labels} + log = {"loss": loss, "preds": preds, "labels": labels, "logits": logits} # Append the outputs to the instance attribute self.test_outputs.append(log) @@ -418,12 +428,37 @@ def on_test_epoch_end(self) -> Any: labels = torch.cat([x["labels"] for x in self.test_outputs]).cpu() preds = torch.cat([x["preds"] for x in self.test_outputs]).cpu() loss = torch.stack([x["loss"] for x in self.test_outputs]).mean().cpu() + logits = torch.cat([x["logits"] for x in self.test_outputs]).cpu() + preds_one_hot = np.eye(labels.shape[1])[preds] + + # Update the saved outputs to include all concatanted batches + self.test_outputs = { + "loss": loss, + "preds": preds, + "labels": labels, + "logits": logits, + } + + if self.problem_type == "multi_label_classification": + accuracy = accuracy_score(labels, preds_one_hot) + f1 = f1_score(labels, preds_one_hot, average="micro") + auc = roc_auc_score(labels, preds_one_hot, average="micro") + precision = precision_score(labels, preds_one_hot, average="micro") + recall = recall_score(labels, preds_one_hot, average="micro") + + else: # single_label_classification + accuracy = accuracy_score(labels, preds) + f1 = f1_score(labels, preds) + auc = roc_auc_score(labels, preds) + precision = precision_score(labels, preds) + recall = recall_score(labels, preds) self.log("test_loss", loss) - self.log("test_acc", accuracy_score(labels, preds)) - self.log("test_f1", f1_score(labels, preds)) - self.log("test_auc", roc_auc_score(labels, preds)) - self.log("test_precision", precision_score(labels, preds)) - self.log("test_recall", recall_score(labels, preds)) + self.log("test_acc", accuracy) + self.log("test_f1", f1) + self.log("test_auc", auc) + self.log("test_precision", precision) + self.log("test_recall", recall) + return loss def configure_optimizers( diff --git a/odyssey/models/cehr_big_bird/playground.ipynb b/odyssey/models/cehr_big_bird/playground.ipynb index 5ebf350..1c26964 100644 --- a/odyssey/models/cehr_big_bird/playground.ipynb +++ b/odyssey/models/cehr_big_bird/playground.ipynb @@ -15,18 +15,19 @@ "source": [ "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", "\n", - "from typing import Any, Tuple\n", + "import os\n", + "import pickle\n", + "from typing import Tuple\n", "\n", "import torch\n", - "import pickle\n", - "import os\n", + "\n", "\n", "os.chdir(ROOT)\n", "import pandas as pd\n", "from torch.utils.data import DataLoader\n", "\n", - "from models.big_bird_cehr.model import BigBirdPretrain, BigBirdFinetune\n", - "from models.big_bird_cehr.data import PretrainDataset, FinetuneDataset\n", + "from models.big_bird_cehr.data import FinetuneDataset, PretrainDataset\n", + "from models.big_bird_cehr.model import BigBirdFinetune, BigBirdPretrain\n", "from models.big_bird_cehr.tokenizer import HuggingFaceConceptTokenizer\n", "\n", "\n", @@ -61,13 +62,13 @@ ], "source": [ "data = pd.read_parquet(\n", - " \"/h/afallah/odyssey/odyssey/data/bigbird_data/patient_sequences_2048_labeled.parquet\"\n", + " \"/h/afallah/odyssey/odyssey/data/bigbird_data/patient_sequences_2048_labeled.parquet\",\n", ")\n", "patient_ids = pickle.load(\n", " open(\n", " \"/h/afallah/odyssey/odyssey/data/bigbird_data/dataset_2048_mortality_1month.pkl\",\n", " \"rb\",\n", - " )\n", + " ),\n", ")\n", "\n", "fine_tune = data.loc[\n", @@ -93,7 +94,7 @@ "source": [ "# Train Tokenizer\n", "tokenizer = HuggingFaceConceptTokenizer(\n", - " data_dir=\"/h/afallah/odyssey/odyssey/data/vocab\"\n", + " data_dir=\"/h/afallah/odyssey/odyssey/data/vocab\",\n", ")\n", "tokenizer.fit_on_vocab()" ] @@ -183,8 +184,8 @@ ")\n", "finetuned_model.load_state_dict(\n", " torch.load(\n", - " \"/h/afallah/odyssey/odyssey/checkpoints/bigbird_finetune/mortality_1month_20000_patients/best-v1.ckpt\"\n", - " )[\"state_dict\"]\n", + " \"/h/afallah/odyssey/odyssey/checkpoints/bigbird_finetune/mortality_1month_20000_patients/best-v1.ckpt\",\n", + " )[\"state_dict\"],\n", ")" ] }, @@ -241,13 +242,13 @@ "outputs": [], "source": [ "data = pd.read_parquet(\n", - " \"/h/afallah/odyssey/odyssey/data/bigbird_data/patient_sequences_2048_labeled.parquet\"\n", + " \"/h/afallah/odyssey/odyssey/data/bigbird_data/patient_sequences_2048_labeled.parquet\",\n", ")\n", "patient_ids = pickle.load(\n", " open(\n", " \"/h/afallah/odyssey/odyssey/data/bigbird_data/dataset_2048_mortality_1month.pkl\",\n", " \"rb\",\n", - " )\n", + " ),\n", ")\n", "pre_data = data.loc[data[\"patient_id\"].isin(patient_ids[\"test\"])]\n", "pre_data.rename(columns={\"label_mortality_1month\": \"label\"}, inplace=True)" @@ -348,8 +349,8 @@ ")\n", "pretrained_model.load_state_dict(\n", " torch.load(\n", - " \"/h/afallah/odyssey/odyssey/checkpoints/bigbird_pretraining_a100/best.ckpt\"\n", - " )[\"state_dict\"]\n", + " \"/h/afallah/odyssey/odyssey/checkpoints/bigbird_pretraining_a100/best.ckpt\",\n", + " )[\"state_dict\"],\n", ")" ] }, @@ -450,7 +451,9 @@ ], "source": [ "pretrained_model(\n", - " inputs=tuple(batch.values()), attention_mask=at, labels=None\n", + " inputs=tuple(batch.values()),\n", + " attention_mask=at,\n", + " labels=None,\n", ").logits.shape" ] }, diff --git a/odyssey/models/cehr_big_bird/tokenizer.ipynb b/odyssey/models/cehr_big_bird/tokenizer.ipynb index 5913eb0..c611b76 100644 --- a/odyssey/models/cehr_big_bird/tokenizer.ipynb +++ b/odyssey/models/cehr_big_bird/tokenizer.ipynb @@ -30,6 +30,7 @@ "source": [ "import os\n", "\n", + "\n", "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", "os.chdir(ROOT)\n", "# from models.cehr_bert.tokenizer import ConceptTokenizer\n", @@ -485,8 +486,10 @@ "# Create the tokenizer object\n", "tokenizer_object = Tokenizer(\n", " models.WordPiece(\n", - " vocab=tokenizer_vocab, unk_token=\"[UNK]\", max_input_chars_per_word=1000\n", - " )\n", + " vocab=tokenizer_vocab,\n", + " unk_token=\"[UNK]\",\n", + " max_input_chars_per_word=1000,\n", + " ),\n", ")\n", "tokenizer_object.pre_tokenizer = pre_tokenizers.WhitespaceSplit()" ] diff --git a/odyssey/models/configs/cehr_bigbird.yaml b/odyssey/models/configs/cehr_bigbird.yaml index b94ecfc..7e7bad9 100644 --- a/odyssey/models/configs/cehr_bigbird.yaml +++ b/odyssey/models/configs/cehr_bigbird.yaml @@ -14,28 +14,26 @@ model: dropout_prob: 0.1 model_finetune: - num_labels: 20 - problem_type: 'multi_label_classification' learning_rate: 5.e-5 classifier_dropout: 0.1 train: - batch_size: 26 - num_workers: 3 + batch_size: 32 + num_workers: 5 gpus: 4 nodes: 1 - max_epochs: 10 - acc: 8 + max_epochs: 15 + acc: 1 mask_prob: 0.15 persistent_workers: True pin_memory: False finetune: batch_size: 26 - num_workers: 3 - gpus: 2 + num_workers: 6 + gpus: 4 nodes: 1 - max_epochs: 5 + max_epochs: 3 acc: 1 patience: 5 persistent_workers: True diff --git a/odyssey/models/utils.py b/odyssey/models/utils.py index efce177..72bd126 100644 --- a/odyssey/models/utils.py +++ b/odyssey/models/utils.py @@ -2,6 +2,7 @@ import os import pickle +import random import uuid from os.path import join from typing import Any @@ -44,6 +45,7 @@ def seed_everything(seed: int) -> None: Seed value to use """ + random.seed(seed) torch.manual_seed(seed) np.random.seed(seed) torch.cuda.manual_seed_all(seed) @@ -177,3 +179,19 @@ def get_run_id( with open(run_id_path, "w") as file: file.write(run_id) return run_id + + +def save_object_to_disk(obj: Any, save_path: str) -> None: + """Save an object to disk using pickle. + + Parameters + ---------- + obj: Any + Object to save + save_path: str + Path to save the object + + """ + with open(save_path, "wb") as f: + pickle.dump(obj, f) + print(f"File saved to disk: {save_path}") diff --git a/poetry.lock b/poetry.lock index 9522d73..d2f7861 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2,87 +2,87 @@ [[package]] name = "aiohttp" -version = "3.9.3" +version = "3.9.4" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.8" files = [ - {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:939677b61f9d72a4fa2a042a5eee2a99a24001a67c13da113b2e30396567db54"}, - {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f5cd333fcf7590a18334c90f8c9147c837a6ec8a178e88d90a9b96ea03194cc"}, - {file = "aiohttp-3.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:82e6aa28dd46374f72093eda8bcd142f7771ee1eb9d1e223ff0fa7177a96b4a5"}, - {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f56455b0c2c7cc3b0c584815264461d07b177f903a04481dfc33e08a89f0c26b"}, - {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bca77a198bb6e69795ef2f09a5f4c12758487f83f33d63acde5f0d4919815768"}, - {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e083c285857b78ee21a96ba1eb1b5339733c3563f72980728ca2b08b53826ca5"}, - {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab40e6251c3873d86ea9b30a1ac6d7478c09277b32e14745d0d3c6e76e3c7e29"}, - {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df822ee7feaaeffb99c1a9e5e608800bd8eda6e5f18f5cfb0dc7eeb2eaa6bbec"}, - {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:acef0899fea7492145d2bbaaaec7b345c87753168589cc7faf0afec9afe9b747"}, - {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cd73265a9e5ea618014802ab01babf1940cecb90c9762d8b9e7d2cc1e1969ec6"}, - {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a78ed8a53a1221393d9637c01870248a6f4ea5b214a59a92a36f18151739452c"}, - {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6b0e029353361f1746bac2e4cc19b32f972ec03f0f943b390c4ab3371840aabf"}, - {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7cf5c9458e1e90e3c390c2639f1017a0379a99a94fdfad3a1fd966a2874bba52"}, - {file = "aiohttp-3.9.3-cp310-cp310-win32.whl", hash = "sha256:3e59c23c52765951b69ec45ddbbc9403a8761ee6f57253250c6e1536cacc758b"}, - {file = "aiohttp-3.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:055ce4f74b82551678291473f66dc9fb9048a50d8324278751926ff0ae7715e5"}, - {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6b88f9386ff1ad91ace19d2a1c0225896e28815ee09fc6a8932fded8cda97c3d"}, - {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c46956ed82961e31557b6857a5ca153c67e5476972e5f7190015018760938da2"}, - {file = "aiohttp-3.9.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:07b837ef0d2f252f96009e9b8435ec1fef68ef8b1461933253d318748ec1acdc"}, - {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad46e6f620574b3b4801c68255492e0159d1712271cc99d8bdf35f2043ec266"}, - {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ed3e046ea7b14938112ccd53d91c1539af3e6679b222f9469981e3dac7ba1ce"}, - {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:039df344b45ae0b34ac885ab5b53940b174530d4dd8a14ed8b0e2155b9dddccb"}, - {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7943c414d3a8d9235f5f15c22ace69787c140c80b718dcd57caaade95f7cd93b"}, - {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84871a243359bb42c12728f04d181a389718710129b36b6aad0fc4655a7647d4"}, - {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5eafe2c065df5401ba06821b9a054d9cb2848867f3c59801b5d07a0be3a380ae"}, - {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9d3c9b50f19704552f23b4eaea1fc082fdd82c63429a6506446cbd8737823da3"}, - {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:f033d80bc6283092613882dfe40419c6a6a1527e04fc69350e87a9df02bbc283"}, - {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:2c895a656dd7e061b2fd6bb77d971cc38f2afc277229ce7dd3552de8313a483e"}, - {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1f5a71d25cd8106eab05f8704cd9167b6e5187bcdf8f090a66c6d88b634802b4"}, - {file = "aiohttp-3.9.3-cp311-cp311-win32.whl", hash = "sha256:50fca156d718f8ced687a373f9e140c1bb765ca16e3d6f4fe116e3df7c05b2c5"}, - {file = "aiohttp-3.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:5fe9ce6c09668063b8447f85d43b8d1c4e5d3d7e92c63173e6180b2ac5d46dd8"}, - {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:38a19bc3b686ad55804ae931012f78f7a534cce165d089a2059f658f6c91fa60"}, - {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:770d015888c2a598b377bd2f663adfd947d78c0124cfe7b959e1ef39f5b13869"}, - {file = "aiohttp-3.9.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee43080e75fc92bf36219926c8e6de497f9b247301bbf88c5c7593d931426679"}, - {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52df73f14ed99cee84865b95a3d9e044f226320a87af208f068ecc33e0c35b96"}, - {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc9b311743a78043b26ffaeeb9715dc360335e5517832f5a8e339f8a43581e4d"}, - {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b955ed993491f1a5da7f92e98d5dad3c1e14dc175f74517c4e610b1f2456fb11"}, - {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:504b6981675ace64c28bf4a05a508af5cde526e36492c98916127f5a02354d53"}, - {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6fe5571784af92b6bc2fda8d1925cccdf24642d49546d3144948a6a1ed58ca5"}, - {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ba39e9c8627edc56544c8628cc180d88605df3892beeb2b94c9bc857774848ca"}, - {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e5e46b578c0e9db71d04c4b506a2121c0cb371dd89af17a0586ff6769d4c58c1"}, - {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:938a9653e1e0c592053f815f7028e41a3062e902095e5a7dc84617c87267ebd5"}, - {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:c3452ea726c76e92f3b9fae4b34a151981a9ec0a4847a627c43d71a15ac32aa6"}, - {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ff30218887e62209942f91ac1be902cc80cddb86bf00fbc6783b7a43b2bea26f"}, - {file = "aiohttp-3.9.3-cp312-cp312-win32.whl", hash = "sha256:38f307b41e0bea3294a9a2a87833191e4bcf89bb0365e83a8be3a58b31fb7f38"}, - {file = "aiohttp-3.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:b791a3143681a520c0a17e26ae7465f1b6f99461a28019d1a2f425236e6eedb5"}, - {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0ed621426d961df79aa3b963ac7af0d40392956ffa9be022024cd16297b30c8c"}, - {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7f46acd6a194287b7e41e87957bfe2ad1ad88318d447caf5b090012f2c5bb528"}, - {file = "aiohttp-3.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feeb18a801aacb098220e2c3eea59a512362eb408d4afd0c242044c33ad6d542"}, - {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f734e38fd8666f53da904c52a23ce517f1b07722118d750405af7e4123933511"}, - {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b40670ec7e2156d8e57f70aec34a7216407848dfe6c693ef131ddf6e76feb672"}, - {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fdd215b7b7fd4a53994f238d0f46b7ba4ac4c0adb12452beee724ddd0743ae5d"}, - {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:017a21b0df49039c8f46ca0971b3a7fdc1f56741ab1240cb90ca408049766168"}, - {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e99abf0bba688259a496f966211c49a514e65afa9b3073a1fcee08856e04425b"}, - {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:648056db9a9fa565d3fa851880f99f45e3f9a771dd3ff3bb0c048ea83fb28194"}, - {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8aacb477dc26797ee089721536a292a664846489c49d3ef9725f992449eda5a8"}, - {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:522a11c934ea660ff8953eda090dcd2154d367dec1ae3c540aff9f8a5c109ab4"}, - {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5bce0dc147ca85caa5d33debc4f4d65e8e8b5c97c7f9f660f215fa74fc49a321"}, - {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b4af9f25b49a7be47c0972139e59ec0e8285c371049df1a63b6ca81fdd216a2"}, - {file = "aiohttp-3.9.3-cp38-cp38-win32.whl", hash = "sha256:298abd678033b8571995650ccee753d9458dfa0377be4dba91e4491da3f2be63"}, - {file = "aiohttp-3.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:69361bfdca5468c0488d7017b9b1e5ce769d40b46a9f4a2eed26b78619e9396c"}, - {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0fa43c32d1643f518491d9d3a730f85f5bbaedcbd7fbcae27435bb8b7a061b29"}, - {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:835a55b7ca49468aaaac0b217092dfdff370e6c215c9224c52f30daaa735c1c1"}, - {file = "aiohttp-3.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06a9b2c8837d9a94fae16c6223acc14b4dfdff216ab9b7202e07a9a09541168f"}, - {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abf151955990d23f84205286938796c55ff11bbfb4ccfada8c9c83ae6b3c89a3"}, - {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59c26c95975f26e662ca78fdf543d4eeaef70e533a672b4113dd888bd2423caa"}, - {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f95511dd5d0e05fd9728bac4096319f80615aaef4acbecb35a990afebe953b0e"}, - {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:595f105710293e76b9dc09f52e0dd896bd064a79346234b521f6b968ffdd8e58"}, - {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c8b816c2b5af5c8a436df44ca08258fc1a13b449393a91484225fcb7545533"}, - {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f1088fa100bf46e7b398ffd9904f4808a0612e1d966b4aa43baa535d1b6341eb"}, - {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f59dfe57bb1ec82ac0698ebfcdb7bcd0e99c255bd637ff613760d5f33e7c81b3"}, - {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:361a1026c9dd4aba0109e4040e2aecf9884f5cfe1b1b1bd3d09419c205e2e53d"}, - {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:363afe77cfcbe3a36353d8ea133e904b108feea505aa4792dad6585a8192c55a"}, - {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e2c45c208c62e955e8256949eb225bd8b66a4c9b6865729a786f2aa79b72e9d"}, - {file = "aiohttp-3.9.3-cp39-cp39-win32.whl", hash = "sha256:f7217af2e14da0856e082e96ff637f14ae45c10a5714b63c77f26d8884cf1051"}, - {file = "aiohttp-3.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:27468897f628c627230dba07ec65dc8d0db566923c48f29e084ce382119802bc"}, - {file = "aiohttp-3.9.3.tar.gz", hash = "sha256:90842933e5d1ff760fae6caca4b2b3edba53ba8f4b71e95dacf2818a2aca06f7"}, + {file = "aiohttp-3.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:76d32588ef7e4a3f3adff1956a0ba96faabbdee58f2407c122dd45aa6e34f372"}, + {file = "aiohttp-3.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:56181093c10dbc6ceb8a29dfeea1e815e1dfdc020169203d87fd8d37616f73f9"}, + {file = "aiohttp-3.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7a5b676d3c65e88b3aca41816bf72831898fcd73f0cbb2680e9d88e819d1e4d"}, + {file = "aiohttp-3.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1df528a85fb404899d4207a8d9934cfd6be626e30e5d3a5544a83dbae6d8a7e"}, + {file = "aiohttp-3.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f595db1bceabd71c82e92df212dd9525a8a2c6947d39e3c994c4f27d2fe15b11"}, + {file = "aiohttp-3.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c0b09d76e5a4caac3d27752027fbd43dc987b95f3748fad2b924a03fe8632ad"}, + {file = "aiohttp-3.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:689eb4356649ec9535b3686200b231876fb4cab4aca54e3bece71d37f50c1d13"}, + {file = "aiohttp-3.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3666cf4182efdb44d73602379a66f5fdfd5da0db5e4520f0ac0dcca644a3497"}, + {file = "aiohttp-3.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b65b0f8747b013570eea2f75726046fa54fa8e0c5db60f3b98dd5d161052004a"}, + {file = "aiohttp-3.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a1885d2470955f70dfdd33a02e1749613c5a9c5ab855f6db38e0b9389453dce7"}, + {file = "aiohttp-3.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0593822dcdb9483d41f12041ff7c90d4d1033ec0e880bcfaf102919b715f47f1"}, + {file = "aiohttp-3.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:47f6eb74e1ecb5e19a78f4a4228aa24df7fbab3b62d4a625d3f41194a08bd54f"}, + {file = "aiohttp-3.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c8b04a3dbd54de6ccb7604242fe3ad67f2f3ca558f2d33fe19d4b08d90701a89"}, + {file = "aiohttp-3.9.4-cp310-cp310-win32.whl", hash = "sha256:8a78dfb198a328bfb38e4308ca8167028920fb747ddcf086ce706fbdd23b2926"}, + {file = "aiohttp-3.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:e78da6b55275987cbc89141a1d8e75f5070e577c482dd48bd9123a76a96f0bbb"}, + {file = "aiohttp-3.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c111b3c69060d2bafc446917534150fd049e7aedd6cbf21ba526a5a97b4402a5"}, + {file = "aiohttp-3.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:efbdd51872cf170093998c87ccdf3cb5993add3559341a8e5708bcb311934c94"}, + {file = "aiohttp-3.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7bfdb41dc6e85d8535b00d73947548a748e9534e8e4fddd2638109ff3fb081df"}, + {file = "aiohttp-3.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bd9d334412961125e9f68d5b73c1d0ab9ea3f74a58a475e6b119f5293eee7ba"}, + {file = "aiohttp-3.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:35d78076736f4a668d57ade00c65d30a8ce28719d8a42471b2a06ccd1a2e3063"}, + {file = "aiohttp-3.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:824dff4f9f4d0f59d0fa3577932ee9a20e09edec8a2f813e1d6b9f89ced8293f"}, + {file = "aiohttp-3.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52b8b4e06fc15519019e128abedaeb56412b106ab88b3c452188ca47a25c4093"}, + {file = "aiohttp-3.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eae569fb1e7559d4f3919965617bb39f9e753967fae55ce13454bec2d1c54f09"}, + {file = "aiohttp-3.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:69b97aa5792428f321f72aeb2f118e56893371f27e0b7d05750bcad06fc42ca1"}, + {file = "aiohttp-3.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4d79aad0ad4b980663316f26d9a492e8fab2af77c69c0f33780a56843ad2f89e"}, + {file = "aiohttp-3.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:d6577140cd7db19e430661e4b2653680194ea8c22c994bc65b7a19d8ec834403"}, + {file = "aiohttp-3.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:9860d455847cd98eb67897f5957b7cd69fbcb436dd3f06099230f16a66e66f79"}, + {file = "aiohttp-3.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:69ff36d3f8f5652994e08bd22f093e11cfd0444cea310f92e01b45a4e46b624e"}, + {file = "aiohttp-3.9.4-cp311-cp311-win32.whl", hash = "sha256:e27d3b5ed2c2013bce66ad67ee57cbf614288bda8cdf426c8d8fe548316f1b5f"}, + {file = "aiohttp-3.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d6a67e26daa686a6fbdb600a9af8619c80a332556245fa8e86c747d226ab1a1e"}, + {file = "aiohttp-3.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:c5ff8ff44825736a4065d8544b43b43ee4c6dd1530f3a08e6c0578a813b0aa35"}, + {file = "aiohttp-3.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d12a244627eba4e9dc52cbf924edef905ddd6cafc6513849b4876076a6f38b0e"}, + {file = "aiohttp-3.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:dcad56c8d8348e7e468899d2fb3b309b9bc59d94e6db08710555f7436156097f"}, + {file = "aiohttp-3.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f7e69a7fd4b5ce419238388e55abd220336bd32212c673ceabc57ccf3d05b55"}, + {file = "aiohttp-3.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4870cb049f10d7680c239b55428916d84158798eb8f353e74fa2c98980dcc0b"}, + {file = "aiohttp-3.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b2feaf1b7031ede1bc0880cec4b0776fd347259a723d625357bb4b82f62687b"}, + {file = "aiohttp-3.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:939393e8c3f0a5bcd33ef7ace67680c318dc2ae406f15e381c0054dd658397de"}, + {file = "aiohttp-3.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d2334e387b2adcc944680bebcf412743f2caf4eeebd550f67249c1c3696be04"}, + {file = "aiohttp-3.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e0198ea897680e480845ec0ffc5a14e8b694e25b3f104f63676d55bf76a82f1a"}, + {file = "aiohttp-3.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e40d2cd22914d67c84824045861a5bb0fb46586b15dfe4f046c7495bf08306b2"}, + {file = "aiohttp-3.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:aba80e77c227f4234aa34a5ff2b6ff30c5d6a827a91d22ff6b999de9175d71bd"}, + {file = "aiohttp-3.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:fb68dc73bc8ac322d2e392a59a9e396c4f35cb6fdbdd749e139d1d6c985f2527"}, + {file = "aiohttp-3.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f3460a92638dce7e47062cf088d6e7663adb135e936cb117be88d5e6c48c9d53"}, + {file = "aiohttp-3.9.4-cp312-cp312-win32.whl", hash = "sha256:32dc814ddbb254f6170bca198fe307920f6c1308a5492f049f7f63554b88ef36"}, + {file = "aiohttp-3.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:63f41a909d182d2b78fe3abef557fcc14da50c7852f70ae3be60e83ff64edba5"}, + {file = "aiohttp-3.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:c3770365675f6be220032f6609a8fbad994d6dcf3ef7dbcf295c7ee70884c9af"}, + {file = "aiohttp-3.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:305edae1dea368ce09bcb858cf5a63a064f3bff4767dec6fa60a0cc0e805a1d3"}, + {file = "aiohttp-3.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6f121900131d116e4a93b55ab0d12ad72573f967b100e49086e496a9b24523ea"}, + {file = "aiohttp-3.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b71e614c1ae35c3d62a293b19eface83d5e4d194e3eb2fabb10059d33e6e8cbf"}, + {file = "aiohttp-3.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:419f009fa4cfde4d16a7fc070d64f36d70a8d35a90d71aa27670bba2be4fd039"}, + {file = "aiohttp-3.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7b39476ee69cfe64061fd77a73bf692c40021f8547cda617a3466530ef63f947"}, + {file = "aiohttp-3.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b33f34c9c7decdb2ab99c74be6443942b730b56d9c5ee48fb7df2c86492f293c"}, + {file = "aiohttp-3.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c78700130ce2dcebb1a8103202ae795be2fa8c9351d0dd22338fe3dac74847d9"}, + {file = "aiohttp-3.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:268ba22d917655d1259af2d5659072b7dc11b4e1dc2cb9662fdd867d75afc6a4"}, + {file = "aiohttp-3.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:17e7c051f53a0d2ebf33013a9cbf020bb4e098c4bc5bce6f7b0c962108d97eab"}, + {file = "aiohttp-3.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:7be99f4abb008cb38e144f85f515598f4c2c8932bf11b65add0ff59c9c876d99"}, + {file = "aiohttp-3.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:d58a54d6ff08d2547656356eea8572b224e6f9bbc0cf55fa9966bcaac4ddfb10"}, + {file = "aiohttp-3.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7673a76772bda15d0d10d1aa881b7911d0580c980dbd16e59d7ba1422b2d83cd"}, + {file = "aiohttp-3.9.4-cp38-cp38-win32.whl", hash = "sha256:e4370dda04dc8951012f30e1ce7956a0a226ac0714a7b6c389fb2f43f22a250e"}, + {file = "aiohttp-3.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:eb30c4510a691bb87081192a394fb661860e75ca3896c01c6d186febe7c88530"}, + {file = "aiohttp-3.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:84e90494db7df3be5e056f91412f9fa9e611fbe8ce4aaef70647297f5943b276"}, + {file = "aiohttp-3.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7d4845f8501ab28ebfdbeab980a50a273b415cf69e96e4e674d43d86a464df9d"}, + {file = "aiohttp-3.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:69046cd9a2a17245c4ce3c1f1a4ff8c70c7701ef222fce3d1d8435f09042bba1"}, + {file = "aiohttp-3.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b73a06bafc8dcc508420db43b4dd5850e41e69de99009d0351c4f3007960019"}, + {file = "aiohttp-3.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:418bb0038dfafeac923823c2e63226179976c76f981a2aaad0ad5d51f2229bca"}, + {file = "aiohttp-3.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71a8f241456b6c2668374d5d28398f8e8cdae4cce568aaea54e0f39359cd928d"}, + {file = "aiohttp-3.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:935c369bf8acc2dc26f6eeb5222768aa7c62917c3554f7215f2ead7386b33748"}, + {file = "aiohttp-3.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:74e4e48c8752d14ecfb36d2ebb3d76d614320570e14de0a3aa7a726ff150a03c"}, + {file = "aiohttp-3.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:916b0417aeddf2c8c61291238ce25286f391a6acb6f28005dd9ce282bd6311b6"}, + {file = "aiohttp-3.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9b6787b6d0b3518b2ee4cbeadd24a507756ee703adbac1ab6dc7c4434b8c572a"}, + {file = "aiohttp-3.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:221204dbda5ef350e8db6287937621cf75e85778b296c9c52260b522231940ed"}, + {file = "aiohttp-3.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:10afd99b8251022ddf81eaed1d90f5a988e349ee7d779eb429fb07b670751e8c"}, + {file = "aiohttp-3.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2506d9f7a9b91033201be9ffe7d89c6a54150b0578803cce5cb84a943d075bc3"}, + {file = "aiohttp-3.9.4-cp39-cp39-win32.whl", hash = "sha256:e571fdd9efd65e86c6af2f332e0e95dad259bfe6beb5d15b3c3eca3a6eb5d87b"}, + {file = "aiohttp-3.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:7d29dd5319d20aa3b7749719ac9685fbd926f71ac8c77b2477272725f882072d"}, + {file = "aiohttp-3.9.4.tar.gz", hash = "sha256:6ff71ede6d9a5a58cfb7b6fffc83ab5d4a63138276c771ac91ceaaddf5459644"}, ] [package.dependencies] @@ -588,13 +588,13 @@ test = ["coverage", "pytest (>5.4.0)", "pytest-cov (>=2.10.0)", "pytest-runner"] [[package]] name = "filelock" -version = "3.13.3" +version = "3.13.4" description = "A platform independent file lock." optional = false python-versions = ">=3.8" files = [ - {file = "filelock-3.13.3-py3-none-any.whl", hash = "sha256:5ffa845303983e7a0b7ae17636509bc97997d58afeafa72fb141a17b152284cb"}, - {file = "filelock-3.13.3.tar.gz", hash = "sha256:a79895a25bbefdf55d1a2a0a80968f7dbb28edcd6d4234a0afb3f37ecde4b546"}, + {file = "filelock-3.13.4-py3-none-any.whl", hash = "sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f"}, + {file = "filelock-3.13.4.tar.gz", hash = "sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4"}, ] [package.extras] @@ -758,20 +758,21 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.42" +version = "3.1.43" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" files = [ - {file = "GitPython-3.1.42-py3-none-any.whl", hash = "sha256:1bf9cd7c9e7255f77778ea54359e54ac22a72a5b51288c457c881057b7bb9ecd"}, - {file = "GitPython-3.1.42.tar.gz", hash = "sha256:2d99869e0fef71a73cbd242528105af1d6c1b108c60dfabd994bf292f76c3ceb"}, + {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, + {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, ] [package.dependencies] gitdb = ">=4.0.1,<5" [package.extras] -test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar"] +doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] +test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] [[package]] name = "greenlet" @@ -894,13 +895,13 @@ license = ["ukkonen"] [[package]] name = "idna" -version = "3.6" +version = "3.7" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, - {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] [[package]] @@ -1000,13 +1001,13 @@ testing = ["Django", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] [[package]] name = "joblib" -version = "1.3.2" +version = "1.4.0" description = "Lightweight pipelining with Python functions" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "joblib-1.3.2-py3-none-any.whl", hash = "sha256:ef4331c65f239985f3f2220ecc87db222f08fd22097a3dd5698f693875f8cbb9"}, - {file = "joblib-1.3.2.tar.gz", hash = "sha256:92f865e621e17784e7955080b6d042489e3b8e294949cc44c6eac304f59772b1"}, + {file = "joblib-1.4.0-py3-none-any.whl", hash = "sha256:42942470d4062537be4d54c83511186da1fc14ba354961a2114da91efa9a4ed7"}, + {file = "joblib-1.4.0.tar.gz", hash = "sha256:1eb0dc091919cd384490de890cb5dfd538410a6d4b3b54eef09fb8c50b409b1c"}, ] [[package]] @@ -1210,13 +1211,13 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] [[package]] name = "matplotlib-inline" -version = "0.1.6" +version = "0.1.7" description = "Inline Matplotlib backend for Jupyter" optional = false -python-versions = ">=3.5" +python-versions = ">=3.8" files = [ - {file = "matplotlib-inline-0.1.6.tar.gz", hash = "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304"}, - {file = "matplotlib_inline-0.1.6-py3-none-any.whl", hash = "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311"}, + {file = "matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca"}, + {file = "matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90"}, ] [package.dependencies] @@ -1422,19 +1423,19 @@ files = [ [[package]] name = "nbformat" -version = "5.10.3" +version = "5.10.4" description = "The Jupyter Notebook format" optional = false python-versions = ">=3.8" files = [ - {file = "nbformat-5.10.3-py3-none-any.whl", hash = "sha256:d9476ca28676799af85385f409b49d95e199951477a159a576ef2a675151e5e8"}, - {file = "nbformat-5.10.3.tar.gz", hash = "sha256:60ed5e910ef7c6264b87d644f276b1b49e24011930deef54605188ddeb211685"}, + {file = "nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b"}, + {file = "nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a"}, ] [package.dependencies] -fastjsonschema = "*" +fastjsonschema = ">=2.15" jsonschema = ">=2.6" -jupyter-core = "*" +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" traitlets = ">=5.1" [package.extras] @@ -1617,46 +1618,46 @@ files = [ [[package]] name = "pandas" -version = "2.2.1" +version = "2.2.2" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.9" files = [ - {file = "pandas-2.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8df8612be9cd1c7797c93e1c5df861b2ddda0b48b08f2c3eaa0702cf88fb5f88"}, - {file = "pandas-2.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0f573ab277252ed9aaf38240f3b54cfc90fff8e5cab70411ee1d03f5d51f3944"}, - {file = "pandas-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f02a3a6c83df4026e55b63c1f06476c9aa3ed6af3d89b4f04ea656ccdaaaa359"}, - {file = "pandas-2.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c38ce92cb22a4bea4e3929429aa1067a454dcc9c335799af93ba9be21b6beb51"}, - {file = "pandas-2.2.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c2ce852e1cf2509a69e98358e8458775f89599566ac3775e70419b98615f4b06"}, - {file = "pandas-2.2.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:53680dc9b2519cbf609c62db3ed7c0b499077c7fefda564e330286e619ff0dd9"}, - {file = "pandas-2.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:94e714a1cca63e4f5939cdce5f29ba8d415d85166be3441165edd427dc9f6bc0"}, - {file = "pandas-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f821213d48f4ab353d20ebc24e4faf94ba40d76680642fb7ce2ea31a3ad94f9b"}, - {file = "pandas-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c70e00c2d894cb230e5c15e4b1e1e6b2b478e09cf27cc593a11ef955b9ecc81a"}, - {file = "pandas-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e97fbb5387c69209f134893abc788a6486dbf2f9e511070ca05eed4b930b1b02"}, - {file = "pandas-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101d0eb9c5361aa0146f500773395a03839a5e6ecde4d4b6ced88b7e5a1a6403"}, - {file = "pandas-2.2.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7d2ed41c319c9fb4fd454fe25372028dfa417aacb9790f68171b2e3f06eae8cd"}, - {file = "pandas-2.2.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:af5d3c00557d657c8773ef9ee702c61dd13b9d7426794c9dfeb1dc4a0bf0ebc7"}, - {file = "pandas-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:06cf591dbaefb6da9de8472535b185cba556d0ce2e6ed28e21d919704fef1a9e"}, - {file = "pandas-2.2.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:88ecb5c01bb9ca927ebc4098136038519aa5d66b44671861ffab754cae75102c"}, - {file = "pandas-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:04f6ec3baec203c13e3f8b139fb0f9f86cd8c0b94603ae3ae8ce9a422e9f5bee"}, - {file = "pandas-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a935a90a76c44fe170d01e90a3594beef9e9a6220021acfb26053d01426f7dc2"}, - {file = "pandas-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c391f594aae2fd9f679d419e9a4d5ba4bce5bb13f6a989195656e7dc4b95c8f0"}, - {file = "pandas-2.2.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9d1265545f579edf3f8f0cb6f89f234f5e44ba725a34d86535b1a1d38decbccc"}, - {file = "pandas-2.2.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:11940e9e3056576ac3244baef2fedade891977bcc1cb7e5cc8f8cc7d603edc89"}, - {file = "pandas-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:4acf681325ee1c7f950d058b05a820441075b0dd9a2adf5c4835b9bc056bf4fb"}, - {file = "pandas-2.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9bd8a40f47080825af4317d0340c656744f2bfdb6819f818e6ba3cd24c0e1397"}, - {file = "pandas-2.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:df0c37ebd19e11d089ceba66eba59a168242fc6b7155cba4ffffa6eccdfb8f16"}, - {file = "pandas-2.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:739cc70eaf17d57608639e74d63387b0d8594ce02f69e7a0b046f117974b3019"}, - {file = "pandas-2.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9d3558d263073ed95e46f4650becff0c5e1ffe0fc3a015de3c79283dfbdb3df"}, - {file = "pandas-2.2.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4aa1d8707812a658debf03824016bf5ea0d516afdea29b7dc14cf687bc4d4ec6"}, - {file = "pandas-2.2.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:76f27a809cda87e07f192f001d11adc2b930e93a2b0c4a236fde5429527423be"}, - {file = "pandas-2.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:1ba21b1d5c0e43416218db63037dbe1a01fc101dc6e6024bcad08123e48004ab"}, - {file = "pandas-2.2.1.tar.gz", hash = "sha256:0ab90f87093c13f3e8fa45b48ba9f39181046e8f3317d3aadb2fffbb1b978572"}, + {file = "pandas-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce"}, + {file = "pandas-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7adfc142dac335d8c1e0dcbd37eb8617eac386596eb9e1a1b77791cf2498238"}, + {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08"}, + {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0"}, + {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51"}, + {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8e5a0b00e1e56a842f922e7fae8ae4077aee4af0acb5ae3622bd4b4c30aedf99"}, + {file = "pandas-2.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:ddf818e4e6c7c6f4f7c8a12709696d193976b591cc7dc50588d3d1a6b5dc8772"}, + {file = "pandas-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288"}, + {file = "pandas-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151"}, + {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b"}, + {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee"}, + {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db"}, + {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1"}, + {file = "pandas-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24"}, + {file = "pandas-2.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef"}, + {file = "pandas-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce"}, + {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad"}, + {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"}, + {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76"}, + {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32"}, + {file = "pandas-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23"}, + {file = "pandas-2.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2"}, + {file = "pandas-2.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9057e6aa78a584bc93a13f0a9bf7e753a5e9770a30b4d758b8d5f2a62a9433cd"}, + {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863"}, + {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921"}, + {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a"}, + {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92fd6b027924a7e178ac202cfbe25e53368db90d56872d20ffae94b96c7acc57"}, + {file = "pandas-2.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:640cef9aa381b60e296db324337a554aeeb883ead99dc8f6c18e81a93942f5f4"}, + {file = "pandas-2.2.2.tar.gz", hash = "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54"}, ] [package.dependencies] numpy = [ - {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""}, - {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -1689,18 +1690,18 @@ xml = ["lxml (>=4.9.2)"] [[package]] name = "parso" -version = "0.8.3" +version = "0.8.4" description = "A Python Parser" optional = false python-versions = ">=3.6" files = [ - {file = "parso-0.8.3-py2.py3-none-any.whl", hash = "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"}, - {file = "parso-0.8.3.tar.gz", hash = "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0"}, + {file = "parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18"}, + {file = "parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d"}, ] [package.extras] -qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] -testing = ["docopt", "pytest (<6.0.0)"] +qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] +testing = ["docopt", "pytest"] [[package]] name = "pathspec" @@ -2069,13 +2070,13 @@ six = ">=1.5" [[package]] name = "pytorch-lightning" -version = "2.2.1" +version = "2.2.2" description = "PyTorch Lightning is the lightweight PyTorch wrapper for ML researchers. Scale your models. Write less boilerplate." optional = false python-versions = ">=3.8" files = [ - {file = "pytorch-lightning-2.2.1.tar.gz", hash = "sha256:aa3be30c9109239a371748565a7f4b7b41ea1395725c30e04426cf946b3e2745"}, - {file = "pytorch_lightning-2.2.1-py3-none-any.whl", hash = "sha256:b7efdd46a7ede4d66814a6afc28d0dfadd8eea1bb8bddab4fd2ea36c099af685"}, + {file = "pytorch-lightning-2.2.2.tar.gz", hash = "sha256:db4035785022aeeccc9163469d703c605078df0115175e14035690ad49b0ae76"}, + {file = "pytorch_lightning-2.2.2-py3-none-any.whl", hash = "sha256:28ca1e2d2dd7b99cbf6e250a990b0ae67e8a67b40af1595ed77c59761efac9e5"}, ] [package.dependencies] @@ -2090,11 +2091,11 @@ tqdm = ">=4.57.0" typing-extensions = ">=4.4.0" [package.extras] -all = ["bitsandbytes (==0.41.0)", "deepspeed (>=0.8.2,<=0.9.3)", "gym[classic-control] (>=0.17.0)", "hydra-core (>=1.0.5)", "ipython[all] (<8.15.0)", "jsonargparse[signatures] (>=4.26.1)", "lightning-utilities (>=0.8.0)", "matplotlib (>3.1)", "omegaconf (>=2.0.5)", "requests (<2.32.0)", "rich (>=12.3.0)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.14.0)"] +all = ["bitsandbytes (==0.41.0)", "deepspeed (>=0.8.2,<=0.9.3)", "gym[classic-control] (>=0.17.0)", "hydra-core (>=1.0.5)", "ipython[all] (<8.15.0)", "jsonargparse[signatures] (>=4.27.7)", "lightning-utilities (>=0.8.0)", "matplotlib (>3.1)", "omegaconf (>=2.0.5)", "requests (<2.32.0)", "rich (>=12.3.0)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.14.0)"] deepspeed = ["deepspeed (>=0.8.2,<=0.9.3)"] -dev = ["bitsandbytes (==0.41.0)", "cloudpickle (>=1.3)", "coverage (==7.3.1)", "deepspeed (>=0.8.2,<=0.9.3)", "fastapi", "gym[classic-control] (>=0.17.0)", "hydra-core (>=1.0.5)", "ipython[all] (<8.15.0)", "jsonargparse[signatures] (>=4.26.1)", "lightning-utilities (>=0.8.0)", "matplotlib (>3.1)", "omegaconf (>=2.0.5)", "onnx (>=0.14.0)", "onnxruntime (>=0.15.0)", "pandas (>1.0)", "psutil (<5.9.6)", "pytest (==7.4.0)", "pytest-cov (==4.1.0)", "pytest-random-order (==1.1.0)", "pytest-rerunfailures (==12.0)", "pytest-timeout (==2.1.0)", "requests (<2.32.0)", "rich (>=12.3.0)", "scikit-learn (>0.22.1)", "tensorboard (>=2.9.1)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.14.0)", "uvicorn"] +dev = ["bitsandbytes (==0.41.0)", "cloudpickle (>=1.3)", "coverage (==7.3.1)", "deepspeed (>=0.8.2,<=0.9.3)", "fastapi", "gym[classic-control] (>=0.17.0)", "hydra-core (>=1.0.5)", "ipython[all] (<8.15.0)", "jsonargparse[signatures] (>=4.27.7)", "lightning-utilities (>=0.8.0)", "matplotlib (>3.1)", "omegaconf (>=2.0.5)", "onnx (>=0.14.0)", "onnxruntime (>=0.15.0)", "pandas (>1.0)", "psutil (<5.9.6)", "pytest (==7.4.0)", "pytest-cov (==4.1.0)", "pytest-random-order (==1.1.0)", "pytest-rerunfailures (==12.0)", "pytest-timeout (==2.1.0)", "requests (<2.32.0)", "rich (>=12.3.0)", "scikit-learn (>0.22.1)", "tensorboard (>=2.9.1)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.14.0)", "uvicorn"] examples = ["gym[classic-control] (>=0.17.0)", "ipython[all] (<8.15.0)", "lightning-utilities (>=0.8.0)", "requests (<2.32.0)", "torchmetrics (>=0.10.0)", "torchvision (>=0.14.0)"] -extra = ["bitsandbytes (==0.41.0)", "hydra-core (>=1.0.5)", "jsonargparse[signatures] (>=4.26.1)", "matplotlib (>3.1)", "omegaconf (>=2.0.5)", "rich (>=12.3.0)", "tensorboardX (>=2.2)"] +extra = ["bitsandbytes (==0.41.0)", "hydra-core (>=1.0.5)", "jsonargparse[signatures] (>=4.27.7)", "matplotlib (>3.1)", "omegaconf (>=2.0.5)", "rich (>=12.3.0)", "tensorboardX (>=2.2)"] strategies = ["deepspeed (>=0.8.2,<=0.9.3)"] test = ["cloudpickle (>=1.3)", "coverage (==7.3.1)", "fastapi", "onnx (>=0.14.0)", "onnxruntime (>=0.15.0)", "pandas (>1.0)", "psutil (<5.9.6)", "pytest (==7.4.0)", "pytest-cov (==4.1.0)", "pytest-random-order (==1.1.0)", "pytest-rerunfailures (==12.0)", "pytest-timeout (==2.1.0)", "scikit-learn (>0.22.1)", "tensorboard (>=2.9.1)", "uvicorn"] @@ -2453,147 +2454,137 @@ files = [ [[package]] name = "ruff" -version = "0.3.4" +version = "0.3.7" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.3.4-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:60c870a7d46efcbc8385d27ec07fe534ac32f3b251e4fc44b3cbfd9e09609ef4"}, - {file = "ruff-0.3.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6fc14fa742e1d8f24910e1fff0bd5e26d395b0e0e04cc1b15c7c5e5fe5b4af91"}, - {file = "ruff-0.3.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3ee7880f653cc03749a3bfea720cf2a192e4f884925b0cf7eecce82f0ce5854"}, - {file = "ruff-0.3.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf133dd744f2470b347f602452a88e70dadfbe0fcfb5fd46e093d55da65f82f7"}, - {file = "ruff-0.3.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f3860057590e810c7ffea75669bdc6927bfd91e29b4baa9258fd48b540a4365"}, - {file = "ruff-0.3.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:986f2377f7cf12efac1f515fc1a5b753c000ed1e0a6de96747cdf2da20a1b369"}, - {file = "ruff-0.3.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fd98e85869603e65f554fdc5cddf0712e352fe6e61d29d5a6fe087ec82b76c"}, - {file = "ruff-0.3.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64abeed785dad51801b423fa51840b1764b35d6c461ea8caef9cf9e5e5ab34d9"}, - {file = "ruff-0.3.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df52972138318bc7546d92348a1ee58449bc3f9eaf0db278906eb511889c4b50"}, - {file = "ruff-0.3.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:98e98300056445ba2cc27d0b325fd044dc17fcc38e4e4d2c7711585bd0a958ed"}, - {file = "ruff-0.3.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:519cf6a0ebed244dce1dc8aecd3dc99add7a2ee15bb68cf19588bb5bf58e0488"}, - {file = "ruff-0.3.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:bb0acfb921030d00070539c038cd24bb1df73a2981e9f55942514af8b17be94e"}, - {file = "ruff-0.3.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:cf187a7e7098233d0d0c71175375c5162f880126c4c716fa28a8ac418dcf3378"}, - {file = "ruff-0.3.4-py3-none-win32.whl", hash = "sha256:af27ac187c0a331e8ef91d84bf1c3c6a5dea97e912a7560ac0cef25c526a4102"}, - {file = "ruff-0.3.4-py3-none-win_amd64.whl", hash = "sha256:de0d5069b165e5a32b3c6ffbb81c350b1e3d3483347196ffdf86dc0ef9e37dd6"}, - {file = "ruff-0.3.4-py3-none-win_arm64.whl", hash = "sha256:6810563cc08ad0096b57c717bd78aeac888a1bfd38654d9113cb3dc4d3f74232"}, - {file = "ruff-0.3.4.tar.gz", hash = "sha256:f0f4484c6541a99862b693e13a151435a279b271cff20e37101116a21e2a1ad1"}, + {file = "ruff-0.3.7-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:0e8377cccb2f07abd25e84fc5b2cbe48eeb0fea9f1719cad7caedb061d70e5ce"}, + {file = "ruff-0.3.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:15a4d1cc1e64e556fa0d67bfd388fed416b7f3b26d5d1c3e7d192c897e39ba4b"}, + {file = "ruff-0.3.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d28bdf3d7dc71dd46929fafeec98ba89b7c3550c3f0978e36389b5631b793663"}, + {file = "ruff-0.3.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:379b67d4f49774ba679593b232dcd90d9e10f04d96e3c8ce4a28037ae473f7bb"}, + {file = "ruff-0.3.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c060aea8ad5ef21cdfbbe05475ab5104ce7827b639a78dd55383a6e9895b7c51"}, + {file = "ruff-0.3.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:ebf8f615dde968272d70502c083ebf963b6781aacd3079081e03b32adfe4d58a"}, + {file = "ruff-0.3.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d48098bd8f5c38897b03604f5428901b65e3c97d40b3952e38637b5404b739a2"}, + {file = "ruff-0.3.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da8a4fda219bf9024692b1bc68c9cff4b80507879ada8769dc7e985755d662ea"}, + {file = "ruff-0.3.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c44e0149f1d8b48c4d5c33d88c677a4aa22fd09b1683d6a7ff55b816b5d074f"}, + {file = "ruff-0.3.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3050ec0af72b709a62ecc2aca941b9cd479a7bf2b36cc4562f0033d688e44fa1"}, + {file = "ruff-0.3.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a29cc38e4c1ab00da18a3f6777f8b50099d73326981bb7d182e54a9a21bb4ff7"}, + {file = "ruff-0.3.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5b15cc59c19edca917f51b1956637db47e200b0fc5e6e1878233d3a938384b0b"}, + {file = "ruff-0.3.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e491045781b1e38b72c91247cf4634f040f8d0cb3e6d3d64d38dcf43616650b4"}, + {file = "ruff-0.3.7-py3-none-win32.whl", hash = "sha256:bc931de87593d64fad3a22e201e55ad76271f1d5bfc44e1a1887edd0903c7d9f"}, + {file = "ruff-0.3.7-py3-none-win_amd64.whl", hash = "sha256:5ef0e501e1e39f35e03c2acb1d1238c595b8bb36cf7a170e7c1df1b73da00e74"}, + {file = "ruff-0.3.7-py3-none-win_arm64.whl", hash = "sha256:789e144f6dc7019d1f92a812891c645274ed08af6037d11fc65fcbc183b7d59f"}, + {file = "ruff-0.3.7.tar.gz", hash = "sha256:d5c1aebee5162c2226784800ae031f660c350e7a3402c4d1f8ea4e97e232e3ba"}, ] [[package]] name = "safetensors" -version = "0.4.2" +version = "0.4.3" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "safetensors-0.4.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:69d8bb8384dc2cb5b72c36c4d6980771b293d1a1377b378763f5e37b6bb8d133"}, - {file = "safetensors-0.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3d420e19fcef96d0067f4de4699682b4bbd85fc8fea0bd45fcd961fdf3e8c82c"}, - {file = "safetensors-0.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ca54742122fa3c4821754adb67318e1cd25c3a22bbf0c5520d5176e77a099ac"}, - {file = "safetensors-0.4.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8b47aa643afdfd66cf7ce4c184092ae734e15d10aba2c2948f24270211801c3c"}, - {file = "safetensors-0.4.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d88a16bbc330f27e7f2d4caaf6fb061ad0b8a756ecc4033260b0378e128ce8a2"}, - {file = "safetensors-0.4.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9223b8ac21085db614a510eb3445e7083cae915a9202357555fa939695d4f57"}, - {file = "safetensors-0.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce6cb86133dc8930a7ab5e7438545a7f205f7a1cdd5aaf108c1d0da6bdcfbc2b"}, - {file = "safetensors-0.4.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b8a628e0ae2bbc334b62952c384aa5f41621d01850f8d67b04a96b9c39dd7326"}, - {file = "safetensors-0.4.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:88d6beb7f811a081e0e5f1d9669fdac816c45340c04b1eaf7ebfda0ce93ea403"}, - {file = "safetensors-0.4.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b57fc5b1b54cb12d8690a58a4cf4b7144730d4bde9d98aa0e1dab6295a1cd579"}, - {file = "safetensors-0.4.2-cp310-none-win32.whl", hash = "sha256:9d87a1c98803c16cf113b9ba03f07b2dce5e8eabfd1811a7f7323fcaa2a1bf47"}, - {file = "safetensors-0.4.2-cp310-none-win_amd64.whl", hash = "sha256:18930ec1d1ecb526d3d9835abc2489b8f1530877518f0c541e77ef0b7abcbd99"}, - {file = "safetensors-0.4.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:c5dd2ed788730ed56b415d1a11c62026b8cc8c573f55a2092afb3ab383e94fff"}, - {file = "safetensors-0.4.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cc41791b33efb9c83a59b731619f3d15f543dfe71f3a793cb8fbf9bd5d0d5d71"}, - {file = "safetensors-0.4.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c888bf71d5ca12a720f1ed87d407c4918afa022fb247a6546d8fac15b1f112b"}, - {file = "safetensors-0.4.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e6b2feb4b47226a16a792e6fac3f49442714884a3d4c1008569d5068a3941be9"}, - {file = "safetensors-0.4.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f41cc0ee4b838ae8f4d8364a1b162067693d11a3893f0863be8c228d40e4d0ee"}, - {file = "safetensors-0.4.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:51b7228e46c0a483c40ba4b9470dea00fb1ff8685026bb4766799000f6328ac2"}, - {file = "safetensors-0.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02697f8f2be8ca3c37a4958702dbdb1864447ef765e18b5328a1617022dcf164"}, - {file = "safetensors-0.4.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:27fd8f65cf7c80e4280cae1ee6bcd85c483882f6580821abe71ee1a0d3dcfca7"}, - {file = "safetensors-0.4.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c487b5f113b0924c9534a07dc034830fb4ef05ce9bb6d78cfe016a7dedfe281f"}, - {file = "safetensors-0.4.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:da7f6483f3fe67ff39b3a55552552c67930ea10a36e9f2539d36fc205273d767"}, - {file = "safetensors-0.4.2-cp311-none-win32.whl", hash = "sha256:52a7012f6cb9cb4a132760b6308daede18a9f5f8952ce08adc7c67a7d865c2d8"}, - {file = "safetensors-0.4.2-cp311-none-win_amd64.whl", hash = "sha256:4d1361a097ac430b310ce9eed8ed4746edee33ddafdfbb965debc8966fc34dc2"}, - {file = "safetensors-0.4.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:77af8aa0edcc2863760fd6febbfdb82e88fd75d0e60c1ce4ba57208ba5e4a89b"}, - {file = "safetensors-0.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846666c1c5a8c8888d2dfda8d3921cb9cb8e2c5f78365be756c11021e75a0a2a"}, - {file = "safetensors-0.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f4bfc7ea19b446bfad41510d4b4c76101698c00caaa8a332c8edd8090a412ef"}, - {file = "safetensors-0.4.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:233436fd30f27ffeb3c3780d0b84f496518868445c7a8db003639a649cc98453"}, - {file = "safetensors-0.4.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7a09237a795d11cd11f9dae505d170a29b5616151db1e10c14f892b11caadc7d"}, - {file = "safetensors-0.4.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de01c9a3a3b7b69627d624ff69d9f11d28ce9908eea2fb6245adafa4b1d43df6"}, - {file = "safetensors-0.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c1f25c5069ee42a5bcffdc66c300a407941edd73f3239e9fdefd26216407391"}, - {file = "safetensors-0.4.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7a73b3649456d09ca8506140d44484b63154a7378434cc1e8719f8056550b224"}, - {file = "safetensors-0.4.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e1625a8d07d046e968bd5c4961810aba1225984e4fb9243626f9d04a06ed3fee"}, - {file = "safetensors-0.4.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f74c86b25615cb24ad4cff765a2eefc09d71bf0fed97588cf585aad9c38fbb4"}, - {file = "safetensors-0.4.2-cp312-none-win32.whl", hash = "sha256:8523b9c5777d771bcde5c2389c03f1cdf7ebe8797432a1bd5e345efe25c55987"}, - {file = "safetensors-0.4.2-cp312-none-win_amd64.whl", hash = "sha256:dcff0243e1737a21f83d664c63fed89d1f532c23fc6830d0427279fabd789ccb"}, - {file = "safetensors-0.4.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:96ad3d7d472612e26cbe413922b4fb13933310f0511d346ea5cc9a1e856e52eb"}, - {file = "safetensors-0.4.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:88250922401b5ae4e37de929178caf46be47ed16c817b2237b81679bec07c120"}, - {file = "safetensors-0.4.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d40443554142fc0ab30652d5cc8554c4b7a613513bde00373e18afd5de8cbe4b"}, - {file = "safetensors-0.4.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:27f53f70106224d32d874aacecbeb4a6e4c5b16a1d2006d0e876d97229086d71"}, - {file = "safetensors-0.4.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cc068afe23734dfb26ce19db0a7877499ddf73b1d55ceb762417e8da4a1b05fb"}, - {file = "safetensors-0.4.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9be1918eb8d43a11a6f8806759fccfa0eeb0542b12924caba66af8a7800ad01a"}, - {file = "safetensors-0.4.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41911087d20a7bbd78cb4ad4f98aab0c431533107584df6635d8b54b99945573"}, - {file = "safetensors-0.4.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:50771c662aab909f31e94d048e76861fd027d66076ea773eef2e66c717766e24"}, - {file = "safetensors-0.4.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:13f2e57be007b7ea9329133d2399e6bdfcf1910f655440a4da17df3a45afcd30"}, - {file = "safetensors-0.4.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c772147e6395bc829842e0a98e1b30c67fe25d816299c28196488511d5a5e951"}, - {file = "safetensors-0.4.2-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:36239a0060b537a3e8c473df78cffee14c3ec4f51d5f1a853af99371a2fb2a35"}, - {file = "safetensors-0.4.2-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:d0cbb7664fad2c307f95195f951b7059e95dc23e0e1822e5978c8b500098543c"}, - {file = "safetensors-0.4.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b3e55adb6bd9dc1c2a341e72f48f075953fa35d173dd8e29a95b3b02d0d1462"}, - {file = "safetensors-0.4.2-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42f743b3cca863fba53ca57a193f510e5ec359b97f38c282437716b6768e4a25"}, - {file = "safetensors-0.4.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04e6af4a6dbeb06c4e6e7d46cf9c716cbc4cc5ef62584fd8a7c0fe558562df45"}, - {file = "safetensors-0.4.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a492ba21b5c8f14ee5ec9b20f42ba969e53ca1f909a4d04aad736b66a341dcc2"}, - {file = "safetensors-0.4.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b25b8233a1a85dc67e39838951cfb01595d792f3b7b644add63edb652992e030"}, - {file = "safetensors-0.4.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fd27e063fbdafe776f7b1714da59110e88f270e86db00788a8fd65f4eacfeba7"}, - {file = "safetensors-0.4.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1b6fa399f251bbeb52029bf5a0ac2878d7705dd3612a2f8895b48e9c11f0367d"}, - {file = "safetensors-0.4.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:de642d46b459e4afd5c2020b26c0d6d869a171ea00411897d5776c127cac74f0"}, - {file = "safetensors-0.4.2-cp37-none-win32.whl", hash = "sha256:77b72d17754c93bb68f3598182f14d78776e0b9b31682ca5bb2c7c5bd9a75267"}, - {file = "safetensors-0.4.2-cp37-none-win_amd64.whl", hash = "sha256:d36ee3244d461cd655aeef493792c3bccf4875282f8407fd9af99e9a41cf2530"}, - {file = "safetensors-0.4.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:16b6b3884f7876c6b3b23a742428223a7170a5a9dac819d8c12a1569422c4b5a"}, - {file = "safetensors-0.4.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ee25d311493fbbe0be9d395faee46e9d79e8948f461e388ff39e59875ed9a350"}, - {file = "safetensors-0.4.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eed8097968585cd752a1171f86fce9aa1d89a29033e5cd8bec5a502e29f6b7af"}, - {file = "safetensors-0.4.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:880e6865cf72cb67f9ab8d04a3c4b49dd95ae92fb1583929ce65aed94e1f685f"}, - {file = "safetensors-0.4.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91290f83daf80ce6d1a7f629b244443c200060a80f908b29d879021409e5ea94"}, - {file = "safetensors-0.4.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3517d568486ab3508a7acc360b82d7a4a3e26b86efdf210a9ecd9d233c40708a"}, - {file = "safetensors-0.4.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1f43a77eb38540f782999e5dc5645164fe9027d3f0194f6c9a5126168017efa"}, - {file = "safetensors-0.4.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b684d9818aa5d63fddc65f7d0151968037d255d91adf74eba82125b41c680aaa"}, - {file = "safetensors-0.4.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ab1f5d84185f9fefaf21413efb764e4908057b8a9a0b987ede890c353490fd70"}, - {file = "safetensors-0.4.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2bd979642e6c3a517ef4b84ff36c2fee4015664fea05a61154fc565978347553"}, - {file = "safetensors-0.4.2-cp38-none-win32.whl", hash = "sha256:11be6e7afed29e5a5628f0aa6214e34bc194da73f558dc69fc7d56e07037422a"}, - {file = "safetensors-0.4.2-cp38-none-win_amd64.whl", hash = "sha256:2f7a6e5d29bd2cc340cffaa391fa437b1be9d21a2bd8b8724d2875d13a6ef2a9"}, - {file = "safetensors-0.4.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a5a921b4fe6925f9942adff3ebae8c16e0487908c54586a5a42f35b59fd69794"}, - {file = "safetensors-0.4.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b691727228c28f2d82d8a92b2bc26e7a1f129ee40b2f2a3185b5974e038ed47c"}, - {file = "safetensors-0.4.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91ca1056decc4e981248786e87b2a202d4841ee5f99d433f1adf3d44d4bcfa0e"}, - {file = "safetensors-0.4.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:55969fd2e6fdb38dc221b0ab380668c21b0efa12a7562db9924759faa3c51757"}, - {file = "safetensors-0.4.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6ae429bfaecc10ab5fe78c93009b3d1656c1581da560041e700eadb497dbe7a4"}, - {file = "safetensors-0.4.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ff88f194fe4ac50b463a4a6f0c03af9ad72eb5d24ec6d6730af59522e37fedb"}, - {file = "safetensors-0.4.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a80cb48d0a447f8dd18e61813efa7d3f8f8d52edf0f05806abc0c59b83431f57"}, - {file = "safetensors-0.4.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b286fb7adfee70a4189898ac2342b8a67d5f493e6b21b0af89ca8eac1b967cbf"}, - {file = "safetensors-0.4.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0ceeff9ddbab4f78738489eb6682867ae946178776f33699737b2129b5394dc1"}, - {file = "safetensors-0.4.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a26fae748a7488cb3aac381eddfa818c42052c87b5e689fb4c6e82ed58cec209"}, - {file = "safetensors-0.4.2-cp39-none-win32.whl", hash = "sha256:039a42ab33c9d68b39706fd38f1922ace26866eff246bf20271edb619f5f848b"}, - {file = "safetensors-0.4.2-cp39-none-win_amd64.whl", hash = "sha256:b3a3e1f5b85859e398773f064943b62a4059f225008a2a8ee6add1edcf77cacf"}, - {file = "safetensors-0.4.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4e70d442ad17e8b153ef9095bf48ea64f15a66bf26dc2b6ca94660c154edbc24"}, - {file = "safetensors-0.4.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b90f1d9809caf4ff395951b4703295a68d12907f6945bbc3129e934ff8ae46f6"}, - {file = "safetensors-0.4.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c7ac9ad3728838006598e296b3ae9f27d80b489effd4685b92d97b3fc4c98f6"}, - {file = "safetensors-0.4.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5730d77e6ff7f4c7039e20913661ad0ea2f86c09e71c039e73dfdd1f394f08"}, - {file = "safetensors-0.4.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:44feb8cb156d6803dcd19fc6b81b27235f29b877660605a6ac35e1da7d64f0e4"}, - {file = "safetensors-0.4.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:523a241c33e7c827ab9a3a23760d75c7d062f43dfe55b6b019409f89b0fb52d1"}, - {file = "safetensors-0.4.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:fb18300e8eb74291225214f26c9a8ae2110fd61a6c9b5a2ff4c4e0eb1bb9a998"}, - {file = "safetensors-0.4.2-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:fe5437ff9fb116e44f2ab558981249ae63f978392b4576e62fcfe167d353edbc"}, - {file = "safetensors-0.4.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9304a0934ced5a5d272f39de36291dc141dfc152d277f03fb4d65f2fb2ffa7c"}, - {file = "safetensors-0.4.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:160ba1b1e11cf874602c233ab80a14f588571d09556cbc3586900121d622b5ed"}, - {file = "safetensors-0.4.2-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:04fcd6fcf7d9c13c7e5dc7e08de5e492ee4daa8f4ad74b4d8299d3eb0224292f"}, - {file = "safetensors-0.4.2-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:906d14c4a677d35834fb0f3a5455ef8305e1bba10a5e0f2e0f357b3d1ad989f2"}, - {file = "safetensors-0.4.2-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:df3fcdec0cd543084610d1f09c65cdb10fb3079f79bceddc092b0d187c6a265b"}, - {file = "safetensors-0.4.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5ca76f13fb1cef242ea3ad2cb37388e7d005994f42af8b44bee56ba48b2d45ce"}, - {file = "safetensors-0.4.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:278a1a3414c020785decdcd741c578725721274d2f9f787fcc930882e83b89cc"}, - {file = "safetensors-0.4.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05b5a461cc68ecd42d9d546e5e1268a39d8ede7934a68d1ce17c3c659cb829d6"}, - {file = "safetensors-0.4.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2341411412a41671d25e26bed59ec121e46bf4fadb8132895e610411c4b9681"}, - {file = "safetensors-0.4.2-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3497ac3895acf17c5f98197f1fa4769f09c5e7ede07fcb102f1c201e663e052c"}, - {file = "safetensors-0.4.2-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:01b5e71d3754d2201294f1eb7a6d59cce3a5702ff96d83d226571b2ca2183837"}, - {file = "safetensors-0.4.2-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:3627dbd1ea488dd8046a0491de5087f3c0d641e7acc80c0189a33c69398f1cd1"}, - {file = "safetensors-0.4.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9d56f0ef53afad26ec54ceede78a43e9a23a076dadbbda7b44d304c591abf4c1"}, - {file = "safetensors-0.4.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:b259ca73d42daf658a1bda463f1f83885ae4d93a60869be80d7f7dfcc9d8bbb5"}, - {file = "safetensors-0.4.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ebc3cd401e4eb54e7c0a70346be565e81942d9a41fafd5f4bf7ab3a55d10378"}, - {file = "safetensors-0.4.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5bc384a0309b706aa0425c93abb0390508a61bf029ce99c7d9df4220f25871a5"}, - {file = "safetensors-0.4.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:af2d8f7235d8a08fbccfb8394387890e7fa38942b349a94e6eff13c52ac98087"}, - {file = "safetensors-0.4.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:0911315bbcc5289087d063c2c2c7ccd711ea97a7e557a7bce005ac2cf80146aa"}, - {file = "safetensors-0.4.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1efe31673be91832d73439a2af426743e1395fc9ef7b081914e9e1d567bd7b5f"}, - {file = "safetensors-0.4.2.tar.gz", hash = "sha256:acc85dcb09ec5e8aa787f588d7ad4d55c103f31e4ff060e17d92cc0e8b8cac73"}, + {file = "safetensors-0.4.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:dcf5705cab159ce0130cd56057f5f3425023c407e170bca60b4868048bae64fd"}, + {file = "safetensors-0.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bb4f8c5d0358a31e9a08daeebb68f5e161cdd4018855426d3f0c23bb51087055"}, + {file = "safetensors-0.4.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70a5319ef409e7f88686a46607cbc3c428271069d8b770076feaf913664a07ac"}, + {file = "safetensors-0.4.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fb9c65bd82f9ef3ce4970dc19ee86be5f6f93d032159acf35e663c6bea02b237"}, + {file = "safetensors-0.4.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:edb5698a7bc282089f64c96c477846950358a46ede85a1c040e0230344fdde10"}, + {file = "safetensors-0.4.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:efcc860be094b8d19ac61b452ec635c7acb9afa77beb218b1d7784c6d41fe8ad"}, + {file = "safetensors-0.4.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d88b33980222085dd6001ae2cad87c6068e0991d4f5ccf44975d216db3b57376"}, + {file = "safetensors-0.4.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5fc6775529fb9f0ce2266edd3e5d3f10aab068e49f765e11f6f2a63b5367021d"}, + {file = "safetensors-0.4.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9c6ad011c1b4e3acff058d6b090f1da8e55a332fbf84695cf3100c649cc452d1"}, + {file = "safetensors-0.4.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8c496c5401c1b9c46d41a7688e8ff5b0310a3b9bae31ce0f0ae870e1ea2b8caf"}, + {file = "safetensors-0.4.3-cp310-none-win32.whl", hash = "sha256:38e2a8666178224a51cca61d3cb4c88704f696eac8f72a49a598a93bbd8a4af9"}, + {file = "safetensors-0.4.3-cp310-none-win_amd64.whl", hash = "sha256:393e6e391467d1b2b829c77e47d726f3b9b93630e6a045b1d1fca67dc78bf632"}, + {file = "safetensors-0.4.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:22f3b5d65e440cec0de8edaa672efa888030802e11c09b3d6203bff60ebff05a"}, + {file = "safetensors-0.4.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c4fa560ebd4522adddb71dcd25d09bf211b5634003f015a4b815b7647d62ebe"}, + {file = "safetensors-0.4.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9afd5358719f1b2cf425fad638fc3c887997d6782da317096877e5b15b2ce93"}, + {file = "safetensors-0.4.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d8c5093206ef4b198600ae484230402af6713dab1bd5b8e231905d754022bec7"}, + {file = "safetensors-0.4.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e0b2104df1579d6ba9052c0ae0e3137c9698b2d85b0645507e6fd1813b70931a"}, + {file = "safetensors-0.4.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8cf18888606dad030455d18f6c381720e57fc6a4170ee1966adb7ebc98d4d6a3"}, + {file = "safetensors-0.4.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0bf4f9d6323d9f86eef5567eabd88f070691cf031d4c0df27a40d3b4aaee755b"}, + {file = "safetensors-0.4.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:585c9ae13a205807b63bef8a37994f30c917ff800ab8a1ca9c9b5d73024f97ee"}, + {file = "safetensors-0.4.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:faefeb3b81bdfb4e5a55b9bbdf3d8d8753f65506e1d67d03f5c851a6c87150e9"}, + {file = "safetensors-0.4.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:befdf0167ad626f22f6aac6163477fcefa342224a22f11fdd05abb3995c1783c"}, + {file = "safetensors-0.4.3-cp311-none-win32.whl", hash = "sha256:a7cef55929dcbef24af3eb40bedec35d82c3c2fa46338bb13ecf3c5720af8a61"}, + {file = "safetensors-0.4.3-cp311-none-win_amd64.whl", hash = "sha256:840b7ac0eff5633e1d053cc9db12fdf56b566e9403b4950b2dc85393d9b88d67"}, + {file = "safetensors-0.4.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:22d21760dc6ebae42e9c058d75aa9907d9f35e38f896e3c69ba0e7b213033856"}, + {file = "safetensors-0.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d22c1a10dff3f64d0d68abb8298a3fd88ccff79f408a3e15b3e7f637ef5c980"}, + {file = "safetensors-0.4.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1648568667f820b8c48317c7006221dc40aced1869908c187f493838a1362bc"}, + {file = "safetensors-0.4.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:446e9fe52c051aeab12aac63d1017e0f68a02a92a027b901c4f8e931b24e5397"}, + {file = "safetensors-0.4.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fef5d70683643618244a4f5221053567ca3e77c2531e42ad48ae05fae909f542"}, + {file = "safetensors-0.4.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a1f4430cc0c9d6afa01214a4b3919d0a029637df8e09675ceef1ca3f0dfa0df"}, + {file = "safetensors-0.4.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d603846a8585b9432a0fd415db1d4c57c0f860eb4aea21f92559ff9902bae4d"}, + {file = "safetensors-0.4.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a844cdb5d7cbc22f5f16c7e2a0271170750763c4db08381b7f696dbd2c78a361"}, + {file = "safetensors-0.4.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:88887f69f7a00cf02b954cdc3034ffb383b2303bc0ab481d4716e2da51ddc10e"}, + {file = "safetensors-0.4.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ee463219d9ec6c2be1d331ab13a8e0cd50d2f32240a81d498266d77d07b7e71e"}, + {file = "safetensors-0.4.3-cp312-none-win32.whl", hash = "sha256:d0dd4a1db09db2dba0f94d15addc7e7cd3a7b0d393aa4c7518c39ae7374623c3"}, + {file = "safetensors-0.4.3-cp312-none-win_amd64.whl", hash = "sha256:d14d30c25897b2bf19b6fb5ff7e26cc40006ad53fd4a88244fdf26517d852dd7"}, + {file = "safetensors-0.4.3-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d1456f814655b224d4bf6e7915c51ce74e389b413be791203092b7ff78c936dd"}, + {file = "safetensors-0.4.3-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:455d538aa1aae4a8b279344a08136d3f16334247907b18a5c3c7fa88ef0d3c46"}, + {file = "safetensors-0.4.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf476bca34e1340ee3294ef13e2c625833f83d096cfdf69a5342475602004f95"}, + {file = "safetensors-0.4.3-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:02ef3a24face643456020536591fbd3c717c5abaa2737ec428ccbbc86dffa7a4"}, + {file = "safetensors-0.4.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7de32d0d34b6623bb56ca278f90db081f85fb9c5d327e3c18fd23ac64f465768"}, + {file = "safetensors-0.4.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a0deb16a1d3ea90c244ceb42d2c6c276059616be21a19ac7101aa97da448faf"}, + {file = "safetensors-0.4.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c59d51f182c729f47e841510b70b967b0752039f79f1de23bcdd86462a9b09ee"}, + {file = "safetensors-0.4.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1f598b713cc1a4eb31d3b3203557ac308acf21c8f41104cdd74bf640c6e538e3"}, + {file = "safetensors-0.4.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5757e4688f20df083e233b47de43845d1adb7e17b6cf7da5f8444416fc53828d"}, + {file = "safetensors-0.4.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:fe746d03ed8d193674a26105e4f0fe6c726f5bb602ffc695b409eaf02f04763d"}, + {file = "safetensors-0.4.3-cp37-none-win32.whl", hash = "sha256:0d5ffc6a80f715c30af253e0e288ad1cd97a3d0086c9c87995e5093ebc075e50"}, + {file = "safetensors-0.4.3-cp37-none-win_amd64.whl", hash = "sha256:a11c374eb63a9c16c5ed146457241182f310902bd2a9c18255781bb832b6748b"}, + {file = "safetensors-0.4.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:b1e31be7945f66be23f4ec1682bb47faa3df34cb89fc68527de6554d3c4258a4"}, + {file = "safetensors-0.4.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:03a4447c784917c9bf01d8f2ac5080bc15c41692202cd5f406afba16629e84d6"}, + {file = "safetensors-0.4.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d244bcafeb1bc06d47cfee71727e775bca88a8efda77a13e7306aae3813fa7e4"}, + {file = "safetensors-0.4.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53c4879b9c6bd7cd25d114ee0ef95420e2812e676314300624594940a8d6a91f"}, + {file = "safetensors-0.4.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:74707624b81f1b7f2b93f5619d4a9f00934d5948005a03f2c1845ffbfff42212"}, + {file = "safetensors-0.4.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0d52c958dc210265157573f81d34adf54e255bc2b59ded6218500c9b15a750eb"}, + {file = "safetensors-0.4.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f9568f380f513a60139971169c4a358b8731509cc19112369902eddb33faa4d"}, + {file = "safetensors-0.4.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0d9cd8e1560dfc514b6d7859247dc6a86ad2f83151a62c577428d5102d872721"}, + {file = "safetensors-0.4.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:89f9f17b0dacb913ed87d57afbc8aad85ea42c1085bd5de2f20d83d13e9fc4b2"}, + {file = "safetensors-0.4.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1139eb436fd201c133d03c81209d39ac57e129f5e74e34bb9ab60f8d9b726270"}, + {file = "safetensors-0.4.3-cp38-none-win32.whl", hash = "sha256:d9c289f140a9ae4853fc2236a2ffc9a9f2d5eae0cb673167e0f1b8c18c0961ac"}, + {file = "safetensors-0.4.3-cp38-none-win_amd64.whl", hash = "sha256:622afd28968ef3e9786562d352659a37de4481a4070f4ebac883f98c5836563e"}, + {file = "safetensors-0.4.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:8651c7299cbd8b4161a36cd6a322fa07d39cd23535b144d02f1c1972d0c62f3c"}, + {file = "safetensors-0.4.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e375d975159ac534c7161269de24ddcd490df2157b55c1a6eeace6cbb56903f0"}, + {file = "safetensors-0.4.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:084fc436e317f83f7071fc6a62ca1c513b2103db325cd09952914b50f51cf78f"}, + {file = "safetensors-0.4.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:41a727a7f5e6ad9f1db6951adee21bbdadc632363d79dc434876369a17de6ad6"}, + {file = "safetensors-0.4.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e7dbbde64b6c534548696808a0e01276d28ea5773bc9a2dfb97a88cd3dffe3df"}, + {file = "safetensors-0.4.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bbae3b4b9d997971431c346edbfe6e41e98424a097860ee872721e176040a893"}, + {file = "safetensors-0.4.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01e4b22e3284cd866edeabe4f4d896229495da457229408d2e1e4810c5187121"}, + {file = "safetensors-0.4.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dd37306546b58d3043eb044c8103a02792cc024b51d1dd16bd3dd1f334cb3ed"}, + {file = "safetensors-0.4.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d8815b5e1dac85fc534a97fd339e12404db557878c090f90442247e87c8aeaea"}, + {file = "safetensors-0.4.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e011cc162503c19f4b1fd63dfcddf73739c7a243a17dac09b78e57a00983ab35"}, + {file = "safetensors-0.4.3-cp39-none-win32.whl", hash = "sha256:01feb3089e5932d7e662eda77c3ecc389f97c0883c4a12b5cfdc32b589a811c3"}, + {file = "safetensors-0.4.3-cp39-none-win_amd64.whl", hash = "sha256:3f9cdca09052f585e62328c1c2923c70f46814715c795be65f0b93f57ec98a02"}, + {file = "safetensors-0.4.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1b89381517891a7bb7d1405d828b2bf5d75528299f8231e9346b8eba092227f9"}, + {file = "safetensors-0.4.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:cd6fff9e56df398abc5866b19a32124815b656613c1c5ec0f9350906fd798aac"}, + {file = "safetensors-0.4.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:840caf38d86aa7014fe37ade5d0d84e23dcfbc798b8078015831996ecbc206a3"}, + {file = "safetensors-0.4.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9650713b2cfa9537a2baf7dd9fee458b24a0aaaa6cafcea8bdd5fb2b8efdc34"}, + {file = "safetensors-0.4.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e4119532cd10dba04b423e0f86aecb96cfa5a602238c0aa012f70c3a40c44b50"}, + {file = "safetensors-0.4.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e066e8861eef6387b7c772344d1fe1f9a72800e04ee9a54239d460c400c72aab"}, + {file = "safetensors-0.4.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:90964917f5b0fa0fa07e9a051fbef100250c04d150b7026ccbf87a34a54012e0"}, + {file = "safetensors-0.4.3-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c41e1893d1206aa7054029681778d9a58b3529d4c807002c156d58426c225173"}, + {file = "safetensors-0.4.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae7613a119a71a497d012ccc83775c308b9c1dab454806291427f84397d852fd"}, + {file = "safetensors-0.4.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9bac020faba7f5dc481e881b14b6425265feabb5bfc552551d21189c0eddc3"}, + {file = "safetensors-0.4.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:420a98f593ff9930f5822560d14c395ccbc57342ddff3b463bc0b3d6b1951550"}, + {file = "safetensors-0.4.3-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f5e6883af9a68c0028f70a4c19d5a6ab6238a379be36ad300a22318316c00cb0"}, + {file = "safetensors-0.4.3-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:cdd0a3b5da66e7f377474599814dbf5cbf135ff059cc73694de129b58a5e8a2c"}, + {file = "safetensors-0.4.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9bfb92f82574d9e58401d79c70c716985dc049b635fef6eecbb024c79b2c46ad"}, + {file = "safetensors-0.4.3-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:3615a96dd2dcc30eb66d82bc76cda2565f4f7bfa89fcb0e31ba3cea8a1a9ecbb"}, + {file = "safetensors-0.4.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:868ad1b6fc41209ab6bd12f63923e8baeb1a086814cb2e81a65ed3d497e0cf8f"}, + {file = "safetensors-0.4.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7ffba80aa49bd09195145a7fd233a7781173b422eeb995096f2b30591639517"}, + {file = "safetensors-0.4.3-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c0acbe31340ab150423347e5b9cc595867d814244ac14218932a5cf1dd38eb39"}, + {file = "safetensors-0.4.3-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:19bbdf95de2cf64f25cd614c5236c8b06eb2cfa47cbf64311f4b5d80224623a3"}, + {file = "safetensors-0.4.3-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b852e47eb08475c2c1bd8131207b405793bfc20d6f45aff893d3baaad449ed14"}, + {file = "safetensors-0.4.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5d07cbca5b99babb692d76d8151bec46f461f8ad8daafbfd96b2fca40cadae65"}, + {file = "safetensors-0.4.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:1ab6527a20586d94291c96e00a668fa03f86189b8a9defa2cdd34a1a01acc7d5"}, + {file = "safetensors-0.4.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02318f01e332cc23ffb4f6716e05a492c5f18b1d13e343c49265149396284a44"}, + {file = "safetensors-0.4.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec4b52ce9a396260eb9731eb6aea41a7320de22ed73a1042c2230af0212758ce"}, + {file = "safetensors-0.4.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:018b691383026a2436a22b648873ed11444a364324e7088b99cd2503dd828400"}, + {file = "safetensors-0.4.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:309b10dbcab63269ecbf0e2ca10ce59223bb756ca5d431ce9c9eeabd446569da"}, + {file = "safetensors-0.4.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b277482120df46e27a58082df06a15aebda4481e30a1c21eefd0921ae7e03f65"}, + {file = "safetensors-0.4.3.tar.gz", hash = "sha256:2f85fc50c4e07a21e95c24e07460fe6f7e2859d0ce88092838352b798ce711c2"}, ] [package.extras] @@ -2606,42 +2597,42 @@ paddlepaddle = ["paddlepaddle (>=2.4.1)", "safetensors[numpy]"] pinned-tf = ["safetensors[numpy]", "tensorflow (==2.11.0)"] quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] tensorflow = ["safetensors[numpy]", "tensorflow (>=2.11.0)"] -testing = ["h5py (>=3.7.0)", "huggingface_hub (>=0.12.1)", "hypothesis (>=6.70.2)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "safetensors[numpy]", "setuptools_rust (>=1.5.2)"] +testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "hypothesis (>=6.70.2)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "safetensors[numpy]", "setuptools-rust (>=1.5.2)"] torch = ["safetensors[numpy]", "torch (>=1.10)"] [[package]] name = "scikit-learn" -version = "1.4.1.post1" +version = "1.4.2" description = "A set of python modules for machine learning and data mining" optional = false python-versions = ">=3.9" files = [ - {file = "scikit-learn-1.4.1.post1.tar.gz", hash = "sha256:93d3d496ff1965470f9977d05e5ec3376fb1e63b10e4fda5e39d23c2d8969a30"}, - {file = "scikit_learn-1.4.1.post1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c540aaf44729ab5cd4bd5e394f2b375e65ceaea9cdd8c195788e70433d91bbc5"}, - {file = "scikit_learn-1.4.1.post1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:4310bff71aa98b45b46cd26fa641309deb73a5d1c0461d181587ad4f30ea3c36"}, - {file = "scikit_learn-1.4.1.post1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f43dd527dabff5521af2786a2f8de5ba381e182ec7292663508901cf6ceaf6e"}, - {file = "scikit_learn-1.4.1.post1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c02e27d65b0c7dc32f2c5eb601aaf5530b7a02bfbe92438188624524878336f2"}, - {file = "scikit_learn-1.4.1.post1-cp310-cp310-win_amd64.whl", hash = "sha256:629e09f772ad42f657ca60a1a52342eef786218dd20cf1369a3b8d085e55ef8f"}, - {file = "scikit_learn-1.4.1.post1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6145dfd9605b0b50ae72cdf72b61a2acd87501369a763b0d73d004710ebb76b5"}, - {file = "scikit_learn-1.4.1.post1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:1afed6951bc9d2053c6ee9a518a466cbc9b07c6a3f9d43bfe734192b6125d508"}, - {file = "scikit_learn-1.4.1.post1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce03506ccf5f96b7e9030fea7eb148999b254c44c10182ac55857bc9b5d4815f"}, - {file = "scikit_learn-1.4.1.post1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ba516fcdc73d60e7f48cbb0bccb9acbdb21807de3651531208aac73c758e3ab"}, - {file = "scikit_learn-1.4.1.post1-cp311-cp311-win_amd64.whl", hash = "sha256:78cd27b4669513b50db4f683ef41ea35b5dddc797bd2bbd990d49897fd1c8a46"}, - {file = "scikit_learn-1.4.1.post1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a1e289f33f613cefe6707dead50db31930530dc386b6ccff176c786335a7b01c"}, - {file = "scikit_learn-1.4.1.post1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:0df87de9ce1c0140f2818beef310fb2e2afdc1e66fc9ad587965577f17733649"}, - {file = "scikit_learn-1.4.1.post1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:712c1c69c45b58ef21635360b3d0a680ff7d83ac95b6f9b82cf9294070cda710"}, - {file = "scikit_learn-1.4.1.post1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1754b0c2409d6ed5a3380512d0adcf182a01363c669033a2b55cca429ed86a81"}, - {file = "scikit_learn-1.4.1.post1-cp312-cp312-win_amd64.whl", hash = "sha256:1d491ef66e37f4e812db7e6c8286520c2c3fc61b34bf5e59b67b4ce528de93af"}, - {file = "scikit_learn-1.4.1.post1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:aa0029b78ef59af22cfbd833e8ace8526e4df90212db7ceccbea582ebb5d6794"}, - {file = "scikit_learn-1.4.1.post1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:14e4c88436ac96bf69eb6d746ac76a574c314a23c6961b7d344b38877f20fee1"}, - {file = "scikit_learn-1.4.1.post1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7cd3a77c32879311f2aa93466d3c288c955ef71d191503cf0677c3340ae8ae0"}, - {file = "scikit_learn-1.4.1.post1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a3ee19211ded1a52ee37b0a7b373a8bfc66f95353af058a210b692bd4cda0dd"}, - {file = "scikit_learn-1.4.1.post1-cp39-cp39-win_amd64.whl", hash = "sha256:234b6bda70fdcae9e4abbbe028582ce99c280458665a155eed0b820599377d25"}, + {file = "scikit-learn-1.4.2.tar.gz", hash = "sha256:daa1c471d95bad080c6e44b4946c9390a4842adc3082572c20e4f8884e39e959"}, + {file = "scikit_learn-1.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8539a41b3d6d1af82eb629f9c57f37428ff1481c1e34dddb3b9d7af8ede67ac5"}, + {file = "scikit_learn-1.4.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:68b8404841f944a4a1459b07198fa2edd41a82f189b44f3e1d55c104dbc2e40c"}, + {file = "scikit_learn-1.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81bf5d8bbe87643103334032dd82f7419bc8c8d02a763643a6b9a5c7288c5054"}, + {file = "scikit_learn-1.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36f0ea5d0f693cb247a073d21a4123bdf4172e470e6d163c12b74cbb1536cf38"}, + {file = "scikit_learn-1.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:87440e2e188c87db80ea4023440923dccbd56fbc2d557b18ced00fef79da0727"}, + {file = "scikit_learn-1.4.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:45dee87ac5309bb82e3ea633955030df9bbcb8d2cdb30383c6cd483691c546cc"}, + {file = "scikit_learn-1.4.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:1d0b25d9c651fd050555aadd57431b53d4cf664e749069da77f3d52c5ad14b3b"}, + {file = "scikit_learn-1.4.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0203c368058ab92efc6168a1507d388d41469c873e96ec220ca8e74079bf62e"}, + {file = "scikit_learn-1.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44c62f2b124848a28fd695db5bc4da019287abf390bfce602ddc8aa1ec186aae"}, + {file = "scikit_learn-1.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:5cd7b524115499b18b63f0c96f4224eb885564937a0b3477531b2b63ce331904"}, + {file = "scikit_learn-1.4.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:90378e1747949f90c8f385898fff35d73193dfcaec3dd75d6b542f90c4e89755"}, + {file = "scikit_learn-1.4.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:ff4effe5a1d4e8fed260a83a163f7dbf4f6087b54528d8880bab1d1377bd78be"}, + {file = "scikit_learn-1.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:671e2f0c3f2c15409dae4f282a3a619601fa824d2c820e5b608d9d775f91780c"}, + {file = "scikit_learn-1.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d36d0bc983336bbc1be22f9b686b50c964f593c8a9a913a792442af9bf4f5e68"}, + {file = "scikit_learn-1.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:d762070980c17ba3e9a4a1e043ba0518ce4c55152032f1af0ca6f39b376b5928"}, + {file = "scikit_learn-1.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9993d5e78a8148b1d0fdf5b15ed92452af5581734129998c26f481c46586d68"}, + {file = "scikit_learn-1.4.2-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:426d258fddac674fdf33f3cb2d54d26f49406e2599dbf9a32b4d1696091d4256"}, + {file = "scikit_learn-1.4.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5460a1a5b043ae5ae4596b3126a4ec33ccba1b51e7ca2c5d36dac2169f62ab1d"}, + {file = "scikit_learn-1.4.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49d64ef6cb8c093d883e5a36c4766548d974898d378e395ba41a806d0e824db8"}, + {file = "scikit_learn-1.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:c97a50b05c194be9146d61fe87dbf8eac62b203d9e87a3ccc6ae9aed2dfaf361"}, ] [package.dependencies] joblib = ">=1.2.0" -numpy = ">=1.19.5,<2.0" +numpy = ">=1.19.5" scipy = ">=1.6.0" threadpoolctl = ">=2.0.0" @@ -2653,55 +2644,55 @@ tests = ["black (>=23.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.3)", "numpydoc ( [[package]] name = "scipy" -version = "1.12.0" +version = "1.13.0" description = "Fundamental algorithms for scientific computing in Python" optional = false python-versions = ">=3.9" files = [ - {file = "scipy-1.12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:78e4402e140879387187f7f25d91cc592b3501a2e51dfb320f48dfb73565f10b"}, - {file = "scipy-1.12.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:f5f00ebaf8de24d14b8449981a2842d404152774c1a1d880c901bf454cb8e2a1"}, - {file = "scipy-1.12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e53958531a7c695ff66c2e7bb7b79560ffdc562e2051644c5576c39ff8efb563"}, - {file = "scipy-1.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e32847e08da8d895ce09d108a494d9eb78974cf6de23063f93306a3e419960c"}, - {file = "scipy-1.12.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4c1020cad92772bf44b8e4cdabc1df5d87376cb219742549ef69fc9fd86282dd"}, - {file = "scipy-1.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:75ea2a144096b5e39402e2ff53a36fecfd3b960d786b7efd3c180e29c39e53f2"}, - {file = "scipy-1.12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:408c68423f9de16cb9e602528be4ce0d6312b05001f3de61fe9ec8b1263cad08"}, - {file = "scipy-1.12.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5adfad5dbf0163397beb4aca679187d24aec085343755fcdbdeb32b3679f254c"}, - {file = "scipy-1.12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3003652496f6e7c387b1cf63f4bb720951cfa18907e998ea551e6de51a04467"}, - {file = "scipy-1.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b8066bce124ee5531d12a74b617d9ac0ea59245246410e19bca549656d9a40a"}, - {file = "scipy-1.12.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8bee4993817e204d761dba10dbab0774ba5a8612e57e81319ea04d84945375ba"}, - {file = "scipy-1.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:a24024d45ce9a675c1fb8494e8e5244efea1c7a09c60beb1eeb80373d0fecc70"}, - {file = "scipy-1.12.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e7e76cc48638228212c747ada851ef355c2bb5e7f939e10952bc504c11f4e372"}, - {file = "scipy-1.12.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f7ce148dffcd64ade37b2df9315541f9adad6efcaa86866ee7dd5db0c8f041c3"}, - {file = "scipy-1.12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c39f92041f490422924dfdb782527a4abddf4707616e07b021de33467f917bc"}, - {file = "scipy-1.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7ebda398f86e56178c2fa94cad15bf457a218a54a35c2a7b4490b9f9cb2676c"}, - {file = "scipy-1.12.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:95e5c750d55cf518c398a8240571b0e0782c2d5a703250872f36eaf737751338"}, - {file = "scipy-1.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:e646d8571804a304e1da01040d21577685ce8e2db08ac58e543eaca063453e1c"}, - {file = "scipy-1.12.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:913d6e7956c3a671de3b05ccb66b11bc293f56bfdef040583a7221d9e22a2e35"}, - {file = "scipy-1.12.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba1b0c7256ad75401c73e4b3cf09d1f176e9bd4248f0d3112170fb2ec4db067"}, - {file = "scipy-1.12.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:730badef9b827b368f351eacae2e82da414e13cf8bd5051b4bdfd720271a5371"}, - {file = "scipy-1.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6546dc2c11a9df6926afcbdd8a3edec28566e4e785b915e849348c6dd9f3f490"}, - {file = "scipy-1.12.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:196ebad3a4882081f62a5bf4aeb7326aa34b110e533aab23e4374fcccb0890dc"}, - {file = "scipy-1.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:b360f1b6b2f742781299514e99ff560d1fe9bd1bff2712894b52abe528d1fd1e"}, - {file = "scipy-1.12.0.tar.gz", hash = "sha256:4bf5abab8a36d20193c698b0f1fc282c1d083c94723902c447e5d2f1780936a3"}, -] - -[package.dependencies] -numpy = ">=1.22.4,<1.29.0" + {file = "scipy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ba419578ab343a4e0a77c0ef82f088238a93eef141b2b8017e46149776dfad4d"}, + {file = "scipy-1.13.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:22789b56a999265431c417d462e5b7f2b487e831ca7bef5edeb56efe4c93f86e"}, + {file = "scipy-1.13.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05f1432ba070e90d42d7fd836462c50bf98bd08bed0aa616c359eed8a04e3922"}, + {file = "scipy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8434f6f3fa49f631fae84afee424e2483289dfc30a47755b4b4e6b07b2633a4"}, + {file = "scipy-1.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:dcbb9ea49b0167de4167c40eeee6e167caeef11effb0670b554d10b1e693a8b9"}, + {file = "scipy-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:1d2f7bb14c178f8b13ebae93f67e42b0a6b0fc50eba1cd8021c9b6e08e8fb1cd"}, + {file = "scipy-1.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0fbcf8abaf5aa2dc8d6400566c1a727aed338b5fe880cde64907596a89d576fa"}, + {file = "scipy-1.13.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5e4a756355522eb60fcd61f8372ac2549073c8788f6114449b37e9e8104f15a5"}, + {file = "scipy-1.13.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5acd8e1dbd8dbe38d0004b1497019b2dbbc3d70691e65d69615f8a7292865d7"}, + {file = "scipy-1.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ff7dad5d24a8045d836671e082a490848e8639cabb3dbdacb29f943a678683d"}, + {file = "scipy-1.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4dca18c3ffee287ddd3bc8f1dabaf45f5305c5afc9f8ab9cbfab855e70b2df5c"}, + {file = "scipy-1.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:a2f471de4d01200718b2b8927f7d76b5d9bde18047ea0fa8bd15c5ba3f26a1d6"}, + {file = "scipy-1.13.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d0de696f589681c2802f9090fff730c218f7c51ff49bf252b6a97ec4a5d19e8b"}, + {file = "scipy-1.13.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:b2a3ff461ec4756b7e8e42e1c681077349a038f0686132d623fa404c0bee2551"}, + {file = "scipy-1.13.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf9fe63e7a4bf01d3645b13ff2aa6dea023d38993f42aaac81a18b1bda7a82a"}, + {file = "scipy-1.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e7626dfd91cdea5714f343ce1176b6c4745155d234f1033584154f60ef1ff42"}, + {file = "scipy-1.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:109d391d720fcebf2fbe008621952b08e52907cf4c8c7efc7376822151820820"}, + {file = "scipy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:8930ae3ea371d6b91c203b1032b9600d69c568e537b7988a3073dfe4d4774f21"}, + {file = "scipy-1.13.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5407708195cb38d70fd2d6bb04b1b9dd5c92297d86e9f9daae1576bd9e06f602"}, + {file = "scipy-1.13.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:ac38c4c92951ac0f729c4c48c9e13eb3675d9986cc0c83943784d7390d540c78"}, + {file = "scipy-1.13.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09c74543c4fbeb67af6ce457f6a6a28e5d3739a87f62412e4a16e46f164f0ae5"}, + {file = "scipy-1.13.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28e286bf9ac422d6beb559bc61312c348ca9b0f0dae0d7c5afde7f722d6ea13d"}, + {file = "scipy-1.13.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:33fde20efc380bd23a78a4d26d59fc8704e9b5fd9b08841693eb46716ba13d86"}, + {file = "scipy-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:45c08bec71d3546d606989ba6e7daa6f0992918171e2a6f7fbedfa7361c2de1e"}, + {file = "scipy-1.13.0.tar.gz", hash = "sha256:58569af537ea29d3f78e5abd18398459f195546bb3be23d16677fb26616cc11e"}, +] + +[package.dependencies] +numpy = ">=1.22.4,<2.3" [package.extras] -dev = ["click", "cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy", "pycodestyle", "pydevtool", "rich-click", "ruff", "types-psutil", "typing_extensions"] -doc = ["jupytext", "matplotlib (>2)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-design (>=0.2.0)"] -test = ["asv", "gmpy2", "hypothesis", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy", "pycodestyle", "pydevtool", "rich-click", "ruff", "types-psutil", "typing_extensions"] +doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.12.0)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0)", "sphinx-design (>=0.4.0)"] +test = ["array-api-strict", "asv", "gmpy2", "hypothesis (>=6.30)", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] [[package]] name = "sentry-sdk" -version = "1.43.0" +version = "1.45.0" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = "*" files = [ - {file = "sentry-sdk-1.43.0.tar.gz", hash = "sha256:41df73af89d22921d8733714fb0fc5586c3461907e06688e6537d01a27e0e0f6"}, - {file = "sentry_sdk-1.43.0-py2.py3-none-any.whl", hash = "sha256:8d768724839ca18d7b4c7463ef7528c40b7aa2bfbf7fe554d5f9a7c044acfd36"}, + {file = "sentry-sdk-1.45.0.tar.gz", hash = "sha256:509aa9678c0512344ca886281766c2e538682f8acfa50fd8d405f8c417ad0625"}, + {file = "sentry_sdk-1.45.0-py2.py3-none-any.whl", hash = "sha256:1ce29e30240cc289a027011103a8c83885b15ef2f316a60bcc7c5300afa144f1"}, ] [package.dependencies] @@ -2842,18 +2833,18 @@ test = ["pytest"] [[package]] name = "setuptools" -version = "69.2.0" +version = "69.5.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, - {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, + {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, + {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -3353,13 +3344,13 @@ vision = ["Pillow (>=10.0.1,<=15.0)"] [[package]] name = "typing-extensions" -version = "4.10.0" +version = "4.11.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, - {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, + {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, + {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, ] [[package]] diff --git a/pretrain.py b/pretrain.py index dc68dbe..0958377 100644 --- a/pretrain.py +++ b/pretrain.py @@ -37,14 +37,12 @@ def main(args: argparse.Namespace, model_config: Dict[str, Any]) -> None: args.sequence_file, args.id_file, ) - # pre_data.rename(columns={args.label_name: "label"}, inplace=True) # noqa: ERA001 # Split data pre_train, pre_val = train_test_split( pre_data, test_size=args.val_size, random_state=args.seed, - # stratify=pre_data["label"], # noqa: ERA001 ) # Train Tokenizer @@ -126,8 +124,10 @@ def main(args: argparse.Namespace, model_config: Dict[str, Any]) -> None: accelerator="gpu", num_nodes=args.nodes, devices=args.gpus, - strategy=DDPStrategy(find_unused_parameters=True) if args.gpus > 1 else "auto", - precision="16-mixed", + strategy=DDPStrategy(find_unused_parameters=True) + if args.gpus > 1 + else "auto", # DeepSpeedStrategy(stage=2, offload_optimizer=False) + precision="16", check_val_every_n_epoch=1, max_epochs=args.max_epochs, callbacks=callbacks, @@ -165,12 +165,6 @@ def main(args: argparse.Namespace, model_config: Dict[str, Any]) -> None: required=True, help="Path to model config file", ) - parser.add_argument( - "--label-name", - type=str, - required=True, - help="Name of the label column", - ) parser.add_argument( "--workspace-name", type=str, @@ -180,7 +174,7 @@ def main(args: argparse.Namespace, model_config: Dict[str, Any]) -> None: parser.add_argument( "--config-dir", type=str, - default="models/configs", + required=True, help="Path to model config file", ) @@ -188,25 +182,25 @@ def main(args: argparse.Namespace, model_config: Dict[str, Any]) -> None: parser.add_argument( "--data-dir", type=str, - default="data_files", + required=True, help="Path to the data directory", ) parser.add_argument( "--sequence-file", type=str, - default="patient_sequences_2048_labeled.parquet", + required=True, help="Path to the patient sequence file", ) parser.add_argument( "--id-file", type=str, - default="dataset_2048_mortality_1month.pkl", + required=True, help="Path to the patient id file", ) parser.add_argument( "--vocab-dir", type=str, - default="data_files/vocab", + required=True, help="Path to the vocabulary directory of json files", ) parser.add_argument( @@ -220,20 +214,20 @@ def main(args: argparse.Namespace, model_config: Dict[str, Any]) -> None: parser.add_argument( "--checkpoint-dir", type=str, - default="checkpoints", + required=True, help="Path to the checkpoint directory", ) parser.add_argument( - "--log-dir", + "--log_dir", type=str, default="logs", help="Path to the log directory", ) parser.add_argument( - "--checkpoint-path", + "--resume_checkpoint", type=str, default=None, - help="Checkpoint to resume training from", + help="Checkpoint to resume pretraining from", ) parser.add_argument( "--log_every_n_steps", diff --git a/pyproject.toml b/pyproject.toml index 5e2642c..5850d21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,6 +100,7 @@ ignore = [ "D213", # Multi-line docstring summary should start at the second line "PLR2004", # Replace magic number with named constant "PLR0913", # Too many arguments + "COM812", ] # Ignore import violations in all `__init__.py` files. From 1dcbb79e7766a76ac11133c0556099b53a90bb0a Mon Sep 17 00:00:00 2001 From: Amrit K Date: Tue, 16 Apr 2024 11:15:09 -0400 Subject: [PATCH 2/9] Fix imports --- evaluation/AttentionVisualization.ipynb | 8 ++++---- odyssey/models/cehr_big_bird/playground.ipynb | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/evaluation/AttentionVisualization.ipynb b/evaluation/AttentionVisualization.ipynb index 2395ee6..d452ffb 100644 --- a/evaluation/AttentionVisualization.ipynb +++ b/evaluation/AttentionVisualization.ipynb @@ -61,10 +61,10 @@ "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", "os.chdir(ROOT)\n", "\n", - "from lib.data import FinetuneDataset\n", - "from lib.prediction import load_finetuned_model, predict_patient_outcomes\n", - "from lib.tokenizer import ConceptTokenizer\n", - "from lib.utils import (\n", + "from odyssey.data.dataset import FinetuneDataset\n", + "from odyssey.models.prediction import load_finetuned_model, predict_patient_outcomes\n", + "from odyssey.data.tokenizer import ConceptTokenizer\n", + "from odyssey.models.utils import (\n", " load_finetune_data,\n", ")" ] diff --git a/odyssey/models/cehr_big_bird/playground.ipynb b/odyssey/models/cehr_big_bird/playground.ipynb index 1c26964..60ffb8f 100644 --- a/odyssey/models/cehr_big_bird/playground.ipynb +++ b/odyssey/models/cehr_big_bird/playground.ipynb @@ -26,9 +26,9 @@ "import pandas as pd\n", "from torch.utils.data import DataLoader\n", "\n", - "from models.big_bird_cehr.data import FinetuneDataset, PretrainDataset\n", - "from models.big_bird_cehr.model import BigBirdFinetune, BigBirdPretrain\n", - "from models.big_bird_cehr.tokenizer import HuggingFaceConceptTokenizer\n", + "from odyssey.data.dataset import FinetuneDataset, PretrainDataset\n", + "from odyssey.models.cehr_big_bird.model import BigBirdFinetune, BigBirdPretrain\n", + "from odyssey.models.cehr_big_bird.tokenizer import HuggingFaceConceptTokenizer\n", "\n", "\n", "DATA_ROOT = f\"{ROOT}/data/slurm_data/2048/one_month\"\n", From 45411f2c874fbcc6bbddfff19492548976ab7ca4 Mon Sep 17 00:00:00 2001 From: Amrit K Date: Tue, 16 Apr 2024 11:37:32 -0400 Subject: [PATCH 3/9] Fix imports in finetune.py, add captum, skmulti deps --- finetune.py | 16 +- poetry.lock | 475 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 2 + 3 files changed, 484 insertions(+), 9 deletions(-) diff --git a/finetune.py b/finetune.py index d67ba61..ae5898e 100644 --- a/finetune.py +++ b/finetune.py @@ -8,14 +8,6 @@ import numpy as np import pytorch_lightning as pl import torch -from lib.data import FinetuneDataset, FinetuneMultiDataset -from lib.tokenizer import ConceptTokenizer -from lib.utils import ( - get_run_id, - load_config, - load_finetune_data, - seed_everything, -) from lightning.pytorch.loggers import WandbLogger from pytorch_lightning.callbacks import ( EarlyStopping, @@ -27,8 +19,16 @@ from skmultilearn.model_selection import iterative_train_test_split from torch.utils.data import DataLoader +from odyssey.data.dataset import FinetuneDataset, FinetuneMultiDataset +from odyssey.data.tokenizer import ConceptTokenizer from odyssey.models.cehr_bert.model import BertFinetune, BertPretrain from odyssey.models.cehr_big_bird.model import BigBirdFinetune, BigBirdPretrain +from odyssey.models.utils import ( + get_run_id, + load_config, + load_finetune_data, + seed_everything, +) def main( diff --git a/poetry.lock b/poetry.lock index d2f7861..012211b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -247,6 +247,29 @@ files = [ [package.dependencies] black = ">=22.1.0" +[[package]] +name = "captum" +version = "0.7.0" +description = "Model interpretability for PyTorch" +optional = false +python-versions = ">=3.6" +files = [ + {file = "captum-0.7.0-py3-none-any.whl", hash = "sha256:2cbec9aa4b6ec325c2fdf369c1fdabb011017122e2314e2af009496d53a0757c"}, + {file = "captum-0.7.0.tar.gz", hash = "sha256:7ca87d0dc67b3b7589a730b970b9536172a5468e0e31bf8657fdd73abc568a33"}, +] + +[package.dependencies] +matplotlib = "*" +numpy = "*" +torch = ">=1.6" +tqdm = "*" + +[package.extras] +dev = ["annoy", "black (==22.3.0)", "flake8", "flask", "flask-compress", "ipython", "ipywidgets", "jupyter", "mypy (>=0.760)", "parameterized", "pytest", "pytest-cov", "scikit-learn", "sphinx", "sphinx-autodoc-typehints", "sphinxcontrib-katex", "ufmt", "usort (==1.0.2)"] +insights = ["flask", "flask-compress", "ipython", "ipywidgets", "jupyter"] +test = ["parameterized", "pytest", "pytest-cov"] +tutorials = ["flask", "flask-compress", "ipython", "ipywidgets", "jupyter", "torchtext", "torchvision"] + [[package]] name = "certifi" version = "2024.2.2" @@ -408,6 +431,69 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "contourpy" +version = "1.2.1" +description = "Python library for calculating contours of 2D quadrilateral grids" +optional = false +python-versions = ">=3.9" +files = [ + {file = "contourpy-1.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bd7c23df857d488f418439686d3b10ae2fbf9bc256cd045b37a8c16575ea1040"}, + {file = "contourpy-1.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5b9eb0ca724a241683c9685a484da9d35c872fd42756574a7cfbf58af26677fd"}, + {file = "contourpy-1.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c75507d0a55378240f781599c30e7776674dbaf883a46d1c90f37e563453480"}, + {file = "contourpy-1.2.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11959f0ce4a6f7b76ec578576a0b61a28bdc0696194b6347ba3f1c53827178b9"}, + {file = "contourpy-1.2.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eb3315a8a236ee19b6df481fc5f997436e8ade24a9f03dfdc6bd490fea20c6da"}, + {file = "contourpy-1.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39f3ecaf76cd98e802f094e0d4fbc6dc9c45a8d0c4d185f0f6c2234e14e5f75b"}, + {file = "contourpy-1.2.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:94b34f32646ca0414237168d68a9157cb3889f06b096612afdd296003fdd32fd"}, + {file = "contourpy-1.2.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:457499c79fa84593f22454bbd27670227874cd2ff5d6c84e60575c8b50a69619"}, + {file = "contourpy-1.2.1-cp310-cp310-win32.whl", hash = "sha256:ac58bdee53cbeba2ecad824fa8159493f0bf3b8ea4e93feb06c9a465d6c87da8"}, + {file = "contourpy-1.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:9cffe0f850e89d7c0012a1fb8730f75edd4320a0a731ed0c183904fe6ecfc3a9"}, + {file = "contourpy-1.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6022cecf8f44e36af10bd9118ca71f371078b4c168b6e0fab43d4a889985dbb5"}, + {file = "contourpy-1.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ef5adb9a3b1d0c645ff694f9bca7702ec2c70f4d734f9922ea34de02294fdf72"}, + {file = "contourpy-1.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6150ffa5c767bc6332df27157d95442c379b7dce3a38dff89c0f39b63275696f"}, + {file = "contourpy-1.2.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c863140fafc615c14a4bf4efd0f4425c02230eb8ef02784c9a156461e62c965"}, + {file = "contourpy-1.2.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:00e5388f71c1a0610e6fe56b5c44ab7ba14165cdd6d695429c5cd94021e390b2"}, + {file = "contourpy-1.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4492d82b3bc7fbb7e3610747b159869468079fe149ec5c4d771fa1f614a14df"}, + {file = "contourpy-1.2.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:49e70d111fee47284d9dd867c9bb9a7058a3c617274900780c43e38d90fe1205"}, + {file = "contourpy-1.2.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b59c0ffceff8d4d3996a45f2bb6f4c207f94684a96bf3d9728dbb77428dd8cb8"}, + {file = "contourpy-1.2.1-cp311-cp311-win32.whl", hash = "sha256:7b4182299f251060996af5249c286bae9361fa8c6a9cda5efc29fe8bfd6062ec"}, + {file = "contourpy-1.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2855c8b0b55958265e8b5888d6a615ba02883b225f2227461aa9127c578a4922"}, + {file = "contourpy-1.2.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:62828cada4a2b850dbef89c81f5a33741898b305db244904de418cc957ff05dc"}, + {file = "contourpy-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:309be79c0a354afff9ff7da4aaed7c3257e77edf6c1b448a779329431ee79d7e"}, + {file = "contourpy-1.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e785e0f2ef0d567099b9ff92cbfb958d71c2d5b9259981cd9bee81bd194c9a4"}, + {file = "contourpy-1.2.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1cac0a8f71a041aa587410424ad46dfa6a11f6149ceb219ce7dd48f6b02b87a7"}, + {file = "contourpy-1.2.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af3f4485884750dddd9c25cb7e3915d83c2db92488b38ccb77dd594eac84c4a0"}, + {file = "contourpy-1.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ce6889abac9a42afd07a562c2d6d4b2b7134f83f18571d859b25624a331c90b"}, + {file = "contourpy-1.2.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a1eea9aecf761c661d096d39ed9026574de8adb2ae1c5bd7b33558af884fb2ce"}, + {file = "contourpy-1.2.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:187fa1d4c6acc06adb0fae5544c59898ad781409e61a926ac7e84b8f276dcef4"}, + {file = "contourpy-1.2.1-cp312-cp312-win32.whl", hash = "sha256:c2528d60e398c7c4c799d56f907664673a807635b857df18f7ae64d3e6ce2d9f"}, + {file = "contourpy-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:1a07fc092a4088ee952ddae19a2b2a85757b923217b7eed584fdf25f53a6e7ce"}, + {file = "contourpy-1.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bb6834cbd983b19f06908b45bfc2dad6ac9479ae04abe923a275b5f48f1a186b"}, + {file = "contourpy-1.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1d59e739ab0e3520e62a26c60707cc3ab0365d2f8fecea74bfe4de72dc56388f"}, + {file = "contourpy-1.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd3db01f59fdcbce5b22afad19e390260d6d0222f35a1023d9adc5690a889364"}, + {file = "contourpy-1.2.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a12a813949e5066148712a0626895c26b2578874e4cc63160bb007e6df3436fe"}, + {file = "contourpy-1.2.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe0ccca550bb8e5abc22f530ec0466136379c01321fd94f30a22231e8a48d985"}, + {file = "contourpy-1.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1d59258c3c67c865435d8fbeb35f8c59b8bef3d6f46c1f29f6123556af28445"}, + {file = "contourpy-1.2.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f32c38afb74bd98ce26de7cc74a67b40afb7b05aae7b42924ea990d51e4dac02"}, + {file = "contourpy-1.2.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d31a63bc6e6d87f77d71e1abbd7387ab817a66733734883d1fc0021ed9bfa083"}, + {file = "contourpy-1.2.1-cp39-cp39-win32.whl", hash = "sha256:ddcb8581510311e13421b1f544403c16e901c4e8f09083c881fab2be80ee31ba"}, + {file = "contourpy-1.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:10a37ae557aabf2509c79715cd20b62e4c7c28b8cd62dd7d99e5ed3ce28c3fd9"}, + {file = "contourpy-1.2.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a31f94983fecbac95e58388210427d68cd30fe8a36927980fab9c20062645609"}, + {file = "contourpy-1.2.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef2b055471c0eb466033760a521efb9d8a32b99ab907fc8358481a1dd29e3bd3"}, + {file = "contourpy-1.2.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b33d2bc4f69caedcd0a275329eb2198f560b325605810895627be5d4b876bf7f"}, + {file = "contourpy-1.2.1.tar.gz", hash = "sha256:4d8908b3bee1c889e547867ca4cdc54e5ab6be6d3e078556814a22457f49423c"}, +] + +[package.dependencies] +numpy = ">=1.20" + +[package.extras] +bokeh = ["bokeh", "selenium"] +docs = ["furo", "sphinx (>=7.2)", "sphinx-copybutton"] +mypy = ["contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.8.0)", "types-Pillow"] +test = ["Pillow", "contourpy[test-no-images]", "matplotlib"] +test-no-images = ["pytest", "pytest-cov", "pytest-xdist", "wurlitzer"] + [[package]] name = "coverage" version = "7.4.4" @@ -475,6 +561,21 @@ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.1 [package.extras] toml = ["tomli"] +[[package]] +name = "cycler" +version = "0.12.1" +description = "Composable style cycles" +optional = false +python-versions = ">=3.8" +files = [ + {file = "cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30"}, + {file = "cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c"}, +] + +[package.extras] +docs = ["ipython", "matplotlib", "numpydoc", "sphinx"] +tests = ["pytest", "pytest-cov", "pytest-xdist"] + [[package]] name = "decorator" version = "5.1.1" @@ -618,6 +719,71 @@ mccabe = ">=0.7.0,<0.8.0" pycodestyle = ">=2.11.0,<2.12.0" pyflakes = ">=3.2.0,<3.3.0" +[[package]] +name = "fonttools" +version = "4.51.0" +description = "Tools to manipulate font files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fonttools-4.51.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:84d7751f4468dd8cdd03ddada18b8b0857a5beec80bce9f435742abc9a851a74"}, + {file = "fonttools-4.51.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b4850fa2ef2cfbc1d1f689bc159ef0f45d8d83298c1425838095bf53ef46308"}, + {file = "fonttools-4.51.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5b48a1121117047d82695d276c2af2ee3a24ffe0f502ed581acc2673ecf1037"}, + {file = "fonttools-4.51.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:180194c7fe60c989bb627d7ed5011f2bef1c4d36ecf3ec64daec8302f1ae0716"}, + {file = "fonttools-4.51.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:96a48e137c36be55e68845fc4284533bda2980f8d6f835e26bca79d7e2006438"}, + {file = "fonttools-4.51.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:806e7912c32a657fa39d2d6eb1d3012d35f841387c8fc6cf349ed70b7c340039"}, + {file = "fonttools-4.51.0-cp310-cp310-win32.whl", hash = "sha256:32b17504696f605e9e960647c5f64b35704782a502cc26a37b800b4d69ff3c77"}, + {file = "fonttools-4.51.0-cp310-cp310-win_amd64.whl", hash = "sha256:c7e91abdfae1b5c9e3a543f48ce96013f9a08c6c9668f1e6be0beabf0a569c1b"}, + {file = "fonttools-4.51.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a8feca65bab31479d795b0d16c9a9852902e3a3c0630678efb0b2b7941ea9c74"}, + {file = "fonttools-4.51.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8ac27f436e8af7779f0bb4d5425aa3535270494d3bc5459ed27de3f03151e4c2"}, + {file = "fonttools-4.51.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e19bd9e9964a09cd2433a4b100ca7f34e34731e0758e13ba9a1ed6e5468cc0f"}, + {file = "fonttools-4.51.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2b92381f37b39ba2fc98c3a45a9d6383bfc9916a87d66ccb6553f7bdd129097"}, + {file = "fonttools-4.51.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5f6bc991d1610f5c3bbe997b0233cbc234b8e82fa99fc0b2932dc1ca5e5afec0"}, + {file = "fonttools-4.51.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9696fe9f3f0c32e9a321d5268208a7cc9205a52f99b89479d1b035ed54c923f1"}, + {file = "fonttools-4.51.0-cp311-cp311-win32.whl", hash = "sha256:3bee3f3bd9fa1d5ee616ccfd13b27ca605c2b4270e45715bd2883e9504735034"}, + {file = "fonttools-4.51.0-cp311-cp311-win_amd64.whl", hash = "sha256:0f08c901d3866a8905363619e3741c33f0a83a680d92a9f0e575985c2634fcc1"}, + {file = "fonttools-4.51.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:4060acc2bfa2d8e98117828a238889f13b6f69d59f4f2d5857eece5277b829ba"}, + {file = "fonttools-4.51.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1250e818b5f8a679ad79660855528120a8f0288f8f30ec88b83db51515411fcc"}, + {file = "fonttools-4.51.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76f1777d8b3386479ffb4a282e74318e730014d86ce60f016908d9801af9ca2a"}, + {file = "fonttools-4.51.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b5ad456813d93b9c4b7ee55302208db2b45324315129d85275c01f5cb7e61a2"}, + {file = "fonttools-4.51.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:68b3fb7775a923be73e739f92f7e8a72725fd333eab24834041365d2278c3671"}, + {file = "fonttools-4.51.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8e2f1a4499e3b5ee82c19b5ee57f0294673125c65b0a1ff3764ea1f9db2f9ef5"}, + {file = "fonttools-4.51.0-cp312-cp312-win32.whl", hash = "sha256:278e50f6b003c6aed19bae2242b364e575bcb16304b53f2b64f6551b9c000e15"}, + {file = "fonttools-4.51.0-cp312-cp312-win_amd64.whl", hash = "sha256:b3c61423f22165541b9403ee39874dcae84cd57a9078b82e1dce8cb06b07fa2e"}, + {file = "fonttools-4.51.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:1621ee57da887c17312acc4b0e7ac30d3a4fb0fec6174b2e3754a74c26bbed1e"}, + {file = "fonttools-4.51.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e9d9298be7a05bb4801f558522adbe2feea1b0b103d5294ebf24a92dd49b78e5"}, + {file = "fonttools-4.51.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee1af4be1c5afe4c96ca23badd368d8dc75f611887fb0c0dac9f71ee5d6f110e"}, + {file = "fonttools-4.51.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c18b49adc721a7d0b8dfe7c3130c89b8704baf599fb396396d07d4aa69b824a1"}, + {file = "fonttools-4.51.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de7c29bdbdd35811f14493ffd2534b88f0ce1b9065316433b22d63ca1cd21f14"}, + {file = "fonttools-4.51.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cadf4e12a608ef1d13e039864f484c8a968840afa0258b0b843a0556497ea9ed"}, + {file = "fonttools-4.51.0-cp38-cp38-win32.whl", hash = "sha256:aefa011207ed36cd280babfaa8510b8176f1a77261833e895a9d96e57e44802f"}, + {file = "fonttools-4.51.0-cp38-cp38-win_amd64.whl", hash = "sha256:865a58b6e60b0938874af0968cd0553bcd88e0b2cb6e588727117bd099eef836"}, + {file = "fonttools-4.51.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:60a3409c9112aec02d5fb546f557bca6efa773dcb32ac147c6baf5f742e6258b"}, + {file = "fonttools-4.51.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f7e89853d8bea103c8e3514b9f9dc86b5b4120afb4583b57eb10dfa5afbe0936"}, + {file = "fonttools-4.51.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56fc244f2585d6c00b9bcc59e6593e646cf095a96fe68d62cd4da53dd1287b55"}, + {file = "fonttools-4.51.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d145976194a5242fdd22df18a1b451481a88071feadf251221af110ca8f00ce"}, + {file = "fonttools-4.51.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c5b8cab0c137ca229433570151b5c1fc6af212680b58b15abd797dcdd9dd5051"}, + {file = "fonttools-4.51.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:54dcf21a2f2d06ded676e3c3f9f74b2bafded3a8ff12f0983160b13e9f2fb4a7"}, + {file = "fonttools-4.51.0-cp39-cp39-win32.whl", hash = "sha256:0118ef998a0699a96c7b28457f15546815015a2710a1b23a7bf6c1be60c01636"}, + {file = "fonttools-4.51.0-cp39-cp39-win_amd64.whl", hash = "sha256:599bdb75e220241cedc6faebfafedd7670335d2e29620d207dd0378a4e9ccc5a"}, + {file = "fonttools-4.51.0-py3-none-any.whl", hash = "sha256:15c94eeef6b095831067f72c825eb0e2d48bb4cea0647c1b05c981ecba2bf39f"}, + {file = "fonttools-4.51.0.tar.gz", hash = "sha256:dc0673361331566d7a663d7ce0f6fdcbfbdc1f59c6e3ed1165ad7202ca183c68"}, +] + +[package.extras] +all = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "fs (>=2.2.0,<3)", "lxml (>=4.0)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres", "pycairo", "scipy", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=15.1.0)", "xattr", "zopfli (>=0.1.4)"] +graphite = ["lz4 (>=1.7.4.2)"] +interpolatable = ["munkres", "pycairo", "scipy"] +lxml = ["lxml (>=4.0)"] +pathops = ["skia-pathops (>=0.5.0)"] +plot = ["matplotlib"] +repacker = ["uharfbuzz (>=0.23.0)"] +symfont = ["sympy"] +type1 = ["xattr"] +ufo = ["fs (>=2.2.0,<3)"] +unicode = ["unicodedata2 (>=15.1.0)"] +woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] + [[package]] name = "frozenlist" version = "1.4.1" @@ -904,6 +1070,24 @@ files = [ {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] +[[package]] +name = "importlib-resources" +version = "6.4.0" +description = "Read resources from Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "importlib_resources-6.4.0-py3-none-any.whl", hash = "sha256:50d10f043df931902d4194ea07ec57960f66a80449ff867bfe782b4c486ba78c"}, + {file = "importlib_resources-6.4.0.tar.gz", hash = "sha256:cdb2b453b8046ca4e3798eb1d84f3cce1446a0e8e7b5ef4efb600f19fc398145"}, +] + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["jaraco.test (>=5.4)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)", "zipp (>=3.17)"] + [[package]] name = "iniconfig" version = "2.0.0" @@ -1105,6 +1289,119 @@ files = [ {file = "keras-2.15.0.tar.gz", hash = "sha256:81871d298c064dc4ac6b58440fdae67bfcf47c8d7ad28580fab401834c06a575"}, ] +[[package]] +name = "kiwisolver" +version = "1.4.5" +description = "A fast implementation of the Cassowary constraint solver" +optional = false +python-versions = ">=3.7" +files = [ + {file = "kiwisolver-1.4.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:05703cf211d585109fcd72207a31bb170a0f22144d68298dc5e61b3c946518af"}, + {file = "kiwisolver-1.4.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:146d14bebb7f1dc4d5fbf74f8a6cb15ac42baadee8912eb84ac0b3b2a3dc6ac3"}, + {file = "kiwisolver-1.4.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6ef7afcd2d281494c0a9101d5c571970708ad911d028137cd558f02b851c08b4"}, + {file = "kiwisolver-1.4.5-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9eaa8b117dc8337728e834b9c6e2611f10c79e38f65157c4c38e9400286f5cb1"}, + {file = "kiwisolver-1.4.5-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ec20916e7b4cbfb1f12380e46486ec4bcbaa91a9c448b97023fde0d5bbf9e4ff"}, + {file = "kiwisolver-1.4.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39b42c68602539407884cf70d6a480a469b93b81b7701378ba5e2328660c847a"}, + {file = "kiwisolver-1.4.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa12042de0171fad672b6c59df69106d20d5596e4f87b5e8f76df757a7c399aa"}, + {file = "kiwisolver-1.4.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a40773c71d7ccdd3798f6489aaac9eee213d566850a9533f8d26332d626b82c"}, + {file = "kiwisolver-1.4.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:19df6e621f6d8b4b9c4d45f40a66839294ff2bb235e64d2178f7522d9170ac5b"}, + {file = "kiwisolver-1.4.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:83d78376d0d4fd884e2c114d0621624b73d2aba4e2788182d286309ebdeed770"}, + {file = "kiwisolver-1.4.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e391b1f0a8a5a10ab3b9bb6afcfd74f2175f24f8975fb87ecae700d1503cdee0"}, + {file = "kiwisolver-1.4.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:852542f9481f4a62dbb5dd99e8ab7aedfeb8fb6342349a181d4036877410f525"}, + {file = "kiwisolver-1.4.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59edc41b24031bc25108e210c0def6f6c2191210492a972d585a06ff246bb79b"}, + {file = "kiwisolver-1.4.5-cp310-cp310-win32.whl", hash = "sha256:a6aa6315319a052b4ee378aa171959c898a6183f15c1e541821c5c59beaa0238"}, + {file = "kiwisolver-1.4.5-cp310-cp310-win_amd64.whl", hash = "sha256:d0ef46024e6a3d79c01ff13801cb19d0cad7fd859b15037aec74315540acc276"}, + {file = "kiwisolver-1.4.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:11863aa14a51fd6ec28688d76f1735f8f69ab1fabf388851a595d0721af042f5"}, + {file = "kiwisolver-1.4.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8ab3919a9997ab7ef2fbbed0cc99bb28d3c13e6d4b1ad36e97e482558a91be90"}, + {file = "kiwisolver-1.4.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fcc700eadbbccbf6bc1bcb9dbe0786b4b1cb91ca0dcda336eef5c2beed37b797"}, + {file = "kiwisolver-1.4.5-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dfdd7c0b105af050eb3d64997809dc21da247cf44e63dc73ff0fd20b96be55a9"}, + {file = "kiwisolver-1.4.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76c6a5964640638cdeaa0c359382e5703e9293030fe730018ca06bc2010c4437"}, + {file = "kiwisolver-1.4.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bbea0db94288e29afcc4c28afbf3a7ccaf2d7e027489c449cf7e8f83c6346eb9"}, + {file = "kiwisolver-1.4.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ceec1a6bc6cab1d6ff5d06592a91a692f90ec7505d6463a88a52cc0eb58545da"}, + {file = "kiwisolver-1.4.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:040c1aebeda72197ef477a906782b5ab0d387642e93bda547336b8957c61022e"}, + {file = "kiwisolver-1.4.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f91de7223d4c7b793867797bacd1ee53bfe7359bd70d27b7b58a04efbb9436c8"}, + {file = "kiwisolver-1.4.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:faae4860798c31530dd184046a900e652c95513796ef51a12bc086710c2eec4d"}, + {file = "kiwisolver-1.4.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:b0157420efcb803e71d1b28e2c287518b8808b7cf1ab8af36718fd0a2c453eb0"}, + {file = "kiwisolver-1.4.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:06f54715b7737c2fecdbf140d1afb11a33d59508a47bf11bb38ecf21dc9ab79f"}, + {file = "kiwisolver-1.4.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fdb7adb641a0d13bdcd4ef48e062363d8a9ad4a182ac7647ec88f695e719ae9f"}, + {file = "kiwisolver-1.4.5-cp311-cp311-win32.whl", hash = "sha256:bb86433b1cfe686da83ce32a9d3a8dd308e85c76b60896d58f082136f10bffac"}, + {file = "kiwisolver-1.4.5-cp311-cp311-win_amd64.whl", hash = "sha256:6c08e1312a9cf1074d17b17728d3dfce2a5125b2d791527f33ffbe805200a355"}, + {file = "kiwisolver-1.4.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:32d5cf40c4f7c7b3ca500f8985eb3fb3a7dfc023215e876f207956b5ea26632a"}, + {file = "kiwisolver-1.4.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f846c260f483d1fd217fe5ed7c173fb109efa6b1fc8381c8b7552c5781756192"}, + {file = "kiwisolver-1.4.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5ff5cf3571589b6d13bfbfd6bcd7a3f659e42f96b5fd1c4830c4cf21d4f5ef45"}, + {file = "kiwisolver-1.4.5-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7269d9e5f1084a653d575c7ec012ff57f0c042258bf5db0954bf551c158466e7"}, + {file = "kiwisolver-1.4.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da802a19d6e15dffe4b0c24b38b3af68e6c1a68e6e1d8f30148c83864f3881db"}, + {file = "kiwisolver-1.4.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3aba7311af82e335dd1e36ffff68aaca609ca6290c2cb6d821a39aa075d8e3ff"}, + {file = "kiwisolver-1.4.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:763773d53f07244148ccac5b084da5adb90bfaee39c197554f01b286cf869228"}, + {file = "kiwisolver-1.4.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2270953c0d8cdab5d422bee7d2007f043473f9d2999631c86a223c9db56cbd16"}, + {file = "kiwisolver-1.4.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d099e745a512f7e3bbe7249ca835f4d357c586d78d79ae8f1dcd4d8adeb9bda9"}, + {file = "kiwisolver-1.4.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:74db36e14a7d1ce0986fa104f7d5637aea5c82ca6326ed0ec5694280942d1162"}, + {file = "kiwisolver-1.4.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e5bab140c309cb3a6ce373a9e71eb7e4873c70c2dda01df6820474f9889d6d4"}, + {file = "kiwisolver-1.4.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0f114aa76dc1b8f636d077979c0ac22e7cd8f3493abbab152f20eb8d3cda71f3"}, + {file = "kiwisolver-1.4.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:88a2df29d4724b9237fc0c6eaf2a1adae0cdc0b3e9f4d8e7dc54b16812d2d81a"}, + {file = "kiwisolver-1.4.5-cp312-cp312-win32.whl", hash = "sha256:72d40b33e834371fd330fb1472ca19d9b8327acb79a5821d4008391db8e29f20"}, + {file = "kiwisolver-1.4.5-cp312-cp312-win_amd64.whl", hash = "sha256:2c5674c4e74d939b9d91dda0fae10597ac7521768fec9e399c70a1f27e2ea2d9"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3a2b053a0ab7a3960c98725cfb0bf5b48ba82f64ec95fe06f1d06c99b552e130"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cd32d6c13807e5c66a7cbb79f90b553642f296ae4518a60d8d76243b0ad2898"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59ec7b7c7e1a61061850d53aaf8e93db63dce0c936db1fda2658b70e4a1be709"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da4cfb373035def307905d05041c1d06d8936452fe89d464743ae7fb8371078b"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2400873bccc260b6ae184b2b8a4fec0e4082d30648eadb7c3d9a13405d861e89"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:1b04139c4236a0f3aff534479b58f6f849a8b351e1314826c2d230849ed48985"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:4e66e81a5779b65ac21764c295087de82235597a2293d18d943f8e9e32746265"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:7931d8f1f67c4be9ba1dd9c451fb0eeca1a25b89e4d3f89e828fe12a519b782a"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:b3f7e75f3015df442238cca659f8baa5f42ce2a8582727981cbfa15fee0ee205"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:bbf1d63eef84b2e8c89011b7f2235b1e0bf7dacc11cac9431fc6468e99ac77fb"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4c380469bd3f970ef677bf2bcba2b6b0b4d5c75e7a020fb863ef75084efad66f"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-win32.whl", hash = "sha256:9408acf3270c4b6baad483865191e3e582b638b1654a007c62e3efe96f09a9a3"}, + {file = "kiwisolver-1.4.5-cp37-cp37m-win_amd64.whl", hash = "sha256:5b94529f9b2591b7af5f3e0e730a4e0a41ea174af35a4fd067775f9bdfeee01a"}, + {file = "kiwisolver-1.4.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:11c7de8f692fc99816e8ac50d1d1aef4f75126eefc33ac79aac02c099fd3db71"}, + {file = "kiwisolver-1.4.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:53abb58632235cd154176ced1ae8f0d29a6657aa1aa9decf50b899b755bc2b93"}, + {file = "kiwisolver-1.4.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:88b9f257ca61b838b6f8094a62418421f87ac2a1069f7e896c36a7d86b5d4c29"}, + {file = "kiwisolver-1.4.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3195782b26fc03aa9c6913d5bad5aeb864bdc372924c093b0f1cebad603dd712"}, + {file = "kiwisolver-1.4.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc579bf0f502e54926519451b920e875f433aceb4624a3646b3252b5caa9e0b6"}, + {file = "kiwisolver-1.4.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a580c91d686376f0f7c295357595c5a026e6cbc3d77b7c36e290201e7c11ecb"}, + {file = "kiwisolver-1.4.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cfe6ab8da05c01ba6fbea630377b5da2cd9bcbc6338510116b01c1bc939a2c18"}, + {file = "kiwisolver-1.4.5-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:d2e5a98f0ec99beb3c10e13b387f8db39106d53993f498b295f0c914328b1333"}, + {file = "kiwisolver-1.4.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a51a263952b1429e429ff236d2f5a21c5125437861baeed77f5e1cc2d2c7c6da"}, + {file = "kiwisolver-1.4.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3edd2fa14e68c9be82c5b16689e8d63d89fe927e56debd6e1dbce7a26a17f81b"}, + {file = "kiwisolver-1.4.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:74d1b44c6cfc897df648cc9fdaa09bc3e7679926e6f96df05775d4fb3946571c"}, + {file = "kiwisolver-1.4.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:76d9289ed3f7501012e05abb8358bbb129149dbd173f1f57a1bf1c22d19ab7cc"}, + {file = "kiwisolver-1.4.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:92dea1ffe3714fa8eb6a314d2b3c773208d865a0e0d35e713ec54eea08a66250"}, + {file = "kiwisolver-1.4.5-cp38-cp38-win32.whl", hash = "sha256:5c90ae8c8d32e472be041e76f9d2f2dbff4d0b0be8bd4041770eddb18cf49a4e"}, + {file = "kiwisolver-1.4.5-cp38-cp38-win_amd64.whl", hash = "sha256:c7940c1dc63eb37a67721b10d703247552416f719c4188c54e04334321351ced"}, + {file = "kiwisolver-1.4.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9407b6a5f0d675e8a827ad8742e1d6b49d9c1a1da5d952a67d50ef5f4170b18d"}, + {file = "kiwisolver-1.4.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:15568384086b6df3c65353820a4473575dbad192e35010f622c6ce3eebd57af9"}, + {file = "kiwisolver-1.4.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0dc9db8e79f0036e8173c466d21ef18e1befc02de8bf8aa8dc0813a6dc8a7046"}, + {file = "kiwisolver-1.4.5-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:cdc8a402aaee9a798b50d8b827d7ecf75edc5fb35ea0f91f213ff927c15f4ff0"}, + {file = "kiwisolver-1.4.5-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6c3bd3cde54cafb87d74d8db50b909705c62b17c2099b8f2e25b461882e544ff"}, + {file = "kiwisolver-1.4.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:955e8513d07a283056b1396e9a57ceddbd272d9252c14f154d450d227606eb54"}, + {file = "kiwisolver-1.4.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:346f5343b9e3f00b8db8ba359350eb124b98c99efd0b408728ac6ebf38173958"}, + {file = "kiwisolver-1.4.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b9098e0049e88c6a24ff64545cdfc50807818ba6c1b739cae221bbbcbc58aad3"}, + {file = "kiwisolver-1.4.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:00bd361b903dc4bbf4eb165f24d1acbee754fce22ded24c3d56eec268658a5cf"}, + {file = "kiwisolver-1.4.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7b8b454bac16428b22560d0a1cf0a09875339cab69df61d7805bf48919415901"}, + {file = "kiwisolver-1.4.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:f1d072c2eb0ad60d4c183f3fb44ac6f73fb7a8f16a2694a91f988275cbf352f9"}, + {file = "kiwisolver-1.4.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:31a82d498054cac9f6d0b53d02bb85811185bcb477d4b60144f915f3b3126342"}, + {file = "kiwisolver-1.4.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6512cb89e334e4700febbffaaa52761b65b4f5a3cf33f960213d5656cea36a77"}, + {file = "kiwisolver-1.4.5-cp39-cp39-win32.whl", hash = "sha256:9db8ea4c388fdb0f780fe91346fd438657ea602d58348753d9fb265ce1bca67f"}, + {file = "kiwisolver-1.4.5-cp39-cp39-win_amd64.whl", hash = "sha256:59415f46a37f7f2efeec758353dd2eae1b07640d8ca0f0c42548ec4125492635"}, + {file = "kiwisolver-1.4.5-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5c7b3b3a728dc6faf3fc372ef24f21d1e3cee2ac3e9596691d746e5a536de920"}, + {file = "kiwisolver-1.4.5-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:620ced262a86244e2be10a676b646f29c34537d0d9cc8eb26c08f53d98013390"}, + {file = "kiwisolver-1.4.5-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:378a214a1e3bbf5ac4a8708304318b4f890da88c9e6a07699c4ae7174c09a68d"}, + {file = "kiwisolver-1.4.5-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf7be1207676ac608a50cd08f102f6742dbfc70e8d60c4db1c6897f62f71523"}, + {file = "kiwisolver-1.4.5-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:ba55dce0a9b8ff59495ddd050a0225d58bd0983d09f87cfe2b6aec4f2c1234e4"}, + {file = "kiwisolver-1.4.5-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fd32ea360bcbb92d28933fc05ed09bffcb1704ba3fc7942e81db0fd4f81a7892"}, + {file = "kiwisolver-1.4.5-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5e7139af55d1688f8b960ee9ad5adafc4ac17c1c473fe07133ac092310d76544"}, + {file = "kiwisolver-1.4.5-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:dced8146011d2bc2e883f9bd68618b8247387f4bbec46d7392b3c3b032640126"}, + {file = "kiwisolver-1.4.5-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9bf3325c47b11b2e51bca0824ea217c7cd84491d8ac4eefd1e409705ef092bd"}, + {file = "kiwisolver-1.4.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5794cf59533bc3f1b1c821f7206a3617999db9fbefc345360aafe2e067514929"}, + {file = "kiwisolver-1.4.5-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e368f200bbc2e4f905b8e71eb38b3c04333bddaa6a2464a6355487b02bb7fb09"}, + {file = "kiwisolver-1.4.5-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5d706eba36b4c4d5bc6c6377bb6568098765e990cfc21ee16d13963fab7b3e7"}, + {file = "kiwisolver-1.4.5-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85267bd1aa8880a9c88a8cb71e18d3d64d2751a790e6ca6c27b8ccc724bcd5ad"}, + {file = "kiwisolver-1.4.5-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210ef2c3a1f03272649aff1ef992df2e724748918c4bc2d5a90352849eb40bea"}, + {file = "kiwisolver-1.4.5-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:11d011a7574eb3b82bcc9c1a1d35c1d7075677fdd15de527d91b46bd35e935ee"}, + {file = "kiwisolver-1.4.5.tar.gz", hash = "sha256:e57e563a57fb22a142da34f38acc2fc1a5c864bc29ca1517a88abc963e60d6ec"}, +] + [[package]] name = "lightning" version = "2.2.0" @@ -1209,6 +1506,55 @@ profiling = ["gprof2dot"] rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] +[[package]] +name = "matplotlib" +version = "3.8.4" +description = "Python plotting package" +optional = false +python-versions = ">=3.9" +files = [ + {file = "matplotlib-3.8.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:abc9d838f93583650c35eca41cfcec65b2e7cb50fd486da6f0c49b5e1ed23014"}, + {file = "matplotlib-3.8.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f65c9f002d281a6e904976007b2d46a1ee2bcea3a68a8c12dda24709ddc9106"}, + {file = "matplotlib-3.8.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce1edd9f5383b504dbc26eeea404ed0a00656c526638129028b758fd43fc5f10"}, + {file = "matplotlib-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecd79298550cba13a43c340581a3ec9c707bd895a6a061a78fa2524660482fc0"}, + {file = "matplotlib-3.8.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:90df07db7b599fe7035d2f74ab7e438b656528c68ba6bb59b7dc46af39ee48ef"}, + {file = "matplotlib-3.8.4-cp310-cp310-win_amd64.whl", hash = "sha256:ac24233e8f2939ac4fd2919eed1e9c0871eac8057666070e94cbf0b33dd9c338"}, + {file = "matplotlib-3.8.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:72f9322712e4562e792b2961971891b9fbbb0e525011e09ea0d1f416c4645661"}, + {file = "matplotlib-3.8.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:232ce322bfd020a434caaffbd9a95333f7c2491e59cfc014041d95e38ab90d1c"}, + {file = "matplotlib-3.8.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6addbd5b488aedb7f9bc19f91cd87ea476206f45d7116fcfe3d31416702a82fa"}, + {file = "matplotlib-3.8.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc4ccdc64e3039fc303defd119658148f2349239871db72cd74e2eeaa9b80b71"}, + {file = "matplotlib-3.8.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b7a2a253d3b36d90c8993b4620183b55665a429da8357a4f621e78cd48b2b30b"}, + {file = "matplotlib-3.8.4-cp311-cp311-win_amd64.whl", hash = "sha256:8080d5081a86e690d7688ffa542532e87f224c38a6ed71f8fbed34dd1d9fedae"}, + {file = "matplotlib-3.8.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6485ac1f2e84676cff22e693eaa4fbed50ef5dc37173ce1f023daef4687df616"}, + {file = "matplotlib-3.8.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c89ee9314ef48c72fe92ce55c4e95f2f39d70208f9f1d9db4e64079420d8d732"}, + {file = "matplotlib-3.8.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50bac6e4d77e4262c4340d7a985c30912054745ec99756ce213bfbc3cb3808eb"}, + {file = "matplotlib-3.8.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f51c4c869d4b60d769f7b4406eec39596648d9d70246428745a681c327a8ad30"}, + {file = "matplotlib-3.8.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b12ba985837e4899b762b81f5b2845bd1a28f4fdd1a126d9ace64e9c4eb2fb25"}, + {file = "matplotlib-3.8.4-cp312-cp312-win_amd64.whl", hash = "sha256:7a6769f58ce51791b4cb8b4d7642489df347697cd3e23d88266aaaee93b41d9a"}, + {file = "matplotlib-3.8.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:843cbde2f0946dadd8c5c11c6d91847abd18ec76859dc319362a0964493f0ba6"}, + {file = "matplotlib-3.8.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1c13f041a7178f9780fb61cc3a2b10423d5e125480e4be51beaf62b172413b67"}, + {file = "matplotlib-3.8.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb44f53af0a62dc80bba4443d9b27f2fde6acfdac281d95bc872dc148a6509cc"}, + {file = "matplotlib-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:606e3b90897554c989b1e38a258c626d46c873523de432b1462f295db13de6f9"}, + {file = "matplotlib-3.8.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9bb0189011785ea794ee827b68777db3ca3f93f3e339ea4d920315a0e5a78d54"}, + {file = "matplotlib-3.8.4-cp39-cp39-win_amd64.whl", hash = "sha256:6209e5c9aaccc056e63b547a8152661324404dd92340a6e479b3a7f24b42a5d0"}, + {file = "matplotlib-3.8.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c7064120a59ce6f64103c9cefba8ffe6fba87f2c61d67c401186423c9a20fd35"}, + {file = "matplotlib-3.8.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0e47eda4eb2614300fc7bb4657fced3e83d6334d03da2173b09e447418d499f"}, + {file = "matplotlib-3.8.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:493e9f6aa5819156b58fce42b296ea31969f2aab71c5b680b4ea7a3cb5c07d94"}, + {file = "matplotlib-3.8.4.tar.gz", hash = "sha256:8aac397d5e9ec158960e31c381c5ffc52ddd52bd9a47717e2a694038167dffea"}, +] + +[package.dependencies] +contourpy = ">=1.0.1" +cycler = ">=0.10" +fonttools = ">=4.22.0" +importlib-resources = {version = ">=3.2.0", markers = "python_version < \"3.10\""} +kiwisolver = ">=1.3.1" +numpy = ">=1.21" +packaging = ">=20.0" +pillow = ">=8" +pyparsing = ">=2.3.1" +python-dateutil = ">=2.7" + [[package]] name = "matplotlib-inline" version = "0.1.7" @@ -1728,6 +2074,92 @@ files = [ [package.dependencies] ptyprocess = ">=0.5" +[[package]] +name = "pillow" +version = "10.3.0" +description = "Python Imaging Library (Fork)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pillow-10.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:90b9e29824800e90c84e4022dd5cc16eb2d9605ee13f05d47641eb183cd73d45"}, + {file = "pillow-10.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a2c405445c79c3f5a124573a051062300936b0281fee57637e706453e452746c"}, + {file = "pillow-10.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78618cdbccaa74d3f88d0ad6cb8ac3007f1a6fa5c6f19af64b55ca170bfa1edf"}, + {file = "pillow-10.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:261ddb7ca91fcf71757979534fb4c128448b5b4c55cb6152d280312062f69599"}, + {file = "pillow-10.3.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:ce49c67f4ea0609933d01c0731b34b8695a7a748d6c8d186f95e7d085d2fe475"}, + {file = "pillow-10.3.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b14f16f94cbc61215115b9b1236f9c18403c15dd3c52cf629072afa9d54c1cbf"}, + {file = "pillow-10.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d33891be6df59d93df4d846640f0e46f1a807339f09e79a8040bc887bdcd7ed3"}, + {file = "pillow-10.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b50811d664d392f02f7761621303eba9d1b056fb1868c8cdf4231279645c25f5"}, + {file = "pillow-10.3.0-cp310-cp310-win32.whl", hash = "sha256:ca2870d5d10d8726a27396d3ca4cf7976cec0f3cb706debe88e3a5bd4610f7d2"}, + {file = "pillow-10.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:f0d0591a0aeaefdaf9a5e545e7485f89910c977087e7de2b6c388aec32011e9f"}, + {file = "pillow-10.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:ccce24b7ad89adb5a1e34a6ba96ac2530046763912806ad4c247356a8f33a67b"}, + {file = "pillow-10.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:5f77cf66e96ae734717d341c145c5949c63180842a545c47a0ce7ae52ca83795"}, + {file = "pillow-10.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e4b878386c4bf293578b48fc570b84ecfe477d3b77ba39a6e87150af77f40c57"}, + {file = "pillow-10.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdcbb4068117dfd9ce0138d068ac512843c52295ed996ae6dd1faf537b6dbc27"}, + {file = "pillow-10.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9797a6c8fe16f25749b371c02e2ade0efb51155e767a971c61734b1bf6293994"}, + {file = "pillow-10.3.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:9e91179a242bbc99be65e139e30690e081fe6cb91a8e77faf4c409653de39451"}, + {file = "pillow-10.3.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:1b87bd9d81d179bd8ab871603bd80d8645729939f90b71e62914e816a76fc6bd"}, + {file = "pillow-10.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:81d09caa7b27ef4e61cb7d8fbf1714f5aec1c6b6c5270ee53504981e6e9121ad"}, + {file = "pillow-10.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:048ad577748b9fa4a99a0548c64f2cb8d672d5bf2e643a739ac8faff1164238c"}, + {file = "pillow-10.3.0-cp311-cp311-win32.whl", hash = "sha256:7161ec49ef0800947dc5570f86568a7bb36fa97dd09e9827dc02b718c5643f09"}, + {file = "pillow-10.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:8eb0908e954d093b02a543dc963984d6e99ad2b5e36503d8a0aaf040505f747d"}, + {file = "pillow-10.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:4e6f7d1c414191c1199f8996d3f2282b9ebea0945693fb67392c75a3a320941f"}, + {file = "pillow-10.3.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:e46f38133e5a060d46bd630faa4d9fa0202377495df1f068a8299fd78c84de84"}, + {file = "pillow-10.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:50b8eae8f7334ec826d6eeffaeeb00e36b5e24aa0b9df322c247539714c6df19"}, + {file = "pillow-10.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d3bea1c75f8c53ee4d505c3e67d8c158ad4df0d83170605b50b64025917f338"}, + {file = "pillow-10.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19aeb96d43902f0a783946a0a87dbdad5c84c936025b8419da0a0cd7724356b1"}, + {file = "pillow-10.3.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:74d28c17412d9caa1066f7a31df8403ec23d5268ba46cd0ad2c50fb82ae40462"}, + {file = "pillow-10.3.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:ff61bfd9253c3915e6d41c651d5f962da23eda633cf02262990094a18a55371a"}, + {file = "pillow-10.3.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d886f5d353333b4771d21267c7ecc75b710f1a73d72d03ca06df49b09015a9ef"}, + {file = "pillow-10.3.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4b5ec25d8b17217d635f8935dbc1b9aa5907962fae29dff220f2659487891cd3"}, + {file = "pillow-10.3.0-cp312-cp312-win32.whl", hash = "sha256:51243f1ed5161b9945011a7360e997729776f6e5d7005ba0c6879267d4c5139d"}, + {file = "pillow-10.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:412444afb8c4c7a6cc11a47dade32982439925537e483be7c0ae0cf96c4f6a0b"}, + {file = "pillow-10.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:798232c92e7665fe82ac085f9d8e8ca98826f8e27859d9a96b41d519ecd2e49a"}, + {file = "pillow-10.3.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:4eaa22f0d22b1a7e93ff0a596d57fdede2e550aecffb5a1ef1106aaece48e96b"}, + {file = "pillow-10.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cd5e14fbf22a87321b24c88669aad3a51ec052eb145315b3da3b7e3cc105b9a2"}, + {file = "pillow-10.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1530e8f3a4b965eb6a7785cf17a426c779333eb62c9a7d1bbcf3ffd5bf77a4aa"}, + {file = "pillow-10.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d512aafa1d32efa014fa041d38868fda85028e3f930a96f85d49c7d8ddc0383"}, + {file = "pillow-10.3.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:339894035d0ede518b16073bdc2feef4c991ee991a29774b33e515f1d308e08d"}, + {file = "pillow-10.3.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:aa7e402ce11f0885305bfb6afb3434b3cd8f53b563ac065452d9d5654c7b86fd"}, + {file = "pillow-10.3.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0ea2a783a2bdf2a561808fe4a7a12e9aa3799b701ba305de596bc48b8bdfce9d"}, + {file = "pillow-10.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c78e1b00a87ce43bb37642c0812315b411e856a905d58d597750eb79802aaaa3"}, + {file = "pillow-10.3.0-cp38-cp38-win32.whl", hash = "sha256:72d622d262e463dfb7595202d229f5f3ab4b852289a1cd09650362db23b9eb0b"}, + {file = "pillow-10.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:2034f6759a722da3a3dbd91a81148cf884e91d1b747992ca288ab88c1de15999"}, + {file = "pillow-10.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:2ed854e716a89b1afcedea551cd85f2eb2a807613752ab997b9974aaa0d56936"}, + {file = "pillow-10.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dc1a390a82755a8c26c9964d457d4c9cbec5405896cba94cf51f36ea0d855002"}, + {file = "pillow-10.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4203efca580f0dd6f882ca211f923168548f7ba334c189e9eab1178ab840bf60"}, + {file = "pillow-10.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3102045a10945173d38336f6e71a8dc71bcaeed55c3123ad4af82c52807b9375"}, + {file = "pillow-10.3.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:6fb1b30043271ec92dc65f6d9f0b7a830c210b8a96423074b15c7bc999975f57"}, + {file = "pillow-10.3.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:1dfc94946bc60ea375cc39cff0b8da6c7e5f8fcdc1d946beb8da5c216156ddd8"}, + {file = "pillow-10.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b09b86b27a064c9624d0a6c54da01c1beaf5b6cadfa609cf63789b1d08a797b9"}, + {file = "pillow-10.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d3b2348a78bc939b4fed6552abfd2e7988e0f81443ef3911a4b8498ca084f6eb"}, + {file = "pillow-10.3.0-cp39-cp39-win32.whl", hash = "sha256:45ebc7b45406febf07fef35d856f0293a92e7417ae7933207e90bf9090b70572"}, + {file = "pillow-10.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:0ba26351b137ca4e0db0342d5d00d2e355eb29372c05afd544ebf47c0956ffeb"}, + {file = "pillow-10.3.0-cp39-cp39-win_arm64.whl", hash = "sha256:50fd3f6b26e3441ae07b7c979309638b72abc1a25da31a81a7fbd9495713ef4f"}, + {file = "pillow-10.3.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:6b02471b72526ab8a18c39cb7967b72d194ec53c1fd0a70b050565a0f366d355"}, + {file = "pillow-10.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:8ab74c06ffdab957d7670c2a5a6e1a70181cd10b727cd788c4dd9005b6a8acd9"}, + {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:048eeade4c33fdf7e08da40ef402e748df113fd0b4584e32c4af74fe78baaeb2"}, + {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e2ec1e921fd07c7cda7962bad283acc2f2a9ccc1b971ee4b216b75fad6f0463"}, + {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c8e73e99da7db1b4cad7f8d682cf6abad7844da39834c288fbfa394a47bbced"}, + {file = "pillow-10.3.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:16563993329b79513f59142a6b02055e10514c1a8e86dca8b48a893e33cf91e3"}, + {file = "pillow-10.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:dd78700f5788ae180b5ee8902c6aea5a5726bac7c364b202b4b3e3ba2d293170"}, + {file = "pillow-10.3.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:aff76a55a8aa8364d25400a210a65ff59d0168e0b4285ba6bf2bd83cf675ba32"}, + {file = "pillow-10.3.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:b7bc2176354defba3edc2b9a777744462da2f8e921fbaf61e52acb95bafa9828"}, + {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:793b4e24db2e8742ca6423d3fde8396db336698c55cd34b660663ee9e45ed37f"}, + {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d93480005693d247f8346bc8ee28c72a2191bdf1f6b5db469c096c0c867ac015"}, + {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c83341b89884e2b2e55886e8fbbf37c3fa5efd6c8907124aeb72f285ae5696e5"}, + {file = "pillow-10.3.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1a1d1915db1a4fdb2754b9de292642a39a7fb28f1736699527bb649484fb966a"}, + {file = "pillow-10.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a0eaa93d054751ee9964afa21c06247779b90440ca41d184aeb5d410f20ff591"}, + {file = "pillow-10.3.0.tar.gz", hash = "sha256:9d2455fbf44c914840c793e89aa82d0e1763a14253a000743719ae5946814b2d"}, +] + +[package.extras] +docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] +fpx = ["olefile"] +mic = ["olefile"] +tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] +typing = ["typing-extensions"] +xmp = ["defusedxml"] + [[package]] name = "platformdirs" version = "4.2.0" @@ -2014,6 +2446,20 @@ typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\"" spelling = ["pyenchant (>=3.2,<4.0)"] testutils = ["gitpython (>3)"] +[[package]] +name = "pyparsing" +version = "3.1.2" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +optional = false +python-versions = ">=3.6.8" +files = [ + {file = "pyparsing-3.1.2-py3-none-any.whl", hash = "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742"}, + {file = "pyparsing-3.1.2.tar.gz", hash = "sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad"}, +] + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + [[package]] name = "pytest" version = "7.4.4" @@ -2642,6 +3088,18 @@ docs = ["Pillow (>=7.1.2)", "matplotlib (>=3.3.4)", "memory-profiler (>=0.57.0)" examples = ["matplotlib (>=3.3.4)", "pandas (>=1.1.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.17.2)", "seaborn (>=0.9.0)"] tests = ["black (>=23.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.3)", "numpydoc (>=1.2.0)", "pandas (>=1.1.5)", "polars (>=0.19.12)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pyarrow (>=12.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.0.272)", "scikit-image (>=0.17.2)"] +[[package]] +name = "scikit-multilearn" +version = "0.2.0" +description = "Scikit-multilearn is a BSD-licensed library for multi-label classification that is built on top of the well-known scikit-learn ecosystem." +optional = false +python-versions = "*" +files = [ + {file = "scikit-multilearn-0.2.0.linux-x86_64.tar.gz", hash = "sha256:3179fed29b1492f6a69600696c23045b9f494d2b89d1796a8bdc43ccbb33712b"}, + {file = "scikit_multilearn-0.2.0-py2-none-any.whl", hash = "sha256:0a389600a6797db6567f2f6ca1d0dca30bebfaaa73f75de62d7ae40f8f03d4fb"}, + {file = "scikit_multilearn-0.2.0-py3-none-any.whl", hash = "sha256:068c652f22704a084ca252d05d21a655e7c9b248d0a4543847b74de5fca2b3f0"}, +] + [[package]] name = "scipy" version = "1.13.0" @@ -3570,7 +4028,22 @@ files = [ idna = ">=2.0" multidict = ">=4.0" +[[package]] +name = "zipp" +version = "3.18.1" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "zipp-3.18.1-py3-none-any.whl", hash = "sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b"}, + {file = "zipp-3.18.1.tar.gz", hash = "sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] + [metadata] lock-version = "2.0" python-versions = ">=3.9, <3.12" -content-hash = "b8a5d0dab5db92cf5088a2aa1179effe1761e8383191728b4ddca27ef8180ce6" +content-hash = "9871b7952b08b72020801755d0f3f08a3aaffc3e271d2465ae7d861e3e742352" diff --git a/pyproject.toml b/pyproject.toml index 5850d21..050690b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,8 @@ pyarrow = "^15.0.1" plotly = "^5.7.0" tokenizers = "^0.15.2" transformers = "^4.39.3" +captum = "^0.7.0" +scikit-multilearn = "^0.2.0" [tool.poetry.group.test] optional = true From bbf694b46b8e61de824dfe912196e4d1ac9b230f Mon Sep 17 00:00:00 2001 From: Amrit K Date: Tue, 16 Apr 2024 12:21:31 -0400 Subject: [PATCH 4/9] Add TestAnalysis notebook --- evaluation/AttentionVisualization.ipynb | 2 +- evaluation/TestAnalysis.ipynb | 266 ++++++++++++++++++ odyssey/models/baseline/Bi-LSTM.ipynb | 7 +- odyssey/models/cehr_big_bird/playground.ipynb | 3 +- 4 files changed, 271 insertions(+), 7 deletions(-) create mode 100644 evaluation/TestAnalysis.ipynb diff --git a/evaluation/AttentionVisualization.ipynb b/evaluation/AttentionVisualization.ipynb index d452ffb..dc0a91c 100644 --- a/evaluation/AttentionVisualization.ipynb +++ b/evaluation/AttentionVisualization.ipynb @@ -62,8 +62,8 @@ "os.chdir(ROOT)\n", "\n", "from odyssey.data.dataset import FinetuneDataset\n", - "from odyssey.models.prediction import load_finetuned_model, predict_patient_outcomes\n", "from odyssey.data.tokenizer import ConceptTokenizer\n", + "from odyssey.models.prediction import load_finetuned_model, predict_patient_outcomes\n", "from odyssey.models.utils import (\n", " load_finetune_data,\n", ")" diff --git a/evaluation/TestAnalysis.ipynb b/evaluation/TestAnalysis.ipynb new file mode 100644 index 0000000..05c9d1d --- /dev/null +++ b/evaluation/TestAnalysis.ipynb @@ -0,0 +1,266 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2024-04-10 12:13:14,754] [INFO] [real_accelerator.py:191:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n" + ] + } + ], + "source": [ + "import os\n", + "\n", + "import torch\n", + "from sklearn.metrics import (\n", + " auc,\n", + " average_precision_score,\n", + " balanced_accuracy_score,\n", + " f1_score,\n", + " precision_recall_curve,\n", + " precision_score,\n", + " recall_score,\n", + " roc_auc_score,\n", + ")\n", + "from transformers import utils\n", + "\n", + "\n", + "utils.logging.set_verbosity_error() # Suppress standard warnings\n", + "\n", + "\n", + "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", + "os.chdir(ROOT)\n", + "\n", + "from lib.tokenizer import ConceptTokenizer" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "class config:\n", + " \"\"\"Save the configuration arguments.\"\"\"\n", + "\n", + " model_path = \"test_epoch_end.ckpt\"\n", + " vocab_dir = \"data/vocab\"\n", + " data_dir = \"data/bigbird_data\"\n", + " sequence_file = \"patient_sequences/patient_sequences_2048_mortality.parquet\"\n", + " id_file = \"patient_id_dict/dataset_2048_mortality_1month.pkl\"\n", + " valid_scheme = \"few_shot\"\n", + " num_finetune_patients = \"20000\"\n", + " label_name = \"label_mortality_1month\"\n", + "\n", + " max_len = 2048\n", + " batch_size = 1\n", + " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "tokenizer = ConceptTokenizer(data_dir=config.vocab_dir)\n", + "tokenizer.fit_on_vocab()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['epoch', 'global_step', 'pytorch-lightning_version', 'state_dict', 'loops', 'callbacks', 'optimizer_states', 'lr_schedulers', 'MixedPrecision'])" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model = torch.load(config.model_path, map_location=config.device)\n", + "model.keys()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'loss': tensor(0.1638, dtype=torch.float64),\n", + " 'preds': tensor([6, 7, 0, ..., 7, 7, 7]),\n", + " 'labels': tensor([[1., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [1., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]], dtype=torch.float64),\n", + " 'logits': tensor([[ 2.3418, -2.0781, 0.2194, ..., -5.3945, -7.1797, -3.4180],\n", + " [-1.6533, -3.4277, -6.8086, ..., -6.8359, -5.2266, -5.6484],\n", + " [ 1.0947, -3.7930, -6.1094, ..., -6.3867, -6.6836, -5.5508],\n", + " ...,\n", + " [-2.8223, -3.7285, -4.6797, ..., -7.9922, -5.6992, -6.7812],\n", + " [-3.7148, -5.6328, -6.7188, ..., -9.4062, -7.6445, -7.6016],\n", + " [-2.5840, -2.2871, -4.6484, ..., -7.8633, -4.7539, -6.4648]],\n", + " dtype=torch.float16)}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "test_outputs = torch.load(\"test_outputs.pt\")\n", + "test_outputs" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'Balanced Accuracy': 0.5, 'F1 Score': 0.0, 'Precision': 0.0, 'Recall': 0.0, 'AUROC': 0.8100258785715974, 'Average Precision Score': 0.001364147006900979, 'AUC-PR': 0.5006820735034505}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/fs01/home/afallah/light/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1497: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n" + ] + } + ], + "source": [ + "def calculate_metrics(y_true, y_pred, y_prob):\n", + " \"\"\"\n", + " Calculate and return performance metrics.\n", + " \"\"\"\n", + " metrics = {\n", + " \"Balanced Accuracy\": balanced_accuracy_score(y_true, y_pred),\n", + " \"F1 Score\": f1_score(y_true, y_pred),\n", + " \"Precision\": precision_score(y_true, y_pred),\n", + " \"Recall\": recall_score(y_true, y_pred),\n", + " \"AUROC\": roc_auc_score(y_true, y_prob),\n", + " \"Average Precision Score\": average_precision_score(y_true, y_pred),\n", + " }\n", + "\n", + " precision, recall, _ = precision_recall_curve(y_true, y_pred)\n", + " metrics[\"AUC-PR\"] = auc(recall, precision)\n", + "\n", + " return metrics\n", + "\n", + "\n", + "targets = [10]\n", + "\n", + "for i in targets:\n", + " labels = test_outputs[\"labels\"][:, i]\n", + " logits = torch.sigmoid(test_outputs[\"logits\"][:, i])\n", + " preds = (logits >= 0.5).int()\n", + "\n", + " print(calculate_metrics(labels, preds, logits))" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(0)" + ] + }, + "execution_count": 68, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preds.sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(34., dtype=torch.float64)" + ] + }, + "execution_count": 69, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "labels.sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([0, 0, 0, ..., 0, 0, 0], dtype=torch.int32)" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preds" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/odyssey/models/baseline/Bi-LSTM.ipynb b/odyssey/models/baseline/Bi-LSTM.ipynb index dabed98..d67b155 100644 --- a/odyssey/models/baseline/Bi-LSTM.ipynb +++ b/odyssey/models/baseline/Bi-LSTM.ipynb @@ -71,6 +71,9 @@ "import numpy as np\n", "import pandas as pd\n", "import torch\n", + "from models.big_bird_cehr.data import FinetuneDataset\n", + "from models.big_bird_cehr.embeddings import Embeddings\n", + "from models.big_bird_cehr.tokenizer import ConceptTokenizer\n", "from sklearn.metrics import (\n", " auc,\n", " average_precision_score,\n", @@ -89,10 +92,6 @@ "from torch.utils.data import DataLoader, Dataset\n", "from tqdm import tqdm\n", "\n", - "from models.big_bird_cehr.data import FinetuneDataset\n", - "from models.big_bird_cehr.embeddings import Embeddings\n", - "from models.big_bird_cehr.tokenizer import ConceptTokenizer\n", - "\n", "\n", "DATA_ROOT = f\"{ROOT}/data/slurm_data/512/one_month\"\n", "DATA_PATH = f\"{DATA_ROOT}/pretrain.parquet\"\n", diff --git a/odyssey/models/cehr_big_bird/playground.ipynb b/odyssey/models/cehr_big_bird/playground.ipynb index 60ffb8f..164ab0f 100644 --- a/odyssey/models/cehr_big_bird/playground.ipynb +++ b/odyssey/models/cehr_big_bird/playground.ipynb @@ -1836,10 +1836,9 @@ "os.chdir(ROOT)\n", "import numpy as np\n", "import pandas as pd\n", - "from tqdm import tqdm\n", - "\n", "from models.big_bird_cehr.data import PretrainDataset\n", "from models.big_bird_cehr.tokenizer import ConceptTokenizer\n", + "from tqdm import tqdm\n", "\n", "\n", "DATA_ROOT = f\"{ROOT}/data/slurm_data/2048/one_month\"\n", From 77ece5146686e963841495894f6bd2dd0c01d17c Mon Sep 17 00:00:00 2001 From: Amrit K Date: Tue, 16 Apr 2024 12:24:00 -0400 Subject: [PATCH 5/9] Fix import --- evaluation/TestAnalysis.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaluation/TestAnalysis.ipynb b/evaluation/TestAnalysis.ipynb index 05c9d1d..f39ce8d 100644 --- a/evaluation/TestAnalysis.ipynb +++ b/evaluation/TestAnalysis.ipynb @@ -36,7 +36,7 @@ "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", "os.chdir(ROOT)\n", "\n", - "from lib.tokenizer import ConceptTokenizer" + "from odyssey.data.tokenizer import ConceptTokenizer" ] }, { From 88f6bb2f24a7a866389e5e2d1363ee92b75715ad Mon Sep 17 00:00:00 2001 From: afallah Date: Tue, 16 Apr 2024 15:26:13 -0400 Subject: [PATCH 6/9] Minor style changes. --- finetune.py | 10 ++++++++++ odyssey/models/cehr_big_bird/model.py | 1 + pretrain.py | 1 + 3 files changed, 12 insertions(+) diff --git a/finetune.py b/finetune.py index ae5898e..628c809 100644 --- a/finetune.py +++ b/finetune.py @@ -191,7 +191,9 @@ def main( problem_type=args.problem_type, **fine_model_config, ) + run_id = get_run_id(args.checkpoint_dir) + wandb_logger = WandbLogger( project=args.exp_name, save_dir=args.log_dir, @@ -199,6 +201,7 @@ def main( id=run_id, resume="allow", ) + # Setup PyTorchLightning trainer trainer = pl.Trainer( accelerator="gpu", @@ -217,6 +220,7 @@ def main( accumulate_grad_batches=args.acc, gradient_clip_val=1.0, ) + # Train the model trainer.fit( model=model, @@ -224,6 +228,7 @@ def main( val_dataloaders=val_loader, ckpt_path=args.resume_checkpoint, ) + # Test the model if args.test_last: trainer.test( @@ -246,6 +251,7 @@ def main( if __name__ == "__main__": parser = argparse.ArgumentParser() + # project configuration parser.add_argument( "--model-type", @@ -288,6 +294,7 @@ def main( default=False, help="Is the model a multimodel like multibird or not", ) + # data-related arguments parser.add_argument( "--data-dir", @@ -355,6 +362,7 @@ def main( default=None, help="Define the positive label ratios for label balancing", ) + # checkpointing and logging arguments parser.add_argument( "--checkpoint-dir", @@ -386,6 +394,7 @@ def main( default=10, help="Number of steps to log the training", ) + # other arguments parser.add_argument( "--test-last", @@ -421,6 +430,7 @@ def main( pre_model_config = config["model"] args.max_len = pre_model_config["max_seq_length"] + # Process the tasks and balance guide arguments args.tasks = args.tasks.strip().split(" ") args.balance_guide = { diff --git a/odyssey/models/cehr_big_bird/model.py b/odyssey/models/cehr_big_bird/model.py index 55fbb3a..5acc07e 100644 --- a/odyssey/models/cehr_big_bird/model.py +++ b/odyssey/models/cehr_big_bird/model.py @@ -452,6 +452,7 @@ def on_test_epoch_end(self) -> Any: auc = roc_auc_score(labels, preds) precision = precision_score(labels, preds) recall = recall_score(labels, preds) + self.log("test_loss", loss) self.log("test_acc", accuracy) self.log("test_f1", f1) diff --git a/pretrain.py b/pretrain.py index 0958377..4627b34 100644 --- a/pretrain.py +++ b/pretrain.py @@ -152,6 +152,7 @@ def main(args: argparse.Namespace, model_config: Dict[str, Any]) -> None: if __name__ == "__main__": parser = argparse.ArgumentParser() + # project configuration parser.add_argument( "--model-type", From c86931650f1e3e54f3d456589cc1e21439de9e2d Mon Sep 17 00:00:00 2001 From: afallah Date: Tue, 16 Apr 2024 16:43:48 -0400 Subject: [PATCH 7/9] Refactored the codebase to remove redundant code and follow modular practices. --- evaluation/AttentionVisualization.ipynb | 3412 ------------------- evaluation/CompareAUROC-Poster.ipynb | 158 - evaluation/TestAnalysis.ipynb | 266 -- finetune.py | 4 +- odyssey/data/bigbird_data/DataChecker.ipynb | 1136 ------ odyssey/data/dataset.py | 1 + odyssey/models/baseline/Bi-LSTM.ipynb | 22 +- odyssey/models/baseline/Bi-LSTM.py | 2 +- odyssey/models/cehr_bert/embeddings.py | 175 - odyssey/models/cehr_bert/model.py | 4 +- odyssey/models/cehr_big_bird/embeddings.py | 363 -- odyssey/models/cehr_big_bird/model.py | 7 +- odyssey/models/prediction.py | 113 - odyssey/models/utils.py | 197 -- pretrain.py | 4 +- 15 files changed, 23 insertions(+), 5841 deletions(-) delete mode 100644 evaluation/AttentionVisualization.ipynb delete mode 100644 evaluation/CompareAUROC-Poster.ipynb delete mode 100644 evaluation/TestAnalysis.ipynb delete mode 100644 odyssey/data/bigbird_data/DataChecker.ipynb delete mode 100644 odyssey/models/cehr_bert/embeddings.py delete mode 100644 odyssey/models/cehr_big_bird/embeddings.py delete mode 100644 odyssey/models/prediction.py delete mode 100644 odyssey/models/utils.py diff --git a/evaluation/AttentionVisualization.ipynb b/evaluation/AttentionVisualization.ipynb deleted file mode 100644 index dc0a91c..0000000 --- a/evaluation/AttentionVisualization.ipynb +++ /dev/null @@ -1,3412 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 13, - "id": "initial_id", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:39:47.301592Z", - "start_time": "2024-03-15T15:39:47.293763Z" - }, - "collapsed": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'\\nFile: AttentionVisualization.ipynb\\n---------------------------------\\nVisualize the attention layers of transformer models for interpretability.\\n'" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "\"\"\"\n", - "File: AttentionVisualization.ipynb\n", - "---------------------------------\n", - "Visualize the attention layers of transformer models for interpretability.\n", - "\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "e24876a0c6020df2", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:49:16.880004Z", - "start_time": "2024-03-15T15:49:16.866092Z" - }, - "collapsed": false - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "import numpy as np\n", - "import plotly.graph_objects as go\n", - "import torch\n", - "from bertviz import head_view, model_view\n", - "from bertviz.neuron_view import show\n", - "from torch.utils.data import DataLoader, Subset\n", - "from transformers import utils\n", - "\n", - "\n", - "utils.logging.set_verbosity_error() # Suppress standard warnings\n", - "\n", - "\n", - "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", - "os.chdir(ROOT)\n", - "\n", - "from odyssey.data.dataset import FinetuneDataset\n", - "from odyssey.data.tokenizer import ConceptTokenizer\n", - "from odyssey.models.prediction import load_finetuned_model, predict_patient_outcomes\n", - "from odyssey.models.utils import (\n", - " load_finetune_data,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "dac09785d1fdc0cc", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:39:54.875569Z", - "start_time": "2024-03-15T15:39:54.852027Z" - }, - "collapsed": false - }, - "outputs": [], - "source": [ - "class args:\n", - " \"\"\"Save the configuration arguments.\"\"\"\n", - "\n", - " model_path = (\n", - " \"checkpoints/bigbird_finetune/mortality_1month_20000_patients/best-v3.ckpt\"\n", - " )\n", - " vocab_dir = \"data/vocab\"\n", - " data_dir = \"data/bigbird_data\"\n", - " sequence_file = (\n", - " \"old_data/patient_sequences/patient_sequences_2048_mortality.parquet\"\n", - " )\n", - " id_file = \"old_data/patient_id_dict/dataset_2048_mortality_1month.pkl\"\n", - " valid_scheme = \"few_shot\"\n", - " num_finetune_patients = \"20000\"\n", - " label_name = \"label_mortality_1month\"\n", - "\n", - " max_len = 2048\n", - " batch_size = 1\n", - " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "6b5e21258ced06d2", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:39:54.921944Z", - "start_time": "2024-03-15T15:39:54.877586Z" - }, - "collapsed": false - }, - "outputs": [], - "source": [ - "tokenizer = ConceptTokenizer(data_dir=args.vocab_dir)\n", - "tokenizer.fit_on_vocab()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "8bf63dc4242a163e", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:40:00.629410Z", - "start_time": "2024-03-15T15:39:54.929550Z" - }, - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "BigBirdFinetune(\n", - " (model): BigBirdForSequenceClassification(\n", - " (bert): BigBirdModel(\n", - " (embeddings): BigBirdEmbeddingsForCEHR(\n", - " (word_embeddings): Embedding(20592, 768, padding_idx=0)\n", - " (position_embeddings): Embedding(2048, 768)\n", - " (token_type_embeddings): Embedding(9, 768)\n", - " (time_embeddings): TimeEmbeddingLayer()\n", - " (age_embeddings): TimeEmbeddingLayer()\n", - " (visit_segment_embeddings): VisitEmbedding(\n", - " (embedding): Embedding(3, 768)\n", - " )\n", - " (scale_back_concat_layer): Linear(in_features=832, out_features=768, bias=True)\n", - " (tanh): Tanh()\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (encoder): BigBirdEncoder(\n", - " (layer): ModuleList(\n", - " (0-5): 6 x BigBirdLayer(\n", - " (attention): BigBirdAttention(\n", - " (self): BigBirdBlockSparseAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " )\n", - " (output): BigBirdSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BigBirdIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): NewGELUActivation()\n", - " )\n", - " (output): BigBirdOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " )\n", - " (pooler): Linear(in_features=768, out_features=768, bias=True)\n", - " (activation): Tanh()\n", - " )\n", - " (classifier): BigBirdClassificationHead(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (out_proj): Linear(in_features=768, out_features=2, bias=True)\n", - " )\n", - " )\n", - " (pretrained_model): BigBirdPretrain(\n", - " (embeddings): BigBirdEmbeddingsForCEHR(\n", - " (word_embeddings): Embedding(20592, 768, padding_idx=0)\n", - " (position_embeddings): Embedding(2048, 768)\n", - " (token_type_embeddings): Embedding(9, 768)\n", - " (time_embeddings): TimeEmbeddingLayer()\n", - " (age_embeddings): TimeEmbeddingLayer()\n", - " (visit_segment_embeddings): VisitEmbedding(\n", - " (embedding): Embedding(3, 768)\n", - " )\n", - " (scale_back_concat_layer): Linear(in_features=832, out_features=768, bias=True)\n", - " (tanh): Tanh()\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (model): BigBirdForMaskedLM(\n", - " (bert): BigBirdModel(\n", - " (embeddings): BigBirdEmbeddingsForCEHR(\n", - " (word_embeddings): Embedding(20592, 768, padding_idx=0)\n", - " (position_embeddings): Embedding(2048, 768)\n", - " (token_type_embeddings): Embedding(9, 768)\n", - " (time_embeddings): TimeEmbeddingLayer()\n", - " (age_embeddings): TimeEmbeddingLayer()\n", - " (visit_segment_embeddings): VisitEmbedding(\n", - " (embedding): Embedding(3, 768)\n", - " )\n", - " (scale_back_concat_layer): Linear(in_features=832, out_features=768, bias=True)\n", - " (tanh): Tanh()\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (encoder): BigBirdEncoder(\n", - " (layer): ModuleList(\n", - " (0-5): 6 x BigBirdLayer(\n", - " (attention): BigBirdAttention(\n", - " (self): BigBirdBlockSparseAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " )\n", - " (output): BigBirdSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BigBirdIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): NewGELUActivation()\n", - " )\n", - " (output): BigBirdOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " )\n", - " (pooler): Linear(in_features=768, out_features=768, bias=True)\n", - " (activation): Tanh()\n", - " )\n", - " (cls): BigBirdOnlyMLMHead(\n", - " (predictions): BigBirdLMPredictionHead(\n", - " (transform): BigBirdPredictionHeadTransform(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (transform_act_fn): NewGELUActivation()\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " )\n", - " (decoder): Linear(in_features=768, out_features=20592, bias=True)\n", - " )\n", - " )\n", - " )\n", - " )\n", - ")" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model = load_finetuned_model(args.model_path, tokenizer)\n", - "model" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "45bb295710f64bc0", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:52:23.262096Z", - "start_time": "2024-03-15T15:52:23.252429Z" - }, - "collapsed": false - }, - "outputs": [], - "source": [ - "fine_tune, fine_test = load_finetune_data(\n", - " args.data_dir,\n", - " args.sequence_file,\n", - " args.id_file,\n", - " args.valid_scheme,\n", - " args.num_finetune_patients,\n", - ")\n", - "\n", - "fine_tune.rename(columns={args.label_name: \"label\"}, inplace=True)\n", - "fine_test.rename(columns={args.label_name: \"label\"}, inplace=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "6c2f9f1f", - "metadata": {}, - "outputs": [], - "source": [ - "test_dataset = FinetuneDataset(\n", - " data=fine_test,\n", - " tokenizer=tokenizer,\n", - " max_len=args.max_len,\n", - ")\n", - "\n", - "test_loader = DataLoader(\n", - " Subset(test_dataset, [89, 90]), # 85 and 88 are small\n", - " batch_size=args.batch_size,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "ead6f3658dda5274", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:52:24.058853Z", - "start_time": "2024-03-15T15:52:24.027868Z" - }, - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'concept_ids': tensor([[ 5, 3, 18065, ..., 0, 0, 0]]),\n", - " 'type_ids': tensor([[1, 2, 6, ..., 0, 0, 0]]),\n", - " 'ages': tensor([[ 0, 67, 67, ..., 0, 0, 0]]),\n", - " 'time_stamps': tensor([[ 0, 5773, 5773, ..., 0, 0, 0]]),\n", - " 'visit_orders': tensor([[0, 1, 1, ..., 0, 0, 0]]),\n", - " 'visit_segments': tensor([[0, 2, 2, ..., 0, 0, 0]]),\n", - " 'labels': tensor([0]),\n", - " 'attention_mask': tensor([[1, 1, 1, ..., 0, 0, 0]])}" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "patient = next(iter(test_loader))\n", - "patient" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "b025c033efc9baab", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:52:25.797704Z", - "start_time": "2024-03-15T15:52:25.476580Z" - }, - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "SequenceClassifierOutput(loss=tensor(0.1007, device='cuda:0', grad_fn=), logits=tensor([[ 1.1001, -1.1447]], device='cuda:0', grad_fn=), hidden_states=(tensor([[[-0.6737, -0.4033, -1.0631, ..., -0.0571, -0.5688, -2.8107],\n", - " [-0.2583, 1.4081, 0.0247, ..., 0.9723, -0.1994, -0.6298],\n", - " [-0.3349, -0.7147, -0.2382, ..., 0.6685, -2.0221, 0.3603],\n", - " ...,\n", - " [-1.1487, -0.3301, -1.2087, ..., -1.4240, 0.6589, -3.1762],\n", - " [-0.5165, 0.4454, -1.0153, ..., -2.3917, -0.0144, -3.3358],\n", - " [-0.1893, -0.1348, 0.2863, ..., -2.4897, 0.7176, -2.4138]]],\n", - " device='cuda:0', grad_fn=), tensor([[[-9.1614e-01, 6.7605e-01, 6.2693e-03, ..., -3.1620e-01,\n", - " -4.6282e-01, -2.0648e+00],\n", - " [-6.0621e-01, 8.6801e-01, 2.6213e-01, ..., 3.4474e-01,\n", - " 1.3363e-01, -7.1667e-01],\n", - " [-7.6982e-01, 2.3106e-03, -5.9440e-01, ..., -4.7166e-01,\n", - " -1.8369e+00, 1.3552e-01],\n", - " ...,\n", - " [-8.6433e-01, 7.3538e-01, -3.3437e-01, ..., -1.2050e+00,\n", - " 1.3254e+00, -2.4991e+00],\n", - " [-7.3595e-01, 1.2425e+00, 5.3519e-03, ..., -1.8429e+00,\n", - " 9.7859e-01, -2.8092e+00],\n", - " [-5.8273e-01, 8.9714e-01, 8.5189e-01, ..., -1.7958e+00,\n", - " 1.6784e+00, -2.0120e+00]]], device='cuda:0',\n", - " grad_fn=), tensor([[[-0.5963, 1.0668, 0.6960, ..., -0.5670, -1.8491, -0.7775],\n", - " [-0.8661, 0.9432, 1.3912, ..., -0.1141, -0.0171, -0.0536],\n", - " [-0.4142, 0.1496, 0.0763, ..., -1.3690, -2.2675, 0.8359],\n", - " ...,\n", - " [-0.5217, 1.3100, 0.3550, ..., -0.8422, 0.7881, -2.2435],\n", - " [-0.6279, 1.6811, 0.6428, ..., -1.1621, 0.4885, -2.3759],\n", - " [-0.5811, 1.5146, 1.1337, ..., -1.0819, 1.0710, -2.0434]]],\n", - " device='cuda:0', grad_fn=), tensor([[[-0.4564, 0.8573, 0.4777, ..., -1.0131, -1.2389, -0.7232],\n", - " [-0.7503, 0.8020, 2.2105, ..., 0.3817, -0.5328, -1.1600],\n", - " [-0.2785, 0.3193, 0.7508, ..., -0.8207, -2.1549, -0.0479],\n", - " ...,\n", - " [-0.5749, 0.9360, 0.3542, ..., -1.2162, 0.5259, -1.9892],\n", - " [-0.8316, 1.2188, 0.4210, ..., -1.5618, 0.3291, -2.1654],\n", - " [-0.7359, 1.2650, 0.9190, ..., -1.4486, 0.7956, -1.8708]]],\n", - " device='cuda:0', grad_fn=), tensor([[[-0.6359, 1.1296, 0.3645, ..., -1.0155, -1.4793, -0.0216],\n", - " [-0.2409, 0.9339, 1.1703, ..., 1.1403, -1.0845, -0.8589],\n", - " [-0.2531, 0.2159, 0.2502, ..., -0.1171, -2.6449, -0.1982],\n", - " ...,\n", - " [-0.4492, 0.9411, 0.7911, ..., -0.9499, 0.3742, -1.3986],\n", - " [-0.7147, 1.2283, 0.7656, ..., -1.0737, 0.1806, -1.6471],\n", - " [-0.6436, 1.0595, 1.1752, ..., -1.1340, 0.4983, -1.3118]]],\n", - " device='cuda:0', grad_fn=), tensor([[[-0.0605, 0.4757, 1.2138, ..., -1.2107, -0.7777, 0.4597],\n", - " [-0.0597, 0.6793, 0.6923, ..., 0.5595, -0.6465, -0.4138],\n", - " [-0.2956, 0.0119, 0.0849, ..., -0.4024, -1.7548, 0.2857],\n", - " ...,\n", - " [-0.0730, 1.2001, 0.8387, ..., -1.5022, 0.3392, -0.9774],\n", - " [-0.2355, 1.2640, 0.8785, ..., -1.6132, 0.2851, -1.2461],\n", - " [-0.2264, 1.1501, 1.2094, ..., -1.7025, 0.5599, -0.9631]]],\n", - " device='cuda:0', grad_fn=), tensor([[[-0.4529, 0.7745, 0.9677, ..., -0.7731, 0.3017, 0.6295],\n", - " [-0.2527, 0.7169, 0.3931, ..., 0.3904, -0.2026, 0.2191],\n", - " [-0.7347, 0.0648, 0.1225, ..., -0.4706, -0.9406, 0.7601],\n", - " ...,\n", - " [-0.1587, 1.8421, 0.7164, ..., -1.1803, 0.3151, -1.0949],\n", - " [-0.4175, 1.9637, 0.7340, ..., -1.2465, 0.3226, -1.4348],\n", - " [-0.4056, 1.7997, 1.0610, ..., -1.3635, 0.5409, -1.1068]]],\n", - " device='cuda:0', grad_fn=)), attentions=(tensor([[[[9.2084e-01, 3.4662e-05, 1.6030e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0152e-04, 1.0245e-03, 5.0193e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1340e-05, 3.4795e-04, 1.1249e-01, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.7594e-01, 6.5887e-04, 2.1206e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.4787e-01, 1.3629e-03, 7.0707e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.9477e-01, 1.2923e-03, 4.1574e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[2.2569e-05, 3.1873e-03, 1.4330e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0942e-04, 7.4366e-03, 1.0470e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.2275e-07, 8.8712e-03, 2.0909e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.3820e-04, 5.4235e-03, 3.9131e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.1625e-05, 6.0716e-03, 9.2111e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0256e-05, 6.8238e-03, 1.3608e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[2.1552e-05, 3.2614e-03, 5.1249e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.0069e-02, 1.7049e-03, 7.4966e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.6985e-04, 7.9826e-05, 1.0739e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [7.2891e-05, 2.2444e-03, 8.0701e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.6524e-04, 2.5353e-03, 1.1610e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.4759e-05, 3.7951e-03, 9.3934e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[9.5957e-07, 1.8916e-05, 1.7706e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.4017e-03, 6.3007e-04, 8.2404e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.7625e-05, 2.4848e-05, 1.9090e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.4380e-05, 1.2709e-04, 2.4316e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.0306e-05, 1.8831e-04, 2.8286e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.8297e-05, 1.9117e-04, 2.4082e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.7399e-03, 1.2051e-03, 9.5375e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.7488e-03, 2.7973e-03, 2.2183e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.6722e-03, 7.9718e-04, 2.1421e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.6540e-03, 2.6361e-03, 1.2919e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.7157e-03, 2.6202e-03, 1.2555e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.6167e-03, 1.7538e-03, 1.1150e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.5929e-03, 3.2053e-03, 2.2594e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.3881e-03, 1.7186e-03, 4.7067e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.7379e-03, 3.9178e-03, 5.7301e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.5827e-03, 2.9827e-03, 2.1182e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.3835e-03, 3.9488e-03, 2.3409e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.8265e-03, 3.3581e-03, 2.8842e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=), tensor([[[[3.4325e-03, 3.6487e-03, 2.1270e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.6434e-03, 2.1695e-02, 6.8258e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.8946e-03, 5.1628e-02, 9.1872e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [6.2476e-03, 3.5812e-03, 3.8917e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.5847e-03, 5.8187e-03, 4.7901e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.7512e-03, 7.3790e-03, 5.0303e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[3.9891e-05, 4.4143e-06, 7.2098e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.8795e-03, 2.2040e-02, 1.3027e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.2756e-02, 1.8536e-02, 1.8427e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.0230e-04, 1.0431e-04, 3.6400e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0183e-04, 1.0241e-04, 4.4847e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.2058e-04, 2.5562e-04, 6.5510e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.7265e-03, 6.2247e-06, 3.0965e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.5288e-05, 1.7324e-02, 6.0294e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.6073e-04, 1.7897e-03, 2.7479e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [4.5983e-03, 2.8453e-04, 4.3261e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.3822e-03, 2.4970e-04, 7.7571e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.9758e-03, 2.7208e-04, 6.9291e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[3.2307e-04, 1.6154e-07, 1.1893e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5087e-04, 7.7591e-02, 4.4232e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.9669e-04, 7.1647e-03, 1.2542e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.6157e-03, 1.9115e-04, 8.7496e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5545e-03, 6.9437e-04, 2.2060e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.5639e-03, 4.7817e-04, 1.0144e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[9.0331e-02, 2.0968e-04, 1.5666e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.2050e-05, 4.9400e-03, 2.9039e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.8098e-04, 7.9459e-03, 1.0302e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [8.0900e-02, 5.0021e-03, 3.6893e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.7062e-02, 5.0122e-03, 5.1272e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.2155e-02, 4.1310e-03, 4.8125e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.2098e-04, 4.5407e-08, 1.1781e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.4916e-05, 1.8071e-03, 2.9574e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.9490e-03, 3.8689e-03, 1.6880e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [4.2944e-03, 1.2684e-05, 3.9500e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5202e-03, 9.5840e-06, 3.9704e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.4998e-03, 1.0348e-05, 4.0015e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=), tensor([[[[9.0947e-03, 1.7104e-04, 7.2082e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0905e-03, 8.2947e-03, 4.7806e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2774e-02, 4.1130e-03, 1.0427e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [7.4360e-03, 2.5052e-03, 3.5916e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.5451e-03, 2.0411e-03, 4.1614e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0480e-02, 1.5095e-03, 3.9783e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.0953e-04, 4.8485e-07, 1.1093e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.7053e-06, 7.0000e-03, 1.3357e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.2624e-03, 4.3490e-03, 8.2277e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [8.3735e-03, 9.6488e-05, 1.6069e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.6108e-03, 6.0472e-05, 1.0010e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.4534e-03, 9.8078e-05, 1.4517e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[3.1851e-03, 3.6355e-09, 4.5110e-06, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.1155e-05, 1.3445e-03, 3.1936e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.9026e-03, 3.4348e-04, 3.4485e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [4.4469e-03, 1.2559e-05, 5.1731e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.1810e-03, 6.3667e-06, 5.4800e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.0399e-03, 6.1432e-06, 4.1856e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[1.9304e-02, 3.4070e-05, 1.5798e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.7734e-05, 1.5453e-02, 6.9360e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.5957e-04, 3.6940e-03, 5.8250e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.6903e-03, 1.8801e-03, 4.4015e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.2805e-03, 8.8522e-04, 5.1555e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.4006e-03, 1.2400e-03, 5.7507e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[3.1703e-04, 4.1161e-05, 7.9415e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.8420e-04, 7.7026e-03, 7.9656e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.5139e-03, 2.4419e-03, 2.6276e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.7707e-03, 6.9963e-04, 1.6709e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.0923e-03, 7.9106e-04, 2.5133e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.4068e-03, 2.6501e-04, 1.1517e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[6.0330e-05, 1.8324e-06, 1.0670e-06, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.0295e-04, 6.8225e-03, 4.1048e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.6344e-03, 2.1405e-03, 1.9475e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.0302e-03, 2.2379e-05, 8.3888e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0699e-03, 3.1538e-05, 9.3927e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1386e-03, 4.4208e-05, 1.2655e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=), tensor([[[[1.0968e-03, 1.1158e-05, 2.8874e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.5882e-04, 1.9889e-02, 2.9087e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.0075e-03, 2.4018e-03, 5.6122e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.2534e-03, 5.0390e-05, 2.8556e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.8193e-04, 5.5810e-05, 3.2226e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.8834e-03, 1.9344e-04, 8.2757e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[2.1752e-04, 1.0848e-03, 5.2472e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.3445e-05, 3.1222e-02, 7.3892e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.9519e-04, 2.3304e-02, 4.0696e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [7.4697e-04, 2.2354e-04, 5.6568e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.7230e-04, 4.3862e-04, 9.0070e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.0728e-04, 3.7051e-04, 8.7622e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[6.4030e-04, 1.0463e-03, 1.2372e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.3409e-04, 2.6108e-02, 1.0480e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.3558e-03, 4.1270e-03, 3.1541e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [8.7033e-04, 1.3328e-04, 4.4750e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.1584e-03, 5.6131e-04, 1.9963e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.3833e-03, 1.7700e-04, 1.2080e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[3.0260e-04, 1.0307e-03, 5.4334e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.5736e-04, 4.4732e-03, 3.1108e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0399e-03, 2.5841e-03, 3.4921e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.2584e-03, 3.3595e-04, 1.9949e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.4056e-03, 7.4162e-04, 3.0422e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.4776e-03, 3.4124e-04, 2.3692e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[3.3945e-05, 2.8038e-06, 7.7244e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.1178e-03, 5.9071e-01, 3.8243e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1613e-03, 9.0446e-03, 1.3656e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [6.1137e-06, 8.0224e-07, 2.9675e-06, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.1845e-05, 4.3952e-06, 1.5539e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.0935e-05, 7.2282e-06, 1.6119e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[7.7009e-03, 9.1021e-04, 4.4472e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0289e-04, 1.2443e-02, 8.9628e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.7645e-04, 5.7296e-03, 3.5397e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.8769e-04, 7.9107e-06, 6.3012e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.2969e-04, 4.7156e-05, 2.6878e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.0934e-04, 3.7384e-05, 2.2858e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=), tensor([[[[6.7381e-04, 1.1823e-02, 7.4050e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.4863e-04, 4.0161e-02, 3.6514e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.6004e-03, 1.9051e-02, 5.0602e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.5047e-03, 1.9051e-03, 1.5601e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.6917e-03, 3.4615e-03, 1.6968e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.7805e-03, 3.3552e-03, 2.2134e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[3.1542e-04, 1.2458e-05, 3.5918e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5336e-03, 2.6060e-03, 9.9650e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.3570e-03, 4.4454e-04, 2.1211e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.8644e-05, 1.1354e-06, 2.5000e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.0465e-05, 9.6188e-07, 2.3662e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.5490e-05, 1.7402e-06, 4.0543e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[4.4482e-03, 4.3810e-07, 2.4999e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.1627e-04, 5.7318e-02, 2.5052e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.0123e-03, 1.0834e-03, 3.6955e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.2926e-03, 2.7957e-07, 6.0144e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.3540e-03, 2.3200e-07, 7.3312e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2753e-03, 5.4792e-07, 9.2235e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[1.8663e-05, 6.3300e-07, 1.4228e-06, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2953e-03, 1.2528e-02, 3.5547e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.9098e-03, 1.1120e-03, 1.3087e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.1750e-04, 1.4822e-05, 2.3723e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.7225e-05, 8.3810e-06, 1.6070e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.6516e-04, 2.6572e-05, 4.2221e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[4.5335e-05, 4.0905e-04, 2.3216e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.7303e-04, 5.1438e-02, 3.8481e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.3572e-04, 1.6108e-02, 4.7815e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.6582e-03, 9.6089e-04, 3.1044e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1655e-03, 1.2231e-03, 3.3471e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5900e-03, 1.6183e-03, 5.6334e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.3592e-03, 9.1240e-05, 1.9648e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.9554e-04, 3.4671e-02, 1.2307e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.1695e-03, 3.3091e-03, 4.3312e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [4.5615e-04, 4.8380e-07, 7.2619e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2036e-03, 3.6028e-06, 3.0705e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2138e-03, 3.5186e-06, 2.6554e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=), tensor([[[[4.0090e-03, 6.7263e-04, 7.5136e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.9121e-04, 1.2857e-02, 3.9387e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5724e-03, 4.5150e-03, 2.8897e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [4.5598e-03, 1.4921e-04, 2.2929e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.6446e-03, 1.8594e-04, 2.9366e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.2847e-03, 2.3700e-04, 3.6638e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.9915e-02, 4.0394e-05, 5.0094e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.3356e-04, 3.0291e-03, 1.2489e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.3452e-03, 8.8680e-04, 1.3824e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.0090e-01, 1.4063e-05, 1.0046e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1156e-01, 1.0029e-05, 8.7870e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.5568e-02, 1.6524e-05, 1.0813e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.8186e-02, 5.9815e-06, 5.0354e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1144e-04, 3.3203e-03, 1.0907e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1076e-03, 7.5293e-04, 1.7252e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.2483e-01, 2.2352e-07, 2.1154e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1819e-01, 1.1773e-07, 1.4315e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0731e-01, 2.1415e-07, 1.9841e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[8.9774e-03, 2.6941e-04, 3.7395e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.7763e-04, 4.7386e-03, 2.2903e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2463e-03, 2.2257e-03, 1.9620e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.1848e-02, 8.8093e-05, 1.7620e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.3670e-02, 8.8600e-05, 1.7867e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2773e-02, 1.0601e-04, 2.1455e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[2.7395e-02, 2.0985e-04, 9.8349e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.3505e-04, 3.5468e-03, 1.4784e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.9124e-03, 1.9014e-03, 2.2491e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.7175e-02, 1.4688e-05, 9.3223e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.9626e-02, 9.9576e-06, 9.4240e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.6579e-02, 1.2380e-05, 9.0343e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[2.5388e-03, 4.6041e-05, 9.6215e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.3881e-05, 1.0036e-02, 3.3659e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.2542e-04, 6.3507e-03, 5.8799e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.5240e-04, 1.0778e-08, 7.3738e-08, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5682e-04, 1.5447e-08, 1.0267e-07, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.6646e-04, 3.5608e-08, 2.2134e-07, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=)))" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "output = predict_patient_outcomes(patient, model)\n", - "output" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "e1aa8309b2650820", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:52:33.578704Z", - "start_time": "2024-03-15T15:52:33.571290Z" - }, - "collapsed": false - }, - "outputs": [], - "source": [ - "tokens = tokenizer.decode(patient[\"concept_ids\"].squeeze(0).cpu().numpy()).split(\" \")\n", - "truncate_at = patient[\"attention_mask\"].sum().numpy()\n", - "attention_matrix = output[\"attentions\"]\n", - "last_attention_matrix = attention_matrix[-1].detach()\n", - "# batch_size x num_heads x max_len x max_len x num_layers" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "e9203f59", - "metadata": {}, - "outputs": [], - "source": [ - "truncated_attention_matrix = []\n", - "\n", - "for i in range(len(attention_matrix)):\n", - " truncated_attention_matrix.append(\n", - " attention_matrix[i][:, :, :truncate_at, :truncate_at],\n", - " )\n", - "\n", - "truncated_attention_matrix = tuple(truncated_attention_matrix)\n", - "truncated_tokens = tokens[:truncate_at]" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "id": "7fb27b941602401d91542211134fc71a", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Token [CLS] with Token 50893_0: Attention Value 0.154\n", - "Token [CLS] with Token 51251_1: Attention Value 0.026\n", - "Token [CLS] with Token 51279_3: Attention Value 0.018\n", - "Token [CLS] with Token 50983_0: Attention Value 0.014\n", - "Token [CLS] with Token 51257_0: Attention Value 0.013\n", - "Token [CLS] with Token 52069_0: Attention Value 0.012\n", - "Token [CLS] with Token 50902_0: Attention Value 0.011\n", - "Token [CLS] with Token 51277_1: Attention Value 0.010\n", - "Token [CLS] with Token 50878_4: Attention Value 0.009\n", - "Token [CLS] with Token 00641607825: Attention Value 0.009\n", - "Token [CLS] with Token 00641607825: Attention Value 0.009\n", - "Token [CLS] with Token 51133_2: Attention Value 0.009\n", - "Token [CLS] with Token [CLS]: Attention Value 0.008\n", - "Token [CLS] with Token 51255_1: Attention Value 0.008\n", - "Token [CLS] with Token 61958040101: Attention Value 0.008\n" - ] - }, - { - "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "colorscale": [ - [ - 0, - "rgb(255,255,217)" - ], - [ - 0.125, - "rgb(237,248,177)" - ], - [ - 0.25, - "rgb(199,233,180)" - ], - [ - 0.375, - "rgb(127,205,187)" - ], - [ - 0.5, - "rgb(65,182,196)" - ], - [ - 0.625, - "rgb(29,145,192)" - ], - [ - 0.75, - "rgb(34,94,168)" - ], - [ - 0.875, - "rgb(37,52,148)" - ], - [ - 1, - "rgb(8,29,88)" - ] - ], - "hoverinfo": "text", - "hoverongaps": false, - "text": [ - [ - "Token [CLS] with Token [CLS]:Attention Value 0.008", - "Token [CLS] with Token [VS]:Attention Value 0.000", - "Token [CLS] with Token 00121048910:Attention Value 0.001", - "Token [CLS] with Token 00338004904:Attention Value 0.003", - "Token [CLS] with Token 00121048910:Attention Value 0.001", - "Token [CLS] with Token 00006494300:Attention Value 0.000", - "Token [CLS] with Token 00904272561:Attention Value 0.001", - "Token [CLS] with Token 61958040101:Attention Value 0.004", - "Token [CLS] with Token 00487950101:Attention Value 0.002", - "Token [CLS] with Token 00904198261:Attention Value 0.001", - "Token [CLS] with Token 00173071920:Attention Value 0.007", - "Token [CLS] with Token 00121048910:Attention Value 0.001", - "Token [CLS] with Token 60505011300:Attention Value 0.001", - "Token [CLS] with Token 00338004904:Attention Value 0.004", - "Token [CLS] with Token 61958040101:Attention Value 0.008", - "Token [CLS] with Token 00597008717:Attention Value 0.004", - "Token [CLS] with Token 00338004904:Attention Value 0.004", - "Token [CLS] with Token 42292000120:Attention Value 0.002", - "Token [CLS] with Token 00904639161:Attention Value 0.001", - "Token [CLS] with Token 00338004904:Attention Value 0.005", - "Token [CLS] with Token 00472030680:Attention Value 0.003", - "Token [CLS] with Token 00143989701:Attention Value 0.001", - "Token [CLS] with Token 00641607825:Attention Value 0.009", - "Token [CLS] with Token 00338004904:Attention Value 0.004", - "Token [CLS] with Token 51079096620:Attention Value 0.001", - "Token [CLS] with Token 00409128331:Attention Value 0.002", - "Token [CLS] with Token 63323031461:Attention Value 0.000", - "Token [CLS] with Token 00009513503:Attention Value 0.001", - "Token [CLS] with Token 00056051030:Attention Value 0.004", - "Token [CLS] with Token 66758016013:Attention Value 0.004", - "Token [CLS] with Token 00338004904:Attention Value 0.003", - "Token [CLS] with Token 00338004904:Attention Value 0.003", - "Token [CLS] with Token 00071015892:Attention Value 0.004", - "Token [CLS] with Token 00338004904:Attention Value 0.006", - "Token [CLS] with Token 11523726808:Attention Value 0.001", - "Token [CLS] with Token 00904198861:Attention Value 0.000", - "Token [CLS] with Token 00071015892:Attention Value 0.003", - "Token [CLS] with Token 00487020101:Attention Value 0.002", - "Token [CLS] with Token 63739035810:Attention Value 0.001", - "Token [CLS] with Token 00904635361:Attention Value 0.003", - "Token [CLS] with Token 76439034310:Attention Value 0.005", - "Token [CLS] with Token 57896042101:Attention Value 0.003", - "Token [CLS] with Token 00056051030:Attention Value 0.005", - "Token [CLS] with Token 42292000120:Attention Value 0.002", - "Token [CLS] with Token 51079054720:Attention Value 0.004", - "Token [CLS] with Token 00904224461:Attention Value 0.000", - "Token [CLS] with Token 51079098320:Attention Value 0.002", - "Token [CLS] with Token 61958060101:Attention Value 0.003", - "Token [CLS] with Token 51079098320:Attention Value 0.001", - "Token [CLS] with Token 00641607825:Attention Value 0.009", - "Token [CLS] with Token 49281041688:Attention Value 0.005", - "Token [CLS] with Token 00904516561:Attention Value 0.002", - "Token [CLS] with Token 00121048910:Attention Value 0.001", - "Token [CLS] with Token 00173068224:Attention Value 0.005", - "Token [CLS] with Token 00121048910:Attention Value 0.001", - "Token [CLS] with Token 63323026201:Attention Value 0.000", - "Token [CLS] with Token 57896045208:Attention Value 0.004", - "Token [CLS] with Token 63323031461:Attention Value 0.001", - "Token [CLS] with Token 00487020101:Attention Value 0.002", - "Token [CLS] with Token 52074_3:Attention Value 0.000", - "Token [CLS] with Token 52073_2:Attention Value 0.000", - "Token [CLS] with Token 52069_2:Attention Value 0.000", - "Token [CLS] with Token 51301_1:Attention Value 0.000", - "Token [CLS] with Token 51279_3:Attention Value 0.001", - "Token [CLS] with Token 51277_1:Attention Value 0.001", - "Token [CLS] with Token 51265_2:Attention Value 0.000", - "Token [CLS] with Token 51256_1:Attention Value 0.001", - "Token [CLS] with Token 51254_4:Attention Value 0.001", - "Token [CLS] with Token 51250_2:Attention Value 0.001", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51244_3:Attention Value 0.000", - "Token [CLS] with Token 51222_4:Attention Value 0.001", - "Token [CLS] with Token 51221_3:Attention Value 0.002", - "Token [CLS] with Token 51200_2:Attention Value 0.001", - "Token [CLS] with Token 51146_2:Attention Value 0.001", - "Token [CLS] with Token 51133_3:Attention Value 0.001", - "Token [CLS] with Token 52075_2:Attention Value 0.001", - "Token [CLS] with Token 52074_2:Attention Value 0.001", - "Token [CLS] with Token 52073_2:Attention Value 0.001", - "Token [CLS] with Token 52069_1:Attention Value 0.001", - "Token [CLS] with Token 51301_1:Attention Value 0.001", - "Token [CLS] with Token 51279_3:Attention Value 0.002", - "Token [CLS] with Token 51277_1:Attention Value 0.001", - "Token [CLS] with Token 51265_1:Attention Value 0.000", - "Token [CLS] with Token 51256_3:Attention Value 0.001", - "Token [CLS] with Token 51254_3:Attention Value 0.001", - "Token [CLS] with Token 51250_3:Attention Value 0.001", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51244_1:Attention Value 0.000", - "Token [CLS] with Token 51222_3:Attention Value 0.001", - "Token [CLS] with Token 51221_3:Attention Value 0.001", - "Token [CLS] with Token 51200_2:Attention Value 0.001", - "Token [CLS] with Token 51146_1:Attention Value 0.001", - "Token [CLS] with Token 51133_1:Attention Value 0.001", - "Token [CLS] with Token 51006_0:Attention Value 0.001", - "Token [CLS] with Token 50983_0:Attention Value 0.002", - "Token [CLS] with Token 50971_0:Attention Value 0.001", - "Token [CLS] with Token 50970_1:Attention Value 0.001", - "Token [CLS] with Token 50960_2:Attention Value 0.001", - "Token [CLS] with Token 50931_2:Attention Value 0.001", - "Token [CLS] with Token 50912_3:Attention Value 0.002", - "Token [CLS] with Token 50902_1:Attention Value 0.001", - "Token [CLS] with Token 50893_0:Attention Value 0.003", - "Token [CLS] with Token 50885_1:Attention Value 0.001", - "Token [CLS] with Token 50882_0:Attention Value 0.001", - "Token [CLS] with Token 50878_3:Attention Value 0.001", - "Token [CLS] with Token 50868_4:Attention Value 0.001", - "Token [CLS] with Token 50863_1:Attention Value 0.001", - "Token [CLS] with Token 50861_3:Attention Value 0.001", - "Token [CLS] with Token 51516_1:Attention Value 0.001", - "Token [CLS] with Token 51498_4:Attention Value 0.001", - "Token [CLS] with Token 51493_2:Attention Value 0.000", - "Token [CLS] with Token 51492_1:Attention Value 0.001", - "Token [CLS] with Token 51491_2:Attention Value 0.001", - "Token [CLS] with Token 51009_0:Attention Value 0.001", - "Token [CLS] with Token 51006_1:Attention Value 0.002", - "Token [CLS] with Token 50983_0:Attention Value 0.004", - "Token [CLS] with Token 50971_0:Attention Value 0.002", - "Token [CLS] with Token 50970_1:Attention Value 0.001", - "Token [CLS] with Token 50960_0:Attention Value 0.001", - "Token [CLS] with Token 50931_2:Attention Value 0.001", - "Token [CLS] with Token 50912_3:Attention Value 0.003", - "Token [CLS] with Token 50902_2:Attention Value 0.002", - "Token [CLS] with Token 50893_0:Attention Value 0.003", - "Token [CLS] with Token 50882_0:Attention Value 0.002", - "Token [CLS] with Token 50868_4:Attention Value 0.001", - "Token [CLS] with Token 51301_2:Attention Value 0.002", - "Token [CLS] with Token 51279_3:Attention Value 0.002", - "Token [CLS] with Token 51277_1:Attention Value 0.002", - "Token [CLS] with Token 51265_1:Attention Value 0.001", - "Token [CLS] with Token 51250_3:Attention Value 0.002", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51222_3:Attention Value 0.002", - "Token [CLS] with Token 51221_3:Attention Value 0.001", - "Token [CLS] with Token 50813_3:Attention Value 0.002", - "Token [CLS] with Token 51006_1:Attention Value 0.002", - "Token [CLS] with Token 50983_1:Attention Value 0.002", - "Token [CLS] with Token 50971_1:Attention Value 0.002", - "Token [CLS] with Token 50970_3:Attention Value 0.001", - "Token [CLS] with Token 50960_0:Attention Value 0.001", - "Token [CLS] with Token 50931_2:Attention Value 0.001", - "Token [CLS] with Token 50912_3:Attention Value 0.003", - "Token [CLS] with Token 50911_0:Attention Value 0.001", - "Token [CLS] with Token 50902_1:Attention Value 0.001", - "Token [CLS] with Token 50893_0:Attention Value 0.002", - "Token [CLS] with Token 50882_0:Attention Value 0.002", - "Token [CLS] with Token 50868_4:Attention Value 0.002", - "Token [CLS] with Token 51301_3:Attention Value 0.002", - "Token [CLS] with Token 51279_4:Attention Value 0.002", - "Token [CLS] with Token 51277_1:Attention Value 0.002", - "Token [CLS] with Token 51265_2:Attention Value 0.001", - "Token [CLS] with Token 51250_3:Attention Value 0.002", - "Token [CLS] with Token 51248_4:Attention Value 0.001", - "Token [CLS] with Token 51222_4:Attention Value 0.002", - "Token [CLS] with Token 51221_4:Attention Value 0.001", - "Token [CLS] with Token 51516_3:Attention Value 0.001", - "Token [CLS] with Token 51498_4:Attention Value 0.001", - "Token [CLS] with Token 51493_2:Attention Value 0.001", - "Token [CLS] with Token 51491_2:Attention Value 0.001", - "Token [CLS] with Token 51482_2:Attention Value 0.001", - "Token [CLS] with Token 50813_4:Attention Value 0.003", - "Token [CLS] with Token 51301_4:Attention Value 0.002", - "Token [CLS] with Token 51279_4:Attention Value 0.002", - "Token [CLS] with Token 51277_1:Attention Value 0.002", - "Token [CLS] with Token 51265_2:Attention Value 0.001", - "Token [CLS] with Token 51250_3:Attention Value 0.002", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51222_4:Attention Value 0.002", - "Token [CLS] with Token 51221_4:Attention Value 0.001", - "Token [CLS] with Token 51006_1:Attention Value 0.002", - "Token [CLS] with Token 50983_1:Attention Value 0.003", - "Token [CLS] with Token 50971_1:Attention Value 0.003", - "Token [CLS] with Token 50970_1:Attention Value 0.002", - "Token [CLS] with Token 50960_0:Attention Value 0.001", - "Token [CLS] with Token 50931_4:Attention Value 0.001", - "Token [CLS] with Token 50912_3:Attention Value 0.005", - "Token [CLS] with Token 50902_2:Attention Value 0.003", - "Token [CLS] with Token 50893_2:Attention Value 0.002", - "Token [CLS] with Token 50882_0:Attention Value 0.002", - "Token [CLS] with Token 50868_4:Attention Value 0.002", - "Token [CLS] with Token 52135_4:Attention Value 0.003", - "Token [CLS] with Token 52135_2:Attention Value 0.002", - "Token [CLS] with Token 52075_4:Attention Value 0.002", - "Token [CLS] with Token 52074_4:Attention Value 0.002", - "Token [CLS] with Token 52073_3:Attention Value 0.001", - "Token [CLS] with Token 52069_3:Attention Value 0.002", - "Token [CLS] with Token 51301_4:Attention Value 0.002", - "Token [CLS] with Token 51279_4:Attention Value 0.003", - "Token [CLS] with Token 51277_0:Attention Value 0.004", - "Token [CLS] with Token 51265_4:Attention Value 0.001", - "Token [CLS] with Token 51256_3:Attention Value 0.003", - "Token [CLS] with Token 51254_3:Attention Value 0.002", - "Token [CLS] with Token 51250_3:Attention Value 0.001", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51244_1:Attention Value 0.001", - "Token [CLS] with Token 51222_4:Attention Value 0.001", - "Token [CLS] with Token 51221_4:Attention Value 0.001", - "Token [CLS] with Token 51200_2:Attention Value 0.001", - "Token [CLS] with Token 51146_2:Attention Value 0.001", - "Token [CLS] with Token 51133_2:Attention Value 0.001", - "Token [CLS] with Token 51006_1:Attention Value 0.002", - "Token [CLS] with Token 50983_1:Attention Value 0.002", - "Token [CLS] with Token 50971_3:Attention Value 0.002", - "Token [CLS] with Token 50970_3:Attention Value 0.000", - "Token [CLS] with Token 50960_3:Attention Value 0.001", - "Token [CLS] with Token 50931_2:Attention Value 0.001", - "Token [CLS] with Token 50912_2:Attention Value 0.003", - "Token [CLS] with Token 50902_2:Attention Value 0.002", - "Token [CLS] with Token 50893_3:Attention Value 0.001", - "Token [CLS] with Token 50882_0:Attention Value 0.001", - "Token [CLS] with Token 50868_4:Attention Value 0.001", - "Token [CLS] with Token 51301_2:Attention Value 0.001", - "Token [CLS] with Token 51279_4:Attention Value 0.001", - "Token [CLS] with Token 51277_0:Attention Value 0.001", - "Token [CLS] with Token 51265_2:Attention Value 0.000", - "Token [CLS] with Token 51250_3:Attention Value 0.001", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51222_4:Attention Value 0.001", - "Token [CLS] with Token 51221_4:Attention Value 0.001", - "Token [CLS] with Token 51006_1:Attention Value 0.001", - "Token [CLS] with Token 50983_1:Attention Value 0.001", - "Token [CLS] with Token 50971_1:Attention Value 0.001", - "Token [CLS] with Token 50970_2:Attention Value 0.000", - "Token [CLS] with Token 50960_3:Attention Value 0.001", - "Token [CLS] with Token 50931_2:Attention Value 0.000", - "Token [CLS] with Token 50912_2:Attention Value 0.001", - "Token [CLS] with Token 50902_2:Attention Value 0.001", - "Token [CLS] with Token 50893_1:Attention Value 0.001", - "Token [CLS] with Token 50882_0:Attention Value 0.001", - "Token [CLS] with Token 50868_4:Attention Value 0.001", - "Token [CLS] with Token 51301_2:Attention Value 0.001", - "Token [CLS] with Token 51279_4:Attention Value 0.001", - "Token [CLS] with Token 51277_0:Attention Value 0.001", - "Token [CLS] with Token 51265_2:Attention Value 0.001", - "Token [CLS] with Token 51221_4:Attention Value 0.001", - "Token [CLS] with Token 51222_4:Attention Value 0.001", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51250_3:Attention Value 0.001", - "Token [CLS] with Token 51265_2:Attention Value 0.001", - "Token [CLS] with Token 51277_0:Attention Value 0.001", - "Token [CLS] with Token 51279_4:Attention Value 0.001", - "Token [CLS] with Token 51301_2:Attention Value 0.001", - "Token [CLS] with Token 51250_3:Attention Value 0.001", - "Token [CLS] with Token 50861_1:Attention Value 0.001", - "Token [CLS] with Token 50863_1:Attention Value 0.001", - "Token [CLS] with Token 50868_4:Attention Value 0.001", - "Token [CLS] with Token 50878_0:Attention Value 0.001", - "Token [CLS] with Token 50882_0:Attention Value 0.001", - "Token [CLS] with Token 50885_1:Attention Value 0.001", - "Token [CLS] with Token 50902_3:Attention Value 0.003", - "Token [CLS] with Token 50912_2:Attention Value 0.002", - "Token [CLS] with Token 50931_1:Attention Value 0.001", - "Token [CLS] with Token 50971_2:Attention Value 0.002", - "Token [CLS] with Token 50983_3:Attention Value 0.002", - "Token [CLS] with Token 51006_1:Attention Value 0.002", - "Token [CLS] with Token 50868_4:Attention Value 0.002", - "Token [CLS] with Token 50882_0:Attention Value 0.003", - "Token [CLS] with Token 50893_1:Attention Value 0.003", - "Token [CLS] with Token 50902_2:Attention Value 0.005", - "Token [CLS] with Token 50912_2:Attention Value 0.004", - "Token [CLS] with Token 50931_2:Attention Value 0.002", - "Token [CLS] with Token 50960_2:Attention Value 0.002", - "Token [CLS] with Token 50970_2:Attention Value 0.002", - "Token [CLS] with Token 50971_1:Attention Value 0.003", - "Token [CLS] with Token 50983_1:Attention Value 0.004", - "Token [CLS] with Token 51006_1:Attention Value 0.004", - "Token [CLS] with Token 51009_2:Attention Value 0.002", - "Token [CLS] with Token 51221_4:Attention Value 0.003", - "Token [CLS] with Token 51222_4:Attention Value 0.002", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51476_0:Attention Value 0.002", - "Token [CLS] with Token 52135_4:Attention Value 0.003", - "Token [CLS] with Token 52135_4:Attention Value 0.003", - "Token [CLS] with Token 51492_1:Attention Value 0.002", - "Token [CLS] with Token 51476_0:Attention Value 0.002", - "Token [CLS] with Token 51275_0:Attention Value 0.002", - "Token [CLS] with Token 51274_0:Attention Value 0.002", - "Token [CLS] with Token 51237_1:Attention Value 0.001", - "Token [CLS] with Token 51006_0:Attention Value 0.003", - "Token [CLS] with Token 50983_0:Attention Value 0.005", - "Token [CLS] with Token 50971_1:Attention Value 0.003", - "Token [CLS] with Token 50970_0:Attention Value 0.002", - "Token [CLS] with Token 50960_4:Attention Value 0.002", - "Token [CLS] with Token 50931_2:Attention Value 0.001", - "Token [CLS] with Token 50912_3:Attention Value 0.005", - "Token [CLS] with Token 50902_0:Attention Value 0.003", - "Token [CLS] with Token 50893_2:Attention Value 0.002", - "Token [CLS] with Token 50885_1:Attention Value 0.001", - "Token [CLS] with Token 50882_0:Attention Value 0.004", - "Token [CLS] with Token 50878_4:Attention Value 0.002", - "Token [CLS] with Token 50868_4:Attention Value 0.002", - "Token [CLS] with Token 50863_2:Attention Value 0.001", - "Token [CLS] with Token 50861_4:Attention Value 0.002", - "Token [CLS] with Token 52075_3:Attention Value 0.002", - "Token [CLS] with Token 52074_3:Attention Value 0.002", - "Token [CLS] with Token 52073_2:Attention Value 0.001", - "Token [CLS] with Token 52069_3:Attention Value 0.002", - "Token [CLS] with Token 51301_4:Attention Value 0.002", - "Token [CLS] with Token 51279_3:Attention Value 0.002", - "Token [CLS] with Token 51277_1:Attention Value 0.002", - "Token [CLS] with Token 51265_3:Attention Value 0.001", - "Token [CLS] with Token 51256_3:Attention Value 0.003", - "Token [CLS] with Token 51254_2:Attention Value 0.001", - "Token [CLS] with Token 51250_3:Attention Value 0.001", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51244_1:Attention Value 0.001", - "Token [CLS] with Token 51222_4:Attention Value 0.002", - "Token [CLS] with Token 51221_4:Attention Value 0.001", - "Token [CLS] with Token 51200_2:Attention Value 0.002", - "Token [CLS] with Token 51146_2:Attention Value 0.002", - "Token [CLS] with Token 51133_3:Attention Value 0.001", - "Token [CLS] with Token 51275_0:Attention Value 0.002", - "Token [CLS] with Token 51274_1:Attention Value 0.003", - "Token [CLS] with Token 51237_2:Attention Value 0.002", - "Token [CLS] with Token 51006_0:Attention Value 0.003", - "Token [CLS] with Token 50983_0:Attention Value 0.004", - "Token [CLS] with Token 50976_2:Attention Value 0.002", - "Token [CLS] with Token 50971_0:Attention Value 0.002", - "Token [CLS] with Token 50970_2:Attention Value 0.001", - "Token [CLS] with Token 50960_2:Attention Value 0.001", - "Token [CLS] with Token 50951_1:Attention Value 0.001", - "Token [CLS] with Token 50950_2:Attention Value 0.002", - "Token [CLS] with Token 50949_1:Attention Value 0.004", - "Token [CLS] with Token 50931_3:Attention Value 0.001", - "Token [CLS] with Token 50930_3:Attention Value 0.002", - "Token [CLS] with Token 50912_3:Attention Value 0.003", - "Token [CLS] with Token 50902_0:Attention Value 0.002", - "Token [CLS] with Token 50893_0:Attention Value 0.004", - "Token [CLS] with Token 50885_1:Attention Value 0.001", - "Token [CLS] with Token 50882_0:Attention Value 0.002", - "Token [CLS] with Token 50878_4:Attention Value 0.002", - "Token [CLS] with Token 50868_4:Attention Value 0.002", - "Token [CLS] with Token 50863_2:Attention Value 0.001", - "Token [CLS] with Token 50862_3:Attention Value 0.001", - "Token [CLS] with Token 50861_3:Attention Value 0.001", - "Token [CLS] with Token 50853_1:Attention Value 0.006", - "Token [CLS] with Token 52075_2:Attention Value 0.004", - "Token [CLS] with Token 52074_3:Attention Value 0.002", - "Token [CLS] with Token 52073_3:Attention Value 0.001", - "Token [CLS] with Token 52069_0:Attention Value 0.012", - "Token [CLS] with Token 51301_1:Attention Value 0.006", - "Token [CLS] with Token 51279_3:Attention Value 0.018", - "Token [CLS] with Token 51277_1:Attention Value 0.010", - "Token [CLS] with Token 51265_2:Attention Value 0.001", - "Token [CLS] with Token 51257_0:Attention Value 0.013", - "Token [CLS] with Token 51256_1:Attention Value 0.003", - "Token [CLS] with Token 51255_1:Attention Value 0.008", - "Token [CLS] with Token 51254_4:Attention Value 0.003", - "Token [CLS] with Token 51251_1:Attention Value 0.026", - "Token [CLS] with Token 51250_2:Attention Value 0.003", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51244_2:Attention Value 0.002", - "Token [CLS] with Token 51222_4:Attention Value 0.003", - "Token [CLS] with Token 51221_3:Attention Value 0.002", - "Token [CLS] with Token 51200_3:Attention Value 0.003", - "Token [CLS] with Token 51146_0:Attention Value 0.003", - "Token [CLS] with Token 51144_2:Attention Value 0.002", - "Token [CLS] with Token 51143_0:Attention Value 0.002", - "Token [CLS] with Token 51133_2:Attention Value 0.009", - "Token [CLS] with Token 51006_0:Attention Value 0.007", - "Token [CLS] with Token 50983_0:Attention Value 0.014", - "Token [CLS] with Token 50971_0:Attention Value 0.006", - "Token [CLS] with Token 50970_1:Attention Value 0.002", - "Token [CLS] with Token 50960_2:Attention Value 0.004", - "Token [CLS] with Token 50950_2:Attention Value 0.005", - "Token [CLS] with Token 50931_2:Attention Value 0.003", - "Token [CLS] with Token 50912_3:Attention Value 0.008", - "Token [CLS] with Token 50902_0:Attention Value 0.011", - "Token [CLS] with Token 50893_0:Attention Value 0.154", - "Token [CLS] with Token 50885_1:Attention Value 0.002", - "Token [CLS] with Token 50882_0:Attention Value 0.004", - "Token [CLS] with Token 50878_4:Attention Value 0.009", - "Token [CLS] with Token 50868_4:Attention Value 0.007", - "Token [CLS] with Token 50863_2:Attention Value 0.005", - "Token [CLS] with Token 50861_3:Attention Value 0.002", - "Token [CLS] with Token 52075_2:Attention Value 0.004", - "Token [CLS] with Token [VE]:Attention Value 0.006", - "Token [CLS] with Token [REG]:Attention Value 0.003" - ] - ], - "type": "heatmap", - "x": [ - "[CLS]", - "[VS]", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "10", - "11", - "12", - "13", - "14", - "15", - "16", - "17", - "18", - "19", - "20", - "21", - "22", - "23", - "24", - "25", - "26", - "27", - "28", - "29", - "30", - "31", - "32", - "33", - "34", - "35", - "36", - "37", - "38", - "39", - "40", - "41", - "42", - "43", - "44", - "45", - "46", - "47", - "48", - "49", - "50", - "51", - "52", - "53", - "54", - "55", - "56", - "57", - "58", - "59", - "60", - "61", - "62", - "63", - "64", - "65", - "66", - "67", - "68", - "69", - "70", - "71", - "72", - "73", - "74", - "75", - "76", - "77", - "78", - "79", - "80", - "81", - "82", - "83", - "84", - "85", - "86", - "87", - "88", - "89", - "90", - "91", - "92", - "93", - "94", - "95", - "96", - "97", - "98", - "99", - "100", - "101", - "102", - "103", - "104", - "105", - "106", - "107", - "108", - "109", - "110", - "111", - "112", - "113", - "114", - "115", - "116", - "117", - "118", - "119", - "120", - "121", - "122", - "123", - "124", - "125", - "126", - "127", - "128", - "129", - "130", - "131", - "132", - "133", - "134", - "135", - "136", - "137", - "138", - "139", - "140", - "141", - "142", - "143", - "144", - "145", - "146", - "147", - "148", - "149", - "150", - "151", - "152", - "153", - "154", - "155", - "156", - "157", - "158", - "159", - "160", - "161", - "162", - "163", - "164", - "165", - "166", - "167", - "168", - "169", - "170", - "171", - "172", - "173", - "174", - "175", - "176", - "177", - "178", - "179", - "180", - "181", - "182", - "183", - "184", - "185", - "186", - "187", - "188", - "189", - "190", - "191", - "192", - "193", - "194", - "195", - "196", - "197", - "198", - "199", - "200", - "201", - "202", - "203", - "204", - "205", - "206", - "207", - "208", - "209", - "210", - "211", - "212", - "213", - "214", - "215", - "216", - "217", - "218", - "219", - "220", - "221", - "222", - "223", - "224", - "225", - "226", - "227", - "228", - "229", - "230", - "231", - "232", - "233", - "234", - "235", - "236", - "237", - "238", - "239", - "240", - "241", - "242", - "243", - "244", - "245", - "246", - "247", - "248", - "249", - "250", - "251", - "252", - "253", - "254", - "255", - "256", - "257", - "258", - "259", - "260", - "261", - "262", - "263", - "264", - "265", - "266", - "267", - "268", - "269", - "270", - "271", - "272", - "273", - "274", - "275", - "276", - "277", - "278", - "279", - "280", - "281", - "282", - "283", - "284", - "285", - "286", - "287", - "288", - "289", - "290", - "291", - "292", - "293", - "294", - "295", - "296", - "297", - "298", - "299", - "300", - "301", - "302", - "303", - "304", - "305", - "306", - "307", - "308", - "309", - "310", - "311", - "312", - "313", - "314", - "315", - "316", - "317", - "318", - "319", - "320", - "321", - "322", - "323", - "324", - "325", - "326", - "327", - "328", - "329", - "330", - "331", - "332", - "333", - "334", - "335", - "336", - "337", - "338", - "339", - "340", - "341", - "342", - "343", - "344", - "345", - "346", - "347", - "348", - "349", - "350", - "351", - "352", - "353", - "354", - "355", - "356", - "357", - "358", - "359", - "360", - "361", - "362", - "363", - "364", - "365", - "366", - "367", - "368", - "369", - "370", - "371", - "372", - "373", - "374", - "375", - "[VE]", - "[REG]" - ], - "y": [ - "[CLS]" - ], - "z": [ - [ - 0.008451767265796661, - 0.00047753742546774447, - 0.001319802482612431, - 0.002615319797769189, - 0.0005728917894884944, - 0.0004854540165979415, - 0.0007460298365913332, - 0.004369142930954695, - 0.0021512836683541536, - 0.0010651483898982406, - 0.006857313681393862, - 0.0009411114733666182, - 0.001146065187640488, - 0.003502822481095791, - 0.008377399295568466, - 0.00369816436432302, - 0.0036342681851238017, - 0.001818674267269671, - 0.0011760754277929664, - 0.005098414141684771, - 0.003414525417611003, - 0.0005102005670778453, - 0.009316006675362589, - 0.0036000509280711412, - 0.0010369595838710666, - 0.0019885266665369272, - 0.0004615136131178588, - 0.0008582020527683198, - 0.004428019281476736, - 0.0036287240218371153, - 0.003203341970220208, - 0.003022358985617757, - 0.004367131274193525, - 0.005930684972554445, - 0.0006912555545568466, - 0.0003829135384876281, - 0.002646421082317829, - 0.001659494242630899, - 0.0008906491566449404, - 0.0034986960235983133, - 0.004725936334580183, - 0.003188258036971092, - 0.004636996425688267, - 0.0017536324448883531, - 0.003574026981368661, - 0.0004167505830992013, - 0.001605302095413208, - 0.0034259206149727106, - 0.0010352873941883445, - 0.008988478220999241, - 0.004783918149769306, - 0.0020494854543358088, - 0.0008181784651242197, - 0.005045545753091574, - 0.0008356395992450416, - 0.0003578341274987906, - 0.004442144185304642, - 0.0007386466022580862, - 0.0015157737070694566, - 0.0003436319238971919, - 0.00029586421442218125, - 0.0004052982258144766, - 0.0004121463280171156, - 0.0006444680620916188, - 0.0006272507016547024, - 0.0003953258565161377, - 0.0005427289288491011, - 0.000674175564199686, - 0.0005934844375588, - 0.0005067584570497274, - 0.00048668019007891417, - 0.0006641843356192112, - 0.001736009493470192, - 0.0006450659711845219, - 0.0006109003443270922, - 0.0014823578530922532, - 0.0007514643366448581, - 0.0010008058743551371, - 0.0005770500865764916, - 0.0009993098210543394, - 0.0013827277580276132, - 0.0017192121595144272, - 0.0011489095631986856, - 0.0004682957660406828, - 0.0008927697781473398, - 0.000683685124386102, - 0.0006812589126639068, - 0.0005206714267842472, - 0.0004224051663186401, - 0.0008084981818683445, - 0.001108424854464829, - 0.0009732283069752156, - 0.0007592075853608549, - 0.0005552212824113667, - 0.00127204111777246, - 0.002062402432784438, - 0.00112050655297935, - 0.0008525322191417217, - 0.0008413223549723625, - 0.000753440021071583, - 0.0019989104475826025, - 0.0010396834695711732, - 0.00325021636672318, - 0.0006715958588756621, - 0.0012460827128961682, - 0.000999152078293264, - 0.0009655823814682662, - 0.0005084593431092799, - 0.0005497524398379028, - 0.0006103277555666864, - 0.001034797285683453, - 0.0004598199157044291, - 0.0008794588502496481, - 0.000696350762154907, - 0.001486221794039011, - 0.0017987970495596528, - 0.003589141182601452, - 0.001990983495488763, - 0.0010251256171613932, - 0.001364864525385201, - 0.0008277010056190193, - 0.0025306493043899536, - 0.0021720887161791325, - 0.003414546838030219, - 0.0019456190057098863, - 0.0013443160569295287, - 0.001797817298211157, - 0.0022308507468551397, - 0.0018264207756146789, - 0.0006515602581202984, - 0.0015126297948881984, - 0.0009584605577401816, - 0.0015974263660609722, - 0.0012428780319169164, - 0.0018666594987735152, - 0.0018526790663599968, - 0.001926859957166016, - 0.001790812355466187, - 0.0006914885598234832, - 0.0009147145319730044, - 0.0006848545162938535, - 0.003264268860220909, - 0.001066222321242094, - 0.0014806516701355577, - 0.0019578184001147747, - 0.001972360536456108, - 0.001908112782984972, - 0.0017941169207915664, - 0.002000534441322088, - 0.00232343259267509, - 0.0006864610477350652, - 0.0015883835731074214, - 0.001452704076655209, - 0.0015260959044098854, - 0.001217573182657361, - 0.0007630472537130117, - 0.0009102648473344744, - 0.0010583768598735332, - 0.0010495075257495046, - 0.0013496936298906803, - 0.002790031023323536, - 0.0016460781916975975, - 0.001800714642740786, - 0.001588907209224999, - 0.0006608786643482745, - 0.0015428924234583974, - 0.00130733463447541, - 0.0019498993642628193, - 0.0011935298098251224, - 0.002081812592223286, - 0.0027750423178076744, - 0.002682143822312355, - 0.0017559939296916127, - 0.0014839059440419078, - 0.0011782203800976276, - 0.005421904847025871, - 0.0033844260033220053, - 0.0019421830074861648, - 0.002427612664178014, - 0.0021077985875308514, - 0.002597493352368474, - 0.0019147455459460616, - 0.0022990747820585966, - 0.00196700356900692, - 0.0013677203096449375, - 0.0017343414947390556, - 0.0022338901180773973, - 0.002618071623146534, - 0.003806268097832799, - 0.0009631214197725058, - 0.0029177824035286903, - 0.0015531842363998294, - 0.00118330551777035, - 0.0009476374834775924, - 0.0007008272805251181, - 0.0012339174281805754, - 0.0011241122847422955, - 0.0009876210242509842, - 0.0008716397569514811, - 0.001008612453006208, - 0.0015318739460781217, - 0.0016837789444252849, - 0.0015550237149000168, - 0.00042292437865398824, - 0.0012427183100953698, - 0.0006267700809985399, - 0.0025690405163913965, - 0.002414754591882229, - 0.0008665476925671101, - 0.0007410055841319263, - 0.0011087892344221473, - 0.0014391514705494046, - 0.001120170229114592, - 0.0013088533887639642, - 0.0004123076796531677, - 0.0007661065901629627, - 0.0005386698176153004, - 0.0007872642017900944, - 0.0005083107971586287, - 0.0009654506575316192, - 0.0009281352977268398, - 0.0006618788465857506, - 0.0004174182831775397, - 0.0006613184814341366, - 0.0003955806605517864, - 0.0011703699128702285, - 0.0007652677595615387, - 0.0009616908500902356, - 0.000610732939094305, - 0.0005696555017493665, - 0.000935644842684269, - 0.0005050115869380534, - 0.0008104036678560078, - 0.0005484815337695181, - 0.0005282653728500009, - 0.0006690899026580155, - 0.0006202083895914257, - 0.0011858937796205282, - 0.0006323098205029964, - 0.0012637152103707194, - 0.0010147326393052936, - 0.001434296485967934, - 0.0010879833716899157, - 0.0007398917223326862, - 0.0006289978045970201, - 0.0010291151702404022, - 0.0007981405942700803, - 0.0012479289434850216, - 0.000667006301227957, - 0.0027725808322429657, - 0.002342016203328967, - 0.0006513544940389693, - 0.0018907733028754592, - 0.002159562660381198, - 0.0017471732571721077, - 0.0015442147850990295, - 0.003038904396817088, - 0.003148901043459773, - 0.005466174334287643, - 0.004158675670623779, - 0.001521154772490263, - 0.002499047899618745, - 0.0023495368659496307, - 0.0034855001140385866, - 0.004092938732355833, - 0.003764112712815404, - 0.0024812589399516582, - 0.0029548786114901304, - 0.00236957217566669, - 0.0012237809132784605, - 0.001522903912700713, - 0.0026057965587824583, - 0.0028801767621189356, - 0.0020971631165593863, - 0.0017077381489798429, - 0.002488095546141267, - 0.001942558796145022, - 0.0012060959124937654, - 0.0032189686316996813, - 0.004897533915936947, - 0.002546509262174368, - 0.002436422742903233, - 0.0016161665553227067, - 0.0008353215525858104, - 0.004716587718576193, - 0.003007955150678754, - 0.001906886580400169, - 0.0010407179361209271, - 0.0036604618653655057, - 0.0019616391509771347, - 0.001880012801848352, - 0.0010364892659708858, - 0.0020254242699593306, - 0.001567527768202126, - 0.0018129348754882812, - 0.0011344858212396502, - 0.001787062268704176, - 0.002030221512541175, - 0.002426884835585952, - 0.0018259655917063355, - 0.0006956890574656427, - 0.0026017874479293823, - 0.0008315623854286969, - 0.0014645435148850083, - 0.0009511765674687922, - 0.0009218405466526748, - 0.0015906983753666282, - 0.0014132362557575109, - 0.0017343055224046111, - 0.0019996073096990585, - 0.0013999902876093984, - 0.0021138533484190702, - 0.002869958057999611, - 0.0016324784373864532, - 0.003336323192343116, - 0.004306184593588114, - 0.0022455891594290733, - 0.0018202642677351832, - 0.0014412502059713006, - 0.0014982676366344094, - 0.0010419118916615844, - 0.0016191820614039898, - 0.003920895978808403, - 0.0013921178178861735, - 0.0016874541761353612, - 0.003467828966677189, - 0.0024901803117245436, - 0.003774721873924136, - 0.0008989623747766018, - 0.0018224921077489853, - 0.001697454135864973, - 0.002475267043337226, - 0.001093029393814504, - 0.0014057605294510722, - 0.0009937405120581388, - 0.006045807618647814, - 0.003723071655258536, - 0.0024961901362985373, - 0.0010668792529031634, - 0.012105301953852177, - 0.006213821936398745, - 0.018133636564016346, - 0.010110520757734776, - 0.001374133862555027, - 0.013326204381883144, - 0.002680495148524642, - 0.008382327854633331, - 0.0030930601060390472, - 0.025627313181757927, - 0.0031410318333655596, - 0.001220939448103309, - 0.0015257663326337934, - 0.0025776990223675966, - 0.0020443866960704327, - 0.003117623971775174, - 0.0028798922430723906, - 0.001909662154503167, - 0.0018645658856257796, - 0.00894069205969572, - 0.006883753929287195, - 0.014036208391189575, - 0.0055046118795871735, - 0.0024932369124144316, - 0.0035899977665394545, - 0.004715373273938894, - 0.002608843147754669, - 0.007800393272191286, - 0.011004121042788029, - 0.15411949157714844, - 0.0017875274643301964, - 0.0042082518339157104, - 0.009378299117088318, - 0.007050686050206423, - 0.005348358768969774, - 0.0016888940008357167, - 0.004475145135074854, - 0.006452843081206083, - 0.0025523456279188395 - ] - ] - } - ], - "layout": { - "annotations": [ - { - "bgcolor": "red", - "font": { - "color": "black", - "size": 10 - }, - "opacity": 0.8, - "showarrow": false, - "text": "[CLS]", - "textangle": -90, - "x": 0, - "xref": "x", - "y": 0.5, - "yref": "paper" - }, - { - "bgcolor": "red", - "font": { - "color": "black", - "size": 10 - }, - "opacity": 0.8, - "showarrow": false, - "text": "[VS]", - "textangle": -90, - "x": 1, - "xref": "x", - "y": 0.5, - "yref": "paper" - }, - { - "bgcolor": "red", - "font": { - "color": "black", - "size": 10 - }, - "opacity": 0.8, - "showarrow": false, - "text": "[VE]", - "textangle": -90, - "x": 376, - "xref": "x", - "y": 0.5, - "yref": "paper" - }, - { - "bgcolor": "red", - "font": { - "color": "black", - "size": 10 - }, - "opacity": 0.8, - "showarrow": false, - "text": "[REG]", - "textangle": -90, - "x": 377, - "xref": "x", - "y": 0.5, - "yref": "paper" - } - ], - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "heatmapgl": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmapgl" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "Attention Visualization" - }, - "xaxis": { - "nticks": 378, - "tickangle": -90, - "title": { - "text": "Token in Input Sequence" - } - }, - "yaxis": { - "nticks": 1, - "title": { - "text": "Token in Input Sequence" - } - } - } - }, - "text/html": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "def visualize_attention(\n", - " attention_weights,\n", - " patient,\n", - " special_tokens,\n", - " tokenizer,\n", - " truncate=False,\n", - " only_cls=False,\n", - " top_k=10,\n", - "):\n", - " # Convert attention tensor to numpy array and squeeze the batch dimension\n", - " concept_ids = patient[\"concept_ids\"].squeeze(0).cpu().numpy()\n", - " attention_weights = attention_weights.squeeze(0).cpu().numpy()\n", - "\n", - " # Truncate attention weights if specified\n", - " if truncate:\n", - " truncate_at = patient[\"attention_mask\"].sum().numpy()\n", - " attention_weights = attention_weights[:, :truncate_at, :truncate_at]\n", - " concept_ids = concept_ids[:truncate_at]\n", - "\n", - " if only_cls:\n", - " attention_weights = attention_weights[:, :1, :]\n", - "\n", - " # Average attention weights across heads\n", - " attention_weights = attention_weights.mean(axis=0)\n", - "\n", - " # Generate token labels, marking special tokens with a special symbol\n", - " x_token_labels = [\n", - " f\"{tokenizer.id_to_token(token)}\"\n", - " if tokenizer.id_to_token(token) in special_tokens\n", - " else str(i)\n", - " for i, token in enumerate(concept_ids)\n", - " ]\n", - " y_token_labels = [\"[CLS]\"]\n", - "\n", - " # Generate hover text\n", - " hover_text = [\n", - " [\n", - " f\"Token {tokenizer.id_to_token(concept_ids[row])} with Token {tokenizer.id_to_token(concept_ids[col])}:\"\n", - " f\"Attention Value {attention_weights[row, col]:.3f}\"\n", - " for col in range(attention_weights.shape[1])\n", - " ]\n", - " for row in range(attention_weights.shape[0])\n", - " ]\n", - "\n", - " # Generate annotations for special tokens\n", - " annotations = []\n", - " for i, token in enumerate(concept_ids):\n", - " if tokenizer.id_to_token(token) in special_tokens:\n", - " annotations.append(\n", - " dict(\n", - " x=i,\n", - " y=0.5,\n", - " xref=\"x\",\n", - " yref=\"paper\", # Use 'paper' coordinates for y\n", - " text=tokenizer.id_to_token(token),\n", - " showarrow=False,\n", - " font=dict(color=\"black\", size=10),\n", - " textangle=-90,\n", - " bgcolor=\"red\",\n", - " opacity=0.8,\n", - " ),\n", - " )\n", - "\n", - " # Plot the attention matrix as a heatmap\n", - " fig = go.Figure(\n", - " data=go.Heatmap(\n", - " z=attention_weights,\n", - " x=x_token_labels,\n", - " y=y_token_labels,\n", - " hoverongaps=False,\n", - " hoverinfo=\"text\",\n", - " text=hover_text,\n", - " colorscale=\"YlGnBu\",\n", - " ),\n", - " )\n", - "\n", - " fig.update_layout(\n", - " title=\"Attention Visualization\",\n", - " xaxis_nticks=len(concept_ids),\n", - " yaxis_nticks=len(y_token_labels),\n", - " xaxis_title=\"Token in Input Sequence\",\n", - " yaxis_title=\"Token in Input Sequence\",\n", - " annotations=annotations,\n", - " xaxis_tickangle=-90,\n", - " )\n", - "\n", - " # Print top k tokens with their attention values\n", - " top_k_indices = np.argsort(-attention_weights, axis=None)[:top_k]\n", - " top_k_values = attention_weights.flatten()[top_k_indices]\n", - " top_k_indices = np.unravel_index(top_k_indices, attention_weights.shape)\n", - "\n", - " for idx in range(len(top_k_indices[0])):\n", - " token1 = top_k_indices[0][idx]\n", - " token2 = top_k_indices[1][idx]\n", - " attention_value = top_k_values[idx]\n", - " print(\n", - " f\"Token {tokenizer.id_to_token(concept_ids[token1])} \"\n", - " f\"with Token {tokenizer.id_to_token(concept_ids[token2])}: \"\n", - " f\"Attention Value {attention_value:.3f}\",\n", - " )\n", - "\n", - " fig.show()\n", - "\n", - "\n", - "# Visualize the attention matrix with special tokens\n", - "special_tokens = [\"[CLS]\", \"[VS]\", \"[VE]\", \"[REG]\"]\n", - "visualize_attention(\n", - " last_attention_matrix,\n", - " patient=patient,\n", - " special_tokens=special_tokens,\n", - " tokenizer=tokenizer,\n", - " truncate=True,\n", - " only_cls=True,\n", - " top_k=15,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "92da688f", - "metadata": {}, - "outputs": [], - "source": [ - "# def visualize_attention(attention_weights, patient, special_tokens, tokenizer, truncate=False, only_cls=False, top_k=10):\n", - "# # Convert attention tensor to numpy array and squeeze the batch dimension\n", - "# concept_ids = patient['concept_ids'].squeeze(0).cpu().numpy()\n", - "# attention_weights = attention_weights.squeeze(0).cpu().numpy()\n", - "\n", - "# # Truncate attention weights if specified\n", - "# if truncate:\n", - "# truncate_at = patient['attention_mask'].sum().numpy()\n", - "# attention_weights = attention_weights[:, :truncate_at, :truncate_at]\n", - "# concept_ids = concept_ids[:truncate_at]\n", - "\n", - "# if only_cls:\n", - "# attention_weights = attention_weights[:, :1, :]\n", - "\n", - "# # Average attention weights across heads\n", - "# attention_weights = attention_weights.mean(axis=0)\n", - "\n", - "# # Generate token labels, replacing special tokens with their names\n", - "# token_labels = [tokenizer.id_to_token(token) if tokenizer.id_to_token(token) in special_tokens else '' for token in concept_ids]\n", - "\n", - "# # Plot the attention matrix as a heatmap\n", - "# sns.set_theme(font_scale=1.2)\n", - "# plt.figure(figsize=(15, 12))\n", - "# ax = sns.heatmap(attention_weights, cmap=\"YlGnBu\", linewidths=.5, annot=False, cbar=True)\n", - "# ax.set_title('Attention Visualization')\n", - "\n", - "# # Set custom tick labels\n", - "# # ax.set_xticks(np.arange(len(token_labels)) + 0.5)\n", - "# # ax.set_xticklabels(token_labels, rotation=45, ha='right', fontsize=10)\n", - "# # ax.set_yticks(np.arange(len(token_labels)) + 0.5)\n", - "# # ax.set_yticklabels(token_labels, rotation=0, ha='right', fontsize=10)\n", - "\n", - "# ax.set_xlabel('Token in Input Sequence')\n", - "# ax.set_ylabel('Token in Input Sequence')\n", - "\n", - "# # Print top k tokens with their attention values\n", - "# top_k_indices = np.argsort(-attention_weights, axis=None)[:top_k]\n", - "# top_k_values = attention_weights.flatten()[top_k_indices]\n", - "# top_k_indices = np.unravel_index(top_k_indices, attention_weights.shape)\n", - "\n", - "# for idx in range(len(top_k_indices[0])):\n", - "# token1 = top_k_indices[0][idx]\n", - "# token2 = top_k_indices[1][idx]\n", - "# attention_value = top_k_values[idx]\n", - "# print(f\"Token {tokenizer.id_to_token(concept_ids[token1])} \"\n", - "# f\"with Token {tokenizer.id_to_token(concept_ids[token2])}: \"\n", - "# f\"Attention Value {attention_value}\")\n", - "\n", - "# plt.show()\n", - "\n", - "\n", - "# # Visualize the attention matrix with special tokens\n", - "# special_tokens = ['[CLS]', '[VS]', '[VE]', '[REG]'] # Update this list with your actual special tokens\n", - "# visualize_attention(last_attention_matrix, patient=patient, special_tokens=special_tokens, tokenizer=tokenizer, truncate=True, only_cls=True, top_k=25)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cf327908", - "metadata": {}, - "outputs": [], - "source": [ - "# Model view\n", - "html_model_view = model_view(\n", - " truncated_attention_matrix,\n", - " truncated_tokens,\n", - " include_layers=[5],\n", - " include_heads=[0, 1, 2, 3, 4, 5],\n", - " display_mode=\"light\",\n", - " html_action=\"return\",\n", - ")\n", - "\n", - "with open(\"model_view.html\", \"w\") as file:\n", - " file.write(html_model_view.data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "acae54e37e7d407bbb7b55eff062a284", - "metadata": {}, - "outputs": [], - "source": [ - "# Head View\n", - "html_head_view = head_view(\n", - " truncated_attention_matrix,\n", - " truncated_tokens,\n", - " # include_layers=[5],\n", - " html_action=\"return\",\n", - ")\n", - "\n", - "with open(\"head_view.html\", \"w\") as file:\n", - " file.write(html_head_view.data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1c13d4bf", - "metadata": {}, - "outputs": [], - "source": [ - "# Neuron View\n", - "model_type = \"bert\"\n", - "\n", - "show(model, model_type, tokenizer, display_mode=\"dark\", layer=5, head=0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2e4ee069c3f083f", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T13:59:33.204805Z", - "start_time": "2024-03-15T13:59:33.193223Z" - }, - "collapsed": false - }, - "outputs": [], - "source": [ - "# Visualize REG token -> Tricky?\n", - "# DONE Why the row vs column attention differs? -> What the matrix actually represents\n", - "# Include one example patient and visualize the attention matrix -> Include the exact concept token\n", - "# Some sort of markers to separate visits and special tokens\n", - "# Libraries used for attention visualization -> Amrit suggestion\n", - "# Visualize the gradients" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.9" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/evaluation/CompareAUROC-Poster.ipynb b/evaluation/CompareAUROC-Poster.ipynb deleted file mode 100644 index 9e92592..0000000 --- a/evaluation/CompareAUROC-Poster.ipynb +++ /dev/null @@ -1,158 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "\"\"\"\n", - "File: CompareAUROC-Poster.ipynb\n", - "---------------------------------\n", - "Compare performance of XGBoost to BigBird & Bi-LSTM using the AUROC curve for all three models on the same test set\n", - "Used to generate the AUROC curves on the poster showcased in Vector Institute's Research Symposium, on February 9\n", - "\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# Import dependencies and define useful constants\n", - "import os\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "from scipy.special import expit\n", - "from sklearn.metrics import (\n", - " roc_auc_score,\n", - " roc_curve,\n", - ")\n", - "\n", - "\n", - "plt.style.use(\"seaborn-v0_8\")\n", - "%matplotlib inline\n", - "\n", - "TEST_SIZE = \"512\"\n", - "TEST_GROUP = \"two_weeks\"\n", - "TRANSFORMER_TEST_GROUP = \"week\" if TEST_GROUP == \"two_weeks\" else \"month\"\n", - "\n", - "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", - "DATA_ROOT = f\"{ROOT}/data/slurm_data/{TEST_SIZE}/{TEST_GROUP}\"\n", - "os.chdir(ROOT)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# Load predictions, labels, and probabilities of different models\n", - "y_xgboost_pred = np.load(f\"{ROOT}/xgboost_y_test_pred_{TEST_GROUP}.npy\")\n", - "y_xgboost_labels = np.load(f\"{ROOT}/xgboost_y_test_pred_{TEST_GROUP}_labels.npy\")\n", - "y_xgboost_prob = np.load(f\"{ROOT}/xgboost_y_test_pred_{TEST_GROUP}_prob.npy\")\n", - "y_xgboost_prob = y_xgboost_prob[:, 1]\n", - "\n", - "y_lstm_pred = np.load(f\"{ROOT}/lstm_y_test_pred_{TEST_GROUP}.npy\")\n", - "y_lstm_labels = np.load(f\"{ROOT}/lstm_y_test_pred_{TEST_GROUP}_labels.npy\")\n", - "y_lstm_prob = np.load(f\"{ROOT}/lstm_y_test_pred_{TEST_GROUP}_prob.npy\")\n", - "\n", - "y_transformer_pred = np.load(\n", - " f\"/ssd003/projects/aieng/public/odyssey/results/test_preds_{TRANSFORMER_TEST_GROUP}.npy\",\n", - ")\n", - "y_transformer_labels = np.load(\n", - " f\"/ssd003/projects/aieng/public/odyssey/results/test_labels_{TRANSFORMER_TEST_GROUP}.npy\",\n", - ")\n", - "y_transformer_prob = np.load(\n", - " f\"/ssd003/projects/aieng/public/odyssey/results/test_prob_{TRANSFORMER_TEST_GROUP}.npy\",\n", - ")\n", - "y_transformer_prob = expit(y_transformer_prob[:, 1])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# Plot ROC Curve for XGBoost, Bi-LSTM, and Transformer\n", - "fpr_xgboost, tpr_xgboost, _ = roc_curve(y_xgboost_labels, y_xgboost_prob)\n", - "fpr_lstm, tpr_lstm, _ = roc_curve(y_lstm_labels, y_lstm_prob)\n", - "fpr_transformer, tpr_transformer, _ = roc_curve(\n", - " y_transformer_labels,\n", - " y_transformer_prob,\n", - ")\n", - "\n", - "# AUROC\n", - "y_xgboost_auroc = roc_auc_score(y_xgboost_labels, y_xgboost_prob)\n", - "y_lstm_auroc = roc_auc_score(y_lstm_labels, y_lstm_prob)\n", - "transformer_auroc = roc_auc_score(y_transformer_labels, y_transformer_prob)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# Plot Information\n", - "plt.figure(figsize=(8, 10))\n", - "\n", - "plt.plot(\n", - " fpr_transformer,\n", - " tpr_transformer,\n", - " label=f\"BigBird = {transformer_auroc:.2f}\",\n", - " color=\"red\",\n", - ")\n", - "plt.plot(\n", - " fpr_xgboost,\n", - " tpr_xgboost,\n", - " label=f\"XGBoost = {y_xgboost_auroc:.2f}\",\n", - " color=\"green\",\n", - ")\n", - "plt.plot(fpr_lstm, tpr_lstm, label=f\"Bi-LSTM = {y_lstm_auroc:.2f}\", color=\"blue\")\n", - "plt.plot([0, 1], [0, 1], linestyle=\"--\", color=\"gray\", label=\"Random\")\n", - "\n", - "plt.xlabel(\"False Positive Rate\")\n", - "plt.ylabel(\"True Positive Rate\")\n", - "plt.title(\"ROC Curve - Two-Weeks Mortality Prediction\")\n", - "plt.legend(loc=\"lower right\", fontsize=\"large\", facecolor=\"white\", frameon=True)\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/evaluation/TestAnalysis.ipynb b/evaluation/TestAnalysis.ipynb deleted file mode 100644 index f39ce8d..0000000 --- a/evaluation/TestAnalysis.ipynb +++ /dev/null @@ -1,266 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2024-04-10 12:13:14,754] [INFO] [real_accelerator.py:191:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n" - ] - } - ], - "source": [ - "import os\n", - "\n", - "import torch\n", - "from sklearn.metrics import (\n", - " auc,\n", - " average_precision_score,\n", - " balanced_accuracy_score,\n", - " f1_score,\n", - " precision_recall_curve,\n", - " precision_score,\n", - " recall_score,\n", - " roc_auc_score,\n", - ")\n", - "from transformers import utils\n", - "\n", - "\n", - "utils.logging.set_verbosity_error() # Suppress standard warnings\n", - "\n", - "\n", - "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", - "os.chdir(ROOT)\n", - "\n", - "from odyssey.data.tokenizer import ConceptTokenizer" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "class config:\n", - " \"\"\"Save the configuration arguments.\"\"\"\n", - "\n", - " model_path = \"test_epoch_end.ckpt\"\n", - " vocab_dir = \"data/vocab\"\n", - " data_dir = \"data/bigbird_data\"\n", - " sequence_file = \"patient_sequences/patient_sequences_2048_mortality.parquet\"\n", - " id_file = \"patient_id_dict/dataset_2048_mortality_1month.pkl\"\n", - " valid_scheme = \"few_shot\"\n", - " num_finetune_patients = \"20000\"\n", - " label_name = \"label_mortality_1month\"\n", - "\n", - " max_len = 2048\n", - " batch_size = 1\n", - " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "tokenizer = ConceptTokenizer(data_dir=config.vocab_dir)\n", - "tokenizer.fit_on_vocab()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_keys(['epoch', 'global_step', 'pytorch-lightning_version', 'state_dict', 'loops', 'callbacks', 'optimizer_states', 'lr_schedulers', 'MixedPrecision'])" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model = torch.load(config.model_path, map_location=config.device)\n", - "model.keys()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'loss': tensor(0.1638, dtype=torch.float64),\n", - " 'preds': tensor([6, 7, 0, ..., 7, 7, 7]),\n", - " 'labels': tensor([[1., 0., 0., ..., 0., 0., 0.],\n", - " [0., 0., 0., ..., 0., 0., 0.],\n", - " [1., 0., 0., ..., 0., 0., 0.],\n", - " ...,\n", - " [0., 0., 0., ..., 0., 0., 0.],\n", - " [0., 0., 0., ..., 0., 0., 0.],\n", - " [0., 0., 0., ..., 0., 0., 0.]], dtype=torch.float64),\n", - " 'logits': tensor([[ 2.3418, -2.0781, 0.2194, ..., -5.3945, -7.1797, -3.4180],\n", - " [-1.6533, -3.4277, -6.8086, ..., -6.8359, -5.2266, -5.6484],\n", - " [ 1.0947, -3.7930, -6.1094, ..., -6.3867, -6.6836, -5.5508],\n", - " ...,\n", - " [-2.8223, -3.7285, -4.6797, ..., -7.9922, -5.6992, -6.7812],\n", - " [-3.7148, -5.6328, -6.7188, ..., -9.4062, -7.6445, -7.6016],\n", - " [-2.5840, -2.2871, -4.6484, ..., -7.8633, -4.7539, -6.4648]],\n", - " dtype=torch.float16)}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "test_outputs = torch.load(\"test_outputs.pt\")\n", - "test_outputs" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'Balanced Accuracy': 0.5, 'F1 Score': 0.0, 'Precision': 0.0, 'Recall': 0.0, 'AUROC': 0.8100258785715974, 'Average Precision Score': 0.001364147006900979, 'AUC-PR': 0.5006820735034505}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/fs01/home/afallah/light/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1497: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n" - ] - } - ], - "source": [ - "def calculate_metrics(y_true, y_pred, y_prob):\n", - " \"\"\"\n", - " Calculate and return performance metrics.\n", - " \"\"\"\n", - " metrics = {\n", - " \"Balanced Accuracy\": balanced_accuracy_score(y_true, y_pred),\n", - " \"F1 Score\": f1_score(y_true, y_pred),\n", - " \"Precision\": precision_score(y_true, y_pred),\n", - " \"Recall\": recall_score(y_true, y_pred),\n", - " \"AUROC\": roc_auc_score(y_true, y_prob),\n", - " \"Average Precision Score\": average_precision_score(y_true, y_pred),\n", - " }\n", - "\n", - " precision, recall, _ = precision_recall_curve(y_true, y_pred)\n", - " metrics[\"AUC-PR\"] = auc(recall, precision)\n", - "\n", - " return metrics\n", - "\n", - "\n", - "targets = [10]\n", - "\n", - "for i in targets:\n", - " labels = test_outputs[\"labels\"][:, i]\n", - " logits = torch.sigmoid(test_outputs[\"logits\"][:, i])\n", - " preds = (logits >= 0.5).int()\n", - "\n", - " print(calculate_metrics(labels, preds, logits))" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor(0)" - ] - }, - "execution_count": 68, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "preds.sum()" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor(34., dtype=torch.float64)" - ] - }, - "execution_count": 69, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "labels.sum()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([0, 0, 0, ..., 0, 0, 0], dtype=torch.int32)" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "preds" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.9" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/finetune.py b/finetune.py index 628c809..ab4e1a3 100644 --- a/finetune.py +++ b/finetune.py @@ -19,15 +19,15 @@ from skmultilearn.model_selection import iterative_train_test_split from torch.utils.data import DataLoader +from odyssey.utils.utils import seed_everything from odyssey.data.dataset import FinetuneDataset, FinetuneMultiDataset from odyssey.data.tokenizer import ConceptTokenizer from odyssey.models.cehr_bert.model import BertFinetune, BertPretrain from odyssey.models.cehr_big_bird.model import BigBirdFinetune, BigBirdPretrain -from odyssey.models.utils import ( +from odyssey.models.model_utils import ( get_run_id, load_config, load_finetune_data, - seed_everything, ) diff --git a/odyssey/data/bigbird_data/DataChecker.ipynb b/odyssey/data/bigbird_data/DataChecker.ipynb deleted file mode 100644 index 4944995..0000000 --- a/odyssey/data/bigbird_data/DataChecker.ipynb +++ /dev/null @@ -1,1136 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T16:14:45.546088300Z", - "start_time": "2024-03-13T16:14:43.587090300Z" - }, - "collapsed": true - }, - "outputs": [], - "source": [ - "import os\n", - "import pickle\n", - "import random\n", - "import sys\n", - "from typing import Any, Dict, List, Optional\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "from sklearn.model_selection import train_test_split\n", - "from skmultilearn.model_selection import iterative_train_test_split\n", - "\n", - "\n", - "sys.path.append(\"/h/afallah/odyssey/odyssey/lib\")\n", - "from utils import save_object_to_disk\n", - "\n", - "\n", - "DATA_ROOT = \"/h/afallah/odyssey/odyssey/data/bigbird_data\"\n", - "DATASET = f\"{DATA_ROOT}/patient_sequences/patient_sequences_2048.parquet\"\n", - "MAX_LEN = 2048\n", - "\n", - "SEED = 23\n", - "os.chdir(DATA_ROOT)\n", - "random.seed(SEED)\n", - "np.random.seed(SEED)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T16:15:12.321718600Z", - "start_time": "2024-03-13T16:14:45.553089800Z" - }, - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Current columns: Index(['patient_id', 'num_visits', 'deceased', 'death_after_start',\n", - " 'death_after_end', 'length', 'token_length', 'event_tokens_2048',\n", - " 'type_tokens_2048', 'age_tokens_2048', 'time_tokens_2048',\n", - " 'visit_tokens_2048', 'position_tokens_2048', 'elapsed_tokens_2048',\n", - " 'common_conditions', 'rare_conditions'],\n", - " dtype='object')\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
patient_idnum_visitsdeceaseddeath_after_startdeath_after_endlengthtoken_lengthevent_tokens_2048type_tokens_2048age_tokens_2048time_tokens_2048visit_tokens_2048position_tokens_2048elapsed_tokens_2048common_conditionsrare_conditions
035581927-9c95-5ae9-af76-7d74870a349c10NaNNaN5054[[CLS], [VS], 00006473900, 00904516561, 510790...[1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, ...[0, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85...[0, 5902, 5902, 5902, 5902, 5902, 5902, 5902, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 1.97, 2.02, 2.02, 2.02, 2.02, 2.0...[1, 0, 0, 0, 0, 0, 0, 0, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
1f5bba8dd-25c0-5336-8d3d-37424c18502620NaNNaN148156[[CLS], [VS], 52135_2, 52075_2, 52074_2, 52073...[1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...[0, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83...[0, 6594, 6594, 6594, 6594, 6594, 6594, 6594, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...[0, 0, 0, 0, 0, 0, 0, 1, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
2f4938f91-cadb-5133-8541-a52fb0916cea20NaNNaN7886[[CLS], [VS], 0RB30ZZ, 0RG10A0, 00071101441, 0...[1, 2, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...[0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44...[0, 8150, 8150, 8150, 8150, 8150, 8150, 8150, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 0.0, 0.0, 1.08, 1.08, 13.89, 13.8...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
36fe2371b-a6f0-5436-aade-7795005b0c6620NaNNaN8694[[CLS], [VS], 63739057310, 49281041688, 005970...[1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...[0, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72...[0, 6093, 6093, 6093, 6093, 6093, 6093, 6093, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 0.75, 0.75, 0.75, 0.75, 0.75, 0.7...[1, 0, 0, 0, 0, 0, 0, 1, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
46f7590ae-f3b9-50e5-9e41-d4bb1000887a10NaNNaN7276[[CLS], [VS], 50813_0, 52135_0, 52075_3, 52074...[1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...[0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47...[0, 6379, 6379, 6379, 6379, 6379, 6379, 6379, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...[1, 0, 0, 0, 0, 0, 0, 0, 0, 1][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
\n", - "
" - ], - "text/plain": [ - " patient_id num_visits deceased \\\n", - "0 35581927-9c95-5ae9-af76-7d74870a349c 1 0 \n", - "1 f5bba8dd-25c0-5336-8d3d-37424c185026 2 0 \n", - "2 f4938f91-cadb-5133-8541-a52fb0916cea 2 0 \n", - "3 6fe2371b-a6f0-5436-aade-7795005b0c66 2 0 \n", - "4 6f7590ae-f3b9-50e5-9e41-d4bb1000887a 1 0 \n", - "\n", - " death_after_start death_after_end length token_length \\\n", - "0 NaN NaN 50 54 \n", - "1 NaN NaN 148 156 \n", - "2 NaN NaN 78 86 \n", - "3 NaN NaN 86 94 \n", - "4 NaN NaN 72 76 \n", - "\n", - " event_tokens_2048 \\\n", - "0 [[CLS], [VS], 00006473900, 00904516561, 510790... \n", - "1 [[CLS], [VS], 52135_2, 52075_2, 52074_2, 52073... \n", - "2 [[CLS], [VS], 0RB30ZZ, 0RG10A0, 00071101441, 0... \n", - "3 [[CLS], [VS], 63739057310, 49281041688, 005970... \n", - "4 [[CLS], [VS], 50813_0, 52135_0, 52075_3, 52074... \n", - "\n", - " type_tokens_2048 \\\n", - "0 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, ... \n", - "1 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", - "2 [1, 2, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "3 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "4 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", - "\n", - " age_tokens_2048 \\\n", - "0 [0, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85... \n", - "1 [0, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83... \n", - "2 [0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44... \n", - "3 [0, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72... \n", - "4 [0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47... \n", - "\n", - " time_tokens_2048 \\\n", - "0 [0, 5902, 5902, 5902, 5902, 5902, 5902, 5902, ... \n", - "1 [0, 6594, 6594, 6594, 6594, 6594, 6594, 6594, ... \n", - "2 [0, 8150, 8150, 8150, 8150, 8150, 8150, 8150, ... \n", - "3 [0, 6093, 6093, 6093, 6093, 6093, 6093, 6093, ... \n", - "4 [0, 6379, 6379, 6379, 6379, 6379, 6379, 6379, ... \n", - "\n", - " visit_tokens_2048 \\\n", - "0 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "1 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "2 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "3 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "4 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "\n", - " position_tokens_2048 \\\n", - "0 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "1 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "2 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "3 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "4 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "\n", - " elapsed_tokens_2048 \\\n", - "0 [-2.0, -1.0, 1.97, 2.02, 2.02, 2.02, 2.02, 2.0... \n", - "1 [-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n", - "2 [-2.0, -1.0, 0.0, 0.0, 1.08, 1.08, 13.89, 13.8... \n", - "3 [-2.0, -1.0, 0.75, 0.75, 0.75, 0.75, 0.75, 0.7... \n", - "4 [-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n", - "\n", - " common_conditions rare_conditions \n", - "0 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "1 [0, 0, 0, 0, 0, 0, 0, 1, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "2 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "3 [1, 0, 0, 0, 0, 0, 0, 1, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "4 [1, 0, 0, 0, 0, 0, 0, 0, 0, 1] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] " - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Load complete dataset\n", - "dataset_2048 = pd.read_parquet(DATASET)\n", - "\n", - "print(f\"Current columns: {dataset_2048.columns}\")\n", - "dataset_2048.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "def filter_by_num_visit(dataset: pd.DataFrame, minimum_num_visits: int) -> pd.DataFrame:\n", - " \"\"\"Filter the patients based on num_visits threshold.\n", - "\n", - " Args:\n", - " dataset (pd.DataFrame): The input dataset.\n", - " minimum_num_visits (int): The threshold num_visits\n", - "\n", - " Returns\n", - " -------\n", - " pd.DataFrame: The filtered dataset.\n", - " \"\"\"\n", - " filtered_dataset = dataset.loc[dataset[\"num_visits\"] >= minimum_num_visits]\n", - " filtered_dataset.reset_index(drop=True, inplace=True)\n", - " return filtered_dataset\n", - "\n", - "\n", - "def filter_by_length_of_stay(dataset: pd.DataFrame, threshold: int = 1) -> pd.DataFrame:\n", - " \"\"\"Filter the patients based on length of stay threshold.\n", - "\n", - " Args:\n", - " dataset (pd.DataFrame): The input dataset.\n", - " minimum_num_visits (int): The threshold length of stay\n", - "\n", - " Returns\n", - " -------\n", - " pd.DataFrame: The filtered dataset.\n", - " \"\"\"\n", - " filtered_dataset = dataset.loc[dataset[\"length_of_stay\"] >= threshold]\n", - "\n", - " # Only keep the patients that their first event happens within threshold\n", - " # TODO: Check how many patients get removed here?\n", - " filtered_dataset = filtered_dataset[\n", - " filtered_dataset.apply(\n", - " lambda row: row[\"elapsed_tokens_2048\"][row[\"last_VS_index\"] + 1]\n", - " < threshold * 24,\n", - " axis=1,\n", - " )\n", - " ]\n", - "\n", - " filtered_dataset.reset_index(drop=True, inplace=True)\n", - " return filtered_dataset\n", - "\n", - "\n", - "def get_last_occurence_index(seq: List[str], target: str) -> int:\n", - " \"\"\"Return the index of the last occurrence of target in seq.\n", - "\n", - " Args:\n", - " seq (List[str]): The input sequence.\n", - " target (str): The target string to find.\n", - "\n", - " Returns\n", - " -------\n", - " int: The index of the last occurrence of target in seq.\n", - " \"\"\"\n", - " return len(seq) - (seq[::-1].index(target) + 1)\n", - "\n", - "\n", - "def check_readmission_label(row: pd.Series) -> int:\n", - " \"\"\"Check if the label indicates readmission within one month.\n", - "\n", - " Args:\n", - " row (pd.Series): The input row.\n", - "\n", - " Returns\n", - " -------\n", - " bool: True if readmission label is present, False otherwise.\n", - " \"\"\"\n", - " last_vs_index = row[\"last_VS_index\"]\n", - " return int(\n", - " row[\"event_tokens_2048\"][last_vs_index - 1]\n", - " in (\"[W_0]\", \"[W_1]\", \"[W_2]\", \"[W_3]\", \"[M_1]\"),\n", - " )\n", - "\n", - "\n", - "def get_length_of_stay(row: pd.Series) -> pd.Series:\n", - " \"\"\"Determine the length of a given visit.\n", - "\n", - " Args:\n", - " row (pd.Series): The input row.\n", - "\n", - " Returns\n", - " -------\n", - " pd.Series: The preprocessed row.\n", - " \"\"\"\n", - " admission_time = row[\"last_VS_index\"] + 1\n", - " discharge_time = row[\"last_VE_index\"] - 1\n", - " return (discharge_time - admission_time) / 24\n", - "\n", - "\n", - "def get_visit_cutoff_at_threshold(row: pd.Series, threshold: int = 24) -> int:\n", - " \"\"\"Get the index of the first event token of last visit that occurs after threshold hours.\n", - "\n", - " Args:\n", - " row (pd.Series): The input row.\n", - " threshold (int): The number of hours to consider.\n", - "\n", - " Returns\n", - " -------\n", - " cutoff_index (int): The corrosponding cutoff index.\n", - " \"\"\"\n", - " last_vs_index = row[\"last_VS_index\"]\n", - " last_ve_index = row[\"last_VE_index\"]\n", - "\n", - " for i in range(last_vs_index + 1, last_ve_index):\n", - " if row[\"elapsed_tokens_2048\"][i] > threshold:\n", - " return i\n", - "\n", - " return len(row[\"event_tokens_2048\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def process_length_of_stay_dataset(\n", - " dataset: pd.DataFrame,\n", - " threshold: int = 7,\n", - ") -> pd.DataFrame:\n", - " \"\"\"Process the length of stay dataset to extract required features.\n", - "\n", - " Args:\n", - " dataset (pd.DataFrame): The input dataset.\n", - " threshold (int): The threshold length of stay.\n", - "\n", - " Returns\n", - " -------\n", - " pd.DataFrame: The processed dataset.\n", - " \"\"\"\n", - " dataset[\"last_VS_index\"] = dataset[\"event_tokens_2048\"].transform(\n", - " lambda seq: get_last_occurence_index(list(seq), \"[VS]\"),\n", - " )\n", - " dataset[\"last_VE_index\"] = dataset[\"event_tokens_2048\"].transform(\n", - " lambda seq: get_last_occurence_index(list(seq), \"[VE]\"),\n", - " )\n", - " dataset[\"length_of_stay\"] = dataset.apply(get_length_of_stay, axis=1)\n", - "\n", - " dataset = filter_by_length_of_stay(dataset, threshold=1)\n", - " dataset[\"label_los_1week\"] = (dataset[\"length_of_stay\"] >= threshold).astype(int)\n", - "\n", - " dataset[\"cutoff_los\"] = dataset.apply(\n", - " lambda row: get_visit_cutoff_at_threshold(row, threshold=24),\n", - " axis=1,\n", - " )\n", - " dataset[\"token_length\"] = dataset[\"event_tokens_2048\"].apply(len)\n", - "\n", - " return dataset\n", - "\n", - "\n", - "# Process the dataset for length of stay prediction above a threshold\n", - "dataset_2048_los = process_length_of_stay_dataset(dataset_2048.copy(), threshold=7)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def process_condition_dataset(dataset: pd.DataFrame) -> pd.DataFrame:\n", - " \"\"\"Process the condition dataset to extract required features.\n", - "\n", - " Args:\n", - " dataset (pd.DataFrame): The input condition dataset.\n", - "\n", - " Returns\n", - " -------\n", - " pd.DataFrame: The processed condition dataset.\n", - " \"\"\"\n", - " dataset[\"all_conditions\"] = dataset.apply(\n", - " lambda row: np.concatenate(\n", - " [row[\"common_conditions\"], row[\"rare_conditions\"]],\n", - " dtype=np.int64,\n", - " ),\n", - " axis=1,\n", - " )\n", - "\n", - " return dataset\n", - "\n", - "\n", - "# Process the dataset for conditions including rare and common\n", - "dataset_2048_condition = process_condition_dataset(dataset_2048.copy())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T16:15:16.075719400Z", - "start_time": "2024-03-13T16:15:12.335721100Z" - }, - "collapsed": false - }, - "outputs": [], - "source": [ - "def process_mortality_dataset(dataset: pd.DataFrame) -> pd.DataFrame:\n", - " \"\"\"Process the mortality dataset to extract required features.\n", - "\n", - " Args:\n", - " dataset (pd.DataFrame): The input mortality dataset.\n", - "\n", - " Returns\n", - " -------\n", - " pd.DataFrame: The processed mortality dataset.\n", - " \"\"\"\n", - " dataset[\"label_mortality_2weeks\"] = (\n", - " (dataset[\"death_after_start\"] >= 0) & (dataset[\"death_after_end\"] <= 15)\n", - " ).astype(int)\n", - " dataset[\"label_mortality_1month\"] = (\n", - " (dataset[\"death_after_start\"] >= 0) & (dataset[\"death_after_end\"] <= 32)\n", - " ).astype(int)\n", - "\n", - " return dataset\n", - "\n", - "\n", - "# Process the dataset for mortality in two weeks or one month task\n", - "dataset_2048_mortality = process_mortality_dataset(dataset_2048.copy())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T16:15:47.326996100Z", - "start_time": "2024-03-13T16:15:16.094719300Z" - }, - "collapsed": false - }, - "outputs": [], - "source": [ - "def process_readmission_dataset(dataset: pd.DataFrame) -> pd.DataFrame:\n", - " \"\"\"Process the readmission dataset to extract required features.\n", - "\n", - " Args:\n", - " dataset (pd.DataFrame): The input dataset.\n", - "\n", - " Returns\n", - " -------\n", - " pd.DataFrame: The processed dataset.\n", - " \"\"\"\n", - " dataset[\"last_VS_index\"] = dataset[\"event_tokens_2048\"].transform(\n", - " lambda seq: get_last_occurence_index(list(seq), \"[VS]\"),\n", - " )\n", - " dataset[\"cutoff_readmission\"] = dataset[\"last_VS_index\"] - 1\n", - " dataset[\"label_readmission_1month\"] = dataset.apply(check_readmission_label, axis=1)\n", - "\n", - " dataset[\"num_visits\"] -= 1\n", - " dataset[\"token_length\"] = dataset[\"event_tokens_2048\"].apply(len)\n", - "\n", - " return dataset\n", - "\n", - "\n", - "# Process the dataset for hospital readmission in one month task\n", - "dataset_2048_readmission = filter_by_num_visit(\n", - " dataset_2048.copy(),\n", - " minimum_num_visits=2,\n", - ")\n", - "dataset_2048_readmission = process_readmission_dataset(dataset_2048_readmission)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def process_multi_dataset(datasets: Dict[str, pd.DataFrame]):\n", - " \"\"\"\n", - " Process the multi-task dataset by merging the original dataset with the other datasets.\n", - "\n", - " Args:\n", - " datasets (Dict): Dictionary mapping each task to its respective dataframe\n", - "\n", - " Returns\n", - " -------\n", - " pd.DataFrame: The processed multi-task dataset\n", - " \"\"\"\n", - " # Merging datasets on 'patient_id'\n", - " multi_dataset = datasets[\"original\"].merge(\n", - " datasets[\"condition\"][[\"patient_id\", \"all_conditions\"]],\n", - " on=\"patient_id\",\n", - " how=\"left\",\n", - " )\n", - " multi_dataset = multi_dataset.merge(\n", - " datasets[\"mortality\"][[\"patient_id\", \"label_mortality_1month\"]],\n", - " on=\"patient_id\",\n", - " how=\"left\",\n", - " )\n", - " multi_dataset = multi_dataset.merge(\n", - " datasets[\"readmission\"][\n", - " [\"patient_id\", \"cutoff_readmission\", \"label_readmission_1month\"]\n", - " ],\n", - " on=\"patient_id\",\n", - " how=\"left\",\n", - " )\n", - " multi_dataset = multi_dataset.merge(\n", - " datasets[\"los\"][[\"patient_id\", \"cutoff_los\", \"label_los_1week\"]],\n", - " on=\"patient_id\",\n", - " how=\"left\",\n", - " )\n", - "\n", - " # Selecting the required columns\n", - " multi_dataset = multi_dataset[\n", - " [\n", - " \"patient_id\",\n", - " \"num_visits\",\n", - " \"event_tokens_2048\",\n", - " \"type_tokens_2048\",\n", - " \"age_tokens_2048\",\n", - " \"time_tokens_2048\",\n", - " \"visit_tokens_2048\",\n", - " \"position_tokens_2048\",\n", - " \"elapsed_tokens_2048\",\n", - " \"cutoff_los\",\n", - " \"cutoff_readmission\",\n", - " \"all_conditions\",\n", - " \"label_mortality_1month\",\n", - " \"label_readmission_1month\",\n", - " \"label_los_1week\",\n", - " ]\n", - " ]\n", - "\n", - " # Transform conditions from a vector of numbers to binary classes\n", - " conditions_expanded = multi_dataset[\"all_conditions\"].apply(pd.Series)\n", - " conditions_expanded.columns = [f\"condition{i}\" for i in range(20)]\n", - " multi_dataset = multi_dataset.drop(\"all_conditions\", axis=1)\n", - " multi_dataset = pd.concat([multi_dataset, conditions_expanded], axis=1)\n", - "\n", - " # Standardize important column names\n", - " multi_dataset.rename(\n", - " columns={\n", - " \"cutoff_los\": \"cutoff_los_1week\",\n", - " \"cutoff_readmission\": \"cutoff_readmission_1month\",\n", - " },\n", - " inplace=True,\n", - " )\n", - " condition_columns = {f\"condition{i}\": f\"label_c{i}\" for i in range(20)}\n", - " multi_dataset.rename(columns=condition_columns, inplace=True)\n", - "\n", - " numerical_columns = [\n", - " \"cutoff_los_1week\",\n", - " \"cutoff_readmission_1month\",\n", - " \"label_mortality_1month\",\n", - " \"label_readmission_1month\",\n", - " \"label_los_1week\",\n", - " ] + [f\"label_c{i}\" for i in range(20)]\n", - "\n", - " # Fill NaN values and convert to integers\n", - " for column in numerical_columns:\n", - " multi_dataset[column] = multi_dataset[column].fillna(-1).astype(int)\n", - "\n", - " # Reset dataset index\n", - " multi_dataset.reset_index(drop=True, inplace=True)\n", - "\n", - " return multi_dataset\n", - "\n", - "\n", - "multi_dataset = process_multi_dataset(\n", - " datasets={\n", - " \"original\": dataset_2048,\n", - " \"mortality\": dataset_2048_mortality,\n", - " \"condition\": dataset_2048_condition,\n", - " \"readmission\": dataset_2048_readmission,\n", - " \"los\": dataset_2048_los,\n", - " },\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def stratified_train_test_split(\n", - " dataset: pd.DataFrame,\n", - " target: str,\n", - " test_size: float,\n", - " return_test: Optional[bool] = False,\n", - "):\n", - " \"\"\"\n", - " Split the given dataset into training and testing sets using iterative stratification on given multi-label target.\n", - " \"\"\"\n", - " # Convert all_conditions into a format suitable for multi-label stratification\n", - " Y = np.array(dataset[target].values.tolist())\n", - " X = dataset[\"patient_id\"].to_numpy().reshape(-1, 1)\n", - " is_single_label = type(dataset.iloc[0][target]) == np.int64\n", - "\n", - " # Perform stratified split\n", - " if is_single_label:\n", - " X_train, X_test, y_train, y_test = train_test_split(\n", - " X,\n", - " Y,\n", - " stratify=Y,\n", - " test_size=test_size,\n", - " random_state=SEED,\n", - " )\n", - "\n", - " else:\n", - " X_train, y_train, X_test, y_test = iterative_train_test_split(\n", - " X,\n", - " Y,\n", - " test_size=test_size,\n", - " )\n", - "\n", - " X_train = X_train.flatten().tolist()\n", - " X_test = X_test.flatten().tolist()\n", - "\n", - " if return_test:\n", - " return X_test\n", - " else:\n", - " return X_train, X_test\n", - "\n", - "\n", - "def sample_balanced_subset(dataset: pd.DataFrame, target: str, sample_size: int):\n", - " \"\"\"\n", - " Sample a subset of dataset with balanced target labels.\n", - " \"\"\"\n", - " # Sampling positive and negative patients\n", - " pos_patients = dataset[dataset[target] == True].sample(\n", - " n=sample_size // 2,\n", - " random_state=SEED,\n", - " )\n", - " neg_patients = dataset[dataset[target] == False].sample(\n", - " n=sample_size // 2,\n", - " random_state=SEED,\n", - " )\n", - "\n", - " # Combining and shuffling patient IDs\n", - " sample_patients = (\n", - " pos_patients[\"patient_id\"].tolist() + neg_patients[\"patient_id\"].tolist()\n", - " )\n", - " random.shuffle(sample_patients)\n", - "\n", - " return sample_patients\n", - "\n", - "\n", - "def get_pretrain_test_split(\n", - " dataset: pd.DataFrame,\n", - " stratify_target: Optional[str] = None,\n", - " test_size: float = 0.15,\n", - "):\n", - " \"\"\"Split dataset into pretrain and test set. Stratify on a given target column if needed.\"\"\"\n", - " if stratify_target:\n", - " pretrain_ids, test_ids = stratified_train_test_split(\n", - " dataset,\n", - " target=stratify_target,\n", - " test_size=test_size,\n", - " )\n", - "\n", - " else:\n", - " test_patients = dataset.sample(n=test_size, random_state=SEED)\n", - " test_ids = test_patients[\"patient_id\"].tolist()\n", - " pretrain_ids = dataset[~dataset[\"patient_id\"].isin(test_patients)][\n", - " \"patient_id\"\n", - " ].tolist()\n", - "\n", - " random.shuffle(pretrain_ids)\n", - "\n", - " return pretrain_ids, test_ids" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Split data\n", - "patient_ids_dict = {\n", - " \"pretrain\": [],\n", - " \"finetune\": {\"few_shot\": {}, \"kfold\": {}},\n", - " \"test\": [],\n", - "}\n", - "\n", - "# Get train-test split\n", - "# pretrain_ids, test_ids = get_pretrain_test_split(dataset_2048_readmission, stratify_target='label_readmission_1month', test_size=0.2)\n", - "# pretrain_ids, test_ids = get_pretrain_test_split(process_condition_dataset, stratify_target='all_conditions', test_size=0.15)\n", - "# patient_ids_dict['pretrain'] = pretrain_ids\n", - "# patient_ids_dict['test'] = test_ids\n", - "\n", - "# Load pretrain and test patient IDs\n", - "pid = pickle.load(open(\"patient_id_dict/dataset_2048_multi.pkl\", \"rb\"))\n", - "patient_ids_dict[\"pretrain\"] = pid[\"pretrain\"]\n", - "patient_ids_dict[\"test\"] = pid[\"test\"]\n", - "set(pid[\"pretrain\"] + pid[\"test\"]) == set(dataset_2048[\"patient_id\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class config:\n", - " task_splits = {\n", - " \"mortality\": {\n", - " \"dataset\": dataset_2048_mortality,\n", - " \"label_col\": \"label_mortality_1month\",\n", - " \"finetune_size\": [250, 500, 1000, 5000, 20000],\n", - " \"save_path\": \"patient_id_dict/dataset_2048_mortality.pkl\",\n", - " \"split_mode\": \"single_label_balanced\",\n", - " },\n", - " \"readmission\": {\n", - " \"dataset\": dataset_2048_readmission,\n", - " \"label_col\": \"label_readmission_1month\",\n", - " \"finetune_size\": [250, 1000, 5000, 20000, 60000],\n", - " \"save_path\": \"patient_id_dict/dataset_2048_readmission.pkl\",\n", - " \"split_mode\": \"single_label_stratified\",\n", - " },\n", - " \"length_of_stay\": {\n", - " \"dataset\": dataset_2048_los,\n", - " \"label_col\": \"label_los_1week\",\n", - " \"finetune_size\": [250, 1000, 5000, 20000, 50000],\n", - " \"save_path\": \"patient_id_dict/dataset_2048_los.pkl\",\n", - " \"split_mode\": \"single_label_balanced\",\n", - " },\n", - " \"condition\": {\n", - " \"dataset\": dataset_2048_condition,\n", - " \"label_col\": \"all_conditions\",\n", - " \"finetune_size\": [50000],\n", - " \"save_path\": \"patient_id_dict/dataset_2048_condition.pkl\",\n", - " \"split_mode\": \"multi_label_stratified\",\n", - " },\n", - " }\n", - "\n", - " all_tasks = list(task_splits.keys())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T16:15:51.800996200Z", - "start_time": "2024-03-13T16:15:50.494996100Z" - }, - "collapsed": false - }, - "outputs": [], - "source": [ - "def get_finetune_split(\n", - " config: config,\n", - " patient_ids_dict: Dict[str, Any],\n", - ") -> Dict[str, Dict[str, List[str]]]:\n", - " \"\"\"\n", - " Splits the dataset into training and cross-finetuneation sets using k-fold cross-finetuneation\n", - " while ensuring balanced label distribution in each fold. Saves the resulting dictionary to disk.\n", - " \"\"\"\n", - " # Extract task-specific configuration\n", - " task_config = config.task_splits[task]\n", - " dataset = task_config[\"dataset\"]\n", - " label_col = task_config[\"label_col\"]\n", - " finetune_sizes = task_config[\"finetune_size\"]\n", - " save_path = task_config[\"save_path\"]\n", - " split_mode = task_config[\"split_mode\"]\n", - "\n", - " # Get pretrain dataset\n", - " pretrain_ids = patient_ids_dict[\"pretrain\"]\n", - " dataset = dataset[dataset[\"patient_id\"].isin(pretrain_ids)]\n", - "\n", - " # Few-shot finetune patient ids\n", - " for finetune_num in finetune_sizes:\n", - " if split_mode == \"single_label_balanced\":\n", - " finetune_ids = sample_balanced_subset(\n", - " dataset,\n", - " target=label_col,\n", - " sample_size=finetune_num,\n", - " )\n", - "\n", - " elif (\n", - " split_mode == \"single_label_stratified\"\n", - " or split_mode == \"multi_label_stratified\"\n", - " ):\n", - " finetune_ids = stratified_train_test_split(\n", - " dataset,\n", - " target=label_col,\n", - " test_size=finetune_num / len(dataset),\n", - " return_test=True,\n", - " )\n", - "\n", - " patient_ids_dict[\"finetune\"][\"few_shot\"][f\"{finetune_num}\"] = finetune_ids\n", - "\n", - " # Save the dictionary to disk\n", - " save_object_to_disk(patient_ids_dict, save_path)\n", - "\n", - " return patient_ids_dict\n", - "\n", - "\n", - "for task in config.all_tasks:\n", - " patient_ids_dict = get_finetune_split(\n", - " config=config,\n", - " patient_ids_dict=patient_ids_dict,\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T14:14:10.181184300Z", - "start_time": "2024-03-13T14:13:39.154567400Z" - }, - "collapsed": false - }, - "outputs": [], - "source": [ - "dataset_2048_mortality.to_parquet(\n", - " \"patient_sequences/patient_sequences_2048_mortality.parquet\",\n", - ")\n", - "dataset_2048_readmission.to_parquet(\n", - " \"patient_sequences/patient_sequences_2048_readmission.parquet\",\n", - ")\n", - "dataset_2048_los.to_parquet(\"patient_sequences/patient_sequences_2048_los.parquet\")\n", - "dataset_2048_condition.to_parquet(\n", - " \"patient_sequences/patient_sequences_2048_condition.parquet\",\n", - ")\n", - "multi_dataset.to_parquet(\"patient_sequences/patient_sequences_2048_multi.parquet\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Load data\n", - "# multi_dataset = pd.read_parquet('patient_sequences/patient_sequences_2048_multi.parquet')\n", - "# pid = pickle.load(open('patient_id_dict/dataset_2048_multi.pkl', 'rb'))\n", - "# multi_dataset = multi_dataset[multi_dataset['patient_id'].isin(pid['finetune']['few_shot']['all'])]\n", - "\n", - "# # Train Tokenizer\n", - "# tokenizer = ConceptTokenizer(data_dir='/h/afallah/odyssey/odyssey/data/vocab')\n", - "# tokenizer.fit_on_vocab()\n", - "\n", - "# # Load datasets\n", - "# tasks = ['mortality_1month', 'los_1week'] + [f'c{i}' for i in range(5)]\n", - "\n", - "# train_dataset = FinetuneMultiDataset(\n", - "# data=multi_dataset,\n", - "# tokenizer=tokenizer,\n", - "# tasks=tasks,\n", - "# balance_guide={'mortality_1month': 0.5, 'los_1week': 0.5},\n", - "# max_len=2048,\n", - "# )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# dataset_2048_condition = pd.read_parquet('patient_sequences/patient_sequences_2048_condition.parquet')\n", - "# pid = pickle.load(open('patient_id_dict/dataset_2048_condition.pkl', 'rb'))\n", - "# condition_finetune = dataset_2048_condition.loc[dataset_2048_condition['patient_id'].isin(pid['finetune']['few_shot']['50000'])]\n", - "# condition_finetune" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# freq = np.array(condition_finetune['all_conditions'].tolist()).sum(axis=0)\n", - "# weights = np.clip(0, 50, sum(freq) / freq)\n", - "# np.max(np.sqrt(freq)) / np.sqrt(freq)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# sorted(patient_ids_dict['pretrain']) == sorted(pickle.load(open('new_data/patient_id_dict/sample_pretrain_test_patient_ids_with_conditions.pkl', 'rb'))['pretrain'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# merged_df = pd.merge(dataset_2048_mortality, dataset_2048_readmission, how='outer', on='patient_id')\n", - "# final_merged_df = pd.merge(merged_df, dataset_2048_condition, how='outer', on='patient_id')\n", - "# final_merged_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Performing stratified k-fold split\n", - "# skf = StratifiedKFold(n_splits=num_splits, shuffle=True, random_state=SEED)\n", - "\n", - "# for i, (train_index, cv_index) in enumerate(skf.split(dataset, dataset[label_col])):\n", - "\n", - "# dataset_cv = dataset.iloc[cv_index]\n", - "# dataset_finetune = dataset.iloc[train_index]\n", - "\n", - "# # Separate positive and negative labeled patients\n", - "# pos_patients = dataset_cv[dataset_cv[label_col] == True]['patient_id'].tolist()\n", - "# neg_patients = dataset_cv[dataset_cv[label_col] == False]['patient_id'].tolist()\n", - "\n", - "# # Calculate the number of positive and negative patients needed for balanced CV set\n", - "# num_pos_needed = cv_size // 2\n", - "# num_neg_needed = cv_size // 2\n", - "\n", - "# # Select positive and negative patients for CV set ensuring balanced distribution\n", - "# cv_patients = pos_patients[:num_pos_needed] + neg_patients[:num_neg_needed]\n", - "# remaining_finetune_patients = pos_patients[num_pos_needed:] + neg_patients[num_neg_needed:]\n", - "\n", - "# # Extract patient IDs for training set\n", - "# finetune_patients = dataset_finetune['patient_id'].tolist()\n", - "# finetune_patients += remaining_finetune_patients\n", - "\n", - "# # Shuffle each list of patients\n", - "# random.shuffle(cv_patients)\n", - "# random.shuffle(finetune_patients)\n", - "\n", - "# patient_ids_dict['finetune']['kfold'][f'group{i+1}'] = {'finetune': finetune_patients, 'cv': cv_patients}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# Assuming dataset.event_tokens is your DataFrame column\n", - "# dataset.event_tokens.transform(len).plot(kind='hist', bins=100)\n", - "# plt.xlim(1000, 8000) # Limit x-axis to 5000\n", - "# plt.ylim(0, 6000)\n", - "# plt.xlabel('Length of Event Tokens')\n", - "# plt.ylabel('Frequency')\n", - "# plt.title('Histogram of Event Tokens Length')\n", - "# plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# len(patient_ids_dict['group3']['cv'])\n", - "\n", - "# dataset_2048.loc[dataset_2048['patient_id'].isin(patient_ids_dict['group1']['cv'])]['label_mortality_1month']\n", - "\n", - "# s = set()\n", - "# for i in range(1, 6):\n", - "# s = s.union(set(patient_ids_dict[f'group{i}']['cv']))\n", - "#\n", - "# len(s)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "##### DEAD ZONE | DO NOT ENTER #####\n", - "\n", - "# patient_ids = pickle.load(open(join(\"/h/afallah/odyssey/odyssey/data/bigbird_data\", 'dataset_2048_mortality_1month.pkl'), 'rb'))\n", - "# patient_ids['finetune']['few_shot'].keys()\n", - "\n", - "# patient_ids2 = pickle.load(open(join(\"/h/afallah/odyssey/odyssey/data/bigbird_data\", 'dataset_2048_mortality_2weeks.pkl'), 'rb'))['pretrain']\n", - "#\n", - "# patient_ids1.sort()\n", - "# patient_ids2.sort()\n", - "#\n", - "# patient_ids1 == patient_ids2\n", - "# # dataset_2048.loc[dataset_2048['patient_id'].isin(patient_ids['pretrain'])]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# dataset_2048_readmission = dataset_2048.loc[dataset_2048['num_visits'] > 1]\n", - "# dataset_2048_readmission.reset_index(drop=True, inplace=True)\n", - "#\n", - "# dataset_2048_readmission['last_VS_index'] = dataset_2048_readmission['event_tokens_2048'].transform(lambda seq: get_last_occurence_index(list(seq), '[VS]'))\n", - "#\n", - "# dataset_2048_readmission['label_readmission_1month'] = dataset_2048_readmission.apply(\n", - "# lambda row: row['event_tokens_2048'][row['last_VS_index'] - 1] in ('[W_0]', '[W_1]', '[W_2]', '[W_3]', '[M_1]'), axis=1\n", - "# )\n", - "# dataset_2048_readmission['event_tokens_2048'] = dataset_2048_readmission.apply(\n", - "# lambda row: row['event_tokens_2048'][:row['last_VS_index'] - 1], axis=1\n", - "# )\n", - "# dataset_2048_readmission.drop(['deceased', 'death_after_start', 'death_after_end', 'length'], axis=1, inplace=True)\n", - "# dataset_2048_readmission['num_visits'] -= 1\n", - "# dataset_2048_readmission['token_length'] = dataset_2048_readmission['event_tokens_2048'].apply(len)\n", - "# dataset_2048_readmission = dataset_2048_readmission.apply(lambda row: truncate_and_pad(row), axis=1)\n", - "# dataset_2048_readmission['event_tokens_2048'] = dataset_2048_readmission['event_tokens_2048'].transform(\n", - "# lambda token_list: ' '.join(token_list)\n", - "# )\n", - "#\n", - "# dataset_2048_readmission" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.9" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/odyssey/data/dataset.py b/odyssey/data/dataset.py index 59c4d44..b3663f5 100644 --- a/odyssey/data/dataset.py +++ b/odyssey/data/dataset.py @@ -4,6 +4,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union import pandas as pd + import torch from torch.utils.data import Dataset diff --git a/odyssey/models/baseline/Bi-LSTM.ipynb b/odyssey/models/baseline/Bi-LSTM.ipynb index d67b155..99dea78 100644 --- a/odyssey/models/baseline/Bi-LSTM.ipynb +++ b/odyssey/models/baseline/Bi-LSTM.ipynb @@ -61,19 +61,26 @@ "import os\n", "import sys\n", "\n", - "\n", "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", "os.chdir(ROOT)\n", "\n", "from typing import Any, Dict, Tuple\n", + "from tqdm import tqdm\n", "\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", + "\n", "import torch\n", - "from models.big_bird_cehr.data import FinetuneDataset\n", - "from models.big_bird_cehr.embeddings import Embeddings\n", - "from models.big_bird_cehr.tokenizer import ConceptTokenizer\n", + "from torch import nn, optim\n", + "from torch.nn.functional import sigmoid\n", + "from torch.nn.utils.rnn import pack_padded_sequence\n", + "from torch.optim.lr_scheduler import ExponentialLR\n", + "from torch.utils.data import DataLoader, Dataset\n", + "\n", + "from odyssey.data.dataset import FinetuneDataset\n", + "from odyssey.models.cehr_big_bird.embeddings import Embeddings\n", + "from odyssey.data.tokenizer import ConceptTokenizer\n", "from sklearn.metrics import (\n", " auc,\n", " average_precision_score,\n", @@ -85,13 +92,6 @@ " roc_auc_score,\n", " roc_curve,\n", ")\n", - "from torch import nn, optim\n", - "from torch.nn.functional import sigmoid\n", - "from torch.nn.utils.rnn import pack_padded_sequence\n", - "from torch.optim.lr_scheduler import ExponentialLR\n", - "from torch.utils.data import DataLoader, Dataset\n", - "from tqdm import tqdm\n", - "\n", "\n", "DATA_ROOT = f\"{ROOT}/data/slurm_data/512/one_month\"\n", "DATA_PATH = f\"{DATA_ROOT}/pretrain.parquet\"\n", diff --git a/odyssey/models/baseline/Bi-LSTM.py b/odyssey/models/baseline/Bi-LSTM.py index 193a2aa..a231083 100644 --- a/odyssey/models/baseline/Bi-LSTM.py +++ b/odyssey/models/baseline/Bi-LSTM.py @@ -19,7 +19,7 @@ from odyssey.data.dataset import FinetuneDataset from odyssey.models.cehr_big_bird.embeddings import Embeddings -from odyssey.models.cehr_big_bird.tokenizer import HuggingFaceConceptTokenizer +from odyssey.data.tokenizer import HuggingFaceConceptTokenizer ROOT = "/fs01/home/afallah/odyssey/odyssey" diff --git a/odyssey/models/cehr_bert/embeddings.py b/odyssey/models/cehr_bert/embeddings.py deleted file mode 100644 index ecd096b..0000000 --- a/odyssey/models/cehr_bert/embeddings.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Embedding modules.""" - -import math - -import torch -from torch import nn - - -class TimeEmbeddingLayer(nn.Module): - """Embedding layer for time features.""" - - def __init__(self, embedding_size: int, is_time_delta: bool = False): - super().__init__() - self.embedding_size = embedding_size - self.is_time_delta = is_time_delta - - self.w = nn.Parameter(torch.empty(1, self.embedding_size)) - self.phi = nn.Parameter(torch.empty(1, self.embedding_size)) - - nn.init.xavier_uniform_(self.w) - nn.init.xavier_uniform_(self.phi) - - def forward(self, time_stamps: torch.Tensor) -> torch.Tensor: - """Apply time embedding to the input time stamps.""" - if self.is_time_delta: - # If the time_stamps represent time deltas, we calculate the deltas. - # This is equivalent to the difference between consecutive elements. - time_stamps = torch.cat( - (time_stamps[:, 0:1] * 0, time_stamps[:, 1:] - time_stamps[:, :-1]), - dim=-1, - ) - time_stamps = time_stamps.float() - time_stamps_expanded = time_stamps.unsqueeze(-1) - next_input = time_stamps_expanded * self.w + self.phi - - return torch.sin(next_input) - - -class VisitEmbedding(nn.Module): - """Embedding layer for visit segments.""" - - def __init__( - self, - visit_order_size: int, - embedding_size: int, - ): - super().__init__() - self.visit_order_size = visit_order_size - self.embedding_size = embedding_size - self.embedding = nn.Embedding(self.visit_order_size, self.embedding_size) - - def forward(self, visit_segments: torch.Tensor) -> torch.Tensor: - """Apply visit embedding to the input visit segments.""" - return self.embedding(visit_segments) - - -class ConceptEmbedding(nn.Module): - """Embedding layer for event concepts.""" - - def __init__( - self, - num_embeddings: int, - embedding_size: int, - padding_idx: int = None, - ): - super(ConceptEmbedding, self).__init__() - self.embedding = nn.Embedding( - num_embeddings, - embedding_size, - padding_idx=padding_idx, - ) - - def forward(self, inputs: torch.Tensor) -> torch.Tensor: - """Apply concept embedding to the input concepts.""" - return self.embedding(inputs) - - -class PositionalEmbedding(nn.Module): - """Positional embedding layer.""" - - def __init__(self, embedding_size, max_len=512): - super().__init__() - - # Compute the positional encodings once in log space. - pe = torch.zeros(max_len, embedding_size).float() - pe.require_grad = False - - position = torch.arange(0, max_len).float().unsqueeze(1) - div_term = ( - torch.arange(0, embedding_size, 2).float() - * -(math.log(10000.0) / embedding_size) - ).exp() - - pe[:, 0::2] = torch.sin(position * div_term) - pe[:, 1::2] = torch.cos(position * div_term) - - self.register_buffer("pe", pe) - - def forward(self, visit_orders: torch.Tensor) -> torch.Tensor: - """Apply positional embedding to the input visit orders.""" - first_visit_concept_orders = visit_orders[:, 0:1] - normalized_visit_orders = torch.clamp( - visit_orders - first_visit_concept_orders, - 0, - self.pe.size(0) - 1, - ) - return self.pe[normalized_visit_orders] - - -class Embeddings(nn.Module): - """Embeddings for CEHR-BERT.""" - - def __init__( - self, - vocab_size: int, - embedding_size: int = 128, - time_embedding_size: int = 16, - type_vocab_size: int = 8, - visit_order_size: int = 3, - max_len: int = 512, - layer_norm_eps: float = 1e-12, - dropout_prob: float = 0.1, - padding_idx: int = 1, - ): - super().__init__() - self.concept_embedding = ConceptEmbedding( - num_embeddings=vocab_size, - embedding_size=embedding_size, - padding_idx=padding_idx, - ) - self.token_type_embeddings = nn.Embedding(type_vocab_size, embedding_size) - self.time_embedding = TimeEmbeddingLayer( - embedding_size=time_embedding_size, - is_time_delta=True, - ) - self.age_embedding = TimeEmbeddingLayer(embedding_size=time_embedding_size) - self.positional_embedding = PositionalEmbedding( - embedding_size=embedding_size, - max_len=max_len, - ) - self.visit_embedding = VisitEmbedding( - visit_order_size=visit_order_size, - embedding_size=embedding_size, - ) - self.scale_back_concat_layer = nn.Linear( - embedding_size + 2 * time_embedding_size, - embedding_size, - ) # Assuming 4 input features are concatenated - self.tanh = nn.Tanh() - self.LayerNorm = nn.LayerNorm(embedding_size, eps=layer_norm_eps) - self.dropout = nn.Dropout(dropout_prob) - - def forward( - self, - concept_ids: torch.Tensor, - type_ids: torch.Tensor, - time_stamps: torch.Tensor, - ages: torch.Tensor, - visit_orders: torch.Tensor, - visit_segments: torch.Tensor, - ) -> torch.Tensor: - """Apply embeddings to the input features.""" - concept_embed = self.concept_embedding(concept_ids) - type_embed = self.token_type_embeddings(type_ids) - time_embed = self.time_embedding(time_stamps) - age_embed = self.age_embedding(ages) - positional_embed = self.positional_embedding(visit_orders) - visit_segment_embed = self.visit_embedding(visit_segments) - - embeddings = torch.cat((concept_embed, time_embed, age_embed), dim=-1) - embeddings = self.tanh(self.scale_back_concat_layer(embeddings)) - embeddings = embeddings + type_embed + positional_embed + visit_segment_embed - embeddings = self.LayerNorm(embeddings) - - return self.dropout(embeddings) diff --git a/odyssey/models/cehr_bert/model.py b/odyssey/models/cehr_bert/model.py index 9af880d..9e36f59 100644 --- a/odyssey/models/cehr_bert/model.py +++ b/odyssey/models/cehr_bert/model.py @@ -22,7 +22,7 @@ BertPooler, ) -from odyssey.models.cehr_bert.embeddings import Embeddings +from odyssey.models.embeddings import BERTEmbeddingsForCEHR class BertPretrain(pl.LightningModule): @@ -75,7 +75,7 @@ def __init__( ) # BertForMaskedLM ## BertModel - self.embeddings = Embeddings( + self.embeddings = BERTEmbeddingsForCEHR( vocab_size=self.vocab_size, embedding_size=self.embedding_size, time_embedding_size=self.time_embeddings_size, diff --git a/odyssey/models/cehr_big_bird/embeddings.py b/odyssey/models/cehr_big_bird/embeddings.py deleted file mode 100644 index da43b44..0000000 --- a/odyssey/models/cehr_big_bird/embeddings.py +++ /dev/null @@ -1,363 +0,0 @@ -"""Embedding layers for the models.""" - -import math -from typing import Any, Optional - -import torch -from torch import nn -from transformers import BigBirdConfig - - -class TimeEmbeddingLayer(nn.Module): - """Embedding layer for time features.""" - - def __init__(self, embedding_size: int, is_time_delta: bool = False): - super().__init__() - self.embedding_size = embedding_size - self.is_time_delta = is_time_delta - - self.w = nn.Parameter(torch.empty(1, self.embedding_size)) - self.phi = nn.Parameter(torch.empty(1, self.embedding_size)) - - nn.init.xavier_uniform_(self.w) - nn.init.xavier_uniform_(self.phi) - - def forward(self, time_stamps: torch.Tensor) -> Any: - """Apply time embedding to the input time stamps.""" - if self.is_time_delta: - # If the time_stamps represent time deltas, we calculate the deltas. - # This is equivalent to the difference between consecutive elements. - time_stamps = torch.cat( - (time_stamps[:, 0:1] * 0, time_stamps[:, 1:] - time_stamps[:, :-1]), - dim=-1, - ) - time_stamps = time_stamps.float() - time_stamps_expanded = time_stamps.unsqueeze(-1) - next_input = time_stamps_expanded * self.w + self.phi - - return torch.sin(next_input) - - -class VisitEmbedding(nn.Module): - """Embedding layer for visit segments.""" - - def __init__( - self, - visit_order_size: int, - embedding_size: int, - ): - super().__init__() - self.visit_order_size = visit_order_size - self.embedding_size = embedding_size - self.embedding = nn.Embedding(self.visit_order_size, self.embedding_size) - - def forward(self, visit_segments: torch.Tensor) -> Any: - """Apply visit embedding to the input visit segments.""" - return self.embedding(visit_segments) - - -class ConceptEmbedding(nn.Module): - """Embedding layer for event concepts.""" - - def __init__( - self, - num_embeddings: int, - embedding_size: int, - padding_idx: Optional[int] = None, - ): - super(ConceptEmbedding, self).__init__() - self.embedding = nn.Embedding( - num_embeddings, - embedding_size, - padding_idx=padding_idx, - ) - - def forward(self, inputs: torch.Tensor) -> Any: - """Apply concept embedding to the input concepts.""" - return self.embedding(inputs) - - -class PositionalEmbedding(nn.Module): - """Positional embedding layer.""" - - def __init__(self, embedding_size: int, max_len: int = 2048): - super().__init__() - - # Compute the positional encodings once in log space. - pe = torch.zeros(max_len, embedding_size).float() - pe.require_grad = False - - position = torch.arange(0, max_len).float().unsqueeze(1) - div_term = ( - torch.arange(0, embedding_size, 2).float() - * -(math.log(10000.0) / embedding_size) - ).exp() - - pe[:, 0::2] = torch.sin(position * div_term) - pe[:, 1::2] = torch.cos(position * div_term) - - self.register_buffer("pe", pe) - - def forward(self, visit_orders: torch.Tensor) -> Any: - """Apply positional embedding to the input visit orders.""" - first_visit_concept_orders = visit_orders[:, 0:1] - normalized_visit_orders = torch.clamp( - visit_orders - first_visit_concept_orders, - 0, - self.pe.size(0) - 1, - ) - return self.pe[normalized_visit_orders] - - -class Embeddings(nn.Module): - """Embeddings for CEHR-BERT.""" - - def __init__( - self, - vocab_size: int, - embedding_size: int = 128, - time_embeddings_size: int = 16, - type_vocab_size: int = 8, - visit_order_size: int = 3, - max_len: int = 2048, - layer_norm_eps: float = 1e-12, - dropout_prob: float = 0.1, - padding_idx: int = 1, - ): - super().__init__() - self.concept_embedding = ConceptEmbedding( - num_embeddings=vocab_size, - embedding_size=embedding_size, - padding_idx=padding_idx, - ) - self.token_type_embeddings = nn.Embedding( - type_vocab_size, - embedding_size, - ) - self.time_embedding = TimeEmbeddingLayer( - embedding_size=time_embeddings_size, - is_time_delta=True, - ) - self.age_embedding = TimeEmbeddingLayer( - embedding_size=time_embeddings_size, - ) - self.positional_embedding = PositionalEmbedding( - embedding_size=embedding_size, - max_len=max_len, - ) - self.visit_embedding = VisitEmbedding( - visit_order_size=visit_order_size, - embedding_size=embedding_size, - ) - self.scale_back_concat_layer = nn.Linear( - embedding_size + 2 * time_embeddings_size, - embedding_size, - ) # Assuming 4 input features are concatenated - self.tanh = nn.Tanh() - self.LayerNorm = nn.LayerNorm(embedding_size, eps=layer_norm_eps) - self.dropout = nn.Dropout(dropout_prob) - - def forward( - self, - concept_ids: torch.Tensor, - type_ids: torch.Tensor, - time_stamps: torch.Tensor, - ages: torch.Tensor, - visit_orders: torch.Tensor, - visit_segments: torch.Tensor, - ) -> Any: - """Apply embeddings to the input features.""" - concept_embed = self.concept_embedding(concept_ids) - type_embed = self.token_type_embeddings(type_ids) - time_embed = self.time_embedding(time_stamps) - age_embed = self.age_embedding(ages) - positional_embed = self.positional_embedding(visit_orders) - visit_segment_embed = self.visit_embedding(visit_segments) - - embeddings = torch.cat((concept_embed, time_embed, age_embed), dim=-1) - embeddings = self.tanh(self.scale_back_concat_layer(embeddings)) - embeddings = embeddings + type_embed + positional_embed + visit_segment_embed - embeddings = self.LayerNorm(embeddings) - return self.dropout(embeddings) - - -class BigBirdEmbeddingsForCEHR(nn.Module): - """Construct the embeddings from word, position and token_type embeddings.""" - - # Copied from transformers.models.bert.modeling_bert.BertEmbeddings.__init__ - def __init__( - self, - config: BigBirdConfig, - time_embeddings_size: int = 16, - visit_order_size: int = 3, - ) -> None: - """Initiate wrapper class for embeddings used in BigBird CEHR classes.""" - super().__init__() - - self.word_embeddings = nn.Embedding( - config.vocab_size, - config.hidden_size, - padding_idx=config.pad_token_id, - ) - self.position_embeddings = nn.Embedding( - config.max_position_embeddings, - config.hidden_size, - ) - self.token_type_embeddings = nn.Embedding( - config.type_vocab_size, - config.hidden_size, - ) - self.visit_order_embeddings = nn.Embedding( - config.max_position_embeddings, - config.hidden_size, - ) - self.time_embeddings = TimeEmbeddingLayer( - embedding_size=time_embeddings_size, - is_time_delta=True, - ) - self.age_embeddings = TimeEmbeddingLayer( - embedding_size=time_embeddings_size, - ) - self.visit_segment_embeddings = VisitEmbedding( - visit_order_size=visit_order_size, - embedding_size=config.hidden_size, - ) - self.scale_back_concat_layer = nn.Linear( - config.hidden_size + 2 * time_embeddings_size, - config.hidden_size, - ) - - self.time_stamps: Optional[torch.Tensor] = None - self.ages: Optional[torch.Tensor] = None - self.visit_orders: Optional[torch.Tensor] = None - self.visit_segments: Optional[torch.Tensor] = None - - # self.LayerNorm is not snake-cased to stick with TensorFlow model - # variable name and be able to load any TensorFlow checkpoint file. - self.tanh = nn.Tanh() - self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) - self.dropout = nn.Dropout(config.hidden_dropout_prob) - - # position_ids (1, len position emb) is contiguous in memory. - self.position_embedding_type = getattr( - config, - "position_embedding_type", - "absolute", - ) - self.register_buffer( - "position_ids", - torch.arange(config.max_position_embeddings).expand((1, -1)), - persistent=False, - ) - self.register_buffer( - "token_type_ids", - torch.zeros(self.position_ids.size(), dtype=torch.long), - persistent=False, - ) - # End copy - - self.rescale_embeddings = config.rescale_embeddings - self.hidden_size = config.hidden_size - - def cache_input( - self, - time_stamps: torch.Tensor, - ages: torch.Tensor, - visit_orders: torch.Tensor, - visit_segments: torch.Tensor, - ) -> None: - """Cache values for time_stamps, ages, visit_orders & visit_segments. - - These values will be used by the forward pass to change the final embedding. - - Parameters - ---------- - time_stamps : torch.Tensor - Time stamps of the input data. - ages : torch.Tensor - Ages of the input data. - visit_orders : torch.Tensor - Visit orders of the input data. - visit_segments : torch.Tensor - Visit segments of the input data. - - """ - self.time_stamps = time_stamps - self.ages = ages - self.visit_orders = visit_orders - self.visit_segments = visit_segments - - def clear_cache(self) -> None: - """Delete the tensors cached by cache_input method.""" - del self.time_stamps, self.ages, self.visit_orders, self.visit_segments - - def forward( - self, - input_ids: Optional[torch.Tensor] = None, - token_type_ids: Optional[torch.Tensor] = None, - position_ids: Optional[torch.Tensor] = None, - inputs_embeds: Optional[torch.Tensor] = None, - past_key_values_length: int = 0, - ) -> Any: - """Return the final embeddings of concept ids using input and cached values.""" - if input_ids is not None: - input_shape = input_ids.size() - else: - input_shape = inputs_embeds.size()[:-1] - - seq_length = input_shape[1] - - if position_ids is None: - position_ids = self.position_ids[ - :, - past_key_values_length : seq_length + past_key_values_length, - ] - - # Setting the token_type_ids to the registered buffer in constructor - if token_type_ids is None: - if hasattr(self, "token_type_ids"): - buffered_token_type_ids = self.token_type_ids[:, :seq_length] - buffered_token_type_ids_expanded = buffered_token_type_ids.expand( - input_shape[0], - seq_length, - ) - token_type_ids = buffered_token_type_ids_expanded - else: - token_type_ids = torch.zeros( - input_shape, - dtype=torch.long, - device=self.position_ids.device, - ) - - if inputs_embeds is None: - inputs_embeds = self.word_embeddings(input_ids) - - if self.rescale_embeddings: - inputs_embeds = inputs_embeds * (self.hidden_size**0.5) - - # Using cached values from a prior cache_input call - time_stamps_embeds = self.time_embeddings(self.time_stamps) - ages_embeds = self.age_embeddings(self.ages) - visit_segments_embeds = self.visit_segment_embeddings(self.visit_segments) - visit_order_embeds = self.visit_order_embeddings(self.visit_orders) - - position_embeds = self.position_embeddings(position_ids) - token_type_embeds = self.token_type_embeddings(token_type_ids) - - inputs_embeds = torch.cat( - (inputs_embeds, time_stamps_embeds, ages_embeds), - dim=-1, - ) - inputs_embeds = self.tanh(self.scale_back_concat_layer(inputs_embeds)) - embeddings = inputs_embeds + token_type_embeds - embeddings += position_embeds - embeddings += visit_order_embeds - embeddings += visit_segments_embeds - - embeddings = self.dropout(embeddings) - embeddings = self.LayerNorm(embeddings) - - # Clear the cache for next forward call - self.clear_cache() - - return embeddings diff --git a/odyssey/models/cehr_big_bird/model.py b/odyssey/models/cehr_big_bird/model.py index 5acc07e..78ebd9c 100644 --- a/odyssey/models/cehr_big_bird/model.py +++ b/odyssey/models/cehr_big_bird/model.py @@ -1,10 +1,9 @@ -"""Big Bird transformer model.""" +"""BigBird transformer model.""" from typing import Any, Dict, Optional, Tuple, Union import numpy as np import pytorch_lightning as pl -import torch from sklearn.metrics import ( accuracy_score, f1_score, @@ -12,6 +11,8 @@ recall_score, roc_auc_score, ) + +import torch from torch import nn, optim from torch.cuda.amp import autocast from torch.optim import AdamW @@ -23,7 +24,7 @@ BigBirdForSequenceClassification, ) -from odyssey.models.cehr_big_bird.embeddings import BigBirdEmbeddingsForCEHR +from odyssey.models.embeddings import BigBirdEmbeddingsForCEHR class BigBirdPretrain(pl.LightningModule): diff --git a/odyssey/models/prediction.py b/odyssey/models/prediction.py deleted file mode 100644 index 7b8f92c..0000000 --- a/odyssey/models/prediction.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Prediction module for loading and running BigBird models on patient data.""" - -from typing import Any, Dict, Optional - -import torch - -from odyssey.models.cehr_big_bird.model import BigBirdFinetune, BigBirdPretrain -from odyssey.tokenizer import ConceptTokenizer - - -def load_finetuned_model( - model_path: str, - tokenizer: ConceptTokenizer, - pre_model_config: Optional[Dict[str, Any]] = None, - fine_model_config: Optional[Dict[str, Any]] = None, - device: Optional[torch.device] = None, -) -> torch.nn.Module: - """Load a finetuned model from model_path using tokenizer information. - - Return a loaded finetuned model from model_path, using tokenizer information. - If config arguments are not provided, the default configs built into the - PyTorch classes are used. - - Parameters - ---------- - model_path: str - Path to the finetuned model to load - tokenizer: ConceptTokenizer - Loaded tokenizer object - pre_model_config: Dict[str, Any], optional - Optional config to override default values of a pretrained model - fine_model_config: Dict[str, Any], optional - Optional config to override default values of a finetuned model - device: torch.device, optional - CUDA device. By default, GPU is used - - Returns - ------- - torch.nn.Module - Finetuned model loaded from model_path - - """ - # Load GPU or CPU device - if not device: - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - # Create the skeleton of a pretrained and finetuned model - pretrained_model = BigBirdPretrain( - vocab_size=tokenizer.get_vocab_size(), - padding_idx=tokenizer.get_pad_token_id(), - **(pre_model_config or {}), - ) - - model = BigBirdFinetune( - pretrained_model=pretrained_model, - **(fine_model_config or {}), - ) - - # Load the weights using model_path directory - state_dict = torch.load(model_path, map_location=device)["state_dict"] - model.load_state_dict(state_dict) - model.to(device) - model.eval() - - return model - - -def predict_patient_outcomes( - patient: Dict[str, torch.Tensor], - model: torch.nn.Module, - device: Optional[torch.device] = None, -) -> Any: - """Compute model output predictions on given patient data. - - Parameters - ---------- - patient: Dict[str, torch.Tensor] - Patient data as a dictionary of tensors - model: torch.nn.Module - Model to use for prediction - device: torch.device, optional - CUDA device. By default, GPU is used - - Returns - ------- - Any - Model output predictions on the given patient data - - """ - # Load GPU or CPU device - if not device: - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - # Load patient information as a Tuple - patient_inputs = ( - patient["concept_ids"].to(device), - patient["type_ids"].to(device), - patient["time_stamps"].to(device), - patient["ages"].to(device), - patient["visit_orders"].to(device), - patient["visit_segments"].to(device), - ) - patient_labels = patient["labels"].to(device) - patient_attention_mask = patient["attention_mask"].to(device) - - # Get model output predictions - model.to(device) - - return model( - inputs=patient_inputs, - attention_mask=patient_attention_mask, - labels=patient_labels, - ) diff --git a/odyssey/models/utils.py b/odyssey/models/utils.py deleted file mode 100644 index 72bd126..0000000 --- a/odyssey/models/utils.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Utility functions for the model.""" - -import os -import pickle -import random -import uuid -from os.path import join -from typing import Any - -import numpy as np -import pandas as pd -import pytorch_lightning as pl -import torch -import yaml - - -def load_config(config_dir: str, model_type: str) -> Any: - """Load the model configuration. - - Parameters - ---------- - config_dir: str - Directory containing the model configuration files - - model_type: str - Model type to load configuration for - - Returns - ------- - Any - Model configuration - - """ - config_file = join(config_dir, f"{model_type}.yaml") - with open(config_file, "r") as file: - return yaml.safe_load(file) - - -def seed_everything(seed: int) -> None: - """Seed all components of the model. - - Parameters - ---------- - seed: int - Seed value to use - - """ - random.seed(seed) - torch.manual_seed(seed) - np.random.seed(seed) - torch.cuda.manual_seed_all(seed) - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - pl.seed_everything(seed) - - -def load_pretrain_data( - data_dir: str, - sequence_file: str, - id_file: str, -) -> pd.DataFrame: - """Load the pretraining data. - - Parameters - ---------- - data_dir: str - Directory containing the data files - sequence_file: str - Sequence file name - id_file: str - ID file name - - Returns - ------- - pd.DataFrame - Pretraining data - - """ - sequence_path = join(data_dir, sequence_file) - id_path = join(data_dir, id_file) - - if not os.path.exists(sequence_path): - raise FileNotFoundError(f"Sequence file not found: {sequence_path}") - - if not os.path.exists(id_path): - raise FileNotFoundError(f"ID file not found: {id_path}") - - data = pd.read_parquet(sequence_path) - with open(id_path, "rb") as file: - patient_ids = pickle.load(file) - - return data.loc[data["patient_id"].isin(patient_ids["pretrain"])] - - -def load_finetune_data( - data_dir: str, - sequence_file: str, - id_file: str, - valid_scheme: str, - num_finetune_patients: str, -) -> pd.DataFrame: - """Load the finetuning data. - - Parameters - ---------- - data_dir: str - Directory containing the data files - sequence_file: str - Sequence file name - id_file: str - ID file name - valid_scheme: str - Validation scheme - num_finetune_patients: str - Number of finetune patients - - Returns - ------- - pd.DataFrame - Finetuning data - - """ - sequence_path = join(data_dir, sequence_file) - id_path = join(data_dir, id_file) - - if not os.path.exists(sequence_path): - raise FileNotFoundError(f"Sequence file not found: {sequence_path}") - - if not os.path.exists(id_path): - raise FileNotFoundError(f"ID file not found: {id_path}") - - data = pd.read_parquet(sequence_path) - with open(id_path, "rb") as file: - patient_ids = pickle.load(file) - - fine_tune = data.loc[ - data["patient_id"].isin( - patient_ids["finetune"][valid_scheme][num_finetune_patients], - ) - ] - fine_test = data.loc[data["patient_id"].isin(patient_ids["test"])] - return fine_tune, fine_test - - -def get_run_id( - checkpoint_dir: str, - retrieve: bool = False, - run_id_file: str = "wandb_run_id.txt", - length: int = 8, -) -> str: - """Fetch the run ID for the current run. - - If the run ID file exists, retrieve the run ID from the file. - Otherwise, generate a new run ID and save it to the file. - - Parameters - ---------- - checkpoint_dir: str - Directory to store the run ID file - retrieve: bool, optional - Retrieve the run ID from the file, by default False - run_id_file: str, optional - Run ID file name, by default "wandb_run_id.txt" - length: int, optional - String length of the run ID, by default 8 - - Returns - ------- - str - Run ID for the current run - - """ - run_id_path = os.path.join(checkpoint_dir, run_id_file) - if retrieve and os.path.exists(run_id_path): - with open(run_id_path, "r") as file: - run_id = file.read().strip() - else: - run_id = str(uuid.uuid4())[:length] - with open(run_id_path, "w") as file: - file.write(run_id) - return run_id - - -def save_object_to_disk(obj: Any, save_path: str) -> None: - """Save an object to disk using pickle. - - Parameters - ---------- - obj: Any - Object to save - save_path: str - Path to save the object - - """ - with open(save_path, "wb") as f: - pickle.dump(obj, f) - print(f"File saved to disk: {save_path}") diff --git a/pretrain.py b/pretrain.py index 4627b34..143f10c 100644 --- a/pretrain.py +++ b/pretrain.py @@ -13,15 +13,15 @@ from sklearn.model_selection import train_test_split from torch.utils.data import DataLoader +from odyssey.utils.utils import seed_everything from odyssey.data.dataset import PretrainDataset from odyssey.data.tokenizer import ConceptTokenizer from odyssey.models.cehr_bert.model import BertPretrain from odyssey.models.cehr_big_bird.model import BigBirdPretrain -from odyssey.models.utils import ( +from odyssey.models.model_utils import ( get_run_id, load_config, load_pretrain_data, - seed_everything, ) From 6bad1dcd416076bcc4d98ce6f67fa960274892d7 Mon Sep 17 00:00:00 2001 From: afallah Date: Tue, 16 Apr 2024 16:44:44 -0400 Subject: [PATCH 8/9] Moved files in different packages and renamed exisiting files to names that make more sense. --- odyssey/data/bigbird_data/DataProcessor.ipynb | 1136 ++++++ odyssey/evals/AttentionVisualization.ipynb | 3412 +++++++++++++++++ odyssey/evals/CompareAUROC-Poster.ipynb | 158 + odyssey/evals/TestAnalysis.ipynb | 266 ++ odyssey/evals/prediction.py | 113 + odyssey/models/embeddings.py | 368 ++ odyssey/models/model_utils.py | 159 + odyssey/utils/__init__.py | 1 + odyssey/utils/utils.py | 48 + 9 files changed, 5661 insertions(+) create mode 100644 odyssey/data/bigbird_data/DataProcessor.ipynb create mode 100644 odyssey/evals/AttentionVisualization.ipynb create mode 100644 odyssey/evals/CompareAUROC-Poster.ipynb create mode 100644 odyssey/evals/TestAnalysis.ipynb create mode 100644 odyssey/evals/prediction.py create mode 100644 odyssey/models/embeddings.py create mode 100644 odyssey/models/model_utils.py create mode 100644 odyssey/utils/__init__.py create mode 100644 odyssey/utils/utils.py diff --git a/odyssey/data/bigbird_data/DataProcessor.ipynb b/odyssey/data/bigbird_data/DataProcessor.ipynb new file mode 100644 index 0000000..4944995 --- /dev/null +++ b/odyssey/data/bigbird_data/DataProcessor.ipynb @@ -0,0 +1,1136 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-13T16:14:45.546088300Z", + "start_time": "2024-03-13T16:14:43.587090300Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "import os\n", + "import pickle\n", + "import random\n", + "import sys\n", + "from typing import Any, Dict, List, Optional\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn.model_selection import train_test_split\n", + "from skmultilearn.model_selection import iterative_train_test_split\n", + "\n", + "\n", + "sys.path.append(\"/h/afallah/odyssey/odyssey/lib\")\n", + "from utils import save_object_to_disk\n", + "\n", + "\n", + "DATA_ROOT = \"/h/afallah/odyssey/odyssey/data/bigbird_data\"\n", + "DATASET = f\"{DATA_ROOT}/patient_sequences/patient_sequences_2048.parquet\"\n", + "MAX_LEN = 2048\n", + "\n", + "SEED = 23\n", + "os.chdir(DATA_ROOT)\n", + "random.seed(SEED)\n", + "np.random.seed(SEED)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-13T16:15:12.321718600Z", + "start_time": "2024-03-13T16:14:45.553089800Z" + }, + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Current columns: Index(['patient_id', 'num_visits', 'deceased', 'death_after_start',\n", + " 'death_after_end', 'length', 'token_length', 'event_tokens_2048',\n", + " 'type_tokens_2048', 'age_tokens_2048', 'time_tokens_2048',\n", + " 'visit_tokens_2048', 'position_tokens_2048', 'elapsed_tokens_2048',\n", + " 'common_conditions', 'rare_conditions'],\n", + " dtype='object')\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
patient_idnum_visitsdeceaseddeath_after_startdeath_after_endlengthtoken_lengthevent_tokens_2048type_tokens_2048age_tokens_2048time_tokens_2048visit_tokens_2048position_tokens_2048elapsed_tokens_2048common_conditionsrare_conditions
035581927-9c95-5ae9-af76-7d74870a349c10NaNNaN5054[[CLS], [VS], 00006473900, 00904516561, 510790...[1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, ...[0, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85...[0, 5902, 5902, 5902, 5902, 5902, 5902, 5902, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 1.97, 2.02, 2.02, 2.02, 2.02, 2.0...[1, 0, 0, 0, 0, 0, 0, 0, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
1f5bba8dd-25c0-5336-8d3d-37424c18502620NaNNaN148156[[CLS], [VS], 52135_2, 52075_2, 52074_2, 52073...[1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...[0, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83...[0, 6594, 6594, 6594, 6594, 6594, 6594, 6594, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...[0, 0, 0, 0, 0, 0, 0, 1, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
2f4938f91-cadb-5133-8541-a52fb0916cea20NaNNaN7886[[CLS], [VS], 0RB30ZZ, 0RG10A0, 00071101441, 0...[1, 2, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...[0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44...[0, 8150, 8150, 8150, 8150, 8150, 8150, 8150, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 0.0, 0.0, 1.08, 1.08, 13.89, 13.8...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
36fe2371b-a6f0-5436-aade-7795005b0c6620NaNNaN8694[[CLS], [VS], 63739057310, 49281041688, 005970...[1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...[0, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72...[0, 6093, 6093, 6093, 6093, 6093, 6093, 6093, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 0.75, 0.75, 0.75, 0.75, 0.75, 0.7...[1, 0, 0, 0, 0, 0, 0, 1, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
46f7590ae-f3b9-50e5-9e41-d4bb1000887a10NaNNaN7276[[CLS], [VS], 50813_0, 52135_0, 52075_3, 52074...[1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...[0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47...[0, 6379, 6379, 6379, 6379, 6379, 6379, 6379, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...[1, 0, 0, 0, 0, 0, 0, 0, 0, 1][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
\n", + "
" + ], + "text/plain": [ + " patient_id num_visits deceased \\\n", + "0 35581927-9c95-5ae9-af76-7d74870a349c 1 0 \n", + "1 f5bba8dd-25c0-5336-8d3d-37424c185026 2 0 \n", + "2 f4938f91-cadb-5133-8541-a52fb0916cea 2 0 \n", + "3 6fe2371b-a6f0-5436-aade-7795005b0c66 2 0 \n", + "4 6f7590ae-f3b9-50e5-9e41-d4bb1000887a 1 0 \n", + "\n", + " death_after_start death_after_end length token_length \\\n", + "0 NaN NaN 50 54 \n", + "1 NaN NaN 148 156 \n", + "2 NaN NaN 78 86 \n", + "3 NaN NaN 86 94 \n", + "4 NaN NaN 72 76 \n", + "\n", + " event_tokens_2048 \\\n", + "0 [[CLS], [VS], 00006473900, 00904516561, 510790... \n", + "1 [[CLS], [VS], 52135_2, 52075_2, 52074_2, 52073... \n", + "2 [[CLS], [VS], 0RB30ZZ, 0RG10A0, 00071101441, 0... \n", + "3 [[CLS], [VS], 63739057310, 49281041688, 005970... \n", + "4 [[CLS], [VS], 50813_0, 52135_0, 52075_3, 52074... \n", + "\n", + " type_tokens_2048 \\\n", + "0 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, ... \n", + "1 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", + "2 [1, 2, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", + "3 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", + "4 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", + "\n", + " age_tokens_2048 \\\n", + "0 [0, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85... \n", + "1 [0, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83... \n", + "2 [0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44... \n", + "3 [0, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72... \n", + "4 [0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47... \n", + "\n", + " time_tokens_2048 \\\n", + "0 [0, 5902, 5902, 5902, 5902, 5902, 5902, 5902, ... \n", + "1 [0, 6594, 6594, 6594, 6594, 6594, 6594, 6594, ... \n", + "2 [0, 8150, 8150, 8150, 8150, 8150, 8150, 8150, ... \n", + "3 [0, 6093, 6093, 6093, 6093, 6093, 6093, 6093, ... \n", + "4 [0, 6379, 6379, 6379, 6379, 6379, 6379, 6379, ... \n", + "\n", + " visit_tokens_2048 \\\n", + "0 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", + "1 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", + "2 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", + "3 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", + "4 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", + "\n", + " position_tokens_2048 \\\n", + "0 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", + "1 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", + "2 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", + "3 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", + "4 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", + "\n", + " elapsed_tokens_2048 \\\n", + "0 [-2.0, -1.0, 1.97, 2.02, 2.02, 2.02, 2.02, 2.0... \n", + "1 [-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n", + "2 [-2.0, -1.0, 0.0, 0.0, 1.08, 1.08, 13.89, 13.8... \n", + "3 [-2.0, -1.0, 0.75, 0.75, 0.75, 0.75, 0.75, 0.7... \n", + "4 [-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n", + "\n", + " common_conditions rare_conditions \n", + "0 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", + "1 [0, 0, 0, 0, 0, 0, 0, 1, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", + "2 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", + "3 [1, 0, 0, 0, 0, 0, 0, 1, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", + "4 [1, 0, 0, 0, 0, 0, 0, 0, 0, 1] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Load complete dataset\n", + "dataset_2048 = pd.read_parquet(DATASET)\n", + "\n", + "print(f\"Current columns: {dataset_2048.columns}\")\n", + "dataset_2048.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def filter_by_num_visit(dataset: pd.DataFrame, minimum_num_visits: int) -> pd.DataFrame:\n", + " \"\"\"Filter the patients based on num_visits threshold.\n", + "\n", + " Args:\n", + " dataset (pd.DataFrame): The input dataset.\n", + " minimum_num_visits (int): The threshold num_visits\n", + "\n", + " Returns\n", + " -------\n", + " pd.DataFrame: The filtered dataset.\n", + " \"\"\"\n", + " filtered_dataset = dataset.loc[dataset[\"num_visits\"] >= minimum_num_visits]\n", + " filtered_dataset.reset_index(drop=True, inplace=True)\n", + " return filtered_dataset\n", + "\n", + "\n", + "def filter_by_length_of_stay(dataset: pd.DataFrame, threshold: int = 1) -> pd.DataFrame:\n", + " \"\"\"Filter the patients based on length of stay threshold.\n", + "\n", + " Args:\n", + " dataset (pd.DataFrame): The input dataset.\n", + " minimum_num_visits (int): The threshold length of stay\n", + "\n", + " Returns\n", + " -------\n", + " pd.DataFrame: The filtered dataset.\n", + " \"\"\"\n", + " filtered_dataset = dataset.loc[dataset[\"length_of_stay\"] >= threshold]\n", + "\n", + " # Only keep the patients that their first event happens within threshold\n", + " # TODO: Check how many patients get removed here?\n", + " filtered_dataset = filtered_dataset[\n", + " filtered_dataset.apply(\n", + " lambda row: row[\"elapsed_tokens_2048\"][row[\"last_VS_index\"] + 1]\n", + " < threshold * 24,\n", + " axis=1,\n", + " )\n", + " ]\n", + "\n", + " filtered_dataset.reset_index(drop=True, inplace=True)\n", + " return filtered_dataset\n", + "\n", + "\n", + "def get_last_occurence_index(seq: List[str], target: str) -> int:\n", + " \"\"\"Return the index of the last occurrence of target in seq.\n", + "\n", + " Args:\n", + " seq (List[str]): The input sequence.\n", + " target (str): The target string to find.\n", + "\n", + " Returns\n", + " -------\n", + " int: The index of the last occurrence of target in seq.\n", + " \"\"\"\n", + " return len(seq) - (seq[::-1].index(target) + 1)\n", + "\n", + "\n", + "def check_readmission_label(row: pd.Series) -> int:\n", + " \"\"\"Check if the label indicates readmission within one month.\n", + "\n", + " Args:\n", + " row (pd.Series): The input row.\n", + "\n", + " Returns\n", + " -------\n", + " bool: True if readmission label is present, False otherwise.\n", + " \"\"\"\n", + " last_vs_index = row[\"last_VS_index\"]\n", + " return int(\n", + " row[\"event_tokens_2048\"][last_vs_index - 1]\n", + " in (\"[W_0]\", \"[W_1]\", \"[W_2]\", \"[W_3]\", \"[M_1]\"),\n", + " )\n", + "\n", + "\n", + "def get_length_of_stay(row: pd.Series) -> pd.Series:\n", + " \"\"\"Determine the length of a given visit.\n", + "\n", + " Args:\n", + " row (pd.Series): The input row.\n", + "\n", + " Returns\n", + " -------\n", + " pd.Series: The preprocessed row.\n", + " \"\"\"\n", + " admission_time = row[\"last_VS_index\"] + 1\n", + " discharge_time = row[\"last_VE_index\"] - 1\n", + " return (discharge_time - admission_time) / 24\n", + "\n", + "\n", + "def get_visit_cutoff_at_threshold(row: pd.Series, threshold: int = 24) -> int:\n", + " \"\"\"Get the index of the first event token of last visit that occurs after threshold hours.\n", + "\n", + " Args:\n", + " row (pd.Series): The input row.\n", + " threshold (int): The number of hours to consider.\n", + "\n", + " Returns\n", + " -------\n", + " cutoff_index (int): The corrosponding cutoff index.\n", + " \"\"\"\n", + " last_vs_index = row[\"last_VS_index\"]\n", + " last_ve_index = row[\"last_VE_index\"]\n", + "\n", + " for i in range(last_vs_index + 1, last_ve_index):\n", + " if row[\"elapsed_tokens_2048\"][i] > threshold:\n", + " return i\n", + "\n", + " return len(row[\"event_tokens_2048\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def process_length_of_stay_dataset(\n", + " dataset: pd.DataFrame,\n", + " threshold: int = 7,\n", + ") -> pd.DataFrame:\n", + " \"\"\"Process the length of stay dataset to extract required features.\n", + "\n", + " Args:\n", + " dataset (pd.DataFrame): The input dataset.\n", + " threshold (int): The threshold length of stay.\n", + "\n", + " Returns\n", + " -------\n", + " pd.DataFrame: The processed dataset.\n", + " \"\"\"\n", + " dataset[\"last_VS_index\"] = dataset[\"event_tokens_2048\"].transform(\n", + " lambda seq: get_last_occurence_index(list(seq), \"[VS]\"),\n", + " )\n", + " dataset[\"last_VE_index\"] = dataset[\"event_tokens_2048\"].transform(\n", + " lambda seq: get_last_occurence_index(list(seq), \"[VE]\"),\n", + " )\n", + " dataset[\"length_of_stay\"] = dataset.apply(get_length_of_stay, axis=1)\n", + "\n", + " dataset = filter_by_length_of_stay(dataset, threshold=1)\n", + " dataset[\"label_los_1week\"] = (dataset[\"length_of_stay\"] >= threshold).astype(int)\n", + "\n", + " dataset[\"cutoff_los\"] = dataset.apply(\n", + " lambda row: get_visit_cutoff_at_threshold(row, threshold=24),\n", + " axis=1,\n", + " )\n", + " dataset[\"token_length\"] = dataset[\"event_tokens_2048\"].apply(len)\n", + "\n", + " return dataset\n", + "\n", + "\n", + "# Process the dataset for length of stay prediction above a threshold\n", + "dataset_2048_los = process_length_of_stay_dataset(dataset_2048.copy(), threshold=7)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def process_condition_dataset(dataset: pd.DataFrame) -> pd.DataFrame:\n", + " \"\"\"Process the condition dataset to extract required features.\n", + "\n", + " Args:\n", + " dataset (pd.DataFrame): The input condition dataset.\n", + "\n", + " Returns\n", + " -------\n", + " pd.DataFrame: The processed condition dataset.\n", + " \"\"\"\n", + " dataset[\"all_conditions\"] = dataset.apply(\n", + " lambda row: np.concatenate(\n", + " [row[\"common_conditions\"], row[\"rare_conditions\"]],\n", + " dtype=np.int64,\n", + " ),\n", + " axis=1,\n", + " )\n", + "\n", + " return dataset\n", + "\n", + "\n", + "# Process the dataset for conditions including rare and common\n", + "dataset_2048_condition = process_condition_dataset(dataset_2048.copy())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-13T16:15:16.075719400Z", + "start_time": "2024-03-13T16:15:12.335721100Z" + }, + "collapsed": false + }, + "outputs": [], + "source": [ + "def process_mortality_dataset(dataset: pd.DataFrame) -> pd.DataFrame:\n", + " \"\"\"Process the mortality dataset to extract required features.\n", + "\n", + " Args:\n", + " dataset (pd.DataFrame): The input mortality dataset.\n", + "\n", + " Returns\n", + " -------\n", + " pd.DataFrame: The processed mortality dataset.\n", + " \"\"\"\n", + " dataset[\"label_mortality_2weeks\"] = (\n", + " (dataset[\"death_after_start\"] >= 0) & (dataset[\"death_after_end\"] <= 15)\n", + " ).astype(int)\n", + " dataset[\"label_mortality_1month\"] = (\n", + " (dataset[\"death_after_start\"] >= 0) & (dataset[\"death_after_end\"] <= 32)\n", + " ).astype(int)\n", + "\n", + " return dataset\n", + "\n", + "\n", + "# Process the dataset for mortality in two weeks or one month task\n", + "dataset_2048_mortality = process_mortality_dataset(dataset_2048.copy())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-13T16:15:47.326996100Z", + "start_time": "2024-03-13T16:15:16.094719300Z" + }, + "collapsed": false + }, + "outputs": [], + "source": [ + "def process_readmission_dataset(dataset: pd.DataFrame) -> pd.DataFrame:\n", + " \"\"\"Process the readmission dataset to extract required features.\n", + "\n", + " Args:\n", + " dataset (pd.DataFrame): The input dataset.\n", + "\n", + " Returns\n", + " -------\n", + " pd.DataFrame: The processed dataset.\n", + " \"\"\"\n", + " dataset[\"last_VS_index\"] = dataset[\"event_tokens_2048\"].transform(\n", + " lambda seq: get_last_occurence_index(list(seq), \"[VS]\"),\n", + " )\n", + " dataset[\"cutoff_readmission\"] = dataset[\"last_VS_index\"] - 1\n", + " dataset[\"label_readmission_1month\"] = dataset.apply(check_readmission_label, axis=1)\n", + "\n", + " dataset[\"num_visits\"] -= 1\n", + " dataset[\"token_length\"] = dataset[\"event_tokens_2048\"].apply(len)\n", + "\n", + " return dataset\n", + "\n", + "\n", + "# Process the dataset for hospital readmission in one month task\n", + "dataset_2048_readmission = filter_by_num_visit(\n", + " dataset_2048.copy(),\n", + " minimum_num_visits=2,\n", + ")\n", + "dataset_2048_readmission = process_readmission_dataset(dataset_2048_readmission)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def process_multi_dataset(datasets: Dict[str, pd.DataFrame]):\n", + " \"\"\"\n", + " Process the multi-task dataset by merging the original dataset with the other datasets.\n", + "\n", + " Args:\n", + " datasets (Dict): Dictionary mapping each task to its respective dataframe\n", + "\n", + " Returns\n", + " -------\n", + " pd.DataFrame: The processed multi-task dataset\n", + " \"\"\"\n", + " # Merging datasets on 'patient_id'\n", + " multi_dataset = datasets[\"original\"].merge(\n", + " datasets[\"condition\"][[\"patient_id\", \"all_conditions\"]],\n", + " on=\"patient_id\",\n", + " how=\"left\",\n", + " )\n", + " multi_dataset = multi_dataset.merge(\n", + " datasets[\"mortality\"][[\"patient_id\", \"label_mortality_1month\"]],\n", + " on=\"patient_id\",\n", + " how=\"left\",\n", + " )\n", + " multi_dataset = multi_dataset.merge(\n", + " datasets[\"readmission\"][\n", + " [\"patient_id\", \"cutoff_readmission\", \"label_readmission_1month\"]\n", + " ],\n", + " on=\"patient_id\",\n", + " how=\"left\",\n", + " )\n", + " multi_dataset = multi_dataset.merge(\n", + " datasets[\"los\"][[\"patient_id\", \"cutoff_los\", \"label_los_1week\"]],\n", + " on=\"patient_id\",\n", + " how=\"left\",\n", + " )\n", + "\n", + " # Selecting the required columns\n", + " multi_dataset = multi_dataset[\n", + " [\n", + " \"patient_id\",\n", + " \"num_visits\",\n", + " \"event_tokens_2048\",\n", + " \"type_tokens_2048\",\n", + " \"age_tokens_2048\",\n", + " \"time_tokens_2048\",\n", + " \"visit_tokens_2048\",\n", + " \"position_tokens_2048\",\n", + " \"elapsed_tokens_2048\",\n", + " \"cutoff_los\",\n", + " \"cutoff_readmission\",\n", + " \"all_conditions\",\n", + " \"label_mortality_1month\",\n", + " \"label_readmission_1month\",\n", + " \"label_los_1week\",\n", + " ]\n", + " ]\n", + "\n", + " # Transform conditions from a vector of numbers to binary classes\n", + " conditions_expanded = multi_dataset[\"all_conditions\"].apply(pd.Series)\n", + " conditions_expanded.columns = [f\"condition{i}\" for i in range(20)]\n", + " multi_dataset = multi_dataset.drop(\"all_conditions\", axis=1)\n", + " multi_dataset = pd.concat([multi_dataset, conditions_expanded], axis=1)\n", + "\n", + " # Standardize important column names\n", + " multi_dataset.rename(\n", + " columns={\n", + " \"cutoff_los\": \"cutoff_los_1week\",\n", + " \"cutoff_readmission\": \"cutoff_readmission_1month\",\n", + " },\n", + " inplace=True,\n", + " )\n", + " condition_columns = {f\"condition{i}\": f\"label_c{i}\" for i in range(20)}\n", + " multi_dataset.rename(columns=condition_columns, inplace=True)\n", + "\n", + " numerical_columns = [\n", + " \"cutoff_los_1week\",\n", + " \"cutoff_readmission_1month\",\n", + " \"label_mortality_1month\",\n", + " \"label_readmission_1month\",\n", + " \"label_los_1week\",\n", + " ] + [f\"label_c{i}\" for i in range(20)]\n", + "\n", + " # Fill NaN values and convert to integers\n", + " for column in numerical_columns:\n", + " multi_dataset[column] = multi_dataset[column].fillna(-1).astype(int)\n", + "\n", + " # Reset dataset index\n", + " multi_dataset.reset_index(drop=True, inplace=True)\n", + "\n", + " return multi_dataset\n", + "\n", + "\n", + "multi_dataset = process_multi_dataset(\n", + " datasets={\n", + " \"original\": dataset_2048,\n", + " \"mortality\": dataset_2048_mortality,\n", + " \"condition\": dataset_2048_condition,\n", + " \"readmission\": dataset_2048_readmission,\n", + " \"los\": dataset_2048_los,\n", + " },\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def stratified_train_test_split(\n", + " dataset: pd.DataFrame,\n", + " target: str,\n", + " test_size: float,\n", + " return_test: Optional[bool] = False,\n", + "):\n", + " \"\"\"\n", + " Split the given dataset into training and testing sets using iterative stratification on given multi-label target.\n", + " \"\"\"\n", + " # Convert all_conditions into a format suitable for multi-label stratification\n", + " Y = np.array(dataset[target].values.tolist())\n", + " X = dataset[\"patient_id\"].to_numpy().reshape(-1, 1)\n", + " is_single_label = type(dataset.iloc[0][target]) == np.int64\n", + "\n", + " # Perform stratified split\n", + " if is_single_label:\n", + " X_train, X_test, y_train, y_test = train_test_split(\n", + " X,\n", + " Y,\n", + " stratify=Y,\n", + " test_size=test_size,\n", + " random_state=SEED,\n", + " )\n", + "\n", + " else:\n", + " X_train, y_train, X_test, y_test = iterative_train_test_split(\n", + " X,\n", + " Y,\n", + " test_size=test_size,\n", + " )\n", + "\n", + " X_train = X_train.flatten().tolist()\n", + " X_test = X_test.flatten().tolist()\n", + "\n", + " if return_test:\n", + " return X_test\n", + " else:\n", + " return X_train, X_test\n", + "\n", + "\n", + "def sample_balanced_subset(dataset: pd.DataFrame, target: str, sample_size: int):\n", + " \"\"\"\n", + " Sample a subset of dataset with balanced target labels.\n", + " \"\"\"\n", + " # Sampling positive and negative patients\n", + " pos_patients = dataset[dataset[target] == True].sample(\n", + " n=sample_size // 2,\n", + " random_state=SEED,\n", + " )\n", + " neg_patients = dataset[dataset[target] == False].sample(\n", + " n=sample_size // 2,\n", + " random_state=SEED,\n", + " )\n", + "\n", + " # Combining and shuffling patient IDs\n", + " sample_patients = (\n", + " pos_patients[\"patient_id\"].tolist() + neg_patients[\"patient_id\"].tolist()\n", + " )\n", + " random.shuffle(sample_patients)\n", + "\n", + " return sample_patients\n", + "\n", + "\n", + "def get_pretrain_test_split(\n", + " dataset: pd.DataFrame,\n", + " stratify_target: Optional[str] = None,\n", + " test_size: float = 0.15,\n", + "):\n", + " \"\"\"Split dataset into pretrain and test set. Stratify on a given target column if needed.\"\"\"\n", + " if stratify_target:\n", + " pretrain_ids, test_ids = stratified_train_test_split(\n", + " dataset,\n", + " target=stratify_target,\n", + " test_size=test_size,\n", + " )\n", + "\n", + " else:\n", + " test_patients = dataset.sample(n=test_size, random_state=SEED)\n", + " test_ids = test_patients[\"patient_id\"].tolist()\n", + " pretrain_ids = dataset[~dataset[\"patient_id\"].isin(test_patients)][\n", + " \"patient_id\"\n", + " ].tolist()\n", + "\n", + " random.shuffle(pretrain_ids)\n", + "\n", + " return pretrain_ids, test_ids" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Split data\n", + "patient_ids_dict = {\n", + " \"pretrain\": [],\n", + " \"finetune\": {\"few_shot\": {}, \"kfold\": {}},\n", + " \"test\": [],\n", + "}\n", + "\n", + "# Get train-test split\n", + "# pretrain_ids, test_ids = get_pretrain_test_split(dataset_2048_readmission, stratify_target='label_readmission_1month', test_size=0.2)\n", + "# pretrain_ids, test_ids = get_pretrain_test_split(process_condition_dataset, stratify_target='all_conditions', test_size=0.15)\n", + "# patient_ids_dict['pretrain'] = pretrain_ids\n", + "# patient_ids_dict['test'] = test_ids\n", + "\n", + "# Load pretrain and test patient IDs\n", + "pid = pickle.load(open(\"patient_id_dict/dataset_2048_multi.pkl\", \"rb\"))\n", + "patient_ids_dict[\"pretrain\"] = pid[\"pretrain\"]\n", + "patient_ids_dict[\"test\"] = pid[\"test\"]\n", + "set(pid[\"pretrain\"] + pid[\"test\"]) == set(dataset_2048[\"patient_id\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class config:\n", + " task_splits = {\n", + " \"mortality\": {\n", + " \"dataset\": dataset_2048_mortality,\n", + " \"label_col\": \"label_mortality_1month\",\n", + " \"finetune_size\": [250, 500, 1000, 5000, 20000],\n", + " \"save_path\": \"patient_id_dict/dataset_2048_mortality.pkl\",\n", + " \"split_mode\": \"single_label_balanced\",\n", + " },\n", + " \"readmission\": {\n", + " \"dataset\": dataset_2048_readmission,\n", + " \"label_col\": \"label_readmission_1month\",\n", + " \"finetune_size\": [250, 1000, 5000, 20000, 60000],\n", + " \"save_path\": \"patient_id_dict/dataset_2048_readmission.pkl\",\n", + " \"split_mode\": \"single_label_stratified\",\n", + " },\n", + " \"length_of_stay\": {\n", + " \"dataset\": dataset_2048_los,\n", + " \"label_col\": \"label_los_1week\",\n", + " \"finetune_size\": [250, 1000, 5000, 20000, 50000],\n", + " \"save_path\": \"patient_id_dict/dataset_2048_los.pkl\",\n", + " \"split_mode\": \"single_label_balanced\",\n", + " },\n", + " \"condition\": {\n", + " \"dataset\": dataset_2048_condition,\n", + " \"label_col\": \"all_conditions\",\n", + " \"finetune_size\": [50000],\n", + " \"save_path\": \"patient_id_dict/dataset_2048_condition.pkl\",\n", + " \"split_mode\": \"multi_label_stratified\",\n", + " },\n", + " }\n", + "\n", + " all_tasks = list(task_splits.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-13T16:15:51.800996200Z", + "start_time": "2024-03-13T16:15:50.494996100Z" + }, + "collapsed": false + }, + "outputs": [], + "source": [ + "def get_finetune_split(\n", + " config: config,\n", + " patient_ids_dict: Dict[str, Any],\n", + ") -> Dict[str, Dict[str, List[str]]]:\n", + " \"\"\"\n", + " Splits the dataset into training and cross-finetuneation sets using k-fold cross-finetuneation\n", + " while ensuring balanced label distribution in each fold. Saves the resulting dictionary to disk.\n", + " \"\"\"\n", + " # Extract task-specific configuration\n", + " task_config = config.task_splits[task]\n", + " dataset = task_config[\"dataset\"]\n", + " label_col = task_config[\"label_col\"]\n", + " finetune_sizes = task_config[\"finetune_size\"]\n", + " save_path = task_config[\"save_path\"]\n", + " split_mode = task_config[\"split_mode\"]\n", + "\n", + " # Get pretrain dataset\n", + " pretrain_ids = patient_ids_dict[\"pretrain\"]\n", + " dataset = dataset[dataset[\"patient_id\"].isin(pretrain_ids)]\n", + "\n", + " # Few-shot finetune patient ids\n", + " for finetune_num in finetune_sizes:\n", + " if split_mode == \"single_label_balanced\":\n", + " finetune_ids = sample_balanced_subset(\n", + " dataset,\n", + " target=label_col,\n", + " sample_size=finetune_num,\n", + " )\n", + "\n", + " elif (\n", + " split_mode == \"single_label_stratified\"\n", + " or split_mode == \"multi_label_stratified\"\n", + " ):\n", + " finetune_ids = stratified_train_test_split(\n", + " dataset,\n", + " target=label_col,\n", + " test_size=finetune_num / len(dataset),\n", + " return_test=True,\n", + " )\n", + "\n", + " patient_ids_dict[\"finetune\"][\"few_shot\"][f\"{finetune_num}\"] = finetune_ids\n", + "\n", + " # Save the dictionary to disk\n", + " save_object_to_disk(patient_ids_dict, save_path)\n", + "\n", + " return patient_ids_dict\n", + "\n", + "\n", + "for task in config.all_tasks:\n", + " patient_ids_dict = get_finetune_split(\n", + " config=config,\n", + " patient_ids_dict=patient_ids_dict,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-13T14:14:10.181184300Z", + "start_time": "2024-03-13T14:13:39.154567400Z" + }, + "collapsed": false + }, + "outputs": [], + "source": [ + "dataset_2048_mortality.to_parquet(\n", + " \"patient_sequences/patient_sequences_2048_mortality.parquet\",\n", + ")\n", + "dataset_2048_readmission.to_parquet(\n", + " \"patient_sequences/patient_sequences_2048_readmission.parquet\",\n", + ")\n", + "dataset_2048_los.to_parquet(\"patient_sequences/patient_sequences_2048_los.parquet\")\n", + "dataset_2048_condition.to_parquet(\n", + " \"patient_sequences/patient_sequences_2048_condition.parquet\",\n", + ")\n", + "multi_dataset.to_parquet(\"patient_sequences/patient_sequences_2048_multi.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load data\n", + "# multi_dataset = pd.read_parquet('patient_sequences/patient_sequences_2048_multi.parquet')\n", + "# pid = pickle.load(open('patient_id_dict/dataset_2048_multi.pkl', 'rb'))\n", + "# multi_dataset = multi_dataset[multi_dataset['patient_id'].isin(pid['finetune']['few_shot']['all'])]\n", + "\n", + "# # Train Tokenizer\n", + "# tokenizer = ConceptTokenizer(data_dir='/h/afallah/odyssey/odyssey/data/vocab')\n", + "# tokenizer.fit_on_vocab()\n", + "\n", + "# # Load datasets\n", + "# tasks = ['mortality_1month', 'los_1week'] + [f'c{i}' for i in range(5)]\n", + "\n", + "# train_dataset = FinetuneMultiDataset(\n", + "# data=multi_dataset,\n", + "# tokenizer=tokenizer,\n", + "# tasks=tasks,\n", + "# balance_guide={'mortality_1month': 0.5, 'los_1week': 0.5},\n", + "# max_len=2048,\n", + "# )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# dataset_2048_condition = pd.read_parquet('patient_sequences/patient_sequences_2048_condition.parquet')\n", + "# pid = pickle.load(open('patient_id_dict/dataset_2048_condition.pkl', 'rb'))\n", + "# condition_finetune = dataset_2048_condition.loc[dataset_2048_condition['patient_id'].isin(pid['finetune']['few_shot']['50000'])]\n", + "# condition_finetune" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# freq = np.array(condition_finetune['all_conditions'].tolist()).sum(axis=0)\n", + "# weights = np.clip(0, 50, sum(freq) / freq)\n", + "# np.max(np.sqrt(freq)) / np.sqrt(freq)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# sorted(patient_ids_dict['pretrain']) == sorted(pickle.load(open('new_data/patient_id_dict/sample_pretrain_test_patient_ids_with_conditions.pkl', 'rb'))['pretrain'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# merged_df = pd.merge(dataset_2048_mortality, dataset_2048_readmission, how='outer', on='patient_id')\n", + "# final_merged_df = pd.merge(merged_df, dataset_2048_condition, how='outer', on='patient_id')\n", + "# final_merged_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Performing stratified k-fold split\n", + "# skf = StratifiedKFold(n_splits=num_splits, shuffle=True, random_state=SEED)\n", + "\n", + "# for i, (train_index, cv_index) in enumerate(skf.split(dataset, dataset[label_col])):\n", + "\n", + "# dataset_cv = dataset.iloc[cv_index]\n", + "# dataset_finetune = dataset.iloc[train_index]\n", + "\n", + "# # Separate positive and negative labeled patients\n", + "# pos_patients = dataset_cv[dataset_cv[label_col] == True]['patient_id'].tolist()\n", + "# neg_patients = dataset_cv[dataset_cv[label_col] == False]['patient_id'].tolist()\n", + "\n", + "# # Calculate the number of positive and negative patients needed for balanced CV set\n", + "# num_pos_needed = cv_size // 2\n", + "# num_neg_needed = cv_size // 2\n", + "\n", + "# # Select positive and negative patients for CV set ensuring balanced distribution\n", + "# cv_patients = pos_patients[:num_pos_needed] + neg_patients[:num_neg_needed]\n", + "# remaining_finetune_patients = pos_patients[num_pos_needed:] + neg_patients[num_neg_needed:]\n", + "\n", + "# # Extract patient IDs for training set\n", + "# finetune_patients = dataset_finetune['patient_id'].tolist()\n", + "# finetune_patients += remaining_finetune_patients\n", + "\n", + "# # Shuffle each list of patients\n", + "# random.shuffle(cv_patients)\n", + "# random.shuffle(finetune_patients)\n", + "\n", + "# patient_ids_dict['finetune']['kfold'][f'group{i+1}'] = {'finetune': finetune_patients, 'cv': cv_patients}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Assuming dataset.event_tokens is your DataFrame column\n", + "# dataset.event_tokens.transform(len).plot(kind='hist', bins=100)\n", + "# plt.xlim(1000, 8000) # Limit x-axis to 5000\n", + "# plt.ylim(0, 6000)\n", + "# plt.xlabel('Length of Event Tokens')\n", + "# plt.ylabel('Frequency')\n", + "# plt.title('Histogram of Event Tokens Length')\n", + "# plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# len(patient_ids_dict['group3']['cv'])\n", + "\n", + "# dataset_2048.loc[dataset_2048['patient_id'].isin(patient_ids_dict['group1']['cv'])]['label_mortality_1month']\n", + "\n", + "# s = set()\n", + "# for i in range(1, 6):\n", + "# s = s.union(set(patient_ids_dict[f'group{i}']['cv']))\n", + "#\n", + "# len(s)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "##### DEAD ZONE | DO NOT ENTER #####\n", + "\n", + "# patient_ids = pickle.load(open(join(\"/h/afallah/odyssey/odyssey/data/bigbird_data\", 'dataset_2048_mortality_1month.pkl'), 'rb'))\n", + "# patient_ids['finetune']['few_shot'].keys()\n", + "\n", + "# patient_ids2 = pickle.load(open(join(\"/h/afallah/odyssey/odyssey/data/bigbird_data\", 'dataset_2048_mortality_2weeks.pkl'), 'rb'))['pretrain']\n", + "#\n", + "# patient_ids1.sort()\n", + "# patient_ids2.sort()\n", + "#\n", + "# patient_ids1 == patient_ids2\n", + "# # dataset_2048.loc[dataset_2048['patient_id'].isin(patient_ids['pretrain'])]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# dataset_2048_readmission = dataset_2048.loc[dataset_2048['num_visits'] > 1]\n", + "# dataset_2048_readmission.reset_index(drop=True, inplace=True)\n", + "#\n", + "# dataset_2048_readmission['last_VS_index'] = dataset_2048_readmission['event_tokens_2048'].transform(lambda seq: get_last_occurence_index(list(seq), '[VS]'))\n", + "#\n", + "# dataset_2048_readmission['label_readmission_1month'] = dataset_2048_readmission.apply(\n", + "# lambda row: row['event_tokens_2048'][row['last_VS_index'] - 1] in ('[W_0]', '[W_1]', '[W_2]', '[W_3]', '[M_1]'), axis=1\n", + "# )\n", + "# dataset_2048_readmission['event_tokens_2048'] = dataset_2048_readmission.apply(\n", + "# lambda row: row['event_tokens_2048'][:row['last_VS_index'] - 1], axis=1\n", + "# )\n", + "# dataset_2048_readmission.drop(['deceased', 'death_after_start', 'death_after_end', 'length'], axis=1, inplace=True)\n", + "# dataset_2048_readmission['num_visits'] -= 1\n", + "# dataset_2048_readmission['token_length'] = dataset_2048_readmission['event_tokens_2048'].apply(len)\n", + "# dataset_2048_readmission = dataset_2048_readmission.apply(lambda row: truncate_and_pad(row), axis=1)\n", + "# dataset_2048_readmission['event_tokens_2048'] = dataset_2048_readmission['event_tokens_2048'].transform(\n", + "# lambda token_list: ' '.join(token_list)\n", + "# )\n", + "#\n", + "# dataset_2048_readmission" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/odyssey/evals/AttentionVisualization.ipynb b/odyssey/evals/AttentionVisualization.ipynb new file mode 100644 index 0000000..dc0a91c --- /dev/null +++ b/odyssey/evals/AttentionVisualization.ipynb @@ -0,0 +1,3412 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 13, + "id": "initial_id", + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-15T15:39:47.301592Z", + "start_time": "2024-03-15T15:39:47.293763Z" + }, + "collapsed": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\nFile: AttentionVisualization.ipynb\\n---------------------------------\\nVisualize the attention layers of transformer models for interpretability.\\n'" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "File: AttentionVisualization.ipynb\n", + "---------------------------------\n", + "Visualize the attention layers of transformer models for interpretability.\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "e24876a0c6020df2", + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-15T15:49:16.880004Z", + "start_time": "2024-03-15T15:49:16.866092Z" + }, + "collapsed": false + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import numpy as np\n", + "import plotly.graph_objects as go\n", + "import torch\n", + "from bertviz import head_view, model_view\n", + "from bertviz.neuron_view import show\n", + "from torch.utils.data import DataLoader, Subset\n", + "from transformers import utils\n", + "\n", + "\n", + "utils.logging.set_verbosity_error() # Suppress standard warnings\n", + "\n", + "\n", + "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", + "os.chdir(ROOT)\n", + "\n", + "from odyssey.data.dataset import FinetuneDataset\n", + "from odyssey.data.tokenizer import ConceptTokenizer\n", + "from odyssey.models.prediction import load_finetuned_model, predict_patient_outcomes\n", + "from odyssey.models.utils import (\n", + " load_finetune_data,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "dac09785d1fdc0cc", + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-15T15:39:54.875569Z", + "start_time": "2024-03-15T15:39:54.852027Z" + }, + "collapsed": false + }, + "outputs": [], + "source": [ + "class args:\n", + " \"\"\"Save the configuration arguments.\"\"\"\n", + "\n", + " model_path = (\n", + " \"checkpoints/bigbird_finetune/mortality_1month_20000_patients/best-v3.ckpt\"\n", + " )\n", + " vocab_dir = \"data/vocab\"\n", + " data_dir = \"data/bigbird_data\"\n", + " sequence_file = (\n", + " \"old_data/patient_sequences/patient_sequences_2048_mortality.parquet\"\n", + " )\n", + " id_file = \"old_data/patient_id_dict/dataset_2048_mortality_1month.pkl\"\n", + " valid_scheme = \"few_shot\"\n", + " num_finetune_patients = \"20000\"\n", + " label_name = \"label_mortality_1month\"\n", + "\n", + " max_len = 2048\n", + " batch_size = 1\n", + " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "6b5e21258ced06d2", + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-15T15:39:54.921944Z", + "start_time": "2024-03-15T15:39:54.877586Z" + }, + "collapsed": false + }, + "outputs": [], + "source": [ + "tokenizer = ConceptTokenizer(data_dir=args.vocab_dir)\n", + "tokenizer.fit_on_vocab()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "8bf63dc4242a163e", + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-15T15:40:00.629410Z", + "start_time": "2024-03-15T15:39:54.929550Z" + }, + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "BigBirdFinetune(\n", + " (model): BigBirdForSequenceClassification(\n", + " (bert): BigBirdModel(\n", + " (embeddings): BigBirdEmbeddingsForCEHR(\n", + " (word_embeddings): Embedding(20592, 768, padding_idx=0)\n", + " (position_embeddings): Embedding(2048, 768)\n", + " (token_type_embeddings): Embedding(9, 768)\n", + " (time_embeddings): TimeEmbeddingLayer()\n", + " (age_embeddings): TimeEmbeddingLayer()\n", + " (visit_segment_embeddings): VisitEmbedding(\n", + " (embedding): Embedding(3, 768)\n", + " )\n", + " (scale_back_concat_layer): Linear(in_features=832, out_features=768, bias=True)\n", + " (tanh): Tanh()\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (encoder): BigBirdEncoder(\n", + " (layer): ModuleList(\n", + " (0-5): 6 x BigBirdLayer(\n", + " (attention): BigBirdAttention(\n", + " (self): BigBirdBlockSparseAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " )\n", + " (output): BigBirdSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BigBirdIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " (intermediate_act_fn): NewGELUActivation()\n", + " )\n", + " (output): BigBirdOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " )\n", + " (pooler): Linear(in_features=768, out_features=768, bias=True)\n", + " (activation): Tanh()\n", + " )\n", + " (classifier): BigBirdClassificationHead(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " (out_proj): Linear(in_features=768, out_features=2, bias=True)\n", + " )\n", + " )\n", + " (pretrained_model): BigBirdPretrain(\n", + " (embeddings): BigBirdEmbeddingsForCEHR(\n", + " (word_embeddings): Embedding(20592, 768, padding_idx=0)\n", + " (position_embeddings): Embedding(2048, 768)\n", + " (token_type_embeddings): Embedding(9, 768)\n", + " (time_embeddings): TimeEmbeddingLayer()\n", + " (age_embeddings): TimeEmbeddingLayer()\n", + " (visit_segment_embeddings): VisitEmbedding(\n", + " (embedding): Embedding(3, 768)\n", + " )\n", + " (scale_back_concat_layer): Linear(in_features=832, out_features=768, bias=True)\n", + " (tanh): Tanh()\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (model): BigBirdForMaskedLM(\n", + " (bert): BigBirdModel(\n", + " (embeddings): BigBirdEmbeddingsForCEHR(\n", + " (word_embeddings): Embedding(20592, 768, padding_idx=0)\n", + " (position_embeddings): Embedding(2048, 768)\n", + " (token_type_embeddings): Embedding(9, 768)\n", + " (time_embeddings): TimeEmbeddingLayer()\n", + " (age_embeddings): TimeEmbeddingLayer()\n", + " (visit_segment_embeddings): VisitEmbedding(\n", + " (embedding): Embedding(3, 768)\n", + " )\n", + " (scale_back_concat_layer): Linear(in_features=832, out_features=768, bias=True)\n", + " (tanh): Tanh()\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (encoder): BigBirdEncoder(\n", + " (layer): ModuleList(\n", + " (0-5): 6 x BigBirdLayer(\n", + " (attention): BigBirdAttention(\n", + " (self): BigBirdBlockSparseAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " )\n", + " (output): BigBirdSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BigBirdIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " (intermediate_act_fn): NewGELUActivation()\n", + " )\n", + " (output): BigBirdOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " )\n", + " (pooler): Linear(in_features=768, out_features=768, bias=True)\n", + " (activation): Tanh()\n", + " )\n", + " (cls): BigBirdOnlyMLMHead(\n", + " (predictions): BigBirdLMPredictionHead(\n", + " (transform): BigBirdPredictionHeadTransform(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (transform_act_fn): NewGELUActivation()\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " )\n", + " (decoder): Linear(in_features=768, out_features=20592, bias=True)\n", + " )\n", + " )\n", + " )\n", + " )\n", + ")" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model = load_finetuned_model(args.model_path, tokenizer)\n", + "model" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "45bb295710f64bc0", + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-15T15:52:23.262096Z", + "start_time": "2024-03-15T15:52:23.252429Z" + }, + "collapsed": false + }, + "outputs": [], + "source": [ + "fine_tune, fine_test = load_finetune_data(\n", + " args.data_dir,\n", + " args.sequence_file,\n", + " args.id_file,\n", + " args.valid_scheme,\n", + " args.num_finetune_patients,\n", + ")\n", + "\n", + "fine_tune.rename(columns={args.label_name: \"label\"}, inplace=True)\n", + "fine_test.rename(columns={args.label_name: \"label\"}, inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "6c2f9f1f", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset = FinetuneDataset(\n", + " data=fine_test,\n", + " tokenizer=tokenizer,\n", + " max_len=args.max_len,\n", + ")\n", + "\n", + "test_loader = DataLoader(\n", + " Subset(test_dataset, [89, 90]), # 85 and 88 are small\n", + " batch_size=args.batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "ead6f3658dda5274", + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-15T15:52:24.058853Z", + "start_time": "2024-03-15T15:52:24.027868Z" + }, + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'concept_ids': tensor([[ 5, 3, 18065, ..., 0, 0, 0]]),\n", + " 'type_ids': tensor([[1, 2, 6, ..., 0, 0, 0]]),\n", + " 'ages': tensor([[ 0, 67, 67, ..., 0, 0, 0]]),\n", + " 'time_stamps': tensor([[ 0, 5773, 5773, ..., 0, 0, 0]]),\n", + " 'visit_orders': tensor([[0, 1, 1, ..., 0, 0, 0]]),\n", + " 'visit_segments': tensor([[0, 2, 2, ..., 0, 0, 0]]),\n", + " 'labels': tensor([0]),\n", + " 'attention_mask': tensor([[1, 1, 1, ..., 0, 0, 0]])}" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "patient = next(iter(test_loader))\n", + "patient" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "b025c033efc9baab", + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-15T15:52:25.797704Z", + "start_time": "2024-03-15T15:52:25.476580Z" + }, + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "SequenceClassifierOutput(loss=tensor(0.1007, device='cuda:0', grad_fn=), logits=tensor([[ 1.1001, -1.1447]], device='cuda:0', grad_fn=), hidden_states=(tensor([[[-0.6737, -0.4033, -1.0631, ..., -0.0571, -0.5688, -2.8107],\n", + " [-0.2583, 1.4081, 0.0247, ..., 0.9723, -0.1994, -0.6298],\n", + " [-0.3349, -0.7147, -0.2382, ..., 0.6685, -2.0221, 0.3603],\n", + " ...,\n", + " [-1.1487, -0.3301, -1.2087, ..., -1.4240, 0.6589, -3.1762],\n", + " [-0.5165, 0.4454, -1.0153, ..., -2.3917, -0.0144, -3.3358],\n", + " [-0.1893, -0.1348, 0.2863, ..., -2.4897, 0.7176, -2.4138]]],\n", + " device='cuda:0', grad_fn=), tensor([[[-9.1614e-01, 6.7605e-01, 6.2693e-03, ..., -3.1620e-01,\n", + " -4.6282e-01, -2.0648e+00],\n", + " [-6.0621e-01, 8.6801e-01, 2.6213e-01, ..., 3.4474e-01,\n", + " 1.3363e-01, -7.1667e-01],\n", + " [-7.6982e-01, 2.3106e-03, -5.9440e-01, ..., -4.7166e-01,\n", + " -1.8369e+00, 1.3552e-01],\n", + " ...,\n", + " [-8.6433e-01, 7.3538e-01, -3.3437e-01, ..., -1.2050e+00,\n", + " 1.3254e+00, -2.4991e+00],\n", + " [-7.3595e-01, 1.2425e+00, 5.3519e-03, ..., -1.8429e+00,\n", + " 9.7859e-01, -2.8092e+00],\n", + " [-5.8273e-01, 8.9714e-01, 8.5189e-01, ..., -1.7958e+00,\n", + " 1.6784e+00, -2.0120e+00]]], device='cuda:0',\n", + " grad_fn=), tensor([[[-0.5963, 1.0668, 0.6960, ..., -0.5670, -1.8491, -0.7775],\n", + " [-0.8661, 0.9432, 1.3912, ..., -0.1141, -0.0171, -0.0536],\n", + " [-0.4142, 0.1496, 0.0763, ..., -1.3690, -2.2675, 0.8359],\n", + " ...,\n", + " [-0.5217, 1.3100, 0.3550, ..., -0.8422, 0.7881, -2.2435],\n", + " [-0.6279, 1.6811, 0.6428, ..., -1.1621, 0.4885, -2.3759],\n", + " [-0.5811, 1.5146, 1.1337, ..., -1.0819, 1.0710, -2.0434]]],\n", + " device='cuda:0', grad_fn=), tensor([[[-0.4564, 0.8573, 0.4777, ..., -1.0131, -1.2389, -0.7232],\n", + " [-0.7503, 0.8020, 2.2105, ..., 0.3817, -0.5328, -1.1600],\n", + " [-0.2785, 0.3193, 0.7508, ..., -0.8207, -2.1549, -0.0479],\n", + " ...,\n", + " [-0.5749, 0.9360, 0.3542, ..., -1.2162, 0.5259, -1.9892],\n", + " [-0.8316, 1.2188, 0.4210, ..., -1.5618, 0.3291, -2.1654],\n", + " [-0.7359, 1.2650, 0.9190, ..., -1.4486, 0.7956, -1.8708]]],\n", + " device='cuda:0', grad_fn=), tensor([[[-0.6359, 1.1296, 0.3645, ..., -1.0155, -1.4793, -0.0216],\n", + " [-0.2409, 0.9339, 1.1703, ..., 1.1403, -1.0845, -0.8589],\n", + " [-0.2531, 0.2159, 0.2502, ..., -0.1171, -2.6449, -0.1982],\n", + " ...,\n", + " [-0.4492, 0.9411, 0.7911, ..., -0.9499, 0.3742, -1.3986],\n", + " [-0.7147, 1.2283, 0.7656, ..., -1.0737, 0.1806, -1.6471],\n", + " [-0.6436, 1.0595, 1.1752, ..., -1.1340, 0.4983, -1.3118]]],\n", + " device='cuda:0', grad_fn=), tensor([[[-0.0605, 0.4757, 1.2138, ..., -1.2107, -0.7777, 0.4597],\n", + " [-0.0597, 0.6793, 0.6923, ..., 0.5595, -0.6465, -0.4138],\n", + " [-0.2956, 0.0119, 0.0849, ..., -0.4024, -1.7548, 0.2857],\n", + " ...,\n", + " [-0.0730, 1.2001, 0.8387, ..., -1.5022, 0.3392, -0.9774],\n", + " [-0.2355, 1.2640, 0.8785, ..., -1.6132, 0.2851, -1.2461],\n", + " [-0.2264, 1.1501, 1.2094, ..., -1.7025, 0.5599, -0.9631]]],\n", + " device='cuda:0', grad_fn=), tensor([[[-0.4529, 0.7745, 0.9677, ..., -0.7731, 0.3017, 0.6295],\n", + " [-0.2527, 0.7169, 0.3931, ..., 0.3904, -0.2026, 0.2191],\n", + " [-0.7347, 0.0648, 0.1225, ..., -0.4706, -0.9406, 0.7601],\n", + " ...,\n", + " [-0.1587, 1.8421, 0.7164, ..., -1.1803, 0.3151, -1.0949],\n", + " [-0.4175, 1.9637, 0.7340, ..., -1.2465, 0.3226, -1.4348],\n", + " [-0.4056, 1.7997, 1.0610, ..., -1.3635, 0.5409, -1.1068]]],\n", + " device='cuda:0', grad_fn=)), attentions=(tensor([[[[9.2084e-01, 3.4662e-05, 1.6030e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.0152e-04, 1.0245e-03, 5.0193e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.1340e-05, 3.4795e-04, 1.1249e-01, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [3.7594e-01, 6.5887e-04, 2.1206e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.4787e-01, 1.3629e-03, 7.0707e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.9477e-01, 1.2923e-03, 4.1574e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[2.2569e-05, 3.1873e-03, 1.4330e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.0942e-04, 7.4366e-03, 1.0470e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [4.2275e-07, 8.8712e-03, 2.0909e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [3.3820e-04, 5.4235e-03, 3.9131e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.1625e-05, 6.0716e-03, 9.2111e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.0256e-05, 6.8238e-03, 1.3608e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[2.1552e-05, 3.2614e-03, 5.1249e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.0069e-02, 1.7049e-03, 7.4966e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.6985e-04, 7.9826e-05, 1.0739e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [7.2891e-05, 2.2444e-03, 8.0701e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.6524e-04, 2.5353e-03, 1.1610e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [8.4759e-05, 3.7951e-03, 9.3934e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " ...,\n", + "\n", + " [[9.5957e-07, 1.8916e-05, 1.7706e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.4017e-03, 6.3007e-04, 8.2404e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.7625e-05, 2.4848e-05, 1.9090e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [1.4380e-05, 1.2709e-04, 2.4316e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.0306e-05, 1.8831e-04, 2.8286e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.8297e-05, 1.9117e-04, 2.4082e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[1.7399e-03, 1.2051e-03, 9.5375e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.7488e-03, 2.7973e-03, 2.2183e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.6722e-03, 7.9718e-04, 2.1421e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [1.6540e-03, 2.6361e-03, 1.2919e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.7157e-03, 2.6202e-03, 1.2555e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.6167e-03, 1.7538e-03, 1.1150e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[1.5929e-03, 3.2053e-03, 2.2594e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [6.3881e-03, 1.7186e-03, 4.7067e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [8.7379e-03, 3.9178e-03, 5.7301e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [3.5827e-03, 2.9827e-03, 2.1182e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.3835e-03, 3.9488e-03, 2.3409e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.8265e-03, 3.3581e-03, 2.8842e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=), tensor([[[[3.4325e-03, 3.6487e-03, 2.1270e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [8.6434e-03, 2.1695e-02, 6.8258e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [7.8946e-03, 5.1628e-02, 9.1872e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [6.2476e-03, 3.5812e-03, 3.8917e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.5847e-03, 5.8187e-03, 4.7901e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.7512e-03, 7.3790e-03, 5.0303e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[3.9891e-05, 4.4143e-06, 7.2098e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [9.8795e-03, 2.2040e-02, 1.3027e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [7.2756e-02, 1.8536e-02, 1.8427e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [1.0230e-04, 1.0431e-04, 3.6400e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.0183e-04, 1.0241e-04, 4.4847e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.2058e-04, 2.5562e-04, 6.5510e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[1.7265e-03, 6.2247e-06, 3.0965e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [6.5288e-05, 1.7324e-02, 6.0294e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [6.6073e-04, 1.7897e-03, 2.7479e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [4.5983e-03, 2.8453e-04, 4.3261e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.3822e-03, 2.4970e-04, 7.7571e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.9758e-03, 2.7208e-04, 6.9291e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " ...,\n", + "\n", + " [[3.2307e-04, 1.6154e-07, 1.1893e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.5087e-04, 7.7591e-02, 4.4232e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [9.9669e-04, 7.1647e-03, 1.2542e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [1.6157e-03, 1.9115e-04, 8.7496e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.5545e-03, 6.9437e-04, 2.2060e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.5639e-03, 4.7817e-04, 1.0144e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[9.0331e-02, 2.0968e-04, 1.5666e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.2050e-05, 4.9400e-03, 2.9039e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.8098e-04, 7.9459e-03, 1.0302e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [8.0900e-02, 5.0021e-03, 3.6893e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [8.7062e-02, 5.0122e-03, 5.1272e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [9.2155e-02, 4.1310e-03, 4.8125e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[1.2098e-04, 4.5407e-08, 1.1781e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [4.4916e-05, 1.8071e-03, 2.9574e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.9490e-03, 3.8689e-03, 1.6880e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [4.2944e-03, 1.2684e-05, 3.9500e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.5202e-03, 9.5840e-06, 3.9704e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.4998e-03, 1.0348e-05, 4.0015e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=), tensor([[[[9.0947e-03, 1.7104e-04, 7.2082e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.0905e-03, 8.2947e-03, 4.7806e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.2774e-02, 4.1130e-03, 1.0427e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [7.4360e-03, 2.5052e-03, 3.5916e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [9.5451e-03, 2.0411e-03, 4.1614e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.0480e-02, 1.5095e-03, 3.9783e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[1.0953e-04, 4.8485e-07, 1.1093e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [8.7053e-06, 7.0000e-03, 1.3357e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.2624e-03, 4.3490e-03, 8.2277e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [8.3735e-03, 9.6488e-05, 1.6069e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [4.6108e-03, 6.0472e-05, 1.0010e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [8.4534e-03, 9.8078e-05, 1.4517e-02, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[3.1851e-03, 3.6355e-09, 4.5110e-06, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.1155e-05, 1.3445e-03, 3.1936e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [7.9026e-03, 3.4348e-04, 3.4485e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [4.4469e-03, 1.2559e-05, 5.1731e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [5.1810e-03, 6.3667e-06, 5.4800e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [4.0399e-03, 6.1432e-06, 4.1856e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " ...,\n", + "\n", + " [[1.9304e-02, 3.4070e-05, 1.5798e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.7734e-05, 1.5453e-02, 6.9360e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [4.5957e-04, 3.6940e-03, 5.8250e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [1.6903e-03, 1.8801e-03, 4.4015e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.2805e-03, 8.8522e-04, 5.1555e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.4006e-03, 1.2400e-03, 5.7507e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[3.1703e-04, 4.1161e-05, 7.9415e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [4.8420e-04, 7.7026e-03, 7.9656e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.5139e-03, 2.4419e-03, 2.6276e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [2.7707e-03, 6.9963e-04, 1.6709e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.0923e-03, 7.9106e-04, 2.5133e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.4068e-03, 2.6501e-04, 1.1517e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[6.0330e-05, 1.8324e-06, 1.0670e-06, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [7.0295e-04, 6.8225e-03, 4.1048e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.6344e-03, 2.1405e-03, 1.9475e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [1.0302e-03, 2.2379e-05, 8.3888e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.0699e-03, 3.1538e-05, 9.3927e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.1386e-03, 4.4208e-05, 1.2655e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=), tensor([[[[1.0968e-03, 1.1158e-05, 2.8874e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [6.5882e-04, 1.9889e-02, 2.9087e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.0075e-03, 2.4018e-03, 5.6122e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [1.2534e-03, 5.0390e-05, 2.8556e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [7.8193e-04, 5.5810e-05, 3.2226e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.8834e-03, 1.9344e-04, 8.2757e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[2.1752e-04, 1.0848e-03, 5.2472e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [6.3445e-05, 3.1222e-02, 7.3892e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [5.9519e-04, 2.3304e-02, 4.0696e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [7.4697e-04, 2.2354e-04, 5.6568e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [4.7230e-04, 4.3862e-04, 9.0070e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [4.0728e-04, 3.7051e-04, 8.7622e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[6.4030e-04, 1.0463e-03, 1.2372e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.3409e-04, 2.6108e-02, 1.0480e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.3558e-03, 4.1270e-03, 3.1541e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [8.7033e-04, 1.3328e-04, 4.4750e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.1584e-03, 5.6131e-04, 1.9963e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.3833e-03, 1.7700e-04, 1.2080e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " ...,\n", + "\n", + " [[3.0260e-04, 1.0307e-03, 5.4334e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [5.5736e-04, 4.4732e-03, 3.1108e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.0399e-03, 2.5841e-03, 3.4921e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [1.2584e-03, 3.3595e-04, 1.9949e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.4056e-03, 7.4162e-04, 3.0422e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.4776e-03, 3.4124e-04, 2.3692e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[3.3945e-05, 2.8038e-06, 7.7244e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.1178e-03, 5.9071e-01, 3.8243e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.1613e-03, 9.0446e-03, 1.3656e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [6.1137e-06, 8.0224e-07, 2.9675e-06, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.1845e-05, 4.3952e-06, 1.5539e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.0935e-05, 7.2282e-06, 1.6119e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[7.7009e-03, 9.1021e-04, 4.4472e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.0289e-04, 1.2443e-02, 8.9628e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [7.7645e-04, 5.7296e-03, 3.5397e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [2.8769e-04, 7.9107e-06, 6.3012e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [7.2969e-04, 4.7156e-05, 2.6878e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [6.0934e-04, 3.7384e-05, 2.2858e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=), tensor([[[[6.7381e-04, 1.1823e-02, 7.4050e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [6.4863e-04, 4.0161e-02, 3.6514e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [4.6004e-03, 1.9051e-02, 5.0602e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [3.5047e-03, 1.9051e-03, 1.5601e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.6917e-03, 3.4615e-03, 1.6968e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [4.7805e-03, 3.3552e-03, 2.2134e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[3.1542e-04, 1.2458e-05, 3.5918e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.5336e-03, 2.6060e-03, 9.9650e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.3570e-03, 4.4454e-04, 2.1211e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [3.8644e-05, 1.1354e-06, 2.5000e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.0465e-05, 9.6188e-07, 2.3662e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [5.5490e-05, 1.7402e-06, 4.0543e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[4.4482e-03, 4.3810e-07, 2.4999e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.1627e-04, 5.7318e-02, 2.5052e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [4.0123e-03, 1.0834e-03, 3.6955e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [1.2926e-03, 2.7957e-07, 6.0144e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.3540e-03, 2.3200e-07, 7.3312e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.2753e-03, 5.4792e-07, 9.2235e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " ...,\n", + "\n", + " [[1.8663e-05, 6.3300e-07, 1.4228e-06, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.2953e-03, 1.2528e-02, 3.5547e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.9098e-03, 1.1120e-03, 1.3087e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [1.1750e-04, 1.4822e-05, 2.3723e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [6.7225e-05, 8.3810e-06, 1.6070e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.6516e-04, 2.6572e-05, 4.2221e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[4.5335e-05, 4.0905e-04, 2.3216e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.7303e-04, 5.1438e-02, 3.8481e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [6.3572e-04, 1.6108e-02, 4.7815e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [1.6582e-03, 9.6089e-04, 3.1044e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.1655e-03, 1.2231e-03, 3.3471e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.5900e-03, 1.6183e-03, 5.6334e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[1.3592e-03, 9.1240e-05, 1.9648e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [4.9554e-04, 3.4671e-02, 1.2307e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.1695e-03, 3.3091e-03, 4.3312e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [4.5615e-04, 4.8380e-07, 7.2619e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.2036e-03, 3.6028e-06, 3.0705e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.2138e-03, 3.5186e-06, 2.6554e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=), tensor([[[[4.0090e-03, 6.7263e-04, 7.5136e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [3.9121e-04, 1.2857e-02, 3.9387e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.5724e-03, 4.5150e-03, 2.8897e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [4.5598e-03, 1.4921e-04, 2.2929e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [5.6446e-03, 1.8594e-04, 2.9366e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [5.2847e-03, 2.3700e-04, 3.6638e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[1.9915e-02, 4.0394e-05, 5.0094e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [5.3356e-04, 3.0291e-03, 1.2489e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.3452e-03, 8.8680e-04, 1.3824e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [1.0090e-01, 1.4063e-05, 1.0046e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.1156e-01, 1.0029e-05, 8.7870e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [9.5568e-02, 1.6524e-05, 1.0813e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[1.8186e-02, 5.9815e-06, 5.0354e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.1144e-04, 3.3203e-03, 1.0907e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.1076e-03, 7.5293e-04, 1.7252e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [1.2483e-01, 2.2352e-07, 2.1154e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.1819e-01, 1.1773e-07, 1.4315e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.0731e-01, 2.1415e-07, 1.9841e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " ...,\n", + "\n", + " [[8.9774e-03, 2.6941e-04, 3.7395e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.7763e-04, 4.7386e-03, 2.2903e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.2463e-03, 2.2257e-03, 1.9620e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [1.1848e-02, 8.8093e-05, 1.7620e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.3670e-02, 8.8600e-05, 1.7867e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.2773e-02, 1.0601e-04, 2.1455e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[2.7395e-02, 2.0985e-04, 9.8349e-04, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.3505e-04, 3.5468e-03, 1.4784e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.9124e-03, 1.9014e-03, 2.2491e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [2.7175e-02, 1.4688e-05, 9.3223e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.9626e-02, 9.9576e-06, 9.4240e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.6579e-02, 1.2380e-05, 9.0343e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]],\n", + "\n", + " [[2.5388e-03, 4.6041e-05, 9.6215e-05, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.3881e-05, 1.0036e-02, 3.3659e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [9.2542e-04, 6.3507e-03, 5.8799e-03, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " ...,\n", + " [1.5240e-04, 1.0778e-08, 7.3738e-08, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [1.5682e-04, 1.5447e-08, 1.0267e-07, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00],\n", + " [2.6646e-04, 3.5608e-08, 2.2134e-07, ..., 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=)))" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output = predict_patient_outcomes(patient, model)\n", + "output" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "e1aa8309b2650820", + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-15T15:52:33.578704Z", + "start_time": "2024-03-15T15:52:33.571290Z" + }, + "collapsed": false + }, + "outputs": [], + "source": [ + "tokens = tokenizer.decode(patient[\"concept_ids\"].squeeze(0).cpu().numpy()).split(\" \")\n", + "truncate_at = patient[\"attention_mask\"].sum().numpy()\n", + "attention_matrix = output[\"attentions\"]\n", + "last_attention_matrix = attention_matrix[-1].detach()\n", + "# batch_size x num_heads x max_len x max_len x num_layers" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "e9203f59", + "metadata": {}, + "outputs": [], + "source": [ + "truncated_attention_matrix = []\n", + "\n", + "for i in range(len(attention_matrix)):\n", + " truncated_attention_matrix.append(\n", + " attention_matrix[i][:, :, :truncate_at, :truncate_at],\n", + " )\n", + "\n", + "truncated_attention_matrix = tuple(truncated_attention_matrix)\n", + "truncated_tokens = tokens[:truncate_at]" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "7fb27b941602401d91542211134fc71a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Token [CLS] with Token 50893_0: Attention Value 0.154\n", + "Token [CLS] with Token 51251_1: Attention Value 0.026\n", + "Token [CLS] with Token 51279_3: Attention Value 0.018\n", + "Token [CLS] with Token 50983_0: Attention Value 0.014\n", + "Token [CLS] with Token 51257_0: Attention Value 0.013\n", + "Token [CLS] with Token 52069_0: Attention Value 0.012\n", + "Token [CLS] with Token 50902_0: Attention Value 0.011\n", + "Token [CLS] with Token 51277_1: Attention Value 0.010\n", + "Token [CLS] with Token 50878_4: Attention Value 0.009\n", + "Token [CLS] with Token 00641607825: Attention Value 0.009\n", + "Token [CLS] with Token 00641607825: Attention Value 0.009\n", + "Token [CLS] with Token 51133_2: Attention Value 0.009\n", + "Token [CLS] with Token [CLS]: Attention Value 0.008\n", + "Token [CLS] with Token 51255_1: Attention Value 0.008\n", + "Token [CLS] with Token 61958040101: Attention Value 0.008\n" + ] + }, + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "colorscale": [ + [ + 0, + "rgb(255,255,217)" + ], + [ + 0.125, + "rgb(237,248,177)" + ], + [ + 0.25, + "rgb(199,233,180)" + ], + [ + 0.375, + "rgb(127,205,187)" + ], + [ + 0.5, + "rgb(65,182,196)" + ], + [ + 0.625, + "rgb(29,145,192)" + ], + [ + 0.75, + "rgb(34,94,168)" + ], + [ + 0.875, + "rgb(37,52,148)" + ], + [ + 1, + "rgb(8,29,88)" + ] + ], + "hoverinfo": "text", + "hoverongaps": false, + "text": [ + [ + "Token [CLS] with Token [CLS]:Attention Value 0.008", + "Token [CLS] with Token [VS]:Attention Value 0.000", + "Token [CLS] with Token 00121048910:Attention Value 0.001", + "Token [CLS] with Token 00338004904:Attention Value 0.003", + "Token [CLS] with Token 00121048910:Attention Value 0.001", + "Token [CLS] with Token 00006494300:Attention Value 0.000", + "Token [CLS] with Token 00904272561:Attention Value 0.001", + "Token [CLS] with Token 61958040101:Attention Value 0.004", + "Token [CLS] with Token 00487950101:Attention Value 0.002", + "Token [CLS] with Token 00904198261:Attention Value 0.001", + "Token [CLS] with Token 00173071920:Attention Value 0.007", + "Token [CLS] with Token 00121048910:Attention Value 0.001", + "Token [CLS] with Token 60505011300:Attention Value 0.001", + "Token [CLS] with Token 00338004904:Attention Value 0.004", + "Token [CLS] with Token 61958040101:Attention Value 0.008", + "Token [CLS] with Token 00597008717:Attention Value 0.004", + "Token [CLS] with Token 00338004904:Attention Value 0.004", + "Token [CLS] with Token 42292000120:Attention Value 0.002", + "Token [CLS] with Token 00904639161:Attention Value 0.001", + "Token [CLS] with Token 00338004904:Attention Value 0.005", + "Token [CLS] with Token 00472030680:Attention Value 0.003", + "Token [CLS] with Token 00143989701:Attention Value 0.001", + "Token [CLS] with Token 00641607825:Attention Value 0.009", + "Token [CLS] with Token 00338004904:Attention Value 0.004", + "Token [CLS] with Token 51079096620:Attention Value 0.001", + "Token [CLS] with Token 00409128331:Attention Value 0.002", + "Token [CLS] with Token 63323031461:Attention Value 0.000", + "Token [CLS] with Token 00009513503:Attention Value 0.001", + "Token [CLS] with Token 00056051030:Attention Value 0.004", + "Token [CLS] with Token 66758016013:Attention Value 0.004", + "Token [CLS] with Token 00338004904:Attention Value 0.003", + "Token [CLS] with Token 00338004904:Attention Value 0.003", + "Token [CLS] with Token 00071015892:Attention Value 0.004", + "Token [CLS] with Token 00338004904:Attention Value 0.006", + "Token [CLS] with Token 11523726808:Attention Value 0.001", + "Token [CLS] with Token 00904198861:Attention Value 0.000", + "Token [CLS] with Token 00071015892:Attention Value 0.003", + "Token [CLS] with Token 00487020101:Attention Value 0.002", + "Token [CLS] with Token 63739035810:Attention Value 0.001", + "Token [CLS] with Token 00904635361:Attention Value 0.003", + "Token [CLS] with Token 76439034310:Attention Value 0.005", + "Token [CLS] with Token 57896042101:Attention Value 0.003", + "Token [CLS] with Token 00056051030:Attention Value 0.005", + "Token [CLS] with Token 42292000120:Attention Value 0.002", + "Token [CLS] with Token 51079054720:Attention Value 0.004", + "Token [CLS] with Token 00904224461:Attention Value 0.000", + "Token [CLS] with Token 51079098320:Attention Value 0.002", + "Token [CLS] with Token 61958060101:Attention Value 0.003", + "Token [CLS] with Token 51079098320:Attention Value 0.001", + "Token [CLS] with Token 00641607825:Attention Value 0.009", + "Token [CLS] with Token 49281041688:Attention Value 0.005", + "Token [CLS] with Token 00904516561:Attention Value 0.002", + "Token [CLS] with Token 00121048910:Attention Value 0.001", + "Token [CLS] with Token 00173068224:Attention Value 0.005", + "Token [CLS] with Token 00121048910:Attention Value 0.001", + "Token [CLS] with Token 63323026201:Attention Value 0.000", + "Token [CLS] with Token 57896045208:Attention Value 0.004", + "Token [CLS] with Token 63323031461:Attention Value 0.001", + "Token [CLS] with Token 00487020101:Attention Value 0.002", + "Token [CLS] with Token 52074_3:Attention Value 0.000", + "Token [CLS] with Token 52073_2:Attention Value 0.000", + "Token [CLS] with Token 52069_2:Attention Value 0.000", + "Token [CLS] with Token 51301_1:Attention Value 0.000", + "Token [CLS] with Token 51279_3:Attention Value 0.001", + "Token [CLS] with Token 51277_1:Attention Value 0.001", + "Token [CLS] with Token 51265_2:Attention Value 0.000", + "Token [CLS] with Token 51256_1:Attention Value 0.001", + "Token [CLS] with Token 51254_4:Attention Value 0.001", + "Token [CLS] with Token 51250_2:Attention Value 0.001", + "Token [CLS] with Token 51248_3:Attention Value 0.001", + "Token [CLS] with Token 51244_3:Attention Value 0.000", + "Token [CLS] with Token 51222_4:Attention Value 0.001", + "Token [CLS] with Token 51221_3:Attention Value 0.002", + "Token [CLS] with Token 51200_2:Attention Value 0.001", + "Token [CLS] with Token 51146_2:Attention Value 0.001", + "Token [CLS] with Token 51133_3:Attention Value 0.001", + "Token [CLS] with Token 52075_2:Attention Value 0.001", + "Token [CLS] with Token 52074_2:Attention Value 0.001", + "Token [CLS] with Token 52073_2:Attention Value 0.001", + "Token [CLS] with Token 52069_1:Attention Value 0.001", + "Token [CLS] with Token 51301_1:Attention Value 0.001", + "Token [CLS] with Token 51279_3:Attention Value 0.002", + "Token [CLS] with Token 51277_1:Attention Value 0.001", + "Token [CLS] with Token 51265_1:Attention Value 0.000", + "Token [CLS] with Token 51256_3:Attention Value 0.001", + "Token [CLS] with Token 51254_3:Attention Value 0.001", + "Token [CLS] with Token 51250_3:Attention Value 0.001", + "Token [CLS] with Token 51248_3:Attention Value 0.001", + "Token [CLS] with Token 51244_1:Attention Value 0.000", + "Token [CLS] with Token 51222_3:Attention Value 0.001", + "Token [CLS] with Token 51221_3:Attention Value 0.001", + "Token [CLS] with Token 51200_2:Attention Value 0.001", + "Token [CLS] with Token 51146_1:Attention Value 0.001", + "Token [CLS] with Token 51133_1:Attention Value 0.001", + "Token [CLS] with Token 51006_0:Attention Value 0.001", + "Token [CLS] with Token 50983_0:Attention Value 0.002", + "Token [CLS] with Token 50971_0:Attention Value 0.001", + "Token [CLS] with Token 50970_1:Attention Value 0.001", + "Token [CLS] with Token 50960_2:Attention Value 0.001", + "Token [CLS] with Token 50931_2:Attention Value 0.001", + "Token [CLS] with Token 50912_3:Attention Value 0.002", + "Token [CLS] with Token 50902_1:Attention Value 0.001", + "Token [CLS] with Token 50893_0:Attention Value 0.003", + "Token [CLS] with Token 50885_1:Attention Value 0.001", + "Token [CLS] with Token 50882_0:Attention Value 0.001", + "Token [CLS] with Token 50878_3:Attention Value 0.001", + "Token [CLS] with Token 50868_4:Attention Value 0.001", + "Token [CLS] with Token 50863_1:Attention Value 0.001", + "Token [CLS] with Token 50861_3:Attention Value 0.001", + "Token [CLS] with Token 51516_1:Attention Value 0.001", + "Token [CLS] with Token 51498_4:Attention Value 0.001", + "Token [CLS] with Token 51493_2:Attention Value 0.000", + "Token [CLS] with Token 51492_1:Attention Value 0.001", + "Token [CLS] with Token 51491_2:Attention Value 0.001", + "Token [CLS] with Token 51009_0:Attention Value 0.001", + "Token [CLS] with Token 51006_1:Attention Value 0.002", + "Token [CLS] with Token 50983_0:Attention Value 0.004", + "Token [CLS] with Token 50971_0:Attention Value 0.002", + "Token [CLS] with Token 50970_1:Attention Value 0.001", + "Token [CLS] with Token 50960_0:Attention Value 0.001", + "Token [CLS] with Token 50931_2:Attention Value 0.001", + "Token [CLS] with Token 50912_3:Attention Value 0.003", + "Token [CLS] with Token 50902_2:Attention Value 0.002", + "Token [CLS] with Token 50893_0:Attention Value 0.003", + "Token [CLS] with Token 50882_0:Attention Value 0.002", + "Token [CLS] with Token 50868_4:Attention Value 0.001", + "Token [CLS] with Token 51301_2:Attention Value 0.002", + "Token [CLS] with Token 51279_3:Attention Value 0.002", + "Token [CLS] with Token 51277_1:Attention Value 0.002", + "Token [CLS] with Token 51265_1:Attention Value 0.001", + "Token [CLS] with Token 51250_3:Attention Value 0.002", + "Token [CLS] with Token 51248_3:Attention Value 0.001", + "Token [CLS] with Token 51222_3:Attention Value 0.002", + "Token [CLS] with Token 51221_3:Attention Value 0.001", + "Token [CLS] with Token 50813_3:Attention Value 0.002", + "Token [CLS] with Token 51006_1:Attention Value 0.002", + "Token [CLS] with Token 50983_1:Attention Value 0.002", + "Token [CLS] with Token 50971_1:Attention Value 0.002", + "Token [CLS] with Token 50970_3:Attention Value 0.001", + "Token [CLS] with Token 50960_0:Attention Value 0.001", + "Token [CLS] with Token 50931_2:Attention Value 0.001", + "Token [CLS] with Token 50912_3:Attention Value 0.003", + "Token [CLS] with Token 50911_0:Attention Value 0.001", + "Token [CLS] with Token 50902_1:Attention Value 0.001", + "Token [CLS] with Token 50893_0:Attention Value 0.002", + "Token [CLS] with Token 50882_0:Attention Value 0.002", + "Token [CLS] with Token 50868_4:Attention Value 0.002", + "Token [CLS] with Token 51301_3:Attention Value 0.002", + "Token [CLS] with Token 51279_4:Attention Value 0.002", + "Token [CLS] with Token 51277_1:Attention Value 0.002", + "Token [CLS] with Token 51265_2:Attention Value 0.001", + "Token [CLS] with Token 51250_3:Attention Value 0.002", + "Token [CLS] with Token 51248_4:Attention Value 0.001", + "Token [CLS] with Token 51222_4:Attention Value 0.002", + "Token [CLS] with Token 51221_4:Attention Value 0.001", + "Token [CLS] with Token 51516_3:Attention Value 0.001", + "Token [CLS] with Token 51498_4:Attention Value 0.001", + "Token [CLS] with Token 51493_2:Attention Value 0.001", + "Token [CLS] with Token 51491_2:Attention Value 0.001", + "Token [CLS] with Token 51482_2:Attention Value 0.001", + "Token [CLS] with Token 50813_4:Attention Value 0.003", + "Token [CLS] with Token 51301_4:Attention Value 0.002", + "Token [CLS] with Token 51279_4:Attention Value 0.002", + "Token [CLS] with Token 51277_1:Attention Value 0.002", + "Token [CLS] with Token 51265_2:Attention Value 0.001", + "Token [CLS] with Token 51250_3:Attention Value 0.002", + "Token [CLS] with Token 51248_3:Attention Value 0.001", + "Token [CLS] with Token 51222_4:Attention Value 0.002", + "Token [CLS] with Token 51221_4:Attention Value 0.001", + "Token [CLS] with Token 51006_1:Attention Value 0.002", + "Token [CLS] with Token 50983_1:Attention Value 0.003", + "Token [CLS] with Token 50971_1:Attention Value 0.003", + "Token [CLS] with Token 50970_1:Attention Value 0.002", + "Token [CLS] with Token 50960_0:Attention Value 0.001", + "Token [CLS] with Token 50931_4:Attention Value 0.001", + "Token [CLS] with Token 50912_3:Attention Value 0.005", + "Token [CLS] with Token 50902_2:Attention Value 0.003", + "Token [CLS] with Token 50893_2:Attention Value 0.002", + "Token [CLS] with Token 50882_0:Attention Value 0.002", + "Token [CLS] with Token 50868_4:Attention Value 0.002", + "Token [CLS] with Token 52135_4:Attention Value 0.003", + "Token [CLS] with Token 52135_2:Attention Value 0.002", + "Token [CLS] with Token 52075_4:Attention Value 0.002", + "Token [CLS] with Token 52074_4:Attention Value 0.002", + "Token [CLS] with Token 52073_3:Attention Value 0.001", + "Token [CLS] with Token 52069_3:Attention Value 0.002", + "Token [CLS] with Token 51301_4:Attention Value 0.002", + "Token [CLS] with Token 51279_4:Attention Value 0.003", + "Token [CLS] with Token 51277_0:Attention Value 0.004", + "Token [CLS] with Token 51265_4:Attention Value 0.001", + "Token [CLS] with Token 51256_3:Attention Value 0.003", + "Token [CLS] with Token 51254_3:Attention Value 0.002", + "Token [CLS] with Token 51250_3:Attention Value 0.001", + "Token [CLS] with Token 51248_3:Attention Value 0.001", + "Token [CLS] with Token 51244_1:Attention Value 0.001", + "Token [CLS] with Token 51222_4:Attention Value 0.001", + "Token [CLS] with Token 51221_4:Attention Value 0.001", + "Token [CLS] with Token 51200_2:Attention Value 0.001", + "Token [CLS] with Token 51146_2:Attention Value 0.001", + "Token [CLS] with Token 51133_2:Attention Value 0.001", + "Token [CLS] with Token 51006_1:Attention Value 0.002", + "Token [CLS] with Token 50983_1:Attention Value 0.002", + "Token [CLS] with Token 50971_3:Attention Value 0.002", + "Token [CLS] with Token 50970_3:Attention Value 0.000", + "Token [CLS] with Token 50960_3:Attention Value 0.001", + "Token [CLS] with Token 50931_2:Attention Value 0.001", + "Token [CLS] with Token 50912_2:Attention Value 0.003", + "Token [CLS] with Token 50902_2:Attention Value 0.002", + "Token [CLS] with Token 50893_3:Attention Value 0.001", + "Token [CLS] with Token 50882_0:Attention Value 0.001", + "Token [CLS] with Token 50868_4:Attention Value 0.001", + "Token [CLS] with Token 51301_2:Attention Value 0.001", + "Token [CLS] with Token 51279_4:Attention Value 0.001", + "Token [CLS] with Token 51277_0:Attention Value 0.001", + "Token [CLS] with Token 51265_2:Attention Value 0.000", + "Token [CLS] with Token 51250_3:Attention Value 0.001", + "Token [CLS] with Token 51248_3:Attention Value 0.001", + "Token [CLS] with Token 51222_4:Attention Value 0.001", + "Token [CLS] with Token 51221_4:Attention Value 0.001", + "Token [CLS] with Token 51006_1:Attention Value 0.001", + "Token [CLS] with Token 50983_1:Attention Value 0.001", + "Token [CLS] with Token 50971_1:Attention Value 0.001", + "Token [CLS] with Token 50970_2:Attention Value 0.000", + "Token [CLS] with Token 50960_3:Attention Value 0.001", + "Token [CLS] with Token 50931_2:Attention Value 0.000", + "Token [CLS] with Token 50912_2:Attention Value 0.001", + "Token [CLS] with Token 50902_2:Attention Value 0.001", + "Token [CLS] with Token 50893_1:Attention Value 0.001", + "Token [CLS] with Token 50882_0:Attention Value 0.001", + "Token [CLS] with Token 50868_4:Attention Value 0.001", + "Token [CLS] with Token 51301_2:Attention Value 0.001", + "Token [CLS] with Token 51279_4:Attention Value 0.001", + "Token [CLS] with Token 51277_0:Attention Value 0.001", + "Token [CLS] with Token 51265_2:Attention Value 0.001", + "Token [CLS] with Token 51221_4:Attention Value 0.001", + "Token [CLS] with Token 51222_4:Attention Value 0.001", + "Token [CLS] with Token 51248_3:Attention Value 0.001", + "Token [CLS] with Token 51250_3:Attention Value 0.001", + "Token [CLS] with Token 51265_2:Attention Value 0.001", + "Token [CLS] with Token 51277_0:Attention Value 0.001", + "Token [CLS] with Token 51279_4:Attention Value 0.001", + "Token [CLS] with Token 51301_2:Attention Value 0.001", + "Token [CLS] with Token 51250_3:Attention Value 0.001", + "Token [CLS] with Token 50861_1:Attention Value 0.001", + "Token [CLS] with Token 50863_1:Attention Value 0.001", + "Token [CLS] with Token 50868_4:Attention Value 0.001", + "Token [CLS] with Token 50878_0:Attention Value 0.001", + "Token [CLS] with Token 50882_0:Attention Value 0.001", + "Token [CLS] with Token 50885_1:Attention Value 0.001", + "Token [CLS] with Token 50902_3:Attention Value 0.003", + "Token [CLS] with Token 50912_2:Attention Value 0.002", + "Token [CLS] with Token 50931_1:Attention Value 0.001", + "Token [CLS] with Token 50971_2:Attention Value 0.002", + "Token [CLS] with Token 50983_3:Attention Value 0.002", + "Token [CLS] with Token 51006_1:Attention Value 0.002", + "Token [CLS] with Token 50868_4:Attention Value 0.002", + "Token [CLS] with Token 50882_0:Attention Value 0.003", + "Token [CLS] with Token 50893_1:Attention Value 0.003", + "Token [CLS] with Token 50902_2:Attention Value 0.005", + "Token [CLS] with Token 50912_2:Attention Value 0.004", + "Token [CLS] with Token 50931_2:Attention Value 0.002", + "Token [CLS] with Token 50960_2:Attention Value 0.002", + "Token [CLS] with Token 50970_2:Attention Value 0.002", + "Token [CLS] with Token 50971_1:Attention Value 0.003", + "Token [CLS] with Token 50983_1:Attention Value 0.004", + "Token [CLS] with Token 51006_1:Attention Value 0.004", + "Token [CLS] with Token 51009_2:Attention Value 0.002", + "Token [CLS] with Token 51221_4:Attention Value 0.003", + "Token [CLS] with Token 51222_4:Attention Value 0.002", + "Token [CLS] with Token 51248_3:Attention Value 0.001", + "Token [CLS] with Token 51476_0:Attention Value 0.002", + "Token [CLS] with Token 52135_4:Attention Value 0.003", + "Token [CLS] with Token 52135_4:Attention Value 0.003", + "Token [CLS] with Token 51492_1:Attention Value 0.002", + "Token [CLS] with Token 51476_0:Attention Value 0.002", + "Token [CLS] with Token 51275_0:Attention Value 0.002", + "Token [CLS] with Token 51274_0:Attention Value 0.002", + "Token [CLS] with Token 51237_1:Attention Value 0.001", + "Token [CLS] with Token 51006_0:Attention Value 0.003", + "Token [CLS] with Token 50983_0:Attention Value 0.005", + "Token [CLS] with Token 50971_1:Attention Value 0.003", + "Token [CLS] with Token 50970_0:Attention Value 0.002", + "Token [CLS] with Token 50960_4:Attention Value 0.002", + "Token [CLS] with Token 50931_2:Attention Value 0.001", + "Token [CLS] with Token 50912_3:Attention Value 0.005", + "Token [CLS] with Token 50902_0:Attention Value 0.003", + "Token [CLS] with Token 50893_2:Attention Value 0.002", + "Token [CLS] with Token 50885_1:Attention Value 0.001", + "Token [CLS] with Token 50882_0:Attention Value 0.004", + "Token [CLS] with Token 50878_4:Attention Value 0.002", + "Token [CLS] with Token 50868_4:Attention Value 0.002", + "Token [CLS] with Token 50863_2:Attention Value 0.001", + "Token [CLS] with Token 50861_4:Attention Value 0.002", + "Token [CLS] with Token 52075_3:Attention Value 0.002", + "Token [CLS] with Token 52074_3:Attention Value 0.002", + "Token [CLS] with Token 52073_2:Attention Value 0.001", + "Token [CLS] with Token 52069_3:Attention Value 0.002", + "Token [CLS] with Token 51301_4:Attention Value 0.002", + "Token [CLS] with Token 51279_3:Attention Value 0.002", + "Token [CLS] with Token 51277_1:Attention Value 0.002", + "Token [CLS] with Token 51265_3:Attention Value 0.001", + "Token [CLS] with Token 51256_3:Attention Value 0.003", + "Token [CLS] with Token 51254_2:Attention Value 0.001", + "Token [CLS] with Token 51250_3:Attention Value 0.001", + "Token [CLS] with Token 51248_3:Attention Value 0.001", + "Token [CLS] with Token 51244_1:Attention Value 0.001", + "Token [CLS] with Token 51222_4:Attention Value 0.002", + "Token [CLS] with Token 51221_4:Attention Value 0.001", + "Token [CLS] with Token 51200_2:Attention Value 0.002", + "Token [CLS] with Token 51146_2:Attention Value 0.002", + "Token [CLS] with Token 51133_3:Attention Value 0.001", + "Token [CLS] with Token 51275_0:Attention Value 0.002", + "Token [CLS] with Token 51274_1:Attention Value 0.003", + "Token [CLS] with Token 51237_2:Attention Value 0.002", + "Token [CLS] with Token 51006_0:Attention Value 0.003", + "Token [CLS] with Token 50983_0:Attention Value 0.004", + "Token [CLS] with Token 50976_2:Attention Value 0.002", + "Token [CLS] with Token 50971_0:Attention Value 0.002", + "Token [CLS] with Token 50970_2:Attention Value 0.001", + "Token [CLS] with Token 50960_2:Attention Value 0.001", + "Token [CLS] with Token 50951_1:Attention Value 0.001", + "Token [CLS] with Token 50950_2:Attention Value 0.002", + "Token [CLS] with Token 50949_1:Attention Value 0.004", + "Token [CLS] with Token 50931_3:Attention Value 0.001", + "Token [CLS] with Token 50930_3:Attention Value 0.002", + "Token [CLS] with Token 50912_3:Attention Value 0.003", + "Token [CLS] with Token 50902_0:Attention Value 0.002", + "Token [CLS] with Token 50893_0:Attention Value 0.004", + "Token [CLS] with Token 50885_1:Attention Value 0.001", + "Token [CLS] with Token 50882_0:Attention Value 0.002", + "Token [CLS] with Token 50878_4:Attention Value 0.002", + "Token [CLS] with Token 50868_4:Attention Value 0.002", + "Token [CLS] with Token 50863_2:Attention Value 0.001", + "Token [CLS] with Token 50862_3:Attention Value 0.001", + "Token [CLS] with Token 50861_3:Attention Value 0.001", + "Token [CLS] with Token 50853_1:Attention Value 0.006", + "Token [CLS] with Token 52075_2:Attention Value 0.004", + "Token [CLS] with Token 52074_3:Attention Value 0.002", + "Token [CLS] with Token 52073_3:Attention Value 0.001", + "Token [CLS] with Token 52069_0:Attention Value 0.012", + "Token [CLS] with Token 51301_1:Attention Value 0.006", + "Token [CLS] with Token 51279_3:Attention Value 0.018", + "Token [CLS] with Token 51277_1:Attention Value 0.010", + "Token [CLS] with Token 51265_2:Attention Value 0.001", + "Token [CLS] with Token 51257_0:Attention Value 0.013", + "Token [CLS] with Token 51256_1:Attention Value 0.003", + "Token [CLS] with Token 51255_1:Attention Value 0.008", + "Token [CLS] with Token 51254_4:Attention Value 0.003", + "Token [CLS] with Token 51251_1:Attention Value 0.026", + "Token [CLS] with Token 51250_2:Attention Value 0.003", + "Token [CLS] with Token 51248_3:Attention Value 0.001", + "Token [CLS] with Token 51244_2:Attention Value 0.002", + "Token [CLS] with Token 51222_4:Attention Value 0.003", + "Token [CLS] with Token 51221_3:Attention Value 0.002", + "Token [CLS] with Token 51200_3:Attention Value 0.003", + "Token [CLS] with Token 51146_0:Attention Value 0.003", + "Token [CLS] with Token 51144_2:Attention Value 0.002", + "Token [CLS] with Token 51143_0:Attention Value 0.002", + "Token [CLS] with Token 51133_2:Attention Value 0.009", + "Token [CLS] with Token 51006_0:Attention Value 0.007", + "Token [CLS] with Token 50983_0:Attention Value 0.014", + "Token [CLS] with Token 50971_0:Attention Value 0.006", + "Token [CLS] with Token 50970_1:Attention Value 0.002", + "Token [CLS] with Token 50960_2:Attention Value 0.004", + "Token [CLS] with Token 50950_2:Attention Value 0.005", + "Token [CLS] with Token 50931_2:Attention Value 0.003", + "Token [CLS] with Token 50912_3:Attention Value 0.008", + "Token [CLS] with Token 50902_0:Attention Value 0.011", + "Token [CLS] with Token 50893_0:Attention Value 0.154", + "Token [CLS] with Token 50885_1:Attention Value 0.002", + "Token [CLS] with Token 50882_0:Attention Value 0.004", + "Token [CLS] with Token 50878_4:Attention Value 0.009", + "Token [CLS] with Token 50868_4:Attention Value 0.007", + "Token [CLS] with Token 50863_2:Attention Value 0.005", + "Token [CLS] with Token 50861_3:Attention Value 0.002", + "Token [CLS] with Token 52075_2:Attention Value 0.004", + "Token [CLS] with Token [VE]:Attention Value 0.006", + "Token [CLS] with Token [REG]:Attention Value 0.003" + ] + ], + "type": "heatmap", + "x": [ + "[CLS]", + "[VS]", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "43", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "51", + "52", + "53", + "54", + "55", + "56", + "57", + "58", + "59", + "60", + "61", + "62", + "63", + "64", + "65", + "66", + "67", + "68", + "69", + "70", + "71", + "72", + "73", + "74", + "75", + "76", + "77", + "78", + "79", + "80", + "81", + "82", + "83", + "84", + "85", + "86", + "87", + "88", + "89", + "90", + "91", + "92", + "93", + "94", + "95", + "96", + "97", + "98", + "99", + "100", + "101", + "102", + "103", + "104", + "105", + "106", + "107", + "108", + "109", + "110", + "111", + "112", + "113", + "114", + "115", + "116", + "117", + "118", + "119", + "120", + "121", + "122", + "123", + "124", + "125", + "126", + "127", + "128", + "129", + "130", + "131", + "132", + "133", + "134", + "135", + "136", + "137", + "138", + "139", + "140", + "141", + "142", + "143", + "144", + "145", + "146", + "147", + "148", + "149", + "150", + "151", + "152", + "153", + "154", + "155", + "156", + "157", + "158", + "159", + "160", + "161", + "162", + "163", + "164", + "165", + "166", + "167", + "168", + "169", + "170", + "171", + "172", + "173", + "174", + "175", + "176", + "177", + "178", + "179", + "180", + "181", + "182", + "183", + "184", + "185", + "186", + "187", + "188", + "189", + "190", + "191", + "192", + "193", + "194", + "195", + "196", + "197", + "198", + "199", + "200", + "201", + "202", + "203", + "204", + "205", + "206", + "207", + "208", + "209", + "210", + "211", + "212", + "213", + "214", + "215", + "216", + "217", + "218", + "219", + "220", + "221", + "222", + "223", + "224", + "225", + "226", + "227", + "228", + "229", + "230", + "231", + "232", + "233", + "234", + "235", + "236", + "237", + "238", + "239", + "240", + "241", + "242", + "243", + "244", + "245", + "246", + "247", + "248", + "249", + "250", + "251", + "252", + "253", + "254", + "255", + "256", + "257", + "258", + "259", + "260", + "261", + "262", + "263", + "264", + "265", + "266", + "267", + "268", + "269", + "270", + "271", + "272", + "273", + "274", + "275", + "276", + "277", + "278", + "279", + "280", + "281", + "282", + "283", + "284", + "285", + "286", + "287", + "288", + "289", + "290", + "291", + "292", + "293", + "294", + "295", + "296", + "297", + "298", + "299", + "300", + "301", + "302", + "303", + "304", + "305", + "306", + "307", + "308", + "309", + "310", + "311", + "312", + "313", + "314", + "315", + "316", + "317", + "318", + "319", + "320", + "321", + "322", + "323", + "324", + "325", + "326", + "327", + "328", + "329", + "330", + "331", + "332", + "333", + "334", + "335", + "336", + "337", + "338", + "339", + "340", + "341", + "342", + "343", + "344", + "345", + "346", + "347", + "348", + "349", + "350", + "351", + "352", + "353", + "354", + "355", + "356", + "357", + "358", + "359", + "360", + "361", + "362", + "363", + "364", + "365", + "366", + "367", + "368", + "369", + "370", + "371", + "372", + "373", + "374", + "375", + "[VE]", + "[REG]" + ], + "y": [ + "[CLS]" + ], + "z": [ + [ + 0.008451767265796661, + 0.00047753742546774447, + 0.001319802482612431, + 0.002615319797769189, + 0.0005728917894884944, + 0.0004854540165979415, + 0.0007460298365913332, + 0.004369142930954695, + 0.0021512836683541536, + 0.0010651483898982406, + 0.006857313681393862, + 0.0009411114733666182, + 0.001146065187640488, + 0.003502822481095791, + 0.008377399295568466, + 0.00369816436432302, + 0.0036342681851238017, + 0.001818674267269671, + 0.0011760754277929664, + 0.005098414141684771, + 0.003414525417611003, + 0.0005102005670778453, + 0.009316006675362589, + 0.0036000509280711412, + 0.0010369595838710666, + 0.0019885266665369272, + 0.0004615136131178588, + 0.0008582020527683198, + 0.004428019281476736, + 0.0036287240218371153, + 0.003203341970220208, + 0.003022358985617757, + 0.004367131274193525, + 0.005930684972554445, + 0.0006912555545568466, + 0.0003829135384876281, + 0.002646421082317829, + 0.001659494242630899, + 0.0008906491566449404, + 0.0034986960235983133, + 0.004725936334580183, + 0.003188258036971092, + 0.004636996425688267, + 0.0017536324448883531, + 0.003574026981368661, + 0.0004167505830992013, + 0.001605302095413208, + 0.0034259206149727106, + 0.0010352873941883445, + 0.008988478220999241, + 0.004783918149769306, + 0.0020494854543358088, + 0.0008181784651242197, + 0.005045545753091574, + 0.0008356395992450416, + 0.0003578341274987906, + 0.004442144185304642, + 0.0007386466022580862, + 0.0015157737070694566, + 0.0003436319238971919, + 0.00029586421442218125, + 0.0004052982258144766, + 0.0004121463280171156, + 0.0006444680620916188, + 0.0006272507016547024, + 0.0003953258565161377, + 0.0005427289288491011, + 0.000674175564199686, + 0.0005934844375588, + 0.0005067584570497274, + 0.00048668019007891417, + 0.0006641843356192112, + 0.001736009493470192, + 0.0006450659711845219, + 0.0006109003443270922, + 0.0014823578530922532, + 0.0007514643366448581, + 0.0010008058743551371, + 0.0005770500865764916, + 0.0009993098210543394, + 0.0013827277580276132, + 0.0017192121595144272, + 0.0011489095631986856, + 0.0004682957660406828, + 0.0008927697781473398, + 0.000683685124386102, + 0.0006812589126639068, + 0.0005206714267842472, + 0.0004224051663186401, + 0.0008084981818683445, + 0.001108424854464829, + 0.0009732283069752156, + 0.0007592075853608549, + 0.0005552212824113667, + 0.00127204111777246, + 0.002062402432784438, + 0.00112050655297935, + 0.0008525322191417217, + 0.0008413223549723625, + 0.000753440021071583, + 0.0019989104475826025, + 0.0010396834695711732, + 0.00325021636672318, + 0.0006715958588756621, + 0.0012460827128961682, + 0.000999152078293264, + 0.0009655823814682662, + 0.0005084593431092799, + 0.0005497524398379028, + 0.0006103277555666864, + 0.001034797285683453, + 0.0004598199157044291, + 0.0008794588502496481, + 0.000696350762154907, + 0.001486221794039011, + 0.0017987970495596528, + 0.003589141182601452, + 0.001990983495488763, + 0.0010251256171613932, + 0.001364864525385201, + 0.0008277010056190193, + 0.0025306493043899536, + 0.0021720887161791325, + 0.003414546838030219, + 0.0019456190057098863, + 0.0013443160569295287, + 0.001797817298211157, + 0.0022308507468551397, + 0.0018264207756146789, + 0.0006515602581202984, + 0.0015126297948881984, + 0.0009584605577401816, + 0.0015974263660609722, + 0.0012428780319169164, + 0.0018666594987735152, + 0.0018526790663599968, + 0.001926859957166016, + 0.001790812355466187, + 0.0006914885598234832, + 0.0009147145319730044, + 0.0006848545162938535, + 0.003264268860220909, + 0.001066222321242094, + 0.0014806516701355577, + 0.0019578184001147747, + 0.001972360536456108, + 0.001908112782984972, + 0.0017941169207915664, + 0.002000534441322088, + 0.00232343259267509, + 0.0006864610477350652, + 0.0015883835731074214, + 0.001452704076655209, + 0.0015260959044098854, + 0.001217573182657361, + 0.0007630472537130117, + 0.0009102648473344744, + 0.0010583768598735332, + 0.0010495075257495046, + 0.0013496936298906803, + 0.002790031023323536, + 0.0016460781916975975, + 0.001800714642740786, + 0.001588907209224999, + 0.0006608786643482745, + 0.0015428924234583974, + 0.00130733463447541, + 0.0019498993642628193, + 0.0011935298098251224, + 0.002081812592223286, + 0.0027750423178076744, + 0.002682143822312355, + 0.0017559939296916127, + 0.0014839059440419078, + 0.0011782203800976276, + 0.005421904847025871, + 0.0033844260033220053, + 0.0019421830074861648, + 0.002427612664178014, + 0.0021077985875308514, + 0.002597493352368474, + 0.0019147455459460616, + 0.0022990747820585966, + 0.00196700356900692, + 0.0013677203096449375, + 0.0017343414947390556, + 0.0022338901180773973, + 0.002618071623146534, + 0.003806268097832799, + 0.0009631214197725058, + 0.0029177824035286903, + 0.0015531842363998294, + 0.00118330551777035, + 0.0009476374834775924, + 0.0007008272805251181, + 0.0012339174281805754, + 0.0011241122847422955, + 0.0009876210242509842, + 0.0008716397569514811, + 0.001008612453006208, + 0.0015318739460781217, + 0.0016837789444252849, + 0.0015550237149000168, + 0.00042292437865398824, + 0.0012427183100953698, + 0.0006267700809985399, + 0.0025690405163913965, + 0.002414754591882229, + 0.0008665476925671101, + 0.0007410055841319263, + 0.0011087892344221473, + 0.0014391514705494046, + 0.001120170229114592, + 0.0013088533887639642, + 0.0004123076796531677, + 0.0007661065901629627, + 0.0005386698176153004, + 0.0007872642017900944, + 0.0005083107971586287, + 0.0009654506575316192, + 0.0009281352977268398, + 0.0006618788465857506, + 0.0004174182831775397, + 0.0006613184814341366, + 0.0003955806605517864, + 0.0011703699128702285, + 0.0007652677595615387, + 0.0009616908500902356, + 0.000610732939094305, + 0.0005696555017493665, + 0.000935644842684269, + 0.0005050115869380534, + 0.0008104036678560078, + 0.0005484815337695181, + 0.0005282653728500009, + 0.0006690899026580155, + 0.0006202083895914257, + 0.0011858937796205282, + 0.0006323098205029964, + 0.0012637152103707194, + 0.0010147326393052936, + 0.001434296485967934, + 0.0010879833716899157, + 0.0007398917223326862, + 0.0006289978045970201, + 0.0010291151702404022, + 0.0007981405942700803, + 0.0012479289434850216, + 0.000667006301227957, + 0.0027725808322429657, + 0.002342016203328967, + 0.0006513544940389693, + 0.0018907733028754592, + 0.002159562660381198, + 0.0017471732571721077, + 0.0015442147850990295, + 0.003038904396817088, + 0.003148901043459773, + 0.005466174334287643, + 0.004158675670623779, + 0.001521154772490263, + 0.002499047899618745, + 0.0023495368659496307, + 0.0034855001140385866, + 0.004092938732355833, + 0.003764112712815404, + 0.0024812589399516582, + 0.0029548786114901304, + 0.00236957217566669, + 0.0012237809132784605, + 0.001522903912700713, + 0.0026057965587824583, + 0.0028801767621189356, + 0.0020971631165593863, + 0.0017077381489798429, + 0.002488095546141267, + 0.001942558796145022, + 0.0012060959124937654, + 0.0032189686316996813, + 0.004897533915936947, + 0.002546509262174368, + 0.002436422742903233, + 0.0016161665553227067, + 0.0008353215525858104, + 0.004716587718576193, + 0.003007955150678754, + 0.001906886580400169, + 0.0010407179361209271, + 0.0036604618653655057, + 0.0019616391509771347, + 0.001880012801848352, + 0.0010364892659708858, + 0.0020254242699593306, + 0.001567527768202126, + 0.0018129348754882812, + 0.0011344858212396502, + 0.001787062268704176, + 0.002030221512541175, + 0.002426884835585952, + 0.0018259655917063355, + 0.0006956890574656427, + 0.0026017874479293823, + 0.0008315623854286969, + 0.0014645435148850083, + 0.0009511765674687922, + 0.0009218405466526748, + 0.0015906983753666282, + 0.0014132362557575109, + 0.0017343055224046111, + 0.0019996073096990585, + 0.0013999902876093984, + 0.0021138533484190702, + 0.002869958057999611, + 0.0016324784373864532, + 0.003336323192343116, + 0.004306184593588114, + 0.0022455891594290733, + 0.0018202642677351832, + 0.0014412502059713006, + 0.0014982676366344094, + 0.0010419118916615844, + 0.0016191820614039898, + 0.003920895978808403, + 0.0013921178178861735, + 0.0016874541761353612, + 0.003467828966677189, + 0.0024901803117245436, + 0.003774721873924136, + 0.0008989623747766018, + 0.0018224921077489853, + 0.001697454135864973, + 0.002475267043337226, + 0.001093029393814504, + 0.0014057605294510722, + 0.0009937405120581388, + 0.006045807618647814, + 0.003723071655258536, + 0.0024961901362985373, + 0.0010668792529031634, + 0.012105301953852177, + 0.006213821936398745, + 0.018133636564016346, + 0.010110520757734776, + 0.001374133862555027, + 0.013326204381883144, + 0.002680495148524642, + 0.008382327854633331, + 0.0030930601060390472, + 0.025627313181757927, + 0.0031410318333655596, + 0.001220939448103309, + 0.0015257663326337934, + 0.0025776990223675966, + 0.0020443866960704327, + 0.003117623971775174, + 0.0028798922430723906, + 0.001909662154503167, + 0.0018645658856257796, + 0.00894069205969572, + 0.006883753929287195, + 0.014036208391189575, + 0.0055046118795871735, + 0.0024932369124144316, + 0.0035899977665394545, + 0.004715373273938894, + 0.002608843147754669, + 0.007800393272191286, + 0.011004121042788029, + 0.15411949157714844, + 0.0017875274643301964, + 0.0042082518339157104, + 0.009378299117088318, + 0.007050686050206423, + 0.005348358768969774, + 0.0016888940008357167, + 0.004475145135074854, + 0.006452843081206083, + 0.0025523456279188395 + ] + ] + } + ], + "layout": { + "annotations": [ + { + "bgcolor": "red", + "font": { + "color": "black", + "size": 10 + }, + "opacity": 0.8, + "showarrow": false, + "text": "[CLS]", + "textangle": -90, + "x": 0, + "xref": "x", + "y": 0.5, + "yref": "paper" + }, + { + "bgcolor": "red", + "font": { + "color": "black", + "size": 10 + }, + "opacity": 0.8, + "showarrow": false, + "text": "[VS]", + "textangle": -90, + "x": 1, + "xref": "x", + "y": 0.5, + "yref": "paper" + }, + { + "bgcolor": "red", + "font": { + "color": "black", + "size": 10 + }, + "opacity": 0.8, + "showarrow": false, + "text": "[VE]", + "textangle": -90, + "x": 376, + "xref": "x", + "y": 0.5, + "yref": "paper" + }, + { + "bgcolor": "red", + "font": { + "color": "black", + "size": 10 + }, + "opacity": 0.8, + "showarrow": false, + "text": "[REG]", + "textangle": -90, + "x": 377, + "xref": "x", + "y": 0.5, + "yref": "paper" + } + ], + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Attention Visualization" + }, + "xaxis": { + "nticks": 378, + "tickangle": -90, + "title": { + "text": "Token in Input Sequence" + } + }, + "yaxis": { + "nticks": 1, + "title": { + "text": "Token in Input Sequence" + } + } + } + }, + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def visualize_attention(\n", + " attention_weights,\n", + " patient,\n", + " special_tokens,\n", + " tokenizer,\n", + " truncate=False,\n", + " only_cls=False,\n", + " top_k=10,\n", + "):\n", + " # Convert attention tensor to numpy array and squeeze the batch dimension\n", + " concept_ids = patient[\"concept_ids\"].squeeze(0).cpu().numpy()\n", + " attention_weights = attention_weights.squeeze(0).cpu().numpy()\n", + "\n", + " # Truncate attention weights if specified\n", + " if truncate:\n", + " truncate_at = patient[\"attention_mask\"].sum().numpy()\n", + " attention_weights = attention_weights[:, :truncate_at, :truncate_at]\n", + " concept_ids = concept_ids[:truncate_at]\n", + "\n", + " if only_cls:\n", + " attention_weights = attention_weights[:, :1, :]\n", + "\n", + " # Average attention weights across heads\n", + " attention_weights = attention_weights.mean(axis=0)\n", + "\n", + " # Generate token labels, marking special tokens with a special symbol\n", + " x_token_labels = [\n", + " f\"{tokenizer.id_to_token(token)}\"\n", + " if tokenizer.id_to_token(token) in special_tokens\n", + " else str(i)\n", + " for i, token in enumerate(concept_ids)\n", + " ]\n", + " y_token_labels = [\"[CLS]\"]\n", + "\n", + " # Generate hover text\n", + " hover_text = [\n", + " [\n", + " f\"Token {tokenizer.id_to_token(concept_ids[row])} with Token {tokenizer.id_to_token(concept_ids[col])}:\"\n", + " f\"Attention Value {attention_weights[row, col]:.3f}\"\n", + " for col in range(attention_weights.shape[1])\n", + " ]\n", + " for row in range(attention_weights.shape[0])\n", + " ]\n", + "\n", + " # Generate annotations for special tokens\n", + " annotations = []\n", + " for i, token in enumerate(concept_ids):\n", + " if tokenizer.id_to_token(token) in special_tokens:\n", + " annotations.append(\n", + " dict(\n", + " x=i,\n", + " y=0.5,\n", + " xref=\"x\",\n", + " yref=\"paper\", # Use 'paper' coordinates for y\n", + " text=tokenizer.id_to_token(token),\n", + " showarrow=False,\n", + " font=dict(color=\"black\", size=10),\n", + " textangle=-90,\n", + " bgcolor=\"red\",\n", + " opacity=0.8,\n", + " ),\n", + " )\n", + "\n", + " # Plot the attention matrix as a heatmap\n", + " fig = go.Figure(\n", + " data=go.Heatmap(\n", + " z=attention_weights,\n", + " x=x_token_labels,\n", + " y=y_token_labels,\n", + " hoverongaps=False,\n", + " hoverinfo=\"text\",\n", + " text=hover_text,\n", + " colorscale=\"YlGnBu\",\n", + " ),\n", + " )\n", + "\n", + " fig.update_layout(\n", + " title=\"Attention Visualization\",\n", + " xaxis_nticks=len(concept_ids),\n", + " yaxis_nticks=len(y_token_labels),\n", + " xaxis_title=\"Token in Input Sequence\",\n", + " yaxis_title=\"Token in Input Sequence\",\n", + " annotations=annotations,\n", + " xaxis_tickangle=-90,\n", + " )\n", + "\n", + " # Print top k tokens with their attention values\n", + " top_k_indices = np.argsort(-attention_weights, axis=None)[:top_k]\n", + " top_k_values = attention_weights.flatten()[top_k_indices]\n", + " top_k_indices = np.unravel_index(top_k_indices, attention_weights.shape)\n", + "\n", + " for idx in range(len(top_k_indices[0])):\n", + " token1 = top_k_indices[0][idx]\n", + " token2 = top_k_indices[1][idx]\n", + " attention_value = top_k_values[idx]\n", + " print(\n", + " f\"Token {tokenizer.id_to_token(concept_ids[token1])} \"\n", + " f\"with Token {tokenizer.id_to_token(concept_ids[token2])}: \"\n", + " f\"Attention Value {attention_value:.3f}\",\n", + " )\n", + "\n", + " fig.show()\n", + "\n", + "\n", + "# Visualize the attention matrix with special tokens\n", + "special_tokens = [\"[CLS]\", \"[VS]\", \"[VE]\", \"[REG]\"]\n", + "visualize_attention(\n", + " last_attention_matrix,\n", + " patient=patient,\n", + " special_tokens=special_tokens,\n", + " tokenizer=tokenizer,\n", + " truncate=True,\n", + " only_cls=True,\n", + " top_k=15,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "92da688f", + "metadata": {}, + "outputs": [], + "source": [ + "# def visualize_attention(attention_weights, patient, special_tokens, tokenizer, truncate=False, only_cls=False, top_k=10):\n", + "# # Convert attention tensor to numpy array and squeeze the batch dimension\n", + "# concept_ids = patient['concept_ids'].squeeze(0).cpu().numpy()\n", + "# attention_weights = attention_weights.squeeze(0).cpu().numpy()\n", + "\n", + "# # Truncate attention weights if specified\n", + "# if truncate:\n", + "# truncate_at = patient['attention_mask'].sum().numpy()\n", + "# attention_weights = attention_weights[:, :truncate_at, :truncate_at]\n", + "# concept_ids = concept_ids[:truncate_at]\n", + "\n", + "# if only_cls:\n", + "# attention_weights = attention_weights[:, :1, :]\n", + "\n", + "# # Average attention weights across heads\n", + "# attention_weights = attention_weights.mean(axis=0)\n", + "\n", + "# # Generate token labels, replacing special tokens with their names\n", + "# token_labels = [tokenizer.id_to_token(token) if tokenizer.id_to_token(token) in special_tokens else '' for token in concept_ids]\n", + "\n", + "# # Plot the attention matrix as a heatmap\n", + "# sns.set_theme(font_scale=1.2)\n", + "# plt.figure(figsize=(15, 12))\n", + "# ax = sns.heatmap(attention_weights, cmap=\"YlGnBu\", linewidths=.5, annot=False, cbar=True)\n", + "# ax.set_title('Attention Visualization')\n", + "\n", + "# # Set custom tick labels\n", + "# # ax.set_xticks(np.arange(len(token_labels)) + 0.5)\n", + "# # ax.set_xticklabels(token_labels, rotation=45, ha='right', fontsize=10)\n", + "# # ax.set_yticks(np.arange(len(token_labels)) + 0.5)\n", + "# # ax.set_yticklabels(token_labels, rotation=0, ha='right', fontsize=10)\n", + "\n", + "# ax.set_xlabel('Token in Input Sequence')\n", + "# ax.set_ylabel('Token in Input Sequence')\n", + "\n", + "# # Print top k tokens with their attention values\n", + "# top_k_indices = np.argsort(-attention_weights, axis=None)[:top_k]\n", + "# top_k_values = attention_weights.flatten()[top_k_indices]\n", + "# top_k_indices = np.unravel_index(top_k_indices, attention_weights.shape)\n", + "\n", + "# for idx in range(len(top_k_indices[0])):\n", + "# token1 = top_k_indices[0][idx]\n", + "# token2 = top_k_indices[1][idx]\n", + "# attention_value = top_k_values[idx]\n", + "# print(f\"Token {tokenizer.id_to_token(concept_ids[token1])} \"\n", + "# f\"with Token {tokenizer.id_to_token(concept_ids[token2])}: \"\n", + "# f\"Attention Value {attention_value}\")\n", + "\n", + "# plt.show()\n", + "\n", + "\n", + "# # Visualize the attention matrix with special tokens\n", + "# special_tokens = ['[CLS]', '[VS]', '[VE]', '[REG]'] # Update this list with your actual special tokens\n", + "# visualize_attention(last_attention_matrix, patient=patient, special_tokens=special_tokens, tokenizer=tokenizer, truncate=True, only_cls=True, top_k=25)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cf327908", + "metadata": {}, + "outputs": [], + "source": [ + "# Model view\n", + "html_model_view = model_view(\n", + " truncated_attention_matrix,\n", + " truncated_tokens,\n", + " include_layers=[5],\n", + " include_heads=[0, 1, 2, 3, 4, 5],\n", + " display_mode=\"light\",\n", + " html_action=\"return\",\n", + ")\n", + "\n", + "with open(\"model_view.html\", \"w\") as file:\n", + " file.write(html_model_view.data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "acae54e37e7d407bbb7b55eff062a284", + "metadata": {}, + "outputs": [], + "source": [ + "# Head View\n", + "html_head_view = head_view(\n", + " truncated_attention_matrix,\n", + " truncated_tokens,\n", + " # include_layers=[5],\n", + " html_action=\"return\",\n", + ")\n", + "\n", + "with open(\"head_view.html\", \"w\") as file:\n", + " file.write(html_head_view.data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c13d4bf", + "metadata": {}, + "outputs": [], + "source": [ + "# Neuron View\n", + "model_type = \"bert\"\n", + "\n", + "show(model, model_type, tokenizer, display_mode=\"dark\", layer=5, head=0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e4ee069c3f083f", + "metadata": { + "ExecuteTime": { + "end_time": "2024-03-15T13:59:33.204805Z", + "start_time": "2024-03-15T13:59:33.193223Z" + }, + "collapsed": false + }, + "outputs": [], + "source": [ + "# Visualize REG token -> Tricky?\n", + "# DONE Why the row vs column attention differs? -> What the matrix actually represents\n", + "# Include one example patient and visualize the attention matrix -> Include the exact concept token\n", + "# Some sort of markers to separate visits and special tokens\n", + "# Libraries used for attention visualization -> Amrit suggestion\n", + "# Visualize the gradients" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/odyssey/evals/CompareAUROC-Poster.ipynb b/odyssey/evals/CompareAUROC-Poster.ipynb new file mode 100644 index 0000000..9e92592 --- /dev/null +++ b/odyssey/evals/CompareAUROC-Poster.ipynb @@ -0,0 +1,158 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "\"\"\"\n", + "File: CompareAUROC-Poster.ipynb\n", + "---------------------------------\n", + "Compare performance of XGBoost to BigBird & Bi-LSTM using the AUROC curve for all three models on the same test set\n", + "Used to generate the AUROC curves on the poster showcased in Vector Institute's Research Symposium, on February 9\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Import dependencies and define useful constants\n", + "import os\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from scipy.special import expit\n", + "from sklearn.metrics import (\n", + " roc_auc_score,\n", + " roc_curve,\n", + ")\n", + "\n", + "\n", + "plt.style.use(\"seaborn-v0_8\")\n", + "%matplotlib inline\n", + "\n", + "TEST_SIZE = \"512\"\n", + "TEST_GROUP = \"two_weeks\"\n", + "TRANSFORMER_TEST_GROUP = \"week\" if TEST_GROUP == \"two_weeks\" else \"month\"\n", + "\n", + "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", + "DATA_ROOT = f\"{ROOT}/data/slurm_data/{TEST_SIZE}/{TEST_GROUP}\"\n", + "os.chdir(ROOT)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Load predictions, labels, and probabilities of different models\n", + "y_xgboost_pred = np.load(f\"{ROOT}/xgboost_y_test_pred_{TEST_GROUP}.npy\")\n", + "y_xgboost_labels = np.load(f\"{ROOT}/xgboost_y_test_pred_{TEST_GROUP}_labels.npy\")\n", + "y_xgboost_prob = np.load(f\"{ROOT}/xgboost_y_test_pred_{TEST_GROUP}_prob.npy\")\n", + "y_xgboost_prob = y_xgboost_prob[:, 1]\n", + "\n", + "y_lstm_pred = np.load(f\"{ROOT}/lstm_y_test_pred_{TEST_GROUP}.npy\")\n", + "y_lstm_labels = np.load(f\"{ROOT}/lstm_y_test_pred_{TEST_GROUP}_labels.npy\")\n", + "y_lstm_prob = np.load(f\"{ROOT}/lstm_y_test_pred_{TEST_GROUP}_prob.npy\")\n", + "\n", + "y_transformer_pred = np.load(\n", + " f\"/ssd003/projects/aieng/public/odyssey/results/test_preds_{TRANSFORMER_TEST_GROUP}.npy\",\n", + ")\n", + "y_transformer_labels = np.load(\n", + " f\"/ssd003/projects/aieng/public/odyssey/results/test_labels_{TRANSFORMER_TEST_GROUP}.npy\",\n", + ")\n", + "y_transformer_prob = np.load(\n", + " f\"/ssd003/projects/aieng/public/odyssey/results/test_prob_{TRANSFORMER_TEST_GROUP}.npy\",\n", + ")\n", + "y_transformer_prob = expit(y_transformer_prob[:, 1])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Plot ROC Curve for XGBoost, Bi-LSTM, and Transformer\n", + "fpr_xgboost, tpr_xgboost, _ = roc_curve(y_xgboost_labels, y_xgboost_prob)\n", + "fpr_lstm, tpr_lstm, _ = roc_curve(y_lstm_labels, y_lstm_prob)\n", + "fpr_transformer, tpr_transformer, _ = roc_curve(\n", + " y_transformer_labels,\n", + " y_transformer_prob,\n", + ")\n", + "\n", + "# AUROC\n", + "y_xgboost_auroc = roc_auc_score(y_xgboost_labels, y_xgboost_prob)\n", + "y_lstm_auroc = roc_auc_score(y_lstm_labels, y_lstm_prob)\n", + "transformer_auroc = roc_auc_score(y_transformer_labels, y_transformer_prob)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Plot Information\n", + "plt.figure(figsize=(8, 10))\n", + "\n", + "plt.plot(\n", + " fpr_transformer,\n", + " tpr_transformer,\n", + " label=f\"BigBird = {transformer_auroc:.2f}\",\n", + " color=\"red\",\n", + ")\n", + "plt.plot(\n", + " fpr_xgboost,\n", + " tpr_xgboost,\n", + " label=f\"XGBoost = {y_xgboost_auroc:.2f}\",\n", + " color=\"green\",\n", + ")\n", + "plt.plot(fpr_lstm, tpr_lstm, label=f\"Bi-LSTM = {y_lstm_auroc:.2f}\", color=\"blue\")\n", + "plt.plot([0, 1], [0, 1], linestyle=\"--\", color=\"gray\", label=\"Random\")\n", + "\n", + "plt.xlabel(\"False Positive Rate\")\n", + "plt.ylabel(\"True Positive Rate\")\n", + "plt.title(\"ROC Curve - Two-Weeks Mortality Prediction\")\n", + "plt.legend(loc=\"lower right\", fontsize=\"large\", facecolor=\"white\", frameon=True)\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/odyssey/evals/TestAnalysis.ipynb b/odyssey/evals/TestAnalysis.ipynb new file mode 100644 index 0000000..f39ce8d --- /dev/null +++ b/odyssey/evals/TestAnalysis.ipynb @@ -0,0 +1,266 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2024-04-10 12:13:14,754] [INFO] [real_accelerator.py:191:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n" + ] + } + ], + "source": [ + "import os\n", + "\n", + "import torch\n", + "from sklearn.metrics import (\n", + " auc,\n", + " average_precision_score,\n", + " balanced_accuracy_score,\n", + " f1_score,\n", + " precision_recall_curve,\n", + " precision_score,\n", + " recall_score,\n", + " roc_auc_score,\n", + ")\n", + "from transformers import utils\n", + "\n", + "\n", + "utils.logging.set_verbosity_error() # Suppress standard warnings\n", + "\n", + "\n", + "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", + "os.chdir(ROOT)\n", + "\n", + "from odyssey.data.tokenizer import ConceptTokenizer" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "class config:\n", + " \"\"\"Save the configuration arguments.\"\"\"\n", + "\n", + " model_path = \"test_epoch_end.ckpt\"\n", + " vocab_dir = \"data/vocab\"\n", + " data_dir = \"data/bigbird_data\"\n", + " sequence_file = \"patient_sequences/patient_sequences_2048_mortality.parquet\"\n", + " id_file = \"patient_id_dict/dataset_2048_mortality_1month.pkl\"\n", + " valid_scheme = \"few_shot\"\n", + " num_finetune_patients = \"20000\"\n", + " label_name = \"label_mortality_1month\"\n", + "\n", + " max_len = 2048\n", + " batch_size = 1\n", + " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "tokenizer = ConceptTokenizer(data_dir=config.vocab_dir)\n", + "tokenizer.fit_on_vocab()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['epoch', 'global_step', 'pytorch-lightning_version', 'state_dict', 'loops', 'callbacks', 'optimizer_states', 'lr_schedulers', 'MixedPrecision'])" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model = torch.load(config.model_path, map_location=config.device)\n", + "model.keys()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'loss': tensor(0.1638, dtype=torch.float64),\n", + " 'preds': tensor([6, 7, 0, ..., 7, 7, 7]),\n", + " 'labels': tensor([[1., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [1., 0., 0., ..., 0., 0., 0.],\n", + " ...,\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.],\n", + " [0., 0., 0., ..., 0., 0., 0.]], dtype=torch.float64),\n", + " 'logits': tensor([[ 2.3418, -2.0781, 0.2194, ..., -5.3945, -7.1797, -3.4180],\n", + " [-1.6533, -3.4277, -6.8086, ..., -6.8359, -5.2266, -5.6484],\n", + " [ 1.0947, -3.7930, -6.1094, ..., -6.3867, -6.6836, -5.5508],\n", + " ...,\n", + " [-2.8223, -3.7285, -4.6797, ..., -7.9922, -5.6992, -6.7812],\n", + " [-3.7148, -5.6328, -6.7188, ..., -9.4062, -7.6445, -7.6016],\n", + " [-2.5840, -2.2871, -4.6484, ..., -7.8633, -4.7539, -6.4648]],\n", + " dtype=torch.float16)}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "test_outputs = torch.load(\"test_outputs.pt\")\n", + "test_outputs" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'Balanced Accuracy': 0.5, 'F1 Score': 0.0, 'Precision': 0.0, 'Recall': 0.0, 'AUROC': 0.8100258785715974, 'Average Precision Score': 0.001364147006900979, 'AUC-PR': 0.5006820735034505}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/fs01/home/afallah/light/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1497: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n" + ] + } + ], + "source": [ + "def calculate_metrics(y_true, y_pred, y_prob):\n", + " \"\"\"\n", + " Calculate and return performance metrics.\n", + " \"\"\"\n", + " metrics = {\n", + " \"Balanced Accuracy\": balanced_accuracy_score(y_true, y_pred),\n", + " \"F1 Score\": f1_score(y_true, y_pred),\n", + " \"Precision\": precision_score(y_true, y_pred),\n", + " \"Recall\": recall_score(y_true, y_pred),\n", + " \"AUROC\": roc_auc_score(y_true, y_prob),\n", + " \"Average Precision Score\": average_precision_score(y_true, y_pred),\n", + " }\n", + "\n", + " precision, recall, _ = precision_recall_curve(y_true, y_pred)\n", + " metrics[\"AUC-PR\"] = auc(recall, precision)\n", + "\n", + " return metrics\n", + "\n", + "\n", + "targets = [10]\n", + "\n", + "for i in targets:\n", + " labels = test_outputs[\"labels\"][:, i]\n", + " logits = torch.sigmoid(test_outputs[\"logits\"][:, i])\n", + " preds = (logits >= 0.5).int()\n", + "\n", + " print(calculate_metrics(labels, preds, logits))" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(0)" + ] + }, + "execution_count": 68, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preds.sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(34., dtype=torch.float64)" + ] + }, + "execution_count": 69, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "labels.sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([0, 0, 0, ..., 0, 0, 0], dtype=torch.int32)" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preds" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/odyssey/evals/prediction.py b/odyssey/evals/prediction.py new file mode 100644 index 0000000..31194e0 --- /dev/null +++ b/odyssey/evals/prediction.py @@ -0,0 +1,113 @@ +"""Prediction module for loading and running BigBird models on patient data.""" + +from typing import Any, Dict, Optional + +import torch + +from odyssey.models.cehr_big_bird.model import BigBirdFinetune, BigBirdPretrain +from odyssey.data.tokenizer import ConceptTokenizer + + +def load_finetuned_model( + model_path: str, + tokenizer: ConceptTokenizer, + pre_model_config: Optional[Dict[str, Any]] = None, + fine_model_config: Optional[Dict[str, Any]] = None, + device: Optional[torch.device] = None, +) -> torch.nn.Module: + """Load a finetuned model from model_path using tokenizer information. + + Return a loaded finetuned model from model_path, using tokenizer information. + If config arguments are not provided, the default configs built into the + PyTorch classes are used. + + Parameters + ---------- + model_path: str + Path to the finetuned model to load + tokenizer: ConceptTokenizer + Loaded tokenizer object + pre_model_config: Dict[str, Any], optional + Optional config to override default values of a pretrained model + fine_model_config: Dict[str, Any], optional + Optional config to override default values of a finetuned model + device: torch.device, optional + CUDA device. By default, GPU is used + + Returns + ------- + torch.nn.Module + Finetuned model loaded from model_path + + """ + # Load GPU or CPU device + if not device: + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Create the skeleton of a pretrained and finetuned model + pretrained_model = BigBirdPretrain( + vocab_size=tokenizer.get_vocab_size(), + padding_idx=tokenizer.get_pad_token_id(), + **(pre_model_config or {}), + ) + + model = BigBirdFinetune( + pretrained_model=pretrained_model, + **(fine_model_config or {}), + ) + + # Load the weights using model_path directory + state_dict = torch.load(model_path, map_location=device)["state_dict"] + model.load_state_dict(state_dict) + model.to(device) + model.eval() + + return model + + +def predict_patient_outcomes( + patient: Dict[str, torch.Tensor], + model: torch.nn.Module, + device: Optional[torch.device] = None, +) -> Any: + """Compute model output predictions on given patient data. + + Parameters + ---------- + patient: Dict[str, torch.Tensor] + Patient data as a dictionary of tensors + model: torch.nn.Module + Model to use for prediction + device: torch.device, optional + CUDA device. By default, GPU is used + + Returns + ------- + Any + Model output predictions on the given patient data + + """ + # Load GPU or CPU device + if not device: + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Load patient information as a Tuple + patient_inputs = ( + patient["concept_ids"].to(device), + patient["type_ids"].to(device), + patient["time_stamps"].to(device), + patient["ages"].to(device), + patient["visit_orders"].to(device), + patient["visit_segments"].to(device), + ) + patient_labels = patient["labels"].to(device) + patient_attention_mask = patient["attention_mask"].to(device) + + # Get model output predictions + model.to(device) + + return model( + inputs=patient_inputs, + attention_mask=patient_attention_mask, + labels=patient_labels, + ) diff --git a/odyssey/models/embeddings.py b/odyssey/models/embeddings.py new file mode 100644 index 0000000..5de9eb3 --- /dev/null +++ b/odyssey/models/embeddings.py @@ -0,0 +1,368 @@ +""" +File: embeddings.py +--------------------- +Embedding layers for the models. +""" + +import math +from typing import Any, Optional + +import torch +from torch import nn +from transformers import BigBirdConfig + + +class TimeEmbeddingLayer(nn.Module): + """Embedding layer for time features.""" + + def __init__(self, embedding_size: int, is_time_delta: bool = False): + super().__init__() + self.embedding_size = embedding_size + self.is_time_delta = is_time_delta + + self.w = nn.Parameter(torch.empty(1, self.embedding_size)) + self.phi = nn.Parameter(torch.empty(1, self.embedding_size)) + + nn.init.xavier_uniform_(self.w) + nn.init.xavier_uniform_(self.phi) + + def forward(self, time_stamps: torch.Tensor) -> Any: + """Apply time embedding to the input time stamps.""" + if self.is_time_delta: + # If the time_stamps represent time deltas, we calculate the deltas. + # This is equivalent to the difference between consecutive elements. + time_stamps = torch.cat( + (time_stamps[:, 0:1] * 0, time_stamps[:, 1:] - time_stamps[:, :-1]), + dim=-1, + ) + time_stamps = time_stamps.float() + time_stamps_expanded = time_stamps.unsqueeze(-1) + next_input = time_stamps_expanded * self.w + self.phi + + return torch.sin(next_input) + + +class VisitEmbedding(nn.Module): + """Embedding layer for visit segments.""" + + def __init__( + self, + visit_order_size: int, + embedding_size: int, + ): + super().__init__() + self.visit_order_size = visit_order_size + self.embedding_size = embedding_size + self.embedding = nn.Embedding(self.visit_order_size, self.embedding_size) + + def forward(self, visit_segments: torch.Tensor) -> Any: + """Apply visit embedding to the input visit segments.""" + return self.embedding(visit_segments) + + +class ConceptEmbedding(nn.Module): + """Embedding layer for event concepts.""" + + def __init__( + self, + num_embeddings: int, + embedding_size: int, + padding_idx: Optional[int] = None, + ): + super(ConceptEmbedding, self).__init__() + self.embedding = nn.Embedding( + num_embeddings, + embedding_size, + padding_idx=padding_idx, + ) + + def forward(self, inputs: torch.Tensor) -> Any: + """Apply concept embedding to the input concepts.""" + return self.embedding(inputs) + + +class PositionalEmbedding(nn.Module): + """Positional embedding layer.""" + + def __init__(self, embedding_size: int, max_len: int): + super().__init__() + + # Compute the positional encodings once in log space. + pe = torch.zeros(max_len, embedding_size).float() + pe.require_grad = False + + position = torch.arange(0, max_len).float().unsqueeze(1) + div_term = ( + torch.arange(0, embedding_size, 2).float() + * -(math.log(10000.0) / embedding_size) + ).exp() + + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + + self.register_buffer("pe", pe) + + def forward(self, visit_orders: torch.Tensor) -> Any: + """Apply positional embedding to the input visit orders.""" + first_visit_concept_orders = visit_orders[:, 0:1] + normalized_visit_orders = torch.clamp( + visit_orders - first_visit_concept_orders, + 0, + self.pe.size(0) - 1, + ) + return self.pe[normalized_visit_orders] + + +class BERTEmbeddingsForCEHR(nn.Module): + """Embeddings for CEHR-BERT.""" + + def __init__( + self, + vocab_size: int, + embedding_size: int = 128, + time_embeddings_size: int = 16, + type_vocab_size: int = 8, + visit_order_size: int = 3, + max_len: int = 2048, + layer_norm_eps: float = 1e-12, + dropout_prob: float = 0.1, + padding_idx: int = 1, + ): + super().__init__() + self.concept_embedding = ConceptEmbedding( + num_embeddings=vocab_size, + embedding_size=embedding_size, + padding_idx=padding_idx, + ) + self.token_type_embeddings = nn.Embedding( + type_vocab_size, + embedding_size, + ) + self.time_embedding = TimeEmbeddingLayer( + embedding_size=time_embeddings_size, + is_time_delta=True, + ) + self.age_embedding = TimeEmbeddingLayer( + embedding_size=time_embeddings_size, + ) + self.positional_embedding = PositionalEmbedding( + embedding_size=embedding_size, + max_len=max_len, + ) + self.visit_embedding = VisitEmbedding( + visit_order_size=visit_order_size, + embedding_size=embedding_size, + ) + self.scale_back_concat_layer = nn.Linear( + embedding_size + 2 * time_embeddings_size, + embedding_size, + ) # Assuming 4 input features are concatenated + self.tanh = nn.Tanh() + self.LayerNorm = nn.LayerNorm(embedding_size, eps=layer_norm_eps) + self.dropout = nn.Dropout(dropout_prob) + + def forward( + self, + concept_ids: torch.Tensor, + type_ids: torch.Tensor, + time_stamps: torch.Tensor, + ages: torch.Tensor, + visit_orders: torch.Tensor, + visit_segments: torch.Tensor, + ) -> Any: + """Apply embeddings to the input features.""" + concept_embed = self.concept_embedding(concept_ids) + type_embed = self.token_type_embeddings(type_ids) + time_embed = self.time_embedding(time_stamps) + age_embed = self.age_embedding(ages) + positional_embed = self.positional_embedding(visit_orders) + visit_segment_embed = self.visit_embedding(visit_segments) + + embeddings = torch.cat((concept_embed, time_embed, age_embed), dim=-1) + embeddings = self.tanh(self.scale_back_concat_layer(embeddings)) + embeddings = embeddings + type_embed + positional_embed + visit_segment_embed + embeddings = self.LayerNorm(embeddings) + + return self.dropout(embeddings) + + +class BigBirdEmbeddingsForCEHR(nn.Module): + """Construct the embeddings from word, position and token_type embeddings.""" + + # Copied from transformers.models.bert.modeling_bert.BertEmbeddings.__init__ + def __init__( + self, + config: BigBirdConfig, + time_embeddings_size: int = 16, + visit_order_size: int = 3, + ) -> None: + """Initiate wrapper class for embeddings used in BigBird CEHR classes.""" + super().__init__() + + self.word_embeddings = nn.Embedding( + config.vocab_size, + config.hidden_size, + padding_idx=config.pad_token_id, + ) + self.position_embeddings = nn.Embedding( + config.max_position_embeddings, + config.hidden_size, + ) + self.token_type_embeddings = nn.Embedding( + config.type_vocab_size, + config.hidden_size, + ) + self.visit_order_embeddings = nn.Embedding( + config.max_position_embeddings, + config.hidden_size, + ) + self.time_embeddings = TimeEmbeddingLayer( + embedding_size=time_embeddings_size, + is_time_delta=True, + ) + self.age_embeddings = TimeEmbeddingLayer( + embedding_size=time_embeddings_size, + ) + self.visit_segment_embeddings = VisitEmbedding( + visit_order_size=visit_order_size, + embedding_size=config.hidden_size, + ) + self.scale_back_concat_layer = nn.Linear( + config.hidden_size + 2 * time_embeddings_size, + config.hidden_size, + ) + + self.time_stamps: Optional[torch.Tensor] = None + self.ages: Optional[torch.Tensor] = None + self.visit_orders: Optional[torch.Tensor] = None + self.visit_segments: Optional[torch.Tensor] = None + + # self.LayerNorm is not snake-cased to stick with TensorFlow model + # variable name and be able to load any TensorFlow checkpoint file. + self.tanh = nn.Tanh() + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + # position_ids (1, len position emb) is contiguous in memory. + self.position_embedding_type = getattr( + config, + "position_embedding_type", + "absolute", + ) + self.register_buffer( + "position_ids", + torch.arange(config.max_position_embeddings).expand((1, -1)), + persistent=False, + ) + self.register_buffer( + "token_type_ids", + torch.zeros(self.position_ids.size(), dtype=torch.long), + persistent=False, + ) + # End copy + + self.rescale_embeddings = config.rescale_embeddings + self.hidden_size = config.hidden_size + + def cache_input( + self, + time_stamps: torch.Tensor, + ages: torch.Tensor, + visit_orders: torch.Tensor, + visit_segments: torch.Tensor, + ) -> None: + """Cache values for time_stamps, ages, visit_orders & visit_segments. + + These values will be used by the forward pass to change the final embedding. + + Parameters + ---------- + time_stamps : torch.Tensor + Time stamps of the input data. + ages : torch.Tensor + Ages of the input data. + visit_orders : torch.Tensor + Visit orders of the input data. + visit_segments : torch.Tensor + Visit segments of the input data. + + """ + self.time_stamps = time_stamps + self.ages = ages + self.visit_orders = visit_orders + self.visit_segments = visit_segments + + def clear_cache(self) -> None: + """Delete the tensors cached by cache_input method.""" + del self.time_stamps, self.ages, self.visit_orders, self.visit_segments + + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + token_type_ids: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + past_key_values_length: int = 0, + ) -> Any: + """Return the final embeddings of concept ids using input and cached values.""" + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + + if position_ids is None: + position_ids = self.position_ids[ + :, + past_key_values_length : seq_length + past_key_values_length, + ] + + # Setting the token_type_ids to the registered buffer in constructor + if token_type_ids is None: + if hasattr(self, "token_type_ids"): + buffered_token_type_ids = self.token_type_ids[:, :seq_length] + buffered_token_type_ids_expanded = buffered_token_type_ids.expand( + input_shape[0], + seq_length, + ) + token_type_ids = buffered_token_type_ids_expanded + else: + token_type_ids = torch.zeros( + input_shape, + dtype=torch.long, + device=self.position_ids.device, + ) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + + if self.rescale_embeddings: + inputs_embeds = inputs_embeds * (self.hidden_size**0.5) + + # Using cached values from a prior cache_input call + time_stamps_embeds = self.time_embeddings(self.time_stamps) + ages_embeds = self.age_embeddings(self.ages) + visit_segments_embeds = self.visit_segment_embeddings(self.visit_segments) + visit_order_embeds = self.visit_order_embeddings(self.visit_orders) + + position_embeds = self.position_embeddings(position_ids) + token_type_embeds = self.token_type_embeddings(token_type_ids) + + inputs_embeds = torch.cat( + (inputs_embeds, time_stamps_embeds, ages_embeds), + dim=-1, + ) + inputs_embeds = self.tanh(self.scale_back_concat_layer(inputs_embeds)) + embeddings = inputs_embeds + token_type_embeds + embeddings += position_embeds + embeddings += visit_order_embeds + embeddings += visit_segments_embeds + + embeddings = self.dropout(embeddings) + embeddings = self.LayerNorm(embeddings) + + # Clear the cache for next forward call + self.clear_cache() + + return embeddings diff --git a/odyssey/models/model_utils.py b/odyssey/models/model_utils.py new file mode 100644 index 0000000..a1d4f22 --- /dev/null +++ b/odyssey/models/model_utils.py @@ -0,0 +1,159 @@ +"""Utility functions for the model.""" + +import os +import pickle +import uuid +from os.path import join +from typing import Any + +import pandas as pd +import yaml + + +def load_config(config_dir: str, model_type: str) -> Any: + """Load the model configuration. + + Parameters + ---------- + config_dir: str + Directory containing the model configuration files + + model_type: str + Model type to load configuration for + + Returns + ------- + Any + Model configuration + + """ + config_file = join(config_dir, f"{model_type}.yaml") + with open(config_file, "r") as file: + return yaml.safe_load(file) + + +def load_pretrain_data( + data_dir: str, + sequence_file: str, + id_file: str, +) -> pd.DataFrame: + """Load the pretraining data. + + Parameters + ---------- + data_dir: str + Directory containing the data files + sequence_file: str + Sequence file name + id_file: str + ID file name + + Returns + ------- + pd.DataFrame + Pretraining data + + """ + sequence_path = join(data_dir, sequence_file) + id_path = join(data_dir, id_file) + + if not os.path.exists(sequence_path): + raise FileNotFoundError(f"Sequence file not found: {sequence_path}") + + if not os.path.exists(id_path): + raise FileNotFoundError(f"ID file not found: {id_path}") + + data = pd.read_parquet(sequence_path) + with open(id_path, "rb") as file: + patient_ids = pickle.load(file) + + return data.loc[data["patient_id"].isin(patient_ids["pretrain"])] + + +def load_finetune_data( + data_dir: str, + sequence_file: str, + id_file: str, + valid_scheme: str, + num_finetune_patients: str, +) -> pd.DataFrame: + """Load the finetuning data. + + Parameters + ---------- + data_dir: str + Directory containing the data files + sequence_file: str + Sequence file name + id_file: str + ID file name + valid_scheme: str + Validation scheme + num_finetune_patients: str + Number of finetune patients + + Returns + ------- + pd.DataFrame + Finetuning data + + """ + sequence_path = join(data_dir, sequence_file) + id_path = join(data_dir, id_file) + + if not os.path.exists(sequence_path): + raise FileNotFoundError(f"Sequence file not found: {sequence_path}") + + if not os.path.exists(id_path): + raise FileNotFoundError(f"ID file not found: {id_path}") + + data = pd.read_parquet(sequence_path) + with open(id_path, "rb") as file: + patient_ids = pickle.load(file) + + fine_tune = data.loc[ + data["patient_id"].isin( + patient_ids["finetune"][valid_scheme][num_finetune_patients], + ) + ] + fine_test = data.loc[data["patient_id"].isin(patient_ids["test"])] + return fine_tune, fine_test + + +def get_run_id( + checkpoint_dir: str, + retrieve: bool = False, + run_id_file: str = "wandb_run_id.txt", + length: int = 8, +) -> str: + """Fetch the run ID for the current run. + + If the run ID file exists, retrieve the run ID from the file. + Otherwise, generate a new run ID and save it to the file. + + Parameters + ---------- + checkpoint_dir: str + Directory to store the run ID file + retrieve: bool, optional + Retrieve the run ID from the file, by default False + run_id_file: str, optional + Run ID file name, by default "wandb_run_id.txt" + length: int, optional + String length of the run ID, by default 8 + + Returns + ------- + str + Run ID for the current run + + """ + run_id_path = os.path.join(checkpoint_dir, run_id_file) + if retrieve and os.path.exists(run_id_path): + with open(run_id_path, "r") as file: + run_id = file.read().strip() + else: + run_id = str(uuid.uuid4())[:length] + with open(run_id_path, "w") as file: + file.write(run_id) + return run_id diff --git a/odyssey/utils/__init__.py b/odyssey/utils/__init__.py new file mode 100644 index 0000000..2381ea8 --- /dev/null +++ b/odyssey/utils/__init__.py @@ -0,0 +1 @@ +"""Utils sub-package.""" diff --git a/odyssey/utils/utils.py b/odyssey/utils/utils.py new file mode 100644 index 0000000..ee4a748 --- /dev/null +++ b/odyssey/utils/utils.py @@ -0,0 +1,48 @@ +""" +File: utils.py +--------------- +General utility functions and constants for the project. +""" + +from typing import Any + +import pickle +import random + +import numpy as np +import torch +import pytorch_lightning as pl + + +def seed_everything(seed: int) -> None: + """Seed all components of the model. + + Parameters + ---------- + seed: int + Seed value to use + + """ + random.seed(seed) + torch.manual_seed(seed) + np.random.seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + pl.seed_everything(seed) + + +def save_object_to_disk(obj: Any, save_path: str) -> None: + """Save an object to disk using pickle. + + Parameters + ---------- + obj: Any + Object to save + save_path: str + Path to save the object + + """ + with open(save_path, "wb") as f: + pickle.dump(obj, f) + print(f"File saved to disk: {save_path}") From 7cb7c74635aa547f6fbd023f54cb047997d1c1fa Mon Sep 17 00:00:00 2001 From: afallah Date: Tue, 16 Apr 2024 21:31:14 -0400 Subject: [PATCH 9/9] Updated the docstring of Python script. --- odyssey/models/embeddings.py | 6 +----- odyssey/utils/utils.py | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/odyssey/models/embeddings.py b/odyssey/models/embeddings.py index 5de9eb3..0e5a999 100644 --- a/odyssey/models/embeddings.py +++ b/odyssey/models/embeddings.py @@ -1,8 +1,4 @@ -""" -File: embeddings.py ---------------------- -Embedding layers for the models. -""" +"""Embedding layers for the models.""" import math from typing import Any, Optional diff --git a/odyssey/utils/utils.py b/odyssey/utils/utils.py index ee4a748..34f721c 100644 --- a/odyssey/utils/utils.py +++ b/odyssey/utils/utils.py @@ -1,8 +1,4 @@ -""" -File: utils.py ---------------- -General utility functions and constants for the project. -""" +"""General utility functions and constants for the project.""" from typing import Any