treino2-old.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d840f3d0a5174b62983184485cb603f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>content1</th>\n",
       "      <th>content2</th>\n",
       "      <th>target_transformed</th>\n",
       "      <th>target_transformed_sigmoid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://ddschild.com/services/infants-and-todd...</td>\n",
       "      <td>most. Babies are sometimes born with tongue- t...</td>\n",
       "      <td>great experience at the dentist! Experts recom...</td>\n",
       "      <td>7.478170</td>\n",
       "      <td>0.999435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://www.westelm.com/shop/bedding/</td>\n",
       "      <td>modern bedding. Indulge in our organic bedding...</td>\n",
       "      <td>includes duvets, quilts, sheet</td>\n",
       "      <td>-5.181784</td>\n",
       "      <td>0.005587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://ridersinsider.mlblogs.com/2012/07/17/ou...</td>\n",
       "      <td>currently has a record of</td>\n",
       "      <td>to pitch a lot of innings,\" Brigham said. \"I try</td>\n",
       "      <td>4.653960</td>\n",
       "      <td>0.990566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.dmarge.com/2017/02/people-are-awes...</td>\n",
       "      <td>It's got that. Tricking on a beach with yoga</td>\n",
       "      <td>physics and death. Basketball dunks</td>\n",
       "      <td>3.970292</td>\n",
       "      <td>0.981481</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://www.dhammawheel.com/viewtopic.php?p=91540</td>\n",
       "      <td>would lead to a greater</td>\n",
       "      <td>stops me from completely letting</td>\n",
       "      <td>5.981414</td>\n",
       "      <td>0.997481</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>http://www.galwaycycling.org/cyclists-mark-roa...</td>\n",
       "      <td>the environment. Making it safer multiplies th...</td>\n",
       "      <td>that one of the main</td>\n",
       "      <td>5.799093</td>\n",
       "      <td>0.996979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>http://www.claretianassociates.org/calendar.html</td>\n",
       "      <td>the new custom playground. RSVP</td>\n",
       "      <td>They will also use the</td>\n",
       "      <td>4.574711</td>\n",
       "      <td>0.989796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>http://www.conquerclub.com/forum/viewtopic.php...</td>\n",
       "      <td>full sized preview of the following maps just ...</td>\n",
       "      <td>hand their CLA League performance is a poultry...</td>\n",
       "      <td>-9.349754</td>\n",
       "      <td>0.000087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>http://www.computerweekly.com/news/2240050361/...</td>\n",
       "      <td>environment. Meeting the specialised document ...</td>\n",
       "      <td>from MatrixOne to develop a</td>\n",
       "      <td>7.085064</td>\n",
       "      <td>0.999163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>https://www.luxuryyacht.org/marmaris-yacht-cha...</td>\n",
       "      <td>you for a wonderful week</td>\n",
       "      <td>better than any other. The</td>\n",
       "      <td>4.983607</td>\n",
       "      <td>0.993197</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 name  \\\n",
       "0   https://ddschild.com/services/infants-and-todd...   \n",
       "1                http://www.westelm.com/shop/bedding/   \n",
       "2   http://ridersinsider.mlblogs.com/2012/07/17/ou...   \n",
       "3   https://www.dmarge.com/2017/02/people-are-awes...   \n",
       "4    http://www.dhammawheel.com/viewtopic.php?p=91540   \n",
       "..                                                ...   \n",
       "95  http://www.galwaycycling.org/cyclists-mark-roa...   \n",
       "96   http://www.claretianassociates.org/calendar.html   \n",
       "97  http://www.conquerclub.com/forum/viewtopic.php...   \n",
       "98  http://www.computerweekly.com/news/2240050361/...   \n",
       "99  https://www.luxuryyacht.org/marmaris-yacht-cha...   \n",
       "\n",
       "                                             content1  \\\n",
       "0   most. Babies are sometimes born with tongue- t...   \n",
       "1   modern bedding. Indulge in our organic bedding...   \n",
       "2                           currently has a record of   \n",
       "3        It's got that. Tricking on a beach with yoga   \n",
       "4                             would lead to a greater   \n",
       "..                                                ...   \n",
       "95  the environment. Making it safer multiplies th...   \n",
       "96                    the new custom playground. RSVP   \n",
       "97  full sized preview of the following maps just ...   \n",
       "98  environment. Meeting the specialised document ...   \n",
       "99                           you for a wonderful week   \n",
       "\n",
       "                                             content2  target_transformed  \\\n",
       "0   great experience at the dentist! Experts recom...            7.478170   \n",
       "1                      includes duvets, quilts, sheet           -5.181784   \n",
       "2    to pitch a lot of innings,\" Brigham said. \"I try            4.653960   \n",
       "3                 physics and death. Basketball dunks            3.970292   \n",
       "4                    stops me from completely letting            5.981414   \n",
       "..                                                ...                 ...   \n",
       "95                               that one of the main            5.799093   \n",
       "96                             They will also use the            4.574711   \n",
       "97  hand their CLA League performance is a poultry...           -9.349754   \n",
       "98                        from MatrixOne to develop a            7.085064   \n",
       "99                         better than any other. The            4.983607   \n",
       "\n",
       "    target_transformed_sigmoid  \n",
       "0                     0.999435  \n",
       "1                     0.005587  \n",
       "2                     0.990566  \n",
       "3                     0.981481  \n",
       "4                     0.997481  \n",
       "..                         ...  \n",
       "95                    0.996979  \n",
       "96                    0.989796  \n",
       "97                    0.000087  \n",
       "98                    0.999163  \n",
       "99                    0.993197  \n",
       "\n",
       "[100 rows x 5 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import duckdb\n",
    "\n",
    "# Caminhos para os bancos de dados\n",
    "db1_path = 'fineweb.duckdb'  # Substitua pelo caminho do seu primeiro banco de dados\n",
    "db2_path = 'books.duckdb'  # Substitua pelo caminho do seu segundo banco de dados\n",
    "\n",
    "# Alias para o segundo banco de dados\n",
    "alias_db2 = 'db2_alias'\n",
    "\n",
    "# Conectar ao banco de dados principal\n",
    "conn = duckdb.connect(database=db1_path, read_only=False)\n",
    "\n",
    "# Anexar o segundo banco de dados\n",
    "conn.execute(f\"ATTACH DATABASE '{db2_path}' AS {alias_db2}\")\n",
    "\n",
    "# Verificar esquemas (opcional, mas recomendado)\n",
    "schema_main = conn.execute(\"DESCRIBE dataset\").fetchdf()\n",
    "schema_db2 = conn.execute(f\"DESCRIBE {alias_db2}.dataset\").fetchdf()\n",
    "\n",
    "if not schema_main.equals(schema_db2):\n",
    "\traise Exception(\"Os esquemas das tabelas 'dataset' nos dois bancos de dados não são compatíveis.\")\n",
    "\n",
    "# Executar a consulta modificada\n",
    "df_training = conn.execute(f\"\"\"\n",
    "WITH combined_dataset AS (\n",
    "    SELECT * FROM dataset\n",
    "    UNION ALL\n",
    "    SELECT * FROM {alias_db2}.dataset\n",
    "),\n",
    "sampled_a AS (\n",
    "    SELECT id, indice, content, name\n",
    "    FROM (\n",
    "        SELECT\n",
    "            id,\n",
    "            indice,\n",
    "            content,\n",
    "            name,\n",
    "            ROW_NUMBER() OVER (PARTITION BY name ORDER BY RANDOM()) AS rn\n",
    "        FROM combined_dataset\n",
    "    ) sub\n",
    "    WHERE rn <= 1\n",
    "),\n",
    "sampled_b AS (\n",
    "    SELECT id, indice, content, name\n",
    "    FROM (\n",
    "        SELECT\n",
    "            id,\n",
    "            indice,\n",
    "            content,\n",
    "            name,\n",
    "            ROW_NUMBER() OVER (PARTITION BY name ORDER BY RANDOM()) AS rn\n",
    "        FROM combined_dataset\n",
    "    ) sub\n",
    "    WHERE rn <= 1\n",
    ")\n",
    "SELECT\n",
    "    a.name AS name,\n",
    "    a.content AS content1,\n",
    "    b.content AS content2,\n",
    "    -- Cálculo original do target_transformado\n",
    "    SIGN(a.indice - b.indice) * LN(1 + ABS(a.indice - b.indice)) AS target_transformed,\n",
    "    -- Aplicação da função sigmoide no target_transformado\n",
    "    1 / (1 + EXP(- (SIGN(a.indice - b.indice) * LN(1 + ABS(a.indice - b.indice))))) AS target_transformed_sigmoid\n",
    "FROM sampled_a a\n",
    "JOIN sampled_b b\n",
    "    ON a.name = b.name\n",
    "ORDER BY RANDOM()\n",
    "LIMIT 100;\n",
    "\"\"\").df()\n",
    "\n",
    "conn.close()\n",
    "\n",
    "df_training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "# Defina o número de elementos originais em cada coluna\n",
    "num_content1 = len(df_training['content1'])\n",
    "\n",
    "# Combinar as duas colunas de conteúdo em uma lista\n",
    "contents = list(df_training['content1']) + list(df_training['content2'])\n",
    "\n",
    "# Tokenização combinada\n",
    "encoding = embedding_generator.tokenizer(\n",
    "    contents,\n",
    "    return_tensors=\"pt\",\n",
    "    padding=True,  # Padding dinâmico\n",
    "    truncation=True\n",
    ")\n",
    "\n",
    "# Separar os encodings de volta para cada conjunto\n",
    "encoding1 = {key: encoding[key][:num_content1] for key in encoding.keys()}\n",
    "encoding2 = {key: encoding[key][num_content1:] for key in encoding.keys()}\n",
    "\n",
    "# Converter para o dispositivo\n",
    "input_ids1 = encoding1['input_ids'].to(embedding_generator.device)\n",
    "attention_mask1 = encoding1['attention_mask'].to(embedding_generator.device)\n",
    "input_ids2 = encoding2['input_ids'].to(embedding_generator.device)\n",
    "attention_mask2 = encoding2['attention_mask'].to(embedding_generator.device)\n",
    "\n",
    "# Obter embeddings\n",
    "embeddings1 = embedding_generator.weights[input_ids1]\n",
    "embeddings2 = embedding_generator.weights[input_ids2]\n",
    "\n",
    "# Concatenar as duas sequências no eixo 1\n",
    "embeddings = torch.stack([embeddings1, embeddings2], dim=1)  # (batch_size, 2, seq_length, vector_size)\n",
    "\n",
    "# Calcular comprimentos\n",
    "lengths1 = attention_mask1.sum(dim=1)\n",
    "lengths2 = attention_mask2.sum(dim=1)\n",
    "lengths = torch.stack([lengths1, lengths2], dim=1)  # (batch_size, 2)\n",
    "\n",
    "# Converter targets\n",
    "targets = torch.tensor(df_training['target_transformed'], dtype=torch.float16)  # (batch_size,)\n",
    "\n",
    "for i in range(100): \n",
    "\tprint(embeddings[i].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.hist(df_training['target_transformed'], bins=50, alpha=0.5, label='target_transformed')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_fetch_data\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 2, 396, 2048])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  Converter DataLoader para iterador\n",
    "data_iter = iter(train_dataloader)\n",
    "# Pegar o primeiro lote\n",
    "embeddings, lengths, targets = next(data_iter)\n",
    "embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using GPU: NVIDIA GeForce RTX 3050 Laptop GPU\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import gc\n",
    "\n",
    "if not torch.cuda.is_available():\n",
    "    print(\"CUDA is not available. Please ensure you have a compatible GPU and drivers installed.\")\n",
    "else:\n",
    "    print(f\"Using GPU: {torch.cuda.get_device_name(0)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "from transformers import AutoTokenizer\n",
    "import os\n",
    "\n",
    "# Definir paralelismo corretamente\n",
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"\n",
    "\n",
    "class TextToEmbedding:\n",
    "    def __init__(self, weights_path, num_ids=128256, vector_size=2048, device='cpu'):\n",
    "        \"\"\"\n",
    "        Inicializa a classe TextToEmbedding sem padding fixo.\n",
    "\n",
    "        Args:\n",
    "            weights_path (str): Caminho para o arquivo .npy que contém os pesos.\n",
    "            num_ids (int, opcional): Número total de IDs. Padrão é 128256.\n",
    "            vector_size (int, opcional): Tamanho de cada vetor de embedding. Padrão é 2048.\n",
    "            device (str, opcional): Dispositivo para carregar os tensores ('cpu' ou 'cuda'). Padrão é 'cpu'.\n",
    "        \"\"\"\n",
    "\t\t\n",
    "        self.device = device\n",
    "\n",
    "        # Carrega o tokenizer sem padding fixo\n",
    "        self.tokenizer = AutoTokenizer.from_pretrained(\"unsloth/Llama-3.2-1B\", use_fast=True)\n",
    "        \n",
    "        # Carrega os pesos a partir do arquivo .npy\n",
    "        try:\n",
    "            weights_np = np.load(weights_path)\n",
    "            self.weights = torch.from_numpy(weights_np).to(self.device)\n",
    "        except FileNotFoundError:\n",
    "            raise FileNotFoundError(f\"O arquivo de pesos '{weights_path}' não foi encontrado.\")\n",
    "        except Exception as e:\n",
    "            raise RuntimeError(f\"Erro ao carregar os pesos: {e}\")\n",
    "        \n",
    "        # Verifica a forma dos pesos\n",
    "        if self.weights.shape != (num_ids, vector_size):\n",
    "            raise ValueError(f\"O formato do arquivo weights.npy é {self.weights.shape}, mas era esperado {(num_ids, vector_size)}.\")\n",
    "\n",
    "embedding_generator = TextToEmbedding(\"weights_half.npy\", device='cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "\n",
    "class SequenceDataset_val(Dataset):\n",
    "\tdef __init__(self, df):\n",
    "\t\tself.df = df.reset_index(drop=True)\n",
    "\n",
    "\t\t# Defina o número de elementos originais em cada coluna\n",
    "\t\tnum_content1 = len(df['content1'])\n",
    "\n",
    "\t\t# Combinar as duas colunas de conteúdo em uma lista\n",
    "\t\tcontents = list(df['content1']) + list(df['content2'])\n",
    "\n",
    "\t\t# Tokenização combinada\n",
    "\t\tencoding = embedding_generator.tokenizer(\n",
    "\t\t\tcontents,\n",
    "\t\t\treturn_tensors=\"pt\",\n",
    "\t\t\tpadding=True,  # Padding dinâmico\n",
    "\t\t\ttruncation=True\n",
    "\t\t)\n",
    "\n",
    "\t\t# Separar os encodings de volta para cada conjunto\n",
    "\t\tencoding1 = {key: encoding[key][:num_content1] for key in encoding.keys()}\n",
    "\t\tencoding2 = {key: encoding[key][num_content1:] for key in encoding.keys()}\n",
    "\t\t\n",
    "\t\t# Converter para o dispositivo\n",
    "\t\tinput_ids1 = encoding1['input_ids'].to(embedding_generator.device)\n",
    "\t\tattention_mask1 = encoding1['attention_mask'].to(embedding_generator.device)\n",
    "\t\tinput_ids2 = encoding2['input_ids'].to(embedding_generator.device)\n",
    "\t\tattention_mask2 = encoding2['attention_mask'].to(embedding_generator.device)\n",
    "\t\t\n",
    "\t\t# Obter embeddings\n",
    "\t\tembeddings1 = embedding_generator.weights[input_ids1]\n",
    "\t\tembeddings2 = embedding_generator.weights[input_ids2]\n",
    "\t\t\n",
    "\t\t# Concatenar as duas sequências no eixo 1\n",
    "\t\tself.embeddings = torch.stack([embeddings1, embeddings2], dim=1)  # (batch_size, 2, seq_length, vector_size)\n",
    "\t\t\n",
    "\t\t# Calcular comprimentos\n",
    "\t\tlengths1 = attention_mask1.sum(dim=1)\n",
    "\t\tlengths2 = attention_mask2.sum(dim=1)\n",
    "\t\tself.lengths = torch.stack([lengths1, lengths2], dim=1)  # (batch_size, 2)\n",
    "\n",
    "\t\t# Converter targets\n",
    "\t\tself.targets = torch.tensor(df['target_transformed'], dtype=torch.float16)  # (batch_size,)\n",
    "\n",
    "\tdef __len__(self):\n",
    "\t\treturn len(self.df)\n",
    "\t\n",
    "\tdef __getitem__(self,idx):\n",
    "\t\treturn self.embeddings[idx], self.lengths[idx], self.targets[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import IterableDataset, DataLoader\n",
    "import duckdb\n",
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "\n",
    "# Paths to the databases\n",
    "db1_path = 'fineweb.duckdb'\n",
    "db2_path = 'books.duckdb'\n",
    "batch_size = 80\n",
    "\n",
    "class IterableSequenceDataset(IterableDataset):\n",
    "\tdef __init__(self, db1_path, db2_path, batch_size=100):\n",
    "\t\tsuper(IterableSequenceDataset, self).__init__()\n",
    "\t\tself.db1_path = db1_path\n",
    "\t\tself.db2_path = db2_path\n",
    "\t\tself.batch_size = batch_size\n",
    "\n",
    "\tdef _fetch_data(self, conn):\n",
    "\t\tquery = f\"\"\"\n",
    "\t\t\tWITH combined_dataset AS (\n",
    "\t\t\t\tSELECT * FROM main.dataset\n",
    "\t\t\t\tUNION ALL\n",
    "\t\t\t\tSELECT * FROM db2.dataset\n",
    "\t\t\t),\n",
    "\t\t\tsampled_a AS (\n",
    "\t\t\t\tSELECT id, indice, content, name\n",
    "\t\t\t\tFROM (\n",
    "\t\t\t\t\tSELECT\n",
    "\t\t\t\t\t\tid,\n",
    "\t\t\t\t\t\tindice,\n",
    "\t\t\t\t\t\tcontent,\n",
    "\t\t\t\t\t\tname,\n",
    "\t\t\t\t\t\tROW_NUMBER() OVER (PARTITION BY name ORDER BY RANDOM()) AS rn\n",
    "\t\t\t\t\tFROM combined_dataset\n",
    "\t\t\t\t) sub\n",
    "\t\t\t\tWHERE rn <= 1\n",
    "\t\t\t),\n",
    "\t\t\tsampled_b AS (\n",
    "\t\t\t\tSELECT id, indice, content, name\n",
    "\t\t\t\tFROM (\n",
    "\t\t\t\t\tSELECT\n",
    "\t\t\t\t\t\tid,\n",
    "\t\t\t\t\t\tindice,\n",
    "\t\t\t\t\t\tcontent,\n",
    "\t\t\t\t\t\tname,\n",
    "\t\t\t\t\t\tROW_NUMBER() OVER (PARTITION BY name ORDER BY RANDOM()) AS rn\n",
    "\t\t\t\t\tFROM combined_dataset\n",
    "\t\t\t\t) sub\n",
    "\t\t\t\tWHERE rn <= 1\n",
    "\t\t\t)\n",
    "\t\t\tSELECT\n",
    "\t\t\t\ta.name AS name,\n",
    "\t\t\t\ta.content AS content1,\n",
    "\t\t\t\tb.content AS content2,\n",
    "\t\t\t\tSIGN(a.indice - b.indice) * LN(1 + ABS(a.indice - b.indice)) AS target_transformed\n",
    "\t\t\tFROM sampled_a a\n",
    "\t\t\tJOIN sampled_b b\n",
    "\t\t\t\tON a.name = b.name\n",
    "\t\t\tORDER BY RANDOM()\n",
    "\t\t\tLIMIT {self.batch_size};\n",
    "\t\t\"\"\"\n",
    "\t\tdf = conn.execute(query).df()\n",
    "\n",
    "\t\t# Defina o número de elementos originais em cada coluna\n",
    "\t\tnum_content1 = len(df['content1'])\n",
    "\n",
    "\t\t# Combinar as duas colunas de conteúdo em uma lista\n",
    "\t\tcontents = list(df['content1']) + list(df['content2'])\n",
    "\n",
    "\t\t# Tokenização combinada\n",
    "\t\tencoding = embedding_generator.tokenizer(\n",
    "\t\t\tcontents,\n",
    "\t\t\treturn_tensors=\"pt\",\n",
    "\t\t\tpadding=True,  # Padding dinâmico\n",
    "\t\t\ttruncation=True\n",
    "\t\t)\n",
    "\n",
    "\t\t# Separar os encodings de volta para cada conjunto\n",
    "\t\tencoding1 = {key: encoding[key][:num_content1] for key in encoding.keys()}\n",
    "\t\tencoding2 = {key: encoding[key][num_content1:] for key in encoding.keys()}\n",
    "\n",
    "\t\t# Converter para o dispositivo\n",
    "\t\tinput_ids1 = encoding1['input_ids'].to(embedding_generator.device)\n",
    "\t\tattention_mask1 = encoding1['attention_mask'].to(embedding_generator.device)\n",
    "\t\tinput_ids2 = encoding2['input_ids'].to(embedding_generator.device)\n",
    "\t\tattention_mask2 = encoding2['attention_mask'].to(embedding_generator.device)\n",
    "\n",
    "\t\t# Obter embeddings\n",
    "\t\tembeddings1 = embedding_generator.weights[input_ids1]\n",
    "\t\tembeddings2 = embedding_generator.weights[input_ids2]\n",
    "\n",
    "\t\t# Concatenar as duas sequências no eixo 1\n",
    "\t\tembeddings = torch.stack([embeddings1, embeddings2], dim=1)  # (batch_size, 2, seq_length, vector_size)\n",
    "\n",
    "\t\t# Calcular comprimentos\n",
    "\t\tlengths1 = attention_mask1.sum(dim=1)\n",
    "\t\tlengths2 = attention_mask2.sum(dim=1)\n",
    "\t\tlengths = torch.stack([lengths1, lengths2], dim=1)  # (batch_size, 2)\n",
    "\n",
    "\t\t# Converter targets\n",
    "\t\ttargets = torch.tensor(df['target_transformed'], dtype=torch.float16)  # (batch_size,)\n",
    "\n",
    "\t\treturn embeddings,lengths,targets\n",
    "\n",
    "\tdef __iter__(self):\n",
    "\t\t\"\"\"\n",
    "\t\tIterator que gera dados continuamente.\n",
    "\n",
    "\t\tYields:\n",
    "\t\t\ttuple: (content1, content2, target) para cada amostra.\n",
    "\t\t\"\"\"\n",
    "\t\t# Estabelecer conexão com DuckDB\n",
    "\t\tconn = duckdb.connect(database=self.db1_path, read_only=True)\n",
    "\t\tconn.execute(\"SET enable_progress_bar=false\")\n",
    "\t\t\n",
    "\t\t# Anexar o segundo banco de dados com um alias 'db2'\n",
    "\t\tconn.execute(f\"ATTACH '{self.db2_path}' AS db2\")\n",
    "\n",
    "\t\ttry:\n",
    "\t\t\twhile True:\n",
    "\t\t\t\tembeddings,lengths,targets = self._fetch_data(conn)\n",
    "\t\t\t\tfor i in range(batch_size): \n",
    "\t\t\t\t\tyield embeddings[i], lengths[i], targets[i]\n",
    "\t\tfinally:\n",
    "\t\t\t# Garantir que a conexão seja fechada quando o iterador for finalizado\n",
    "\t\t\tconn.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def signed_log_transform(y):\n",
    "    return np.sign(y) * np.log1p(np.abs(y))\n",
    "\n",
    "def inverse_signed_log_transform(y_transformed):\n",
    "    return np.sign(y_transformed) * (np.expm1(np.abs(y_transformed)))\n",
    "\n",
    "def dissimilaridade(S):\n",
    "\tepsilon = 0\n",
    "\tif S == 0:\n",
    "\t\tmax_x = np.log(np.finfo(np.float32).max)\n",
    "\t\tepsilon = (1/(max_x-1))\n",
    "        \n",
    "\treturn (1 - (S+epsilon)) / (S+epsilon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target_transformed</th>\n",
       "      <th>content1</th>\n",
       "      <th>content2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>A tempestade durou várias horas durante a noite.</td>\n",
       "      <td>A tempestade durou várias horas durante a noite.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Ele estudou intensamente para o exame.</td>\n",
       "      <td>Ele estudou intensamente para o exame.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>A fábrica reduziu a emissão de poluentes.</td>\n",
       "      <td>A fábrica reduziu a emissão de poluentes.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>Maria começou a praticar exercícios regularmente.</td>\n",
       "      <td>Maria começou a praticar exercícios regularmente.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>O motor do carro parou de funcionar.</td>\n",
       "      <td>O motor do carro parou de funcionar.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>Isso resultou na redução significativa das emi...</td>\n",
       "      <td>Após anos de pesquisa e desenvolvimento, a equ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>Ela sentiu fome e falta de energia durante a m...</td>\n",
       "      <td>O café da manhã foi esquecido.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>As praias ficaram mais limpas e atraentes para...</td>\n",
       "      <td>A iniciativa comunitária organizou mutirões de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>Finalmente recebeu uma oferta de trabalho em u...</td>\n",
       "      <td>Ele atualizou seu currículo e participou de vá...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>Suas peças ganharam destaque em exposições int...</td>\n",
       "      <td>O artista decidiu experimentar novas técnicas ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>150 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     target_transformed                                           content1  \\\n",
       "0                   0.0   A tempestade durou várias horas durante a noite.   \n",
       "1                   0.0             Ele estudou intensamente para o exame.   \n",
       "2                   0.0          A fábrica reduziu a emissão de poluentes.   \n",
       "3                   0.0  Maria começou a praticar exercícios regularmente.   \n",
       "4                   0.0               O motor do carro parou de funcionar.   \n",
       "..                  ...                                                ...   \n",
       "145                -1.0  Isso resultou na redução significativa das emi...   \n",
       "146                -1.0  Ela sentiu fome e falta de energia durante a m...   \n",
       "147                -1.0  As praias ficaram mais limpas e atraentes para...   \n",
       "148                -1.0  Finalmente recebeu uma oferta de trabalho em u...   \n",
       "149                -1.0  Suas peças ganharam destaque em exposições int...   \n",
       "\n",
       "                                              content2  \n",
       "0     A tempestade durou várias horas durante a noite.  \n",
       "1               Ele estudou intensamente para o exame.  \n",
       "2            A fábrica reduziu a emissão de poluentes.  \n",
       "3    Maria começou a praticar exercícios regularmente.  \n",
       "4                 O motor do carro parou de funcionar.  \n",
       "..                                                 ...  \n",
       "145  Após anos de pesquisa e desenvolvimento, a equ...  \n",
       "146                     O café da manhã foi esquecido.  \n",
       "147  A iniciativa comunitária organizou mutirões de...  \n",
       "148  Ele atualizou seu currículo e participou de vá...  \n",
       "149  O artista decidiu experimentar novas técnicas ...  \n",
       "\n",
       "[150 rows x 3 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "data = {\n",
    "    'target_transformed': [1.0]*50,\n",
    "    \"content1\": [\"A tempestade durou várias horas durante a noite.\", \"Ele estudou intensamente para o exame.\", \"A fábrica reduziu a emissão de poluentes.\", \"Maria começou a praticar exercícios regularmente.\", \"O motor do carro parou de funcionar.\", \"Houve uma forte seca na região.\", \"Ela começou a dormir melhor.\", \"A escola adotou uma alimentação saudável.\", \"O projeto teve apoio governamental.\", \"A internet caiu durante a reunião.\", \"Ele começou a economizar dinheiro mensalmente.\", \"A estrada estava em péssimo estado de conservação.\", \"O sistema de ar condicionado foi desligado no escritório.\", \"A empresa investiu em marketing digital.\", \"Ana não revisou o relatório antes de enviar.\", \"As crianças brincaram no parque até tarde.\", \"O curso de capacitação foi oferecido aos funcionários.\", \"Ele comprou um celular novo com câmera de alta qualidade.\", \"A reforma no prédio foi concluída.\", \"A região teve um crescimento populacional rápido.\", \"Carla começou a meditar diariamente.\", \"O atleta intensificou seu treino antes da competição.\", \"O sinal de celular na região foi ampliado.\", \"As temperaturas caíram drasticamente durante o inverno.\", \"Ele não configurou o alarme antes de dormir.\", \"A empresa ofereceu benefícios extras para seus funcionários.\", \"Houve um vazamento de gás na cozinha do restaurante.\", \"A biblioteca da escola foi modernizada com novos livros e tecnologia.\", \"Marcos passou a fazer pausas regulares durante o expediente.\", \"A empresa desenvolveu um aplicativo intuitivo para clientes.\", \"Devido ao aumento das chuvas nas últimas semanas, o nível dos rios subiu rapidamente e ultrapassou a capacidade das barragens.\", \"Ela decidiu começar uma rotina de alimentação balanceada, exercícios físicos regulares e meditação diária para melhorar sua saúde física e mental.\", \"A empresa, que sofria com baixos índices de produtividade, implementou uma nova estratégia de gestão focada no desenvolvimento dos funcionários e na cultura organizacional.\", \"O governo lançou um programa nacional de reciclagem e incentivou a participação ativa dos cidadãos por meio de campanhas de conscientização em escolas, empresas e residências.\", \"Após uma longa estiagem que afetou grande parte do território agrícola do país, o governo implementou um pacote emergencial de apoio aos agricultores, incluindo subsídios e incentivos para a recuperação das lavouras.\", \"Ele deixou o celular carregando a noite inteira sem usar carregadores de segurança e em um local sem ventilação.\", \"A comunidade local se uniu para limpar e revitalizar a praça abandonada, que estava sem manutenção há anos e havia se tornado um ponto de descarte irregular de lixo.\", \"Durante a reforma do prédio, descobriu-se que a estrutura tinha falhas graves, e o prazo para conclusão foi ampliado em seis meses para garantir a segurança.\", \"Um furacão atingiu a região costeira do país, com ventos de mais de 200 km/h, causando destruição em várias cidades e deixando milhares de pessoas desabrigadas.\", \"Ela esqueceu de regar a planta de sua varanda durante o mês todo, e o clima estava seco.\", \"João quebrou o braço jogando futebol.\", \"Devido a uma combinação de fatores climáticos extremos, incluindo ventos fortes e chuvas intensas, a região sofreu graves danos estruturais e interrupção no fornecimento de energia elétrica por vários dias.\", \"Ela esqueceu de levar o guarda-chuva.\", \"A empresa adotou práticas sustentáveis em sua produção, reduzindo o uso de plástico e implementando programas de reciclagem.\", \"A criança se recusou a comer legumes durante toda a semana.\", \"Após anos de pesquisa e desenvolvimento, a equipe científica finalmente descobriu um método eficiente para produzir energia limpa a partir de fontes renováveis.\", \"O café da manhã foi esquecido.\", \"A iniciativa comunitária organizou mutirões de limpeza nas praias locais regularmente durante o verão.\", \"Ele atualizou seu currículo e participou de várias entrevistas de emprego nos últimos meses.\", \"O artista decidiu experimentar novas técnicas em suas pinturas, incorporando elementos digitais e materiais reciclados em suas obras.\"],\n",
    "    \"content2\": [\"Pela manhã, muitas ruas estavam alagadas.\", \"Conseguiu uma nota alta na prova.\", \"A qualidade do ar na cidade melhorou.\", \"Ela perdeu peso e aumentou sua energia.\", \"Ele teve que chamar o guincho para levar o carro à oficina.\", \"As plantações foram prejudicadas, e a colheita foi menor.\", \"Sua disposição durante o dia melhorou significativamente.\", \"Os alunos passaram a ter mais energia e melhor concentração.\", \"Conseguiu concluir as fases iniciais rapidamente.\", \"A comunicação com a equipe foi interrompida.\", \"Conseguiu juntar uma quantia para uma viagem.\", \"O trânsito ficou mais lento e perigoso para os motoristas.\", \"O ambiente ficou quente e desconfortável para os funcionários.\", \"As vendas aumentaram significativamente.\", \"O documento continha erros e precisou ser corrigido.\", \"Elas ficaram cansadas e dormiram rapidamente ao chegar em casa.\", \"Eles melhoraram suas habilidades e eficiência no trabalho.\", \"Passou a tirar fotos mais nítidas e de melhor resolução.\", \"O local ficou mais seguro e esteticamente agradável.\", \"A demanda por moradias e serviços aumentou consideravelmente.\", \"Ela sentiu uma melhora no seu foco e redução do estresse.\", \"Teve um melhor desempenho e conquistou o primeiro lugar.\", \"A conexão ficou mais estável e acessível para os moradores.\", \"A procura por agasalhos e cobertores aumentou nas lojas.\", \"Acabou se atrasando para o trabalho na manhã seguinte.\", \"A satisfação e motivação dos colaboradores aumentaram.\", \"O local foi evacuado por segurança, e o serviço foi interrompido temporariamente.\", \"Os alunos começaram a frequentá-la mais e a melhorar seu desempenho acadêmico.\", \"Ele se sentiu mais produtivo e menos cansado ao final do dia.\", \"O número de usuários aumentou rapidamente.\", \"Diversas áreas urbanas e rurais foram afetadas por inundações, forçando muitas famílias a deixarem suas casas temporariamente.\", \"Em poucos meses, notou uma grande melhora em sua disposição, concentração e níveis de energia, além de perder peso.\", \"Em menos de um ano, a moral da equipe melhorou, a rotatividade diminuiu, e a produtividade geral aumentou em cerca de 30%.\", \"Como resultado, houve uma redução significativa na quantidade de resíduos sólidos em aterros e uma maior economia de recursos naturais.\", \"Em um ano, a produção agrícola voltou a níveis estáveis, e o impacto econômico negativo foi mitigado, beneficiando a população.\", \"Pela manhã, o dispositivo estava superaquecido e apresentou danos permanentes na bateria, reduzindo sua capacidade de funcionamento.\", \"Com a revitalização, a praça voltou a ser um local de encontro para moradores, e a segurança da área também melhorou.\", \"Apesar do atraso, a reforma resultou em um edifício mais seguro, confortável e com um aumento significativo no valor do imóvel.\", \"A resposta emergencial foi mobilizada rapidamente, com abrigos temporários e ajuda humanitária distribuída para minimizar o impacto nas vítimas.\", \"A planta murchou completamente e, infelizmente, não conseguiu ser recuperada, tendo que ser substituída.\", \"Ele ficou impossibilitado de trabalhar por duas semanas.\", \"A comunidade teve que recorrer a abrigos temporários, e a economia local sofreu uma queda significativa devido à paralisação das atividades comerciais.\", \"Ficou molhada durante o trajeto para o trabalho.\", \"A reputação da empresa melhorou, atraindo consumidores conscientes e aumentando as vendas em 20%.\", \"Sua mãe ficou preocupada com a falta de nutrientes na alimentação dele e decidiu consultar um nutricionista.\", \"Isso resultou na redução significativa das emissões de carbono e no avanço tecnológico sustentável, beneficiando o meio ambiente globalmente.\", \"Ela sentiu fome e falta de energia durante a manhã de trabalho.\", \"As praias ficaram mais limpas e atraentes para turistas, além de promover a conscientização ambiental entre os moradores.\", \"Finalmente recebeu uma oferta de trabalho em uma empresa renomada, melhorando sua estabilidade financeira.\", \"Suas peças ganharam destaque em exposições internacionais, ampliando seu reconhecimento e alcance no mercado de arte.\"]\n",
    "}\n",
    "\n",
    "# Crie o DataFrame\n",
    "df_validation = pd.DataFrame(data)\n",
    "\n",
    "# Adicionar frases iguais\n",
    "df_zero = pd.DataFrame({\n",
    "    'target_transformed': [0.0] * len(data['target_transformed']),\n",
    "    'content1': df_validation['content1'],\n",
    "    'content2': df_validation['content1']\n",
    "})\n",
    "\n",
    "# Adicionar frases com a ordem contrária\n",
    "df_reversed = pd.DataFrame({\n",
    "    'target_transformed': [-1.0] * len(data['target_transformed']),\n",
    "    'content1': df_validation['content2'],\n",
    "    'content2': df_validation['content1']\n",
    "})\n",
    "\n",
    "# Concatenar os dois DataFrames\n",
    "df_validation = pd.concat([df_zero, df_validation, df_reversed], ignore_index=True)\n",
    "\n",
    "df_validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "def collate_fn(batch):\n",
    "    embeddings, lengths, targets = zip(*batch)\n",
    "    return torch.stack(embeddings).to(torch.float32), torch.stack(lengths), torch.stack(targets).to(torch.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_dataset = IterableSequenceDataset(db1_path, db2_path)\n",
    "train_dataloader = DataLoader(\n",
    "  training_dataset, \n",
    "  batch_size=batch_size, \n",
    "  num_workers=0,\n",
    "  collate_fn=collate_fn,\n",
    "  pin_memory=False,\n",
    "  shuffle=False,\n",
    "  #persistent_workers=True\n",
    ")\n",
    "\n",
    "validation_dataset = SequenceDataset_val(df_validation)\n",
    "validation_dataloader = DataLoader(\n",
    "    validation_dataset,\n",
    "    batch_size=150,\n",
    "    shuffle=False,\n",
    "    num_workers=0,\n",
    "    pin_memory=False,\n",
    "    collate_fn=collate_fn\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn.utils.rnn import pack_padded_sequence\n",
    "\n",
    "class SequenceToTwoModel(nn.Module):\n",
    "    def __init__(self, input_size=2048, hidden_size=128, num_layers=3):\n",
    "        super(SequenceToTwoModel, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "\n",
    "        # LSTM compartilhado para ambas as sequências\n",
    "        self.shared_lstm = nn.LSTM(\n",
    "            input_size=input_size,\n",
    "            hidden_size=hidden_size,\n",
    "            num_layers=num_layers,\n",
    "            batch_first=True,\n",
    "            bidirectional=True,\n",
    "            dropout=0.1\n",
    "        )\n",
    "\n",
    "        # Camadas totalmente conectadas após concatenar as saídas do LSTM compartilhado para ambas as sequências\n",
    "        self.fc1 = nn.Linear(hidden_size * 4, 64)  # hidden_size * 2 para cada sequência\n",
    "        self.leaky_relu = nn.LeakyReLU(inplace=True)\n",
    "        self.fc2 = nn.Linear(64, 1)\n",
    "\n",
    "    def forward(self, sequences, lengths):\n",
    "        \"\"\"\n",
    "        Passagem para frente do modelo.\n",
    "\n",
    "        Args:\n",
    "            sequences (Tensor): Tensor de embeddings de forma (batch_size, 2, seq_len, input_size)\n",
    "            lengths (Tensor): Tensor de comprimentos de forma (batch_size, 2)\n",
    "\n",
    "        Returns:\n",
    "            Tensor: Previsões do modelo de forma (batch_size,)\n",
    "        \"\"\"\n",
    "        batch_size = sequences.size(0)\n",
    "        seq_len = sequences.size(2)\n",
    "        input_size = sequences.size(3)\n",
    "\n",
    "        # Combinar as duas sequências ao longo da dimensão do batch\n",
    "        combined_sequences = sequences.view(batch_size * 2, seq_len, input_size)\n",
    "        combined_lengths = lengths.view(batch_size * 2)\n",
    "\n",
    "        # Processar as sequências combinadas através do LSTM compartilhado\n",
    "        lstm_out = self.process_sequence(combined_sequences, combined_lengths)  # (batch_size * 2, hidden_size * 2)\n",
    "\n",
    "        # Reorganizar a saída para separar as sequências\n",
    "        lstm_out = lstm_out.view(batch_size, 2, self.hidden_size * 2)\n",
    "\n",
    "        # Concatenar as saídas das duas sequências\n",
    "        combined = lstm_out.view(batch_size, -1)  # (batch_size, hidden_size * 4)\n",
    "\n",
    "        # Passar pelas camadas totalmente conectadas\n",
    "        out = self.fc1(combined)\n",
    "        out = self.leaky_relu(out)\n",
    "        out = self.fc2(out)\n",
    "\n",
    "        return out.squeeze()\n",
    "        \n",
    "    def process_sequence(self, sequence, length):\n",
    "        \"\"\"\n",
    "        Processa uma única sequência através do LSTM compartilhado e retorna o último estado oculto concatenado.\n",
    "\n",
    "        Args:\n",
    "            sequence (Tensor): Tensor de forma (batch_size, seq_len, input_size)\n",
    "            length (Tensor): Tensor de forma (batch_size,)\n",
    "\n",
    "        Returns:\n",
    "            Tensor: Saída concatenada do último estado oculto de ambas as direções (batch_size, hidden_size * 2)\n",
    "        \"\"\"\n",
    "        # Ordenar as sequências por comprimento decrescente\n",
    "        lengths_sorted, sorted_idx = length.sort(0, descending=True)\n",
    "        sequence_sorted = sequence[sorted_idx]\n",
    "\n",
    "        # Empacotar as sequências\n",
    "        packed_seq = pack_padded_sequence(sequence_sorted, lengths_sorted.cpu(), batch_first=True, enforce_sorted=True)\n",
    "\n",
    "        # Passar pelo LSTM\n",
    "        packed_output, (h_n, c_n) = self.shared_lstm(packed_seq)\n",
    "\n",
    "        # h_n: (num_layers * num_directions, batch, hidden_size)\n",
    "        num_directions = 2  # Bidirecional\n",
    "\n",
    "        # Reorganizar h_n para (num_layers, num_directions, batch, hidden_size)\n",
    "        h_n = h_n.view(self.num_layers, num_directions, sequence.size(0), self.hidden_size)\n",
    "\n",
    "        # Selecionar o último layer\n",
    "        last_layer_h_n = h_n[-1]  # (num_directions, batch, hidden_size)\n",
    "\n",
    "        # Concatenar as direções\n",
    "        gru_out = torch.cat((last_layer_h_n[0], last_layer_h_n[1]), dim=1)  # (batch, hidden_size * 2)\n",
    "\n",
    "        # Restaurar a ordem original do batch\n",
    "        _, original_idx = sorted_idx.sort(0, descending=False)\n",
    "        gru_out = gru_out[original_idx]\n",
    "\n",
    "        return gru_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jadson/anaconda3/envs/pytorch/lib/python3.12/site-packages/torch/optim/lr_scheduler.py:62: UserWarning: The verbose parameter is deprecated. Please use get_last_lr() to access the learning rate.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# Inicializa o modelo, critério de perda e otimizador\n",
    "model = SequenceToTwoModel().to('cuda')\n",
    "#criterion = nn.MSELoss()\n",
    "criterion = nn.HuberLoss()\n",
    "\n",
    "optimizer = torch.optim.Adam(\n",
    "    model.parameters(), lr=1e-3, weight_decay=1e-6)\n",
    "\n",
    "# Inicializar o scheduler ReduceLROnPlateau\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n",
    "    optimizer, mode='min', factor=0.5, patience=1, verbose=True\n",
    ")\n",
    "\n",
    "# Inicializar o contador de passos\n",
    "step = 0\n",
    "steps_per_epoch = 10  # Número de batches por época\n",
    "validation_steps = None  # Se você deseja limitar os passos de validação, defina um número inteiro\n",
    "num_epochs = 10\n",
    "# Inicializar o contador de passos\n",
    "step = 0\n",
    "from torch.utils.tensorboard import SummaryWriter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inicializar o writer do TensorBoard\n",
    "writer = SummaryWriter('runs/treinamento_regressao_rnn')\n",
    "\n",
    "# Envolver o loop de épocas com tqdm para mostrar o progresso das épocas\n",
    "epoch_progress = tqdm(range(num_epochs), desc='Treinamento', unit='época')\n",
    "\n",
    "for epoch in epoch_progress:\n",
    "\tepoch_progress.set_description(f'Época {epoch+1}/{num_epochs}')\n",
    "\n",
    "\tprint(\"Treinamento\")\n",
    "\tmodel.train()\n",
    "\ttrain_loss = 0.0\n",
    "\n",
    "\t# Criar um iterador a partir do dataloader\n",
    "\ttrain_iterator = iter(train_dataloader)\n",
    "\n",
    "\t# Usar tqdm para o progresso dentro da época\n",
    "\tbatch_progress = tqdm(range(steps_per_epoch), desc='Treinando', leave=False, unit='batch')\n",
    "\n",
    "\tfor _ in batch_progress:\n",
    "\t\ttry:\n",
    "\t\t\tembeddings, lengths, targets = next(train_iterator)\n",
    "\t\texcept StopIteration:\n",
    "\t\t\t# Reiniciar o iterador se necessário\n",
    "\t\t\ttrain_iterator = iter(train_dataloader)\n",
    "\t\t\tembeddings, lengths, targets = next(train_iterator)\n",
    "\n",
    "\t\tprint(embeddings.shape)\n",
    "\t\tembeddings = embeddings.to('cuda') # (batch_size, 2, seq_len, vector_size)\n",
    "\t\tlengths = lengths.to('cuda') # (batch_size, 2)\n",
    "\t\ttargets = targets.to('cuda') # (batch_size,)\n",
    "\t\n",
    "\t\toptimizer.zero_grad()\n",
    "\n",
    "\t\t# Forward pass\n",
    "\t\toutputs = model(embeddings, lengths)\n",
    "\t\tloss = criterion(outputs, targets)\n",
    "\n",
    "\t\t# Backward pass e otimização\n",
    "\t\tloss.backward()\n",
    "\t\toptimizer.step()\n",
    "\n",
    "\t\ttrain_loss += loss.item() * targets.size(0)\n",
    "\n",
    "\t\t# Registrar a perda de treinamento a cada passo\n",
    "\t\twriter.add_scalar('Perda_Treinamento_Passo', loss.item(), step)\n",
    "\t\tstep += 1\n",
    "\n",
    "\tavg_train_loss = train_loss / (steps_per_epoch * batch_size)\n",
    "\n",
    "\t# Validação\n",
    "\tmodel.eval()\n",
    "\tvalidation_loss = 0.0\n",
    "\tcorrect_sign = 0\n",
    "\ttotal = 0\n",
    "\n",
    "\twith torch.no_grad():\n",
    "\t\tfor embeddings, lengths, targets in validation_dataloader:\n",
    "\t\t\tprint(embeddings.shape)\n",
    "\t\t\tembeddings = embeddings.to('cuda') # (batch_size, 2, seq_len, vector_size)\n",
    "\t\t\tlengths = lengths.to('cuda') # (batch_size, 2)\n",
    "\t\t\ttargets = targets.to('cuda') # (batch_size,)\n",
    "\n",
    "\t\t\t# Forward pass\n",
    "\t\t\toutputs = model(embeddings, lengths)\n",
    "\t\t\tloss = criterion(outputs, targets)\n",
    "\t\t\tvalidation_loss += loss.item() * targets.size(0)\n",
    "\n",
    "\t\t\t# Considerar apenas os alvos positivos ou negativos\n",
    "\t\t\tnon_zero_mask = targets != 0\n",
    "\t\t\tif non_zero_mask.sum().item() == 0:\n",
    "\t\t\t\tcontinue  # Pular se não houver alvos diferentes de zero\n",
    "\n",
    "\t\t\tpredicted_sign = torch.sign(outputs)[non_zero_mask]\n",
    "\t\t\ttarget_sign = torch.sign(targets)[non_zero_mask]\n",
    "\n",
    "\t\t\t# Calcular acertos onde os sinais correspondem\n",
    "\t\t\tcorrect_sign += (predicted_sign == target_sign).sum().item()\n",
    "\t\t\ttotal += non_zero_mask.sum().item()\n",
    "\n",
    "\t# Evitar divisão por zero\n",
    "\tif total > 0:\n",
    "\t\tsign_accuracy = correct_sign / total\n",
    "\telse:\n",
    "\t\tsign_accuracy = 0.0\n",
    "\n",
    "\tavg_validation_loss = validation_loss / len(validation_dataloader.dataset)\n",
    "\n",
    "\t# Registrar as métricas de validação por época\n",
    "\twriter.add_scalar('Perda_Validação_Epoca', avg_validation_loss, epoch+1)\n",
    "\twriter.add_scalar('Acurácia_Sinal_Validação', sign_accuracy, epoch+1)\n",
    "\n",
    "\t# Passar a perda de validação para o scheduler\n",
    "\tscheduler.step(avg_validation_loss)\n",
    "\n",
    "\t# Registrar a taxa de aprendizado atual (opcional)\n",
    "\tcurrent_lr = optimizer.param_groups[0]['lr']\n",
    "\twriter.add_scalar('Taxa_Aprendizado', current_lr, epoch+1)\n",
    "\n",
    "\t# Atualizar a descrição da barra com as perdas atuais e acurácia do sinal\n",
    "\tepoch_progress.set_postfix({\n",
    "\t\t'Perda Treinamento': f'{avg_train_loss:.3f}',\n",
    "\t\t'Perda Validação': f'{avg_validation_loss:.3f}',\n",
    "\t\t'Acurácia Sinal': f'{sign_accuracy*100:.2f}%',\n",
    "\t\t'LR': f'{current_lr:.8f}'\n",
    "\t})\n",
    "\n",
    "\t# Liberar cache da GPU após cada época\n",
    "\ttorch.cuda.empty_cache()\n",
    "\n",
    "# Fechar o SummaryWriter ao final do treinamento\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pytorch_lightning as pl\n",
    "\n",
    "class SequenceToTwoLightning(pl.LightningModule):\n",
    "\tdef __init__(self, input_size=2048, hidden_size=128, num_layers=3, lr=1e-3):\n",
    "\t\tsuper(SequenceToTwoLightning, self).__init__()\n",
    "\t\tself.save_hyperparameters()\n",
    "\n",
    "\t\t# Instanciar o modelo modificado\n",
    "\t\tself.model = SequenceToTwoModel(\n",
    "\t\t\tinput_size=input_size,\n",
    "\t\t\thidden_size=hidden_size,\n",
    "\t\t\tnum_layers=num_layers\n",
    "\t\t)\n",
    "\n",
    "\t\t# Função de perda\n",
    "\t\tself.criterion = nn.HuberLoss()\n",
    "\n",
    "\tdef forward(self, sequences, lengths):\n",
    "\t\treturn self.model(sequences, lengths)\n",
    "\n",
    "\tdef training_step(self, batch, batch_idx):\n",
    "\t\tembeddings, lengths, targets = batch  # embeddings: (batch_size, 2, seq_len, vector_size)\n",
    "\t\t#print(\"training_step\",embeddings.shape,lengths.shape,targets.shape)\n",
    "\t\t\n",
    "\t\t# Mover tensores para o dispositivo correto\n",
    "\t\tembeddings = embeddings.to(self.device)  # (batch_size, 2, seq_len, vector_size)\n",
    "\t\tlengths = lengths.to(self.device) # (batch_size, 2)\n",
    "\t\ttargets = targets.to(self.device) # (batch_size,)\n",
    "\n",
    "\t\t# Passagem para frente\n",
    "\t\toutputs = self(embeddings, lengths) # (batch_size,)\n",
    "\n",
    "\t\t# Computar a perda\n",
    "\t\tloss = self.criterion(outputs, targets)\n",
    "\n",
    "\t\t# Logar a perda de treinamento\n",
    "\t\tself.log('train_loss', loss, on_step=True, on_epoch=True, prog_bar=True, logger=True)\n",
    "\t\treturn loss\n",
    "\n",
    "\tdef validation_step(self, batch, batch_idx):\n",
    "\t\tembeddings, lengths, targets = batch\n",
    "\t\t#print(\"validation_step\",embeddings.shape,lengths.shape,targets.shape)\n",
    "\n",
    "\t\t# Mover tensores para o dispositivo correto\n",
    "\t\tembeddings = embeddings.to(self.device)\n",
    "\t\tlengths = lengths.to(self.device)\n",
    "\t\ttargets = targets.to(self.device)\n",
    "\n",
    "\t\t# Passagem para frente\n",
    "\t\toutputs = self(embeddings, lengths)\n",
    "\n",
    "\t\t# Computar a perda\n",
    "\t\tloss = self.criterion(outputs, targets)\n",
    "\n",
    "\t\t# Logar a perda de validação\n",
    "\t\tself.log('val_loss', loss, on_step=False, on_epoch=True, prog_bar=True, logger=True)\n",
    "\n",
    "\tdef configure_optimizers(self):\n",
    "\t\toptimizer = torch.optim.Adam(self.parameters(), lr=self.hparams.lr)\n",
    "\t\treturn optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jadson/anaconda3/envs/pytorch/lib/python3.12/site-packages/lightning_fabric/connector.py:571: `precision=16` is supported for historical reasons but its usage is discouraged. Please set your precision to 16-mixed instead!\n",
      "Using 16bit Automatic Mixed Precision (AMP)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "HPU available: False, using: 0 HPUs\n",
      "You are using a CUDA device ('NVIDIA GeForce RTX 3050 Laptop GPU') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "/home/jadson/anaconda3/envs/pytorch/lib/python3.12/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:654: Checkpoint directory /home/jadson/Documentos/Tempo-nas-Narrativas/checkpoints exists and is not empty.\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "  | Name      | Type               | Params | Mode \n",
      "---------------------------------------------------------\n",
      "0 | model     | SequenceToTwoModel | 3.1 M  | train\n",
      "1 | criterion | HuberLoss          | 0      | train\n",
      "---------------------------------------------------------\n",
      "3.1 M     Trainable params\n",
      "0         Non-trainable params\n",
      "3.1 M     Total params\n",
      "12.215    Total estimated model params size (MB)\n",
      "6         Modules in train mode\n",
      "0         Modules in eval mode\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1063843b87ab4d0fac697c73de96a443",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jadson/anaconda3/envs/pytorch/lib/python3.12/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:424: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=19` in the `DataLoader` to improve performance.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_step torch.Size([150, 2, 61, 2048]) torch.Size([150, 2]) torch.Size([150])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jadson/anaconda3/envs/pytorch/lib/python3.12/site-packages/pytorch_lightning/core/module.py:516: You called `self.log('val_loss', ..., logger=True)` but have no logger configured. You can enable one by doing `Trainer(logger=ALogger(...))`\n",
      "/home/jadson/anaconda3/envs/pytorch/lib/python3.12/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:424: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=19` in the `DataLoader` to improve performance.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98ae978deac94c7c8a52dba3f300edb2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_step torch.Size([80, 2, 376, 2048]) torch.Size([80, 2]) torch.Size([80])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jadson/anaconda3/envs/pytorch/lib/python3.12/site-packages/pytorch_lightning/core/module.py:516: You called `self.log('train_loss', ..., logger=True)` but have no logger configured. You can enable one by doing `Trainer(logger=ALogger(...))`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_step torch.Size([80, 2, 180, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 197, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 250, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 211, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 363, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 410, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 246, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 399, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 339, 2048]) torch.Size([80, 2]) torch.Size([80])\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6bb2f156dfba4ec7ab3588fa9939f6a2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_step torch.Size([150, 2, 61, 2048]) torch.Size([150, 2]) torch.Size([150])\n",
      "training_step torch.Size([80, 2, 174, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 408, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 312, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 455, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 289, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 259, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 198, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 207, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 372, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 210, 2048]) torch.Size([80, 2]) torch.Size([80])\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d655f8dd2f1a48358f4df947438426d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_step torch.Size([150, 2, 61, 2048]) torch.Size([150, 2]) torch.Size([150])\n",
      "training_step torch.Size([80, 2, 390, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 383, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 450, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 106, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 421, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 208, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 203, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 407, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 213, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 390, 2048]) torch.Size([80, 2]) torch.Size([80])\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a59e224a177f4947aa723424cb99b7ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_step torch.Size([150, 2, 61, 2048]) torch.Size([150, 2]) torch.Size([150])\n",
      "training_step torch.Size([80, 2, 215, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 402, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 246, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 233, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 437, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 324, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 454, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 202, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 343, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 402, 2048]) torch.Size([80, 2]) torch.Size([80])\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e26526d411484e7e9026fb4fc0e8544c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_step torch.Size([150, 2, 61, 2048]) torch.Size([150, 2]) torch.Size([150])\n",
      "training_step torch.Size([80, 2, 380, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 235, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 340, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 194, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 355, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 216, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 432, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 421, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 381, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 390, 2048]) torch.Size([80, 2]) torch.Size([80])\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0006b6e31f3495db9712bdd4a3636d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_step torch.Size([150, 2, 61, 2048]) torch.Size([150, 2]) torch.Size([150])\n",
      "training_step torch.Size([80, 2, 396, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 424, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 375, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 202, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 398, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 416, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 226, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 375, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 217, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 197, 2048]) torch.Size([80, 2]) torch.Size([80])\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b1614e26edd41bcb5fef58865c44102",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_step torch.Size([150, 2, 61, 2048]) torch.Size([150, 2]) torch.Size([150])\n",
      "training_step torch.Size([80, 2, 411, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 350, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 380, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 225, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 396, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 326, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 413, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 204, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 389, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 226, 2048]) torch.Size([80, 2]) torch.Size([80])\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b08125b3fb074dcd8430926417141eef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_step torch.Size([150, 2, 61, 2048]) torch.Size([150, 2]) torch.Size([150])\n",
      "training_step torch.Size([80, 2, 403, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 368, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 200, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 458, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 201, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 316, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 377, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 178, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 371, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 227, 2048]) torch.Size([80, 2]) torch.Size([80])\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a9532ad83c864d8cb0d2a8146805d6de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_step torch.Size([150, 2, 61, 2048]) torch.Size([150, 2]) torch.Size([150])\n",
      "training_step torch.Size([80, 2, 495, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 391, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 171, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 196, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 388, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 207, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 383, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 390, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 403, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 426, 2048]) torch.Size([80, 2]) torch.Size([80])\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62a3e732d6fc4b4690a9cc062040dd6c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_step torch.Size([150, 2, 61, 2048]) torch.Size([150, 2]) torch.Size([150])\n",
      "training_step torch.Size([80, 2, 198, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 194, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 233, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 224, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 216, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 145, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 216, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 403, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 374, 2048]) torch.Size([80, 2]) torch.Size([80])\n",
      "training_step torch.Size([80, 2, 385, 2048]) torch.Size([80, 2]) torch.Size([80])\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f341ac54a6b44c7885ecfc87f7c5362",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=10` reached.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_step torch.Size([150, 2, 61, 2048]) torch.Size([150, 2]) torch.Size([150])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "FIT Profiler Report\n",
      "\n",
      "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
      "|  Action                                                                                                                                                               \t|  Mean duration (s)\t|  Num calls      \t|  Total time (s) \t|  Percentage %   \t|\n",
      "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
      "|  Total                                                                                                                                                                \t|  -              \t|  4113           \t|  421.04         \t|  100 %          \t|\n",
      "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
      "|  run_training_epoch                                                                                                                                                   \t|  41.716         \t|  10             \t|  417.16         \t|  99.078         \t|\n",
      "|  [_TrainingEpochLoop].train_dataloader_next                                                                                                                           \t|  3.9506         \t|  100            \t|  395.06         \t|  93.828         \t|\n",
      "|  [Strategy]SingleDeviceStrategy.batch_to_device                                                                                                                       \t|  0.079686       \t|  111            \t|  8.8451         \t|  2.1008         \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.transfer_batch_to_device                                                                                                     \t|  0.079624       \t|  111            \t|  8.8382         \t|  2.0991         \t|\n",
      "|  run_training_batch                                                                                                                                                   \t|  0.069731       \t|  100            \t|  6.9731         \t|  1.6562         \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.optimizer_step                                                                                                               \t|  0.069643       \t|  100            \t|  6.9643         \t|  1.6541         \t|\n",
      "|  [Strategy]SingleDeviceStrategy.training_step                                                                                                                         \t|  0.030892       \t|  100            \t|  3.0892         \t|  0.73371        \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_train_epoch_end       \t|  0.27863        \t|  10             \t|  2.7863         \t|  0.66177        \t|\n",
      "|  [Strategy]SingleDeviceStrategy.backward                                                                                                                              \t|  0.022743       \t|  100            \t|  2.2743         \t|  0.54016        \t|\n",
      "|  [_EvaluationLoop].val_next                                                                                                                                           \t|  0.10715        \t|  11             \t|  1.1787         \t|  0.27994        \t|\n",
      "|  [Strategy]SingleDeviceStrategy.validation_step                                                                                                                       \t|  0.0512         \t|  11             \t|  0.5632         \t|  0.13376        \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.optimizer_zero_grad                                                                                                          \t|  0.0037744      \t|  100            \t|  0.37744        \t|  0.089645       \t|\n",
      "|  [Callback]TQDMProgressBar.on_validation_start                                                                                                                        \t|  0.03338        \t|  11             \t|  0.36718        \t|  0.087208       \t|\n",
      "|  [Callback]TQDMProgressBar.on_train_batch_end                                                                                                                         \t|  0.0033648      \t|  100            \t|  0.33648        \t|  0.079916       \t|\n",
      "|  [Callback]TQDMProgressBar.on_validation_batch_start                                                                                                                  \t|  0.0019836      \t|  11             \t|  0.02182        \t|  0.0051823      \t|\n",
      "|  [Callback]TQDMProgressBar.on_train_epoch_start                                                                                                                       \t|  0.0013777      \t|  10             \t|  0.013777       \t|  0.0032722      \t|\n",
      "|  [Callback]TQDMProgressBar.on_validation_end                                                                                                                          \t|  0.001156       \t|  11             \t|  0.012715       \t|  0.00302        \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_validation_model_zero_grad                                                                                                \t|  0.0010443      \t|  10             \t|  0.010443       \t|  0.0024803      \t|\n",
      "|  [Callback]TQDMProgressBar.on_train_epoch_end                                                                                                                         \t|  0.00058178     \t|  10             \t|  0.0058178      \t|  0.0013818      \t|\n",
      "|  [Callback]TQDMProgressBar.on_validation_batch_end                                                                                                                    \t|  0.0005116      \t|  11             \t|  0.0056276      \t|  0.0013366      \t|\n",
      "|  [Callback]TQDMProgressBar.on_sanity_check_start                                                                                                                      \t|  0.0049506      \t|  1              \t|  0.0049506      \t|  0.0011758      \t|\n",
      "|  [Callback]TQDMProgressBar.on_train_start                                                                                                                             \t|  0.003551       \t|  1              \t|  0.003551       \t|  0.00084339     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.configure_gradient_clipping                                                                                                  \t|  2.6014e-05     \t|  100            \t|  0.0026014      \t|  0.00061785     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_validation_model_eval                                                                                                     \t|  0.00015386     \t|  11             \t|  0.0016925      \t|  0.00040198     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_train_batch_end       \t|  1.6865e-05     \t|  100            \t|  0.0016865      \t|  0.00040055     \t|\n",
      "|  [Callback]ModelSummary.on_fit_start                                                                                                                                  \t|  0.00077352     \t|  1              \t|  0.00077352     \t|  0.00018372     \t|\n",
      "|  [Callback]TQDMProgressBar.on_train_end                                                                                                                               \t|  0.00073883     \t|  1              \t|  0.00073883     \t|  0.00017548     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_validation_end        \t|  3.985e-05      \t|  11             \t|  0.00043835     \t|  0.00010411     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.setup                    \t|  0.00041836     \t|  1              \t|  0.00041836     \t|  9.9363e-05     \t|\n",
      "|  [Callback]TQDMProgressBar.teardown                                                                                                                                   \t|  0.00035335     \t|  1              \t|  0.00035335     \t|  8.3922e-05     \t|\n",
      "|  [Callback]TQDMProgressBar.on_after_backward                                                                                                                          \t|  3.483e-06      \t|  100            \t|  0.0003483      \t|  8.2724e-05     \t|\n",
      "|  [Callback]TQDMProgressBar.on_train_batch_start                                                                                                                       \t|  3.4435e-06     \t|  100            \t|  0.00034435     \t|  8.1786e-05     \t|\n",
      "|  [Callback]ModelSummary.on_train_batch_end                                                                                                                            \t|  2.5347e-06     \t|  100            \t|  0.00025347     \t|  6.02e-05       \t|\n",
      "|  [Callback]ModelSummary.on_validation_batch_start                                                                                                                     \t|  2.2437e-05     \t|  11             \t|  0.00024681     \t|  5.8618e-05     \t|\n",
      "|  [Callback]ModelSummary.on_validation_start                                                                                                                           \t|  1.8793e-05     \t|  11             \t|  0.00020673     \t|  4.9099e-05     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.configure_optimizers                                                                                                         \t|  0.00019777     \t|  1              \t|  0.00019777     \t|  4.6971e-05     \t|\n",
      "|  [Callback]TQDMProgressBar.on_before_zero_grad                                                                                                                        \t|  1.8658e-06     \t|  100            \t|  0.00018658     \t|  4.4314e-05     \t|\n",
      "|  [Callback]TQDMProgressBar.on_fit_end                                                                                                                                 \t|  0.00018245     \t|  1              \t|  0.00018245     \t|  4.3334e-05     \t|\n",
      "|  [Callback]TQDMProgressBar.on_validation_epoch_end                                                                                                                    \t|  1.5793e-05     \t|  11             \t|  0.00017372     \t|  4.126e-05      \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_before_batch_transfer                                                                                                     \t|  1.5009e-06     \t|  111            \t|  0.0001666      \t|  3.9569e-05     \t|\n",
      "|  [Callback]TQDMProgressBar.on_before_backward                                                                                                                         \t|  1.5197e-06     \t|  100            \t|  0.00015197     \t|  3.6094e-05     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_validation_start                                                                                                          \t|  1.3609e-05     \t|  11             \t|  0.0001497      \t|  3.5556e-05     \t|\n",
      "|  [Callback]TQDMProgressBar.on_before_optimizer_step                                                                                                                   \t|  1.4034e-06     \t|  100            \t|  0.00014034     \t|  3.3332e-05     \t|\n",
      "|  [Callback]ModelSummary.on_train_end                                                                                                                                  \t|  0.00011171     \t|  1              \t|  0.00011171     \t|  2.6531e-05     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_after_backward        \t|  1.0421e-06     \t|  100            \t|  0.00010421     \t|  2.475e-05      \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_after_batch_transfer                                                                                                      \t|  9.0136e-07     \t|  111            \t|  0.00010005     \t|  2.3763e-05     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_train_batch_start                                                                                                         \t|  9.2195e-07     \t|  100            \t|  9.2195e-05     \t|  2.1897e-05     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_after_backward                                                                                                            \t|  8.4152e-07     \t|  100            \t|  8.4152e-05     \t|  1.9987e-05     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_train_batch_end                                                                                                           \t|  7.3438e-07     \t|  100            \t|  7.3438e-05     \t|  1.7442e-05     \t|\n",
      "|  [Strategy]SingleDeviceStrategy.on_train_batch_start                                                                                                                  \t|  7.0121e-07     \t|  100            \t|  7.0121e-05     \t|  1.6654e-05     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_before_zero_grad                                                                                                          \t|  6.798e-07      \t|  100            \t|  6.798e-05      \t|  1.6146e-05     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_before_backward                                                                                                           \t|  6.0424e-07     \t|  100            \t|  6.0424e-05     \t|  1.4351e-05     \t|\n",
      "|  [Callback]ModelSummary.on_after_backward                                                                                                                             \t|  5.9808e-07     \t|  100            \t|  5.9808e-05     \t|  1.4205e-05     \t|\n",
      "|  [Callback]ModelSummary.on_train_batch_start                                                                                                                          \t|  5.7973e-07     \t|  100            \t|  5.7973e-05     \t|  1.3769e-05     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_before_optimizer_step                                                                                                     \t|  5.6231e-07     \t|  100            \t|  5.6231e-05     \t|  1.3355e-05     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_train_batch_start     \t|  5.5563e-07     \t|  100            \t|  5.5563e-05     \t|  1.3197e-05     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_before_zero_grad      \t|  5.3208e-07     \t|  100            \t|  5.3208e-05     \t|  1.2637e-05     \t|\n",
      "|  [Callback]ModelSummary.on_before_zero_grad                                                                                                                           \t|  4.9854e-07     \t|  100            \t|  4.9854e-05     \t|  1.1841e-05     \t|\n",
      "|  [Callback]ModelSummary.on_before_optimizer_step                                                                                                                      \t|  4.856e-07      \t|  100            \t|  4.856e-05      \t|  1.1533e-05     \t|\n",
      "|  [Callback]ModelSummary.on_before_backward                                                                                                                            \t|  4.6754e-07     \t|  100            \t|  4.6754e-05     \t|  1.1104e-05     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_before_optimizer_step \t|  4.2835e-07     \t|  100            \t|  4.2835e-05     \t|  1.0174e-05     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_before_backward       \t|  4.1846e-07     \t|  100            \t|  4.1846e-05     \t|  9.9387e-06     \t|\n",
      "|  [Callback]ModelSummary.on_validation_batch_end                                                                                                                       \t|  2.3475e-06     \t|  11             \t|  2.5823e-05     \t|  6.1331e-06     \t|\n",
      "|  [Callback]ModelSummary.on_validation_end                                                                                                                             \t|  2.2671e-06     \t|  11             \t|  2.4938e-05     \t|  5.9229e-06     \t|\n",
      "|  [Callback]ModelSummary.on_train_epoch_end                                                                                                                            \t|  2.3268e-06     \t|  10             \t|  2.3268e-05     \t|  5.5263e-06     \t|\n",
      "|  [Callback]ModelSummary.on_train_epoch_start                                                                                                                          \t|  1.8564e-06     \t|  10             \t|  1.8564e-05     \t|  4.4091e-06     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_validation_batch_start\t|  1.0778e-06     \t|  11             \t|  1.1856e-05     \t|  2.8159e-06     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_validation_batch_start                                                                                                    \t|  1.0347e-06     \t|  11             \t|  1.1382e-05     \t|  2.7033e-06     \t|\n",
      "|  [Callback]TQDMProgressBar.on_validation_epoch_start                                                                                                                  \t|  9.3491e-07     \t|  11             \t|  1.0284e-05     \t|  2.4425e-06     \t|\n",
      "|  [Callback]TQDMProgressBar.on_save_checkpoint                                                                                                                         \t|  2.479e-06      \t|  4              \t|  9.916e-06      \t|  2.3551e-06     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_validation_batch_end                                                                                                      \t|  8.8764e-07     \t|  11             \t|  9.764e-06      \t|  2.319e-06      \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_validation_end                                                                                                            \t|  8.3009e-07     \t|  11             \t|  9.131e-06      \t|  2.1687e-06     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_validation_batch_end  \t|  7.79e-07       \t|  11             \t|  8.569e-06      \t|  2.0352e-06     \t|\n",
      "|  [Callback]ModelSummary.on_validation_epoch_end                                                                                                                       \t|  7.6545e-07     \t|  11             \t|  8.42e-06       \t|  1.9998e-06     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_train_epoch_start                                                                                                         \t|  7.832e-07      \t|  10             \t|  7.832e-06      \t|  1.8602e-06     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_validation_epoch_start                                                                                                    \t|  7.05e-07       \t|  11             \t|  7.755e-06      \t|  1.8419e-06     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_train_epoch_end                                                                                                           \t|  7.727e-07      \t|  10             \t|  7.727e-06      \t|  1.8352e-06     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_validation_start      \t|  6.9873e-07     \t|  11             \t|  7.686e-06      \t|  1.8255e-06     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_validation_epoch_end                                                                                                      \t|  6.9591e-07     \t|  11             \t|  7.655e-06      \t|  1.8181e-06     \t|\n",
      "|  [Strategy]SingleDeviceStrategy.on_validation_end                                                                                                                     \t|  6.7536e-07     \t|  11             \t|  7.429e-06      \t|  1.7644e-06     \t|\n",
      "|  [Strategy]SingleDeviceStrategy.on_validation_start                                                                                                                   \t|  6.2536e-07     \t|  11             \t|  6.879e-06      \t|  1.6338e-06     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_validation_epoch_end  \t|  6.1418e-07     \t|  11             \t|  6.756e-06      \t|  1.6046e-06     \t|\n",
      "|  [Callback]TQDMProgressBar.setup                                                                                                                                      \t|  6.294e-06      \t|  1              \t|  6.294e-06      \t|  1.4949e-06     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_train_epoch_start     \t|  6.236e-07      \t|  10             \t|  6.236e-06      \t|  1.4811e-06     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_validation_epoch_start\t|  4.7164e-07     \t|  11             \t|  5.188e-06      \t|  1.2322e-06     \t|\n",
      "|  [Callback]ModelSummary.on_validation_epoch_start                                                                                                                     \t|  4.35e-07       \t|  11             \t|  4.785e-06      \t|  1.1365e-06     \t|\n",
      "|  [Callback]TQDMProgressBar.on_sanity_check_end                                                                                                                        \t|  4.517e-06      \t|  1              \t|  4.517e-06      \t|  1.0728e-06     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.configure_callbacks                                                                                                          \t|  3.507e-06      \t|  1              \t|  3.507e-06      \t|  8.3294e-07     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_save_checkpoint       \t|  8.4825e-07     \t|  4              \t|  3.393e-06      \t|  8.0586e-07     \t|\n",
      "|  [Callback]ModelSummary.on_sanity_check_start                                                                                                                         \t|  3.392e-06      \t|  1              \t|  3.392e-06      \t|  8.0562e-07     \t|\n",
      "|  [Callback]ModelSummary.on_save_checkpoint                                                                                                                            \t|  7.1075e-07     \t|  4              \t|  2.843e-06      \t|  6.7523e-07     \t|\n",
      "|  [Callback]ModelSummary.setup                                                                                                                                         \t|  2.365e-06      \t|  1              \t|  2.365e-06      \t|  5.617e-07      \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_save_checkpoint                                                                                                           \t|  5.5225e-07     \t|  4              \t|  2.209e-06      \t|  5.2465e-07     \t|\n",
      "|  [Callback]ModelSummary.on_train_start                                                                                                                                \t|  2.151e-06      \t|  1              \t|  2.151e-06      \t|  5.1088e-07     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_train_start           \t|  1.972e-06      \t|  1              \t|  1.972e-06      \t|  4.6836e-07     \t|\n",
      "|  [Callback]TQDMProgressBar.on_fit_start                                                                                                                               \t|  1.876e-06      \t|  1              \t|  1.876e-06      \t|  4.4556e-07     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_fit_start             \t|  1.865e-06      \t|  1              \t|  1.865e-06      \t|  4.4295e-07     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_fit_start                                                                                                                 \t|  1.832e-06      \t|  1              \t|  1.832e-06      \t|  4.3511e-07     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.setup                                                                                                                        \t|  1.544e-06      \t|  1              \t|  1.544e-06      \t|  3.6671e-07     \t|\n",
      "|  [Callback]ModelSummary.on_sanity_check_end                                                                                                                           \t|  1.307e-06      \t|  1              \t|  1.307e-06      \t|  3.1042e-07     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_train_start                                                                                                               \t|  9.57e-07       \t|  1              \t|  9.57e-07       \t|  2.2729e-07     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.teardown                                                                                                                     \t|  9.38e-07       \t|  1              \t|  9.38e-07       \t|  2.2278e-07     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_train_end                                                                                                                 \t|  9.06e-07       \t|  1              \t|  9.06e-07       \t|  2.1518e-07     \t|\n",
      "|  [Callback]ModelSummary.teardown                                                                                                                                      \t|  8.04e-07       \t|  1              \t|  8.04e-07       \t|  1.9096e-07     \t|\n",
      "|  [LightningModule]SequenceToTwoLightning.on_fit_end                                                                                                                   \t|  7.96e-07       \t|  1              \t|  7.96e-07       \t|  1.8906e-07     \t|\n",
      "|  [Strategy]SingleDeviceStrategy.on_train_end                                                                                                                          \t|  7.71e-07       \t|  1              \t|  7.71e-07       \t|  1.8312e-07     \t|\n",
      "|  [Strategy]SingleDeviceStrategy.on_train_start                                                                                                                        \t|  6.99e-07       \t|  1              \t|  6.99e-07       \t|  1.6602e-07     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_sanity_check_start    \t|  6.74e-07       \t|  1              \t|  6.74e-07       \t|  1.6008e-07     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_train_end             \t|  6.37e-07       \t|  1              \t|  6.37e-07       \t|  1.5129e-07     \t|\n",
      "|  [Callback]ModelSummary.on_fit_end                                                                                                                                    \t|  5.38e-07       \t|  1              \t|  5.38e-07       \t|  1.2778e-07     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.teardown                 \t|  5.17e-07       \t|  1              \t|  5.17e-07       \t|  1.2279e-07     \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_fit_end               \t|  3.77e-07       \t|  1              \t|  3.77e-07       \t|  8.954e-08      \t|\n",
      "|  [Callback]ModelCheckpoint{'monitor': 'val_loss', 'mode': 'min', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None}.on_sanity_check_end      \t|  3.19e-07       \t|  1              \t|  3.19e-07       \t|  7.5765e-08     \t|\n",
      "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pytorch_lightning.callbacks import ModelCheckpoint\n",
    "\n",
    "model = SequenceToTwoLightning()\n",
    "\n",
    "# Definir callbacks, por exemplo, ModelCheckpoint\n",
    "checkpoint_callback = ModelCheckpoint(\n",
    "    monitor='val_loss',\n",
    "    dirpath='checkpoints',\n",
    "    filename='best-checkpoint',\n",
    "    save_top_k=1,\n",
    "    mode='min'\n",
    ")\n",
    "\n",
    "# Instanciar o Trainer\n",
    "trainer = pl.Trainer(\n",
    "    max_epochs=10,\n",
    "    precision=16,                            # Usa precisão de 16 bits se desejado\n",
    "    accelerator='auto',                      \n",
    "    devices=1 if torch.cuda.is_available() else None,\n",
    "    callbacks=[checkpoint_callback],\n",
    "    accumulate_grad_batches=1,               # Ajuste se usar acumulação de gradientes\n",
    "    log_every_n_steps=50,                    # Ajuste a frequência de log\n",
    "    deterministic=True,                       # Para reprodutibilidade\n",
    "    limit_train_batches=10,\n",
    "    profiler=\"simple\",\n",
    "    logger=False\n",
    ")\n",
    "\n",
    "# Iniciar o treinamento\n",
    "trainer.fit(\n",
    "    model,\n",
    "    train_dataloaders=train_dataloader, #IterableDataset\n",
    "    val_dataloaders=validation_dataloader #Dataset\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Usando dispositivo: cuda\n",
      "\n",
      "Epoch 1/10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Train: 0it [05:23, ?it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[11], line 81\u001b[0m\n\u001b[1;32m     78\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m epoch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m1\u001b[39m, num_epochs \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1\u001b[39m):\n\u001b[1;32m     79\u001b[0m \t\u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mEpoch \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mepoch\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mnum_epochs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 81\u001b[0m \ttrain_loss \u001b[38;5;241m=\u001b[39m \u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_dataloader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moptimizer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcriterion\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     82\u001b[0m \t\u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPerda de Treinamento: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtrain_loss\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.4f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     84\u001b[0m \tval_loss \u001b[38;5;241m=\u001b[39m validate(model, device, validation_dataloader, criterion)\n",
      "Cell \u001b[0;32mIn[11], line 19\u001b[0m, in \u001b[0;36mtrain\u001b[0;34m(model, device, train_loader, optimizer, criterion)\u001b[0m\n\u001b[1;32m     17\u001b[0m model\u001b[38;5;241m.\u001b[39mtrain()\n\u001b[1;32m     18\u001b[0m running_loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0.0\u001b[39m\n\u001b[0;32m---> 19\u001b[0m \u001b[43m\u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mbatch_idx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43membeddings\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlengths\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtargets\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43menumerate\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mtqdm\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_loader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdesc\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mTrain\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n\u001b[1;32m     20\u001b[0m \u001b[43m    \u001b[49m\u001b[43membeddings\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43membeddings\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# (batch_size, 2, seq_len, vector_size)\u001b[39;49;00m\n\u001b[1;32m     21\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlengths\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mlengths\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# (batch_size, 2)\u001b[39;49;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.12/site-packages/tqdm/std.py:1181\u001b[0m, in \u001b[0;36mtqdm.__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1178\u001b[0m time \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_time\n\u001b[1;32m   1180\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1181\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mobj\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43miterable\u001b[49m\u001b[43m:\u001b[49m\n\u001b[1;32m   1182\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43;01myield\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mobj\u001b[49m\n\u001b[1;32m   1183\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;66;43;03m# Update and possibly print the progressbar.\u001b[39;49;00m\n\u001b[1;32m   1184\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;66;43;03m# Note: does not call self.update(1) for speed optimisation.\u001b[39;49;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.12/site-packages/torch/utils/data/dataloader.py:701\u001b[0m, in \u001b[0;36m_BaseDataLoaderIter.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    698\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sampler_iter \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    699\u001b[0m     \u001b[38;5;66;03m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[39;00m\n\u001b[1;32m    700\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reset()  \u001b[38;5;66;03m# type: ignore[call-arg]\u001b[39;00m\n\u001b[0;32m--> 701\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_next_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    702\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m    703\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m    704\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dataset_kind \u001b[38;5;241m==\u001b[39m _DatasetKind\u001b[38;5;241m.\u001b[39mIterable\n\u001b[1;32m    705\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    706\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m>\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called\n\u001b[1;32m    707\u001b[0m ):\n",
      "File \u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.12/site-packages/torch/utils/data/dataloader.py:1448\u001b[0m, in \u001b[0;36m_MultiProcessingDataLoaderIter._next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1445\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_process_data(data)\n\u001b[1;32m   1447\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_shutdown \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tasks_outstanding \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[0;32m-> 1448\u001b[0m idx, data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1449\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tasks_outstanding \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m   1450\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dataset_kind \u001b[38;5;241m==\u001b[39m _DatasetKind\u001b[38;5;241m.\u001b[39mIterable:\n\u001b[1;32m   1451\u001b[0m     \u001b[38;5;66;03m# Check for _IterableDatasetStopIteration\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.12/site-packages/torch/utils/data/dataloader.py:1402\u001b[0m, in \u001b[0;36m_MultiProcessingDataLoaderIter._get_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1400\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory:\n\u001b[1;32m   1401\u001b[0m     \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory_thread\u001b[38;5;241m.\u001b[39mis_alive():\n\u001b[0;32m-> 1402\u001b[0m         success, data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_try_get_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1403\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m success:\n\u001b[1;32m   1404\u001b[0m             \u001b[38;5;28;01mreturn\u001b[39;00m data\n",
      "File \u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.12/site-packages/torch/utils/data/dataloader.py:1243\u001b[0m, in \u001b[0;36m_MultiProcessingDataLoaderIter._try_get_data\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1230\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_try_get_data\u001b[39m(\u001b[38;5;28mself\u001b[39m, timeout\u001b[38;5;241m=\u001b[39m_utils\u001b[38;5;241m.\u001b[39mMP_STATUS_CHECK_INTERVAL):\n\u001b[1;32m   1231\u001b[0m     \u001b[38;5;66;03m# Tries to fetch data from `self._data_queue` once for a given timeout.\u001b[39;00m\n\u001b[1;32m   1232\u001b[0m     \u001b[38;5;66;03m# This can also be used as inner loop of fetching without timeout, with\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1240\u001b[0m     \u001b[38;5;66;03m# Returns a 2-tuple:\u001b[39;00m\n\u001b[1;32m   1241\u001b[0m     \u001b[38;5;66;03m#   (bool: whether successfully get data, any: data if successful else None)\u001b[39;00m\n\u001b[1;32m   1242\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1243\u001b[0m         data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_data_queue\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1244\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m (\u001b[38;5;28;01mTrue\u001b[39;00m, data)\n\u001b[1;32m   1245\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m   1246\u001b[0m         \u001b[38;5;66;03m# At timeout and error, we manually check whether any worker has\u001b[39;00m\n\u001b[1;32m   1247\u001b[0m         \u001b[38;5;66;03m# failed. Note that this is the only mechanism for Windows to detect\u001b[39;00m\n\u001b[1;32m   1248\u001b[0m         \u001b[38;5;66;03m# worker failures.\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.12/queue.py:180\u001b[0m, in \u001b[0;36mQueue.get\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m    178\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m remaining \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0.0\u001b[39m:\n\u001b[1;32m    179\u001b[0m             \u001b[38;5;28;01mraise\u001b[39;00m Empty\n\u001b[0;32m--> 180\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnot_empty\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43mremaining\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    181\u001b[0m item \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get()\n\u001b[1;32m    182\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnot_full\u001b[38;5;241m.\u001b[39mnotify()\n",
      "File \u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.12/threading.py:359\u001b[0m, in \u001b[0;36mCondition.wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    357\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    358\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m--> 359\u001b[0m         gotit \u001b[38;5;241m=\u001b[39m \u001b[43mwaiter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43macquire\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    360\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    361\u001b[0m         gotit \u001b[38;5;241m=\u001b[39m waiter\u001b[38;5;241m.\u001b[39macquire(\u001b[38;5;28;01mFalse\u001b[39;00m)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import os\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Função para Salvar Checkpoints\n",
    "def save_checkpoint(state, is_best, checkpoint_dir='checkpoints', filename='checkpoint.pth'):\n",
    "    os.makedirs(checkpoint_dir, exist_ok=True)\n",
    "    filepath = os.path.join(checkpoint_dir, filename)\n",
    "    torch.save(state, filepath)\n",
    "    if is_best:\n",
    "        best_filepath = os.path.join(checkpoint_dir, 'best_checkpoint.pth')\n",
    "        torch.save(state, best_filepath)\n",
    "\n",
    "# Função de Treinamento\n",
    "def train(model, device, train_loader, optimizer, criterion):\n",
    "    model.train()\n",
    "    running_loss = 0.0\n",
    "    for batch_idx, (embeddings, lengths, targets) in enumerate(tqdm(train_loader, desc=\"Train\")):\n",
    "        embeddings = embeddings.to(device) # (batch_size, 2, seq_len, vector_size)\n",
    "        lengths = lengths.to(device) # (batch_size, 2)\n",
    "        targets = targets.to(device) # (batch_size,)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        outputs = model(embeddings, lengths)          # (batch_size,)\n",
    "        loss = criterion(outputs, targets)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item() * embeddings.size(0)\n",
    "\n",
    "    epoch_loss = running_loss / len(train_loader.dataset)\n",
    "    return epoch_loss\n",
    "\n",
    "# Função de Validação\n",
    "def validate(model, device, val_loader, criterion):\n",
    "    model.eval()\n",
    "    running_loss = 0.0\n",
    "    with torch.no_grad():\n",
    "        for batch_idx, (embeddings, lengths, targets) in enumerate(tqdm(val_loader, desc=\"Validation\")):\n",
    "            embeddings = embeddings.to(device)\n",
    "            lengths = lengths.to(device)\n",
    "            targets = targets.to(device)\n",
    "\n",
    "            outputs = model(embeddings, lengths)\n",
    "            loss = criterion(outputs, targets)\n",
    "\n",
    "            running_loss += loss.item() * embeddings.size(0)\n",
    "\n",
    "    epoch_loss = running_loss / len(val_loader.dataset)\n",
    "    return epoch_loss\n",
    "\n",
    "\n",
    "# Configurações\n",
    "input_size = 2048\n",
    "hidden_size = 128\n",
    "num_layers = 3\n",
    "lr = 1e-3\n",
    "batch_size = 32\n",
    "num_epochs = 10\n",
    "checkpoint_dir = 'checkpoints'\n",
    "best_val_loss = float('inf')\n",
    "\n",
    "# Detectar dispositivo\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(f\"Usando dispositivo: {device}\")\n",
    "\n",
    "# Instanciar o modelo, mover para o dispositivo\n",
    "model = SequenceToTwoModel(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers)\n",
    "model.to(device)\n",
    "\n",
    "# Definir a função de perda e o otimizador\n",
    "criterion = nn.HuberLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "# Treinamento\n",
    "for epoch in range(1, num_epochs + 1):\n",
    "\tprint(f\"\\nEpoch {epoch}/{num_epochs}\")\n",
    "\n",
    "\ttrain_loss = train(model, device, train_dataloader, optimizer, criterion)\n",
    "\tprint(f\"Perda de Treinamento: {train_loss:.4f}\")\n",
    "\n",
    "\tval_loss = validate(model, device, validation_dataloader, criterion)\n",
    "\tprint(f\"Perda de Validação: {val_loss:.4f}\")\n",
    "\n",
    "\t# Checkpointing\n",
    "\tis_best = val_loss < best_val_loss\n",
    "\tif is_best:\n",
    "\t\tbest_val_loss = val_loss\n",
    "\t\tprint(f\"Nova melhor perda de validação: {best_val_loss:.4f}. Salvando modelo...\")\n",
    "\t\tsave_checkpoint({\n",
    "\t\t\t'epoch': epoch,\n",
    "\t\t\t'model_state_dict': model.state_dict(),\n",
    "\t\t\t'optimizer_state_dict': optimizer.state_dict(),\n",
    "\t\t\t'val_loss': val_loss,\n",
    "\t\t}, is_best, checkpoint_dir=checkpoint_dir)\n",
    "\n",
    "\t# (Opcional) Implementar lógica de early stopping, redução de lr, etc.\n",
    "\t# Exemplo de scheduler para reduzir a taxa de aprendizado se a validação não melhorar\n",
    "\t# scheduler.step(val_loss)\n",
    "\n",
    "print(\"Treinamento concluído.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# Diretório onde os modelos serão salvos\n",
    "save_dir = './saved_fineweb2'\n",
    "os.makedirs(save_dir, exist_ok=True)\n",
    "\n",
    "# Salvando o estado do modelo\n",
    "model_path = os.path.join(save_dir, 'regression_rnn_model.pth')\n",
    "torch.save(model_order.state_dict(), model_path)\n",
    "print(f\"Modelo salvo em {model_path}\")\n",
    "\n",
    "# Opcional: Salvando o estado do otimizador\n",
    "optimizer_path = os.path.join(save_dir, 'optimizer_state.pth')\n",
    "torch.save(optimizer.state_dict(), optimizer_path)\n",
    "print(f\"Estado do otimizador salvo em {optimizer_path}\")\n",
    "\n",
    "# Opcional: Salvar a época atual para retomar o treinamento\n",
    "epoch_path = os.path.join(save_dir, 'last_epoch.pth')\n",
    "torch.save({'epoch': epoch, 'model_state_dict': model_order.state_dict(),\n",
    "            'optimizer_state_dict': optimizer.state_dict(),\n",
    "            'loss': avg_validation_loss}, epoch_path)\n",
    "print(f\"Checkpoint salvo em {epoch_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "model_path = os.path.join('./saved_fineweb', 'regression_rnn_model.pth')\n",
    "\n",
    "# Carregar o estado do modelo\n",
    "model_order.load_state_dict(torch.load(model_path, map_location='cuda'))\n",
    "\n",
    "# Colocar o modelo em modo de avaliação\n",
    "#model_order.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation_dataloader torch.Size([150, 2, 61, 2048]) torch.Size([150, 2]) torch.Size([150])\n"
     ]
    }
   ],
   "source": [
    "#  Converter DataLoader para iterador\n",
    "data_iter = iter(validation_dataloader)\n",
    "# Pegar o primeiro lote\n",
    "embeddings, lengths, targets = next(data_iter)\n",
    "print(\"validation_dataloader\",embeddings.shape,lengths.shape,targets.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.3333)\n",
      "Resultado da previsão para o par 1: -5.01230525970459\n",
      "Resultado da previsão para o par 2: -4.987059593200684\n",
      "Resultado da previsão para o par 3: -5.013754367828369\n",
      "Resultado da previsão para o par 4: -5.006444931030273\n",
      "Resultado da previsão para o par 5: -4.987059593200684\n",
      "Resultado da previsão para o par 6: -4.987059593200684\n",
      "Resultado da previsão para o par 7: -4.957428455352783\n",
      "Resultado da previsão para o par 8: -5.010082244873047\n",
      "Resultado da previsão para o par 9: -4.957428455352783\n",
      "Resultado da previsão para o par 10: -4.999967575073242\n",
      "Resultado da previsão para o par 11: -4.999967575073242\n",
      "Resultado da previsão para o par 12: -5.010082244873047\n",
      "Resultado da previsão para o par 13: -5.013754367828369\n",
      "Resultado da previsão para o par 14: -4.878176689147949\n",
      "Resultado da previsão para o par 15: -4.999967575073242\n",
      "Resultado da previsão para o par 16: -5.006444931030273\n",
      "Resultado da previsão para o par 17: -5.010082244873047\n",
      "Resultado da previsão para o par 18: -5.01230525970459\n",
      "Resultado da previsão para o par 19: -5.006444931030273\n",
      "Resultado da previsão para o par 20: -5.006444931030273\n",
      "Resultado da previsão para o par 21: -4.999967575073242\n",
      "Resultado da previsão para o par 22: -5.01230525970459\n",
      "Resultado da previsão para o par 23: -5.006444931030273\n",
      "Resultado da previsão para o par 24: -5.01230525970459\n",
      "Resultado da previsão para o par 25: -5.006444931030273\n",
      "Resultado da previsão para o par 26: -5.010082244873047\n",
      "Resultado da previsão para o par 27: -5.013754367828369\n",
      "Resultado da previsão para o par 28: -5.015959739685059\n",
      "Resultado da previsão para o par 29: -5.014744758605957\n",
      "Resultado da previsão para o par 30: -5.010082244873047\n",
      "Resultado da previsão para o par 31: -5.003296852111816\n",
      "Resultado da previsão para o par 32: -4.5983991622924805\n",
      "Resultado da previsão para o par 33: -4.514101982116699\n",
      "Resultado da previsão para o par 34: -4.508218765258789\n",
      "Resultado da previsão para o par 35: -4.511448860168457\n",
      "Resultado da previsão para o par 36: -5.016793727874756\n",
      "Resultado da previsão para o par 37: -4.508395195007324\n",
      "Resultado da previsão para o par 38: -4.500512599945068\n",
      "Resultado da previsão para o par 39: -4.503727436065674\n",
      "Resultado da previsão para o par 40: -5.01752233505249\n",
      "Resultado da previsão para o par 41: -5.01230525970459\n",
      "Resultado da previsão para o par 42: -4.508588790893555\n",
      "Resultado da previsão para o par 43: -5.01230525970459\n",
      "Resultado da previsão para o par 44: -4.9268035888671875\n",
      "Resultado da previsão para o par 45: -5.014744758605957\n",
      "Resultado da previsão para o par 46: -4.552577972412109\n",
      "Resultado da previsão para o par 47: -4.999967575073242\n",
      "Resultado da previsão para o par 48: -5.01752233505249\n",
      "Resultado da previsão para o par 49: -5.017027854919434\n",
      "Resultado da previsão para o par 50: -4.995327949523926\n",
      "Resultado da previsão para o par 51: -5.009238243103027\n",
      "Resultado da previsão para o par 52: -4.987059593200684\n",
      "Resultado da previsão para o par 53: -5.058990478515625\n",
      "Resultado da previsão para o par 54: -5.006444931030273\n",
      "Resultado da previsão para o par 55: -4.962080001831055\n",
      "Resultado da previsão para o par 56: -4.9636640548706055\n",
      "Resultado da previsão para o par 57: -4.915415287017822\n",
      "Resultado da previsão para o par 58: -5.008018493652344\n",
      "Resultado da previsão para o par 59: -4.911957740783691\n",
      "Resultado da previsão para o par 60: -4.994540691375732\n",
      "Resultado da previsão para o par 61: -4.98786735534668\n",
      "Resultado da previsão para o par 62: -5.004065036773682\n",
      "Resultado da previsão para o par 63: -5.013754367828369\n",
      "Resultado da previsão para o par 64: -4.816554069519043\n",
      "Resultado da previsão para o par 65: -4.989260673522949\n",
      "Resultado da previsão para o par 66: -4.998097896575928\n",
      "Resultado da previsão para o par 67: -5.008018493652344\n",
      "Resultado da previsão para o par 68: -5.009238243103027\n",
      "Resultado da previsão para o par 69: -5.00116491317749\n",
      "Resultado da previsão para o par 70: -4.9997711181640625\n",
      "Resultado da previsão para o par 71: -4.985684871673584\n",
      "Resultado da previsão para o par 72: -5.009936332702637\n",
      "Resultado da previsão para o par 73: -4.998795509338379\n",
      "Resultado da previsão para o par 74: -5.009238243103027\n",
      "Resultado da previsão para o par 75: -4.998097896575928\n",
      "Resultado da previsão para o par 76: -5.006624221801758\n",
      "Resultado da previsão para o par 77: -5.01053524017334\n",
      "Resultado da previsão para o par 78: -5.014922618865967\n",
      "Resultado da previsão para o par 79: -5.014744758605957\n",
      "Resultado da previsão para o par 80: -5.029044151306152\n",
      "Resultado da previsão para o par 81: -2.492607355117798\n",
      "Resultado da previsão para o par 82: -6.832784652709961\n",
      "Resultado da previsão para o par 83: -4.722887992858887\n",
      "Resultado da previsão para o par 84: -6.5199432373046875\n",
      "Resultado da previsão para o par 85: -4.591699600219727\n",
      "Resultado da previsão para o par 86: -4.048611640930176\n",
      "Resultado da previsão para o par 87: -7.069779872894287\n",
      "Resultado da previsão para o par 88: -4.815916061401367\n",
      "Resultado da previsão para o par 89: -6.407536506652832\n",
      "Resultado da previsão para o par 90: -5.017491340637207\n",
      "Resultado da previsão para o par 91: -5.014368534088135\n",
      "Resultado da previsão para o par 92: -4.338381767272949\n",
      "Resultado da previsão para o par 93: -5.01230525970459\n",
      "Resultado da previsão para o par 94: -5.600826740264893\n",
      "Resultado da previsão para o par 95: -5.012055397033691\n",
      "Resultado da previsão para o par 96: -6.668403148651123\n",
      "Resultado da previsão para o par 97: -4.985684871673584\n",
      "Resultado da previsão para o par 98: -4.889004707336426\n",
      "Resultado da previsão para o par 99: -5.016901969909668\n",
      "Resultado da previsão para o par 100: -5.045838356018066\n",
      "Resultado da previsão para o par 101: -5.018514156341553\n",
      "Resultado da previsão para o par 102: -4.987059593200684\n",
      "Resultado da previsão para o par 103: -4.911957740783691\n",
      "Resultado da previsão para o par 104: -5.006444931030273\n",
      "Resultado da previsão para o par 105: -5.041321277618408\n",
      "Resultado da previsão para o par 106: -5.038139820098877\n",
      "Resultado da previsão para o par 107: -5.051929950714111\n",
      "Resultado da previsão para o par 108: -5.014368534088135\n",
      "Resultado da previsão para o par 109: -5.058990478515625\n",
      "Resultado da previsão para o par 110: -5.011871814727783\n",
      "Resultado da previsão para o par 111: -5.025855541229248\n",
      "Resultado da previsão para o par 112: -5.0223588943481445\n",
      "Resultado da previsão para o par 113: -5.013754367828369\n",
      "Resultado da previsão para o par 114: -5.018973350524902\n",
      "Resultado da previsão para o par 115: -5.023012161254883\n",
      "Resultado da previsão para o par 116: -5.023794174194336\n",
      "Resultado da previsão para o par 117: -5.014368534088135\n",
      "Resultado da previsão para o par 118: -5.018514156341553\n",
      "Resultado da previsão para o par 119: -5.017584800720215\n",
      "Resultado da previsão para o par 120: -5.020427703857422\n",
      "Resultado da previsão para o par 121: -5.030243396759033\n",
      "Resultado da previsão para o par 122: -5.01711368560791\n",
      "Resultado da previsão para o par 123: -5.022393226623535\n",
      "Resultado da previsão para o par 124: -5.018514156341553\n",
      "Resultado da previsão para o par 125: -5.023794174194336\n",
      "Resultado da previsão para o par 126: -5.0172119140625\n",
      "Resultado da previsão para o par 127: -5.020246982574463\n",
      "Resultado da previsão para o par 128: -5.018064498901367\n",
      "Resultado da previsão para o par 129: -5.014744758605957\n",
      "Resultado da previsão para o par 130: -4.968096733093262\n",
      "Resultado da previsão para o par 131: -6.814906120300293\n",
      "Resultado da previsão para o par 132: -2.4978058338165283\n",
      "Resultado da previsão para o par 133: -4.339602470397949\n",
      "Resultado da previsão para o par 134: -2.7016990184783936\n",
      "Resultado da previsão para o par 135: -4.445878982543945\n",
      "Resultado da previsão para o par 136: -5.812942981719971\n",
      "Resultado da previsão para o par 137: -2.0306882858276367\n",
      "Resultado da previsão para o par 138: -4.225633144378662\n",
      "Resultado da previsão para o par 139: -2.803224563598633\n",
      "Resultado da previsão para o par 140: -5.017587661743164\n",
      "Resultado da previsão para o par 141: -5.008018493652344\n",
      "Resultado da previsão para o par 142: -4.669567584991455\n",
      "Resultado da previsão para o par 143: -5.01230525970459\n",
      "Resultado da previsão para o par 144: -4.331756114959717\n",
      "Resultado da previsão para o par 145: -5.020211219787598\n",
      "Resultado da previsão para o par 146: -2.6382057666778564\n",
      "Resultado da previsão para o par 147: -5.030243396759033\n",
      "Resultado da previsão para o par 148: -5.135861396789551\n",
      "Resultado da previsão para o par 149: -5.017289638519287\n",
      "Resultado da previsão para o par 150: -4.95112419128418\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "\n",
    "#embeddings = embeddings.to('cuda')\n",
    "#lengths = lengths.to('cuda')\n",
    "#targets = targets.to('cuda')\n",
    "attention_mask2\n",
    "# Realizar a previsão\n",
    "with torch.no_grad():\n",
    "\toutput = model(embeddings, lengths)\n",
    "\t\n",
    "matches = (\n",
    "\t((targets == -1) & (output < -1.71)) |  # Regra para -1\n",
    "\t((targets == 0) & ((-1.71 < output) & (output < 1.71))) |  # Regra para 0\n",
    "\t((targets == 1) & (1.71 < output))     # Regra para 1\n",
    ")\n",
    "\n",
    "print(matches.float().mean())\n",
    "\t\n",
    "# Imprimir os resultados da previsão\n",
    "for i, pred in enumerate(output):\n",
    "\t\tprint(f\"Resultado da previsão para o par {i+1}: {pred.item()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Similaridade Inversa da Diferença Absoluta\n",
      "Correlação de Pearson: -0.0324\n",
      "Correlação de Spearman: 0.0067\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from scipy.stats import pearsonr, spearmanr\n",
    "import torch\n",
    "\n",
    "# Carregar o conjunto de dados de avaliação (STS Benchmark)\n",
    "eval_dataset = load_dataset(\"sentence-transformers/stsb\", split=\"validation\")\n",
    "\n",
    "# Listas para armazenar as pontuações reais e as previsões do modelo\n",
    "true_scores = []\n",
    "pred_scores = []\n",
    "\n",
    "# Iterar sobre os exemplos no conjunto de dados\n",
    "for example in eval_dataset:\n",
    "    # Tokenização combinada\n",
    "    encoding = embedding_generator.tokenizer(\n",
    "        [example['sentence1'], example['sentence2']],\n",
    "        return_tensors=\"pt\",\n",
    "        padding=True,  # Padding dinâmico\n",
    "        truncation=True\n",
    "    )\n",
    "\n",
    "    # Converter para o dispositivo\n",
    "    input_ids1 = encoding['input_ids'][0]\n",
    "    attention_mask1 = encoding['attention_mask'][0]\n",
    "    input_ids2 = encoding['input_ids'][1]\n",
    "    attention_mask2 = encoding['attention_mask'][1]\n",
    "\n",
    "    # Obter embeddings\n",
    "    embeddings1 = embedding_generator.weights[input_ids1]\n",
    "    embeddings2 = embedding_generator.weights[input_ids2]        \n",
    "    \n",
    "    # Concatenar as duas sequências no eixo 1\n",
    "    embeddings = torch.stack([embeddings1, embeddings2]).float().unsqueeze(0)  # (batch_size, 2, seq_length, vector_size)\n",
    "    \n",
    "    # Calcular comprimentos\n",
    "    lengths1 = attention_mask1.sum(dim=0)\n",
    "    lengths2 = attention_mask2.sum(dim=0)\n",
    "    lengths = torch.stack([lengths1, lengths2]).float().unsqueeze(0)  # (batch_size, 2)\n",
    "\n",
    "    # Converter targets\n",
    "    score = torch.tensor(example['score'], dtype=torch.float16).unsqueeze(0)  # (batch_size,)\n",
    "\n",
    "    # Obter a previsão do modelo\n",
    "    prediction = model(embeddings, lengths)\n",
    "    pred_score = 1 / (1 + abs(prediction))\n",
    "\n",
    "    # Armazenar as pontuações\n",
    "    true_scores.append(score.item())\n",
    "    pred_scores.append(pred_score.item())\n",
    "\n",
    "# Calcular as métricas de correlação\n",
    "pearson_corr, _ = pearsonr(true_scores, pred_scores)\n",
    "spearman_corr, _ = spearmanr(true_scores, pred_scores)\n",
    "\n",
    "print(\"Similaridade Inversa da Diferença Absoluta\")\n",
    "print(f\"Correlação de Pearson: {pearson_corr:.4f}\")\n",
    "print(f\"Correlação de Spearman: {spearman_corr:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processando exemplos: 100%|██████████| 1500/1500 [00:08<00:00, 174.15it/s]\n",
      "Buscando o melhor k:   0%|          | 0/10000 [00:00<?, ?it/s]/tmp/ipykernel_52757/2183562453.py:64: ConstantInputWarning: An input array is constant; the correlation coefficient is not defined.\n",
      "  pearson_corr, _ = pearsonr(true_scores, pred_scores)\n",
      "Buscando o melhor k:   0%|          | 0/10000 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[71], line 68\u001b[0m\n\u001b[1;32m     65\u001b[0m spearman_corr, _ \u001b[38;5;241m=\u001b[39m spearmanr(true_scores, pred_scores)\n\u001b[1;32m     67\u001b[0m \u001b[38;5;66;03m# Verificar se este k é o melhor até agora\u001b[39;00m\n\u001b[0;32m---> 68\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m pearson_corr \u001b[38;5;241m>\u001b[39m best_pearson:\n\u001b[1;32m     69\u001b[0m     best_pearson \u001b[38;5;241m=\u001b[39m pearson_corr\n\u001b[1;32m     70\u001b[0m     best_spearman \u001b[38;5;241m=\u001b[39m spearman_corr\n",
      "\u001b[0;31mValueError\u001b[0m: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from datasets import load_dataset\n",
    "from scipy.stats import pearsonr, spearmanr\n",
    "from tqdm import tqdm  # Para acompanhar o progresso\n",
    "\n",
    "# Carregar o conjunto de dados de avaliação (STS Benchmark)\n",
    "eval_dataset = load_dataset(\"sentence-transformers/stsb\", split=\"validation\")\n",
    "\n",
    "# Definir o intervalo de valores de k a serem testados\n",
    "k_values = np.arange(0.001, 100.0, 0.01)\n",
    "best_k = None\n",
    "best_pearson = -1  # Inicialização com um valor baixo\n",
    "best_spearman = -1\n",
    "\n",
    "# Pré-computar todas as predições para evitar recalcular múltiplas vezes\n",
    "true_scores = []\n",
    "predictions = []\n",
    "\n",
    "with torch.no_grad():\n",
    "\tfor example in tqdm(eval_dataset, desc=\"Processando exemplos\"):\n",
    "\t\t# Tokenização combinada\n",
    "\t\tencoding = embedding_generator.tokenizer(\n",
    "\t\t\t[example['sentence1'], example['sentence2']],\n",
    "\t\t\treturn_tensors=\"pt\",\n",
    "\t\t\tpadding=True,  # Padding dinâmico\n",
    "\t\t\ttruncation=True\n",
    "\t\t)\n",
    "\n",
    "\t\t# Converter para o dispositivo\n",
    "\t\tinput_ids1 = encoding['input_ids'][0]\n",
    "\t\tattention_mask1 = encoding['attention_mask'][0]\n",
    "\t\tinput_ids2 = encoding['input_ids'][1]\n",
    "\t\tattention_mask2 = encoding['attention_mask'][1]\n",
    "\n",
    "\t\t# Obter embeddings\n",
    "\t\tembeddings1 = embedding_generator.weights[input_ids1]\n",
    "\t\tembeddings2 = embedding_generator.weights[input_ids2]        \n",
    "\t\t\n",
    "\t\t# Concatenar as duas sequências no eixo 1\n",
    "\t\tembeddings = torch.stack([embeddings1, embeddings2]).float().unsqueeze(0)  # (batch_size, 2, seq_length, vector_size)\n",
    "\t\t\n",
    "\t\t# Calcular comprimentos\n",
    "\t\tlengths1 = attention_mask1.sum(dim=0)\n",
    "\t\tlengths2 = attention_mask2.sum(dim=0)\n",
    "\t\tlengths = torch.stack([lengths1, lengths2]).float().unsqueeze(0)  # (batch_size, 2)\n",
    "\n",
    "\t\t# Converter targets\n",
    "\t\tscore = torch.tensor(example['score'], dtype=torch.float16).unsqueeze(0)  # (batch_size,)\n",
    "\n",
    "\t\t# Obter a previsão do modelo\n",
    "\t\tprediction = model(embeddings, lengths)\n",
    "\t\tpredictions.append(prediction.item())\n",
    "\n",
    "\t\t# Armazenar as pontuações reais\n",
    "\t\ttrue_scores.append(score)\n",
    "\n",
    "true_scores = np.array(true_scores)\n",
    "predictions = np.array(predictions)\n",
    "\n",
    "# Iterar sobre os valores de k para encontrar o melhor\n",
    "for k in tqdm(k_values, desc=\"Buscando o melhor k\"):\n",
    "\tpred_scores = np.exp(-k * np.abs(predictions))\n",
    "\t\n",
    "\tpearson_corr, _ = pearsonr(true_scores, pred_scores)\n",
    "\tspearman_corr, _ = spearmanr(true_scores, pred_scores)\n",
    "\t\n",
    "\t# Verificar se este k é o melhor até agora\n",
    "\tif pearson_corr > best_pearson:\n",
    "\t\tbest_pearson = pearson_corr\n",
    "\t\tbest_spearman = spearman_corr\n",
    "\t\tbest_k = k\n",
    "\n",
    "print(\"Similaridade Baseada em Exponencial\")\n",
    "print(f\"Melhor valor de k: {best_k}\")\n",
    "print(f\"Correlação de Pearson: {best_pearson:.4f}\")\n",
    "print(f\"Correlação de Spearman: {best_spearman:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "from sentence_transformers import SentenceTransformer, util\n",
    "from datasets import load_dataset\n",
    "from scipy.stats import pearsonr, spearmanr\n",
    "\n",
    "# Verificar se CUDA está disponível\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "# Carregar o modelo SentenceTransformer\n",
    "model = SentenceTransformer(\"all-mpnet-base-v2\").to(device)\n",
    "\n",
    "# Carregar o conjunto de dados de avaliação (STS Benchmark)\n",
    "eval_dataset = load_dataset(\"sentence-transformers/stsb\", split=\"validation\")\n",
    "\n",
    "# Listas para armazenar as pontuações reais e as previsões do modelo\n",
    "true_scores = []\n",
    "pred_scores = []\n",
    "\n",
    "# Obter as sentenças e os escores do conjunto de dados\n",
    "sentences1 = eval_dataset['sentence1']\n",
    "sentences2 = eval_dataset['sentence2']\n",
    "scores = eval_dataset['score']\n",
    "\n",
    "# Processar uma frase por vez\n",
    "for i in range(len(eval_dataset)):\n",
    "    sentence1 = sentences1[i]\n",
    "    sentence2 = sentences2[i]\n",
    "    true_score = scores[i]\n",
    "\n",
    "    # Gerar embeddings para ambas as sentenças individualmente\n",
    "    embedding1 = model.encode(sentence1, convert_to_tensor=True, device=device, show_progress_bar=False)\n",
    "    embedding2 = model.encode(sentence2, convert_to_tensor=True, device=device, show_progress_bar=False)\n",
    "\n",
    "    # Calcular a similaridade cosseno\n",
    "    cosine_score = util.cos_sim(embedding1, embedding2).item()\n",
    "\n",
    "    # Escalar a similaridade cosseno de [-1, 1] para [0, 1]\n",
    "    scaled_score = (cosine_score + 1) / 2\n",
    "\n",
    "    # Armazenar as pontuações\n",
    "    true_scores.append(true_score)\n",
    "    pred_scores.append(scaled_score)\n",
    "\n",
    "# Calcular as métricas de correlação\n",
    "pearson_corr, _ = pearsonr(true_scores, pred_scores)\n",
    "spearman_corr, _ = spearmanr(true_scores, pred_scores)\n",
    "\n",
    "print(f\"Correlação de Pearson: {pearson_corr:.4f}\")\n",
    "print(f\"Correlação de Spearman: {spearman_corr:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "model_path = os.path.join('./saved_HuberLoss', 'regression_rnn_model.pth')\n",
    "\n",
    "# Carregar o estado do modelo\n",
    "#model_order.load_state_dict(torch.load(model_path, map_location='cuda'))\n",
    "\n",
    "# Colocar o modelo em modo de avaliação\n",
    "model_order.eval()\n",
    "\n",
    "# Realizar a previsão\n",
    "with torch.no_grad():\n",
    "    output = model_order(\n",
    "        [text_to_embedding(\n",
    "            \"O aluno estudou para o exame durante a noite.\").to('cuda')],\n",
    "        [text_to_embedding(\"O resultado foi excelente devido à dedicação.\").to('cuda')])\n",
    "\n",
    "# Imprimir a saída da regressão\n",
    "print(\"Resultado da previsão:\", output.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del model_order  # Remove a referência ao modelo\n",
    "\n",
    "# Libera a memória da GPU, se o modelo estiver usando CUDA\n",
    "torch.cuda.empty_cache()\n",
    "\n",
    "# Força a coleta de lixo\n",
    "gc.collect()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}