ViT - Cifar 100 Training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "22956b3b",
   "metadata": {},
   "source": [
    "## Vision Transformer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8dc42b90",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: http://ftp.daumkakao.com/pypi/simple\r\n",
      "Requirement already satisfied: ml_collections in /home/ubuntu/miniconda3/lib/python3.8/site-packages (0.1.0)\r\n",
      "Requirement already satisfied: six in /home/ubuntu/miniconda3/lib/python3.8/site-packages (from ml_collections) (1.16.0)\r\n",
      "Requirement already satisfied: PyYAML in /home/ubuntu/miniconda3/lib/python3.8/site-packages (from ml_collections) (5.4.1)\r\n",
      "Requirement already satisfied: contextlib2 in /home/ubuntu/miniconda3/lib/python3.8/site-packages (from ml_collections) (21.6.0)\r\n",
      "Requirement already satisfied: absl-py in /home/ubuntu/miniconda3/lib/python3.8/site-packages (from ml_collections) (0.12.0)\r\n"
     ]
    }
   ],
   "source": [
    "!pip install ml_collections"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4b3d874",
   "metadata": {},
   "source": [
    "### Imporing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "f1a21522",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, copy, logging, math, random, time, typing, io, cv2\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import keras\n",
    "import keras.backend as K\n",
    "from keras.datasets import cifar10, cifar100\n",
    "\n",
    "# import albumentations as albu\n",
    "from sklearn.model_selection import train_test_split, StratifiedShuffleSplit\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from scipy import ndimage\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import CrossEntropyLoss, Dropout, Softmax, Linear, Conv2d, LayerNorm\n",
    "from torch.nn.modules.utils import _pair\n",
    "import torchvision\n",
    "from torchvision import transforms, datasets\n",
    "import torch.distributed as dist\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "from torch.utils.data import DataLoader, RandomSampler, DistributedSampler, SequentialSampler\n",
    "from torch.optim.lr_scheduler import LambdaLR\n",
    "import torchvision.models as models\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "from torchvision import utils\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from matplotlib.colors import LogNorm \n",
    "\n",
    "from urllib.request import urlretrieve\n",
    "from PIL import Image\n",
    "from os.path import join as pjoin\n",
    "\n",
    "from tqdm import tqdm\n",
    "from datetime import timedelta\n",
    "from __future__ import absolute_import, division, print_function \n",
    "import ml_collections"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abdfef3e",
   "metadata": {},
   "source": [
    "### Download Pre-trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fe401bad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2021-11-05 17:55:46--  https://storage.googleapis.com/vit_models/imagenet21k/ViT-B_16.npz\n",
      "Resolving storage.googleapis.com (storage.googleapis.com)... 172.217.174.112, 216.58.220.144, 216.58.220.112, ...\n",
      "Connecting to storage.googleapis.com (storage.googleapis.com)|172.217.174.112|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 412815506 (394M) [application/octet-stream]\n",
      "Saving to: ‘ViT-B_16.npz.1’\n",
      "\n",
      "ViT-B_16.npz.1      100%[===================>] 393.69M  22.0MB/s    in 19s     \n",
      "\n",
      "2021-11-05 17:56:07 (20.5 MB/s) - ‘ViT-B_16.npz.1’ saved [412815506/412815506]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://storage.googleapis.com/vit_models/imagenet21k/ViT-B_16.npz"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97cfef44",
   "metadata": {},
   "source": [
    "### Check GPU Access"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fb9921a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU:  Tesla V100-SXM2-32GB\n"
     ]
    }
   ],
   "source": [
    "if torch.cuda.is_available():    \n",
    "    device = torch.device(\"cuda:1\")\n",
    "    print('GPU: ', torch.cuda.get_device_name(0))\n",
    "\n",
    "else:\n",
    "    device = torch.device(\"cpu\")\n",
    "    print('No GPU available')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d448cc58",
   "metadata": {},
   "source": [
    "### Configurations "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "81e085a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_testing():\n",
    "    '''\n",
    "    Returns a minimal configuration for testing\n",
    "    '''\n",
    "    config = ml_collections.ConfigDict()\n",
    "    config.patches = ml_collections.ConfigDict({'size': (16, 16)})\n",
    "    config.hidden_size = 1\n",
    "    config.transformer = ml_collections.ConfigDict()\n",
    "    config.transformer.mlp_dim = 1\n",
    "    config.transformer.num_heads = 1\n",
    "    config.transformer.num_layers = 1\n",
    "    config.transformer.attention_dropout_rate = 0.0\n",
    "    config.transformer.dropout_rate = 0.1\n",
    "    config.classifier = 'token'\n",
    "    config.representation_size = None\n",
    "    return config\n",
    "\n",
    "def get_b16_config():\n",
    "    '''\n",
    "    Returns the ViT-B/16 configuration\n",
    "    '''\n",
    "    config = ml_collections.ConfigDict()\n",
    "    config.patches = ml_collections.ConfigDict({'size': (16, 16)})\n",
    "    config.hidden_size = 768\n",
    "    config.transformer = ml_collections.ConfigDict()\n",
    "    config.transformer.mlp_dim = 3072\n",
    "    config.transformer.num_heads = 12\n",
    "    config.transformer.num_layers = 12\n",
    "    config.transformer.attention_dropout_rate = 0.0\n",
    "    config.transformer.dropout_rate = 0.1\n",
    "    config.classifier = 'token'\n",
    "    config.representation_size = None\n",
    "    return config\n",
    "\n",
    "def get_b32_config():\n",
    "    '''\n",
    "    Returns the ViT-B/32 configuration\n",
    "    '''\n",
    "    config = get_b16_config()\n",
    "    config.patches.size = (32, 32)\n",
    "    return config\n",
    "\n",
    "def get_l16_config():\n",
    "    '''\n",
    "    Returns the ViT-L/16 configuration\n",
    "    '''\n",
    "    config = ml_collections.ConfigDict()\n",
    "    config.patches = ml_collections.ConfigDict({'size': (16, 16)})\n",
    "    config.hidden_size = 1024\n",
    "    config.transformer = ml_collections.ConfigDict()\n",
    "    config.transformer.mlp_dim = 4096\n",
    "    config.transformer.num_heads = 16\n",
    "    config.transformer.num_layers = 24\n",
    "    config.transformer.attention_dropout_rate = 0.0\n",
    "    config.transformer.dropout_rate = 0.1\n",
    "    config.classifier = 'token'\n",
    "    config.representation_size = None\n",
    "    return config\n",
    "\n",
    "def get_l32_config():\n",
    "    '''\n",
    "    Returns the ViT-L/32 configuration\n",
    "    '''\n",
    "    config = get_l16_config()\n",
    "    config.patches.size = (32, 32)\n",
    "    return config\n",
    "\n",
    "def get_h14_config():\n",
    "    '''\n",
    "    Returns the ViT-L/16 configuration\n",
    "    '''\n",
    "    config = ml_collections.ConfigDict()\n",
    "    config.patches = ml_collections.ConfigDict({'size': (14, 14)})\n",
    "    config.hidden_size = 1280\n",
    "    config.transformer = ml_collections.ConfigDict()\n",
    "    config.transformer.mlp_dim = 5120\n",
    "    config.transformer.num_heads = 16\n",
    "    config.transformer.num_layers = 32\n",
    "    config.transformer.attention_dropout_rate = 0.0\n",
    "    config.transformer.dropout_rate = 0.1\n",
    "    config.classifier = 'token'\n",
    "    config.representation_size = None\n",
    "    return config"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6814ebf0",
   "metadata": {},
   "source": [
    "### Model Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "da02fb0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "ATTENTION_Q = \"MultiHeadDotProductAttention_1/query\"\n",
    "ATTENTION_K = \"MultiHeadDotProductAttention_1/key\"\n",
    "ATTENTION_V = \"MultiHeadDotProductAttention_1/value\"\n",
    "ATTENTION_OUT = \"MultiHeadDotProductAttention_1/out\"\n",
    "FC_0 = \"MlpBlock_3/Dense_0\"\n",
    "FC_1 = \"MlpBlock_3/Dense_1\"\n",
    "ATTENTION_NORM = \"LayerNorm_0\"\n",
    "MLP_NORM = \"LayerNorm_2\"\n",
    "\n",
    "\n",
    "def np2th(weights):\n",
    "    \"\"\"Possibly convert HWIO to OIHW.\"\"\"\n",
    "    if weights.ndim == 4:\n",
    "        weights = weights.transpose([3, 2, 0, 1])\n",
    "    return torch.from_numpy(weights)\n",
    "\n",
    "def swish(x):\n",
    "    return x * torch.sigmoid(x)\n",
    "\n",
    "ACT2FN = {\"gelu\": torch.nn.functional.gelu, \"relu\": torch.nn.functional.relu, \"swish\": swish}\n",
    "\n",
    "\n",
    "class Attention(nn.Module):\n",
    "    def __init__(self, config, vis):\n",
    "        super(Attention, self).__init__()\n",
    "        self.vis = vis\n",
    "        self.num_attention_heads = config.transformer[\"num_heads\"]\n",
    "        self.attention_head_size = int(config.hidden_size / self.num_attention_heads)\n",
    "        self.all_head_size = self.num_attention_heads * self.attention_head_size\n",
    "\n",
    "        self.query = Linear(config.hidden_size, self.all_head_size)\n",
    "        self.key = Linear(config.hidden_size, self.all_head_size)\n",
    "        self.value = Linear(config.hidden_size, self.all_head_size)\n",
    "\n",
    "        self.out = Linear(config.hidden_size, config.hidden_size)\n",
    "        self.attn_dropout = Dropout(config.transformer[\"attention_dropout_rate\"])\n",
    "        self.proj_dropout = Dropout(config.transformer[\"attention_dropout_rate\"])\n",
    "\n",
    "        self.softmax = Softmax(dim=-1)\n",
    "\n",
    "    def transpose_for_scores(self, x):\n",
    "        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)\n",
    "        x = x.view(*new_x_shape)\n",
    "        return x.permute(0, 2, 1, 3)\n",
    "\n",
    "    def forward(self, hidden_states):\n",
    "        mixed_query_layer = self.query(hidden_states)\n",
    "        mixed_key_layer = self.key(hidden_states)\n",
    "        mixed_value_layer = self.value(hidden_states)\n",
    "\n",
    "        query_layer = self.transpose_for_scores(mixed_query_layer)\n",
    "        key_layer = self.transpose_for_scores(mixed_key_layer)\n",
    "        value_layer = self.transpose_for_scores(mixed_value_layer)\n",
    "\n",
    "        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))\n",
    "        attention_scores = attention_scores / math.sqrt(self.attention_head_size)\n",
    "        attention_probs = self.softmax(attention_scores)\n",
    "        weights = attention_probs if self.vis else None\n",
    "        attention_probs = self.attn_dropout(attention_probs)\n",
    "\n",
    "        context_layer = torch.matmul(attention_probs, value_layer)\n",
    "        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()\n",
    "        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)\n",
    "        context_layer = context_layer.view(*new_context_layer_shape)\n",
    "        attention_output = self.out(context_layer)\n",
    "        attention_output = self.proj_dropout(attention_output)\n",
    "        return attention_output, weights\n",
    "\n",
    "\n",
    "class Mlp(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super(Mlp, self).__init__()\n",
    "        self.fc1 = Linear(config.hidden_size, config.transformer[\"mlp_dim\"])\n",
    "        self.fc2 = Linear(config.transformer[\"mlp_dim\"], config.hidden_size)\n",
    "        self.act_fn = ACT2FN[\"gelu\"]\n",
    "        self.dropout = Dropout(config.transformer[\"dropout_rate\"])\n",
    "\n",
    "        self._init_weights()\n",
    "\n",
    "    def _init_weights(self):\n",
    "        nn.init.xavier_uniform_(self.fc1.weight)\n",
    "        nn.init.xavier_uniform_(self.fc2.weight)\n",
    "        nn.init.normal_(self.fc1.bias, std=1e-6)\n",
    "        nn.init.normal_(self.fc2.bias, std=1e-6)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.fc1(x)\n",
    "        x = self.act_fn(x)\n",
    "        x = self.dropout(x)\n",
    "        x = self.fc2(x)\n",
    "        x = self.dropout(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "class Embeddings(nn.Module):\n",
    "    '''\n",
    "    Construct the embeddings from patch, position embeddings\n",
    "    '''\n",
    "    def __init__(self, config, img_size, in_channels=3):\n",
    "        super(Embeddings, self).__init__()\n",
    "        img_size = _pair(img_size)\n",
    "        patch_size = _pair(config.patches[\"size\"])\n",
    "        n_patches = (img_size[0]//patch_size[0]) * (img_size[1]//patch_size[1])\n",
    "\n",
    "        self.patch_embeddings = Conv2d(in_channels=in_channels,\n",
    "                                       out_channels=config.hidden_size,\n",
    "                                       kernel_size=patch_size,\n",
    "                                       stride=patch_size)\n",
    "        self.position_embeddings = nn.Parameter(torch.zeros(1, n_patches+1, config.hidden_size))\n",
    "        self.cls_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size))\n",
    "\n",
    "        self.dropout = Dropout(config.transformer[\"dropout_rate\"])\n",
    "\n",
    "    def forward(self, x):\n",
    "        B = x.shape[0]\n",
    "        cls_tokens = self.cls_token.expand(B, -1, -1)\n",
    "\n",
    "        x = self.patch_embeddings(x)\n",
    "        x = x.flatten(2)\n",
    "        x = x.transpose(-1, -2)\n",
    "        x = torch.cat((cls_tokens, x), dim=1)\n",
    "\n",
    "        embeddings = x + self.position_embeddings\n",
    "        embeddings = self.dropout(embeddings)\n",
    "        return embeddings\n",
    "\n",
    "\n",
    "class Block(nn.Module):\n",
    "    def __init__(self, config, vis):\n",
    "        super(Block, self).__init__()\n",
    "        self.hidden_size = config.hidden_size\n",
    "        self.attention_norm = LayerNorm(config.hidden_size, eps=1e-6)\n",
    "        self.ffn_norm = LayerNorm(config.hidden_size, eps=1e-6)\n",
    "        self.ffn = Mlp(config)\n",
    "        self.attn = Attention(config, vis)\n",
    "\n",
    "    def forward(self, x):\n",
    "        h = x\n",
    "        x = self.attention_norm(x)\n",
    "        x, weights = self.attn(x)\n",
    "        x = x + h\n",
    "\n",
    "        h = x\n",
    "        x = self.ffn_norm(x)\n",
    "        x = self.ffn(x)\n",
    "        x = x + h\n",
    "        return x, weights\n",
    "\n",
    "    def load_from(self, weights, n_block):\n",
    "        ROOT = f\"Transformer/encoderblock_{n_block}\"\n",
    "        with torch.no_grad():\n",
    "            query_weight = np2th(weights[pjoin(ROOT, ATTENTION_Q, \"kernel\")]).view(self.hidden_size, self.hidden_size).t()\n",
    "            key_weight = np2th(weights[pjoin(ROOT, ATTENTION_K, \"kernel\")]).view(self.hidden_size, self.hidden_size).t()\n",
    "            value_weight = np2th(weights[pjoin(ROOT, ATTENTION_V, \"kernel\")]).view(self.hidden_size, self.hidden_size).t()\n",
    "            out_weight = np2th(weights[pjoin(ROOT, ATTENTION_OUT, \"kernel\")]).view(self.hidden_size, self.hidden_size).t()\n",
    "\n",
    "            query_bias = np2th(weights[pjoin(ROOT, ATTENTION_Q, \"bias\")]).view(-1)\n",
    "            key_bias = np2th(weights[pjoin(ROOT, ATTENTION_K, \"bias\")]).view(-1)\n",
    "            value_bias = np2th(weights[pjoin(ROOT, ATTENTION_V, \"bias\")]).view(-1)\n",
    "            out_bias = np2th(weights[pjoin(ROOT, ATTENTION_OUT, \"bias\")]).view(-1)\n",
    "\n",
    "            self.attn.query.weight.copy_(query_weight)\n",
    "            self.attn.key.weight.copy_(key_weight)\n",
    "            self.attn.value.weight.copy_(value_weight)\n",
    "            self.attn.out.weight.copy_(out_weight)\n",
    "            self.attn.query.bias.copy_(query_bias)\n",
    "            self.attn.key.bias.copy_(key_bias)\n",
    "            self.attn.value.bias.copy_(value_bias)\n",
    "            self.attn.out.bias.copy_(out_bias)\n",
    "\n",
    "            mlp_weight_0 = np2th(weights[pjoin(ROOT, FC_0, \"kernel\")]).t()\n",
    "            mlp_weight_1 = np2th(weights[pjoin(ROOT, FC_1, \"kernel\")]).t()\n",
    "            mlp_bias_0 = np2th(weights[pjoin(ROOT, FC_0, \"bias\")]).t()\n",
    "            mlp_bias_1 = np2th(weights[pjoin(ROOT, FC_1, \"bias\")]).t()\n",
    "\n",
    "            self.ffn.fc1.weight.copy_(mlp_weight_0)\n",
    "            self.ffn.fc2.weight.copy_(mlp_weight_1)\n",
    "            self.ffn.fc1.bias.copy_(mlp_bias_0)\n",
    "            self.ffn.fc2.bias.copy_(mlp_bias_1)\n",
    "\n",
    "            self.attention_norm.weight.copy_(np2th(weights[pjoin(ROOT, ATTENTION_NORM, \"scale\")]))\n",
    "            self.attention_norm.bias.copy_(np2th(weights[pjoin(ROOT, ATTENTION_NORM, \"bias\")]))\n",
    "            self.ffn_norm.weight.copy_(np2th(weights[pjoin(ROOT, MLP_NORM, \"scale\")]))\n",
    "            self.ffn_norm.bias.copy_(np2th(weights[pjoin(ROOT, MLP_NORM, \"bias\")]))\n",
    "\n",
    "\n",
    "class Encoder(nn.Module):\n",
    "    def __init__(self, config, vis):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.vis = vis\n",
    "        self.layer = nn.ModuleList()\n",
    "        self.encoder_norm = LayerNorm(config.hidden_size, eps=1e-6)\n",
    "        for _ in range(config.transformer[\"num_layers\"]):\n",
    "            layer = Block(config, vis)\n",
    "            self.layer.append(copy.deepcopy(layer))\n",
    "\n",
    "    def forward(self, hidden_states):\n",
    "        attn_weights = []\n",
    "        for layer_block in self.layer:\n",
    "            hidden_states, weights = layer_block(hidden_states)\n",
    "            if self.vis:\n",
    "                attn_weights.append(weights)\n",
    "        encoded = self.encoder_norm(hidden_states)\n",
    "        return encoded, attn_weights\n",
    "\n",
    "\n",
    "class Transformer(nn.Module):\n",
    "    def __init__(self, config, img_size, vis):\n",
    "        super(Transformer, self).__init__()\n",
    "        self.embeddings = Embeddings(config, img_size=img_size)\n",
    "        self.encoder = Encoder(config, vis)\n",
    "\n",
    "    def forward(self, input_ids):\n",
    "        embedding_output = self.embeddings(input_ids)\n",
    "        encoded, attn_weights = self.encoder(embedding_output)\n",
    "        return encoded, attn_weights\n",
    "\n",
    "\n",
    "class VisionTransformer(nn.Module):\n",
    "    def __init__(self, config, img_size=224, num_classes=21843, zero_head=False, vis=False):\n",
    "        super(VisionTransformer, self).__init__()\n",
    "        self.num_classes = num_classes\n",
    "        self.zero_head = zero_head\n",
    "        self.classifier = config.classifier\n",
    "\n",
    "        self.transformer = Transformer(config, img_size, vis)\n",
    "        self.head = Linear(config.hidden_size, num_classes)\n",
    "\n",
    "    def forward(self, x, labels=None):\n",
    "        x, attn_weights = self.transformer(x)\n",
    "        logits = self.head(x[:, 0])\n",
    "\n",
    "        if labels is not None:\n",
    "            loss_fct = CrossEntropyLoss()\n",
    "            loss = loss_fct(logits.view(-1, self.num_classes), labels.view(-1))\n",
    "            return loss\n",
    "        else:\n",
    "            return logits, attn_weights\n",
    "\n",
    "    def load_from(self, weights):\n",
    "        with torch.no_grad():\n",
    "            if self.zero_head:\n",
    "                nn.init.zeros_(self.head.weight)\n",
    "                nn.init.zeros_(self.head.bias)\n",
    "            else:\n",
    "                self.head.weight.copy_(np2th(weights[\"head/kernel\"]).t())\n",
    "                self.head.bias.copy_(np2th(weights[\"head/bias\"]).t())\n",
    "\n",
    "            self.transformer.embeddings.patch_embeddings.weight.copy_(np2th(weights[\"embedding/kernel\"]))\n",
    "            self.transformer.embeddings.patch_embeddings.bias.copy_(np2th(weights[\"embedding/bias\"]))\n",
    "            self.transformer.embeddings.cls_token.copy_(np2th(weights[\"cls\"]))\n",
    "            self.transformer.encoder.encoder_norm.weight.copy_(np2th(weights[\"Transformer/encoder_norm/scale\"]))\n",
    "            self.transformer.encoder.encoder_norm.bias.copy_(np2th(weights[\"Transformer/encoder_norm/bias\"]))\n",
    "\n",
    "            posemb = np2th(weights[\"Transformer/posembed_input/pos_embedding\"])\n",
    "            posemb_new = self.transformer.embeddings.position_embeddings\n",
    "            if posemb.size() == posemb_new.size():\n",
    "                self.transformer.embeddings.position_embeddings.copy_(posemb)\n",
    "            else:\n",
    "                logger.info(\"load_pretrained: resized variant: %s to %s\" % (posemb.size(), posemb_new.size()))\n",
    "                ntok_new = posemb_new.size(1)\n",
    "\n",
    "                if self.classifier == \"token\":\n",
    "                    posemb_tok, posemb_grid = posemb[:, :1], posemb[0, 1:]\n",
    "                    ntok_new -= 1\n",
    "                else:\n",
    "                    posemb_tok, posemb_grid = posemb[:, :0], posemb[0]\n",
    "\n",
    "                gs_old = int(np.sqrt(len(posemb_grid)))\n",
    "                gs_new = int(np.sqrt(ntok_new))\n",
    "                print('load_pretrained: grid-size from %s to %s' % (gs_old, gs_new))\n",
    "                posemb_grid = posemb_grid.reshape(gs_old, gs_old, -1)\n",
    "\n",
    "                zoom = (gs_new / gs_old, gs_new / gs_old, 1)\n",
    "                posemb_grid = ndimage.zoom(posemb_grid, zoom, order=1)\n",
    "                posemb_grid = posemb_grid.reshape(1, gs_new * gs_new, -1)\n",
    "                posemb = np.concatenate([posemb_tok, posemb_grid], axis=1)\n",
    "                self.transformer.embeddings.position_embeddings.copy_(np2th(posemb))\n",
    "\n",
    "            for bname, block in self.transformer.encoder.named_children():\n",
    "                for uname, unit in block.named_children():\n",
    "                    unit.load_from(weights, n_block=uname)\n",
    "\n",
    "\n",
    "CONFIGS = {\n",
    "    'ViT-B_16': get_b16_config(),\n",
    "    'ViT-B_32': get_b32_config(),\n",
    "    'ViT-L_16': get_l16_config(),\n",
    "    'ViT-L_32': get_l32_config(),\n",
    "    'ViT-H_14': get_h14_config(),\n",
    "    'testing': get_testing(),\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60e03c4e",
   "metadata": {},
   "source": [
    "### Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "cf3cc96c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VisionTransformer(\n",
       "  (transformer): Transformer(\n",
       "    (embeddings): Embeddings(\n",
       "      (patch_embeddings): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16))\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): Encoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (1): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (2): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (3): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (4): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (5): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (6): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (7): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (8): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (9): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (10): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (11): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (encoder_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (head): Linear(in_features=768, out_features=21843, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = CONFIGS[\"ViT-B_16\"]\n",
    "model = VisionTransformer(config, img_size=224, num_classes=21843, zero_head=False, vis=True)\n",
    "model.load_from(np.load(\"ViT-B_16.npz\"))\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97c55912",
   "metadata": {},
   "source": [
    "### Utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "866dd628",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_world_size():\n",
    "    if not dist.is_available():\n",
    "        return 1\n",
    "    if not dist.is_initialized():\n",
    "        return 1\n",
    "    return dist.get_world_size()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "725b6a0d",
   "metadata": {},
   "source": [
    "### Dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "7e9aebc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_loader(local_rank, img_size, dataset, train_batch_size, eval_batch_size):\n",
    "    if local_rank not in [-1, 0]:\n",
    "        torch.distributed.barrier()\n",
    "\n",
    "    transform_train = transforms.Compose([\n",
    "        transforms.RandomResizedCrop((img_size, img_size), scale=(0.05, 1.0)),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),\n",
    "    ])\n",
    "    transform_test = transforms.Compose([\n",
    "        transforms.Resize((img_size, img_size)),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),\n",
    "    ])\n",
    "\n",
    "    if dataset == \"cifar10\":\n",
    "        trainset = datasets.CIFAR10(root=\"./data\",\n",
    "                                    train=True,\n",
    "                                    download=True,\n",
    "                                    transform=transform_train)\n",
    "        testset = datasets.CIFAR10(root=\"./data\",\n",
    "                                   train=False,\n",
    "                                   download=True,\n",
    "                                   transform=transform_test) if local_rank in [-1, 0] else None\n",
    "\n",
    "    else:\n",
    "        trainset = datasets.CIFAR100(root=\"./data\",\n",
    "                                     train=True,\n",
    "                                     download=True,\n",
    "                                     transform=transform_train)\n",
    "        testset = datasets.CIFAR100(root=\"./data\",\n",
    "                                    train=False,\n",
    "                                    download=True,\n",
    "                                    transform=transform_test) if local_rank in [-1, 0] else None\n",
    "    if local_rank == 0:\n",
    "        torch.distributed.barrier()\n",
    "\n",
    "    train_sampler = RandomSampler(trainset) if local_rank == -1 else DistributedSampler(trainset)\n",
    "    test_sampler = SequentialSampler(testset)\n",
    "    train_loader = DataLoader(trainset,\n",
    "                              sampler=train_sampler,\n",
    "                              batch_size=train_batch_size,\n",
    "                              num_workers=4,\n",
    "                              pin_memory=True)\n",
    "    test_loader = DataLoader(testset,\n",
    "                             sampler=test_sampler,\n",
    "                             batch_size=eval_batch_size,\n",
    "                             num_workers=4,\n",
    "                             pin_memory=True) if testset is not None else None\n",
    "\n",
    "    return train_loader, test_loader"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ede994b",
   "metadata": {},
   "source": [
    "### Scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "c9552c64",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConstantLRSchedule(LambdaLR):\n",
    "    \"\"\" Constant learning rate schedule.\n",
    "    \"\"\"\n",
    "    def __init__(self, optimizer, last_epoch=-1):\n",
    "        super(ConstantLRSchedule, self).__init__(optimizer, lambda _: 1.0, last_epoch=last_epoch)\n",
    "\n",
    "\n",
    "class WarmupConstantSchedule(LambdaLR):\n",
    "    \"\"\" Linear warmup and then constant.\n",
    "        Linearly increases learning rate schedule from 0 to 1 over `warmup_steps` training steps.\n",
    "        Keeps learning rate schedule equal to 1. after warmup_steps.\n",
    "    \"\"\"\n",
    "    def __init__(self, optimizer, warmup_steps, last_epoch=-1):\n",
    "        self.warmup_steps = warmup_steps\n",
    "        super(WarmupConstantSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)\n",
    "\n",
    "    def lr_lambda(self, step):\n",
    "        if step < self.warmup_steps:\n",
    "            return float(step) / float(max(1.0, self.warmup_steps))\n",
    "        return 1.\n",
    "\n",
    "\n",
    "class WarmupLinearSchedule(LambdaLR):\n",
    "    \"\"\" Linear warmup and then linear decay.\n",
    "        Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.\n",
    "        Linearly decreases learning rate from 1. to 0. over remaining `t_total - warmup_steps` steps.\n",
    "    \"\"\"\n",
    "    def __init__(self, optimizer, warmup_steps, t_total, last_epoch=-1):\n",
    "        self.warmup_steps = warmup_steps\n",
    "        self.t_total = t_total\n",
    "        super(WarmupLinearSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)\n",
    "\n",
    "    def lr_lambda(self, step):\n",
    "        if step < self.warmup_steps:\n",
    "            return float(step) / float(max(1, self.warmup_steps))\n",
    "        return max(0.0, float(self.t_total - step) / float(max(1.0, self.t_total - self.warmup_steps)))\n",
    "\n",
    "\n",
    "class WarmupCosineSchedule(LambdaLR):\n",
    "    \"\"\" Linear warmup and then cosine decay.\n",
    "        Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.\n",
    "        Decreases learning rate from 1. to 0. over remaining `t_total - warmup_steps` steps following a cosine curve.\n",
    "        If `cycles` (default=0.5) is different from default, learning rate follows cosine function after warmup.\n",
    "    \"\"\"\n",
    "    def __init__(self, optimizer, warmup_steps, t_total, cycles=.5, last_epoch=-1):\n",
    "        self.warmup_steps = warmup_steps\n",
    "        self.t_total = t_total\n",
    "        self.cycles = cycles\n",
    "        super(WarmupCosineSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)\n",
    "\n",
    "    def lr_lambda(self, step):\n",
    "        if step < self.warmup_steps:\n",
    "            return float(step) / float(max(1.0, self.warmup_steps))\n",
    "        # progress after warmup\n",
    "        progress = float(step - self.warmup_steps) / float(max(1, self.t_total - self.warmup_steps))\n",
    "        return max(0.0, 0.5 * (1. + math.cos(math.pi * float(self.cycles) * 2.0 * progress)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6c1900e",
   "metadata": {},
   "source": [
    "### Training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "ecc396ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "class AverageMeter(object):\n",
    "    \"\"\"Computes and stores the average and current value\"\"\"\n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "\n",
    "    def reset(self):\n",
    "        self.val = 0\n",
    "        self.avg = 0\n",
    "        self.sum = 0\n",
    "        self.count = 0\n",
    "\n",
    "    def update(self, val, n=1):\n",
    "        self.val = val\n",
    "        self.sum += val * n\n",
    "        self.count += n\n",
    "        self.avg = self.sum / self.count\n",
    "\n",
    "\n",
    "def simple_accuracy(preds, labels):\n",
    "    return (preds == labels).mean()\n",
    "\n",
    "\n",
    "def save_model(output_dir, name, model):\n",
    "    model_to_save = model.module if hasattr(model, 'module') else model\n",
    "    model_checkpoint = os.path.join(output_dir, \"%s_checkpoint.bin\" % name)\n",
    "    torch.save(model_to_save.state_dict(), model_checkpoint)\n",
    "    logger.info(\"Saved model checkpoint to [DIR: %s]\", output_dir)\n",
    "\n",
    "\n",
    "def setup(model_type, img_size, pretrained_dir, device, dataset):\n",
    "    # Prepare model\n",
    "    config = CONFIGS[model_type]\n",
    "\n",
    "    num_classes = 10 if dataset == \"cifar10\" else 100\n",
    "\n",
    "    model = VisionTransformer(config, img_size, zero_head=True, num_classes=num_classes)\n",
    "    model.load_from(np.load(pretrained_dir))\n",
    "    model.to(device)\n",
    "    num_params = count_parameters(model)\n",
    "\n",
    "    logger.info(\"{}\".format(config))\n",
    "    logger.info(\"Training parameters %s\", model_type, img_size, pretrained_dir, device)\n",
    "    logger.info(\"Total Parameter: \\t%2.1fM\" % num_params)\n",
    "    return model_type, img_size, pretrained_dir, device, model\n",
    "\n",
    "\n",
    "def count_parameters(model):\n",
    "    params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "    return params/1000000\n",
    "\n",
    "\n",
    "def set_seed(seed, n_gpu):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    if n_gpu > 0:\n",
    "        torch.cuda.manual_seed_all(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "4c3b5375",
   "metadata": {},
   "outputs": [],
   "source": [
    "def valid(eval_batch_size, local_rank, device, model, writer, test_loader, global_step):\n",
    "    # Validation!\n",
    "    eval_losses = AverageMeter()\n",
    "\n",
    "    logger.info(\"***** Running Test *****\")\n",
    "    logger.info(\"  Num steps = %d\", len(test_loader))\n",
    "    logger.info(\"  Batch size = %d\", eval_batch_size)\n",
    "\n",
    "    model.eval()\n",
    "    all_preds, all_label = [], []\n",
    "    epoch_iterator = tqdm(test_loader,\n",
    "                          desc=\"Test... (loss=X.X)\",\n",
    "                          bar_format=\"{l_bar}{r_bar}\",\n",
    "                          dynamic_ncols=True,\n",
    "                          disable=local_rank not in [-1, 0])\n",
    "    loss_fct = torch.nn.CrossEntropyLoss()\n",
    "    for step, batch in enumerate(epoch_iterator):\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        x, y = batch\n",
    "        with torch.no_grad():\n",
    "            logits, attn_weights = model(x)\n",
    "\n",
    "            eval_loss = loss_fct(logits, y)\n",
    "            eval_losses.update(eval_loss.item())\n",
    "\n",
    "            preds = torch.argmax(logits, dim=-1)\n",
    "\n",
    "        if len(all_preds) == 0:\n",
    "            all_preds.append(preds.detach().cpu().numpy())\n",
    "            all_label.append(y.detach().cpu().numpy())\n",
    "        else:\n",
    "            all_preds[0] = np.append(\n",
    "                all_preds[0], preds.detach().cpu().numpy(), axis=0\n",
    "            )\n",
    "            all_label[0] = np.append(\n",
    "                all_label[0], y.detach().cpu().numpy(), axis=0\n",
    "            )\n",
    "        epoch_iterator.set_description(\"Testing... (loss=%2.5f)\" % eval_losses.val)\n",
    "\n",
    "    all_preds, all_label = all_preds[0], all_label[0]\n",
    "    accuracy = simple_accuracy(all_preds, all_label)\n",
    "\n",
    "    logger.info(\"\\n\")\n",
    "    logger.info(\"Test Results\")\n",
    "    logger.info(\"Global Steps: %d\" % global_step)\n",
    "    logger.info(\"Test Loss: %2.5f\" % eval_losses.avg)\n",
    "    logger.info(\"Test Accuracy: %2.5f\" % accuracy)\n",
    "\n",
    "    writer.add_scalar(\"test/accuracy\", scalar_value=accuracy, global_step=global_step)\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "04faec58",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(local_rank, output_dir, name, train_batch_size, eval_batch_size, seed, n_gpu, gradient_accumulation_steps, dataset, img_size, learning_rate, weight_decay, num_steps, decay_type, warmup_steps, fp16, fp16_opt_level, device, max_grad_norm, eval_every, model):\n",
    "    \"\"\" Train the model \"\"\"\n",
    "    if local_rank in [-1, 0]:\n",
    "        os.makedirs(output_dir, exist_ok=True)\n",
    "        writer = SummaryWriter(log_dir=os.path.join(\"logs\", name))\n",
    "\n",
    "    train_batch_size = train_batch_size // gradient_accumulation_steps\n",
    "\n",
    "    # Prepare dataset\n",
    "    train_loader, test_loader = get_loader(local_rank, img_size, dataset, train_batch_size, eval_batch_size)\n",
    "\n",
    "    # Prepare optimizer and scheduler\n",
    "    optimizer = torch.optim.SGD(model.parameters(),\n",
    "                                lr=learning_rate,\n",
    "                                momentum=0.9,\n",
    "                                weight_decay=weight_decay)\n",
    "    t_total = num_steps\n",
    "    if decay_type == \"cosine\":\n",
    "        scheduler = WarmupCosineSchedule(optimizer, warmup_steps=warmup_steps, t_total=t_total)\n",
    "    else:\n",
    "        scheduler = WarmupLinearSchedule(optimizer, warmup_steps=warmup_steps, t_total=t_total)\n",
    "\n",
    "    if fp16:\n",
    "        model, optimizer = amp.initialize(models=model,\n",
    "                                          optimizers=optimizer,\n",
    "                                          opt_level=fp16_opt_level)\n",
    "        amp._amp_state.loss_scalers[0]._loss_scale = 2**20\n",
    "\n",
    "    # Distributed training\n",
    "    if local_rank != -1:\n",
    "        model = DDP(model, message_size=250000000, gradient_predivide_factor=get_world_size())\n",
    "\n",
    "    # Train!\n",
    "    logger.info(\"***** Running training *****\")\n",
    "    logger.info(\"  Total optimization steps = %d\", num_steps)\n",
    "    logger.info(\"  Instantaneous batch size per GPU = %d\", train_batch_size)\n",
    "    logger.info(\"  Total train batch size (w. parallel, distributed & accumulation) = %d\",\n",
    "                train_batch_size * gradient_accumulation_steps * (\n",
    "                    torch.distributed.get_world_size() if local_rank != -1 else 1))\n",
    "    logger.info(\"  Gradient Accumulation steps = %d\", gradient_accumulation_steps)\n",
    "\n",
    "    model.zero_grad()\n",
    "    set_seed(seed, n_gpu)  # Added here for reproducibility (even between python 2 and 3)\n",
    "    losses = AverageMeter()\n",
    "    global_step, best_acc = 0, 0\n",
    "    train_accs = []\n",
    "    test_accs = []\n",
    "\n",
    "    while True:\n",
    "        model.train()\n",
    "        epoch_iterator = tqdm(train_loader,\n",
    "                              desc=\"Training (X / X Steps) (loss=X.X)\",\n",
    "                              bar_format=\"{l_bar}{r_bar}\",\n",
    "                              dynamic_ncols=True,\n",
    "                              disable=local_rank not in [-1, 0])\n",
    "        for step, batch in enumerate(epoch_iterator):\n",
    "            batch = tuple(t.to(device) for t in batch)\n",
    "            x, y = batch\n",
    "            loss = model(x, y)\n",
    "\n",
    "            if gradient_accumulation_steps > 1:\n",
    "                loss = loss / gradient_accumulation_steps\n",
    "            if fp16:\n",
    "                with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "                    scaled_loss.backward()\n",
    "            else:\n",
    "                loss.backward()\n",
    "\n",
    "            if (step + 1) % gradient_accumulation_steps == 0:\n",
    "                losses.update(loss.item()*gradient_accumulation_steps)\n",
    "                if fp16:\n",
    "                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), max_grad_norm)\n",
    "                else:\n",
    "                    torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)\n",
    "                scheduler.step()\n",
    "                optimizer.step()\n",
    "                optimizer.zero_grad()\n",
    "                global_step += 1\n",
    "\n",
    "                epoch_iterator.set_description(\n",
    "                    \"Training (%d / %d Steps) (loss=%2.5f)\" % (global_step, t_total, losses.val)\n",
    "                )\n",
    "                if local_rank in [-1, 0]:\n",
    "                    writer.add_scalar(\"train/loss\", scalar_value=losses.val, global_step=global_step)\n",
    "                    writer.add_scalar(\"train/lr\", scalar_value=scheduler.get_lr()[0], global_step=global_step)\n",
    "                if global_step % eval_every == 0 and local_rank in [-1, 0]:\n",
    "                    accuracy = valid(eval_batch_size, local_rank, device, model, writer, test_loader, global_step)\n",
    "                    test_accs.append(accuracy)\n",
    "                    if best_acc < accuracy:\n",
    "                        save_model(output_dir, name, model)\n",
    "                        best_acc = accuracy\n",
    "                    model.train()\n",
    "\n",
    "                if global_step % 5 == 0:\n",
    "                    torch.save({\n",
    "                        'epoch': global_step,\n",
    "                        'model': model,\n",
    "                        'optimizer': optimizer,\n",
    "                        'scheduler': scheduler,\n",
    "                        'test_acc': test_accs\n",
    "                    }, './checkpoint.pt')\n",
    "\n",
    "                if global_step % t_total == 0:\n",
    "                    break\n",
    "        losses.reset()\n",
    "        if global_step % t_total == 0:\n",
    "            break\n",
    "\n",
    "    if local_rank in [-1, 0]:\n",
    "        writer.close()\n",
    "    logger.info(\"Best Accuracy: \\t%f\" % best_acc)\n",
    "    logger.info(\"End Training!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "716d80e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    # Required parameters\n",
    "    name = \"cifar100_500\"                               # Name of this run. Used for monitoring\n",
    "    dataset = \"cifar100\"                                # Which downstream task, choices=[\"cifar10\", \"cifar100\"]\n",
    "    model_type = \"ViT-B_16\"                             # Which variant to use\n",
    "    pretrained_dir = \"ViT-B_16.npz\"                     # Where to search for pretrained ViT models\n",
    "    output_dir = \"output_cifar100\"                      # The output directory where checkpoints will be written\n",
    "    img_size = 224  #224                                 # Resolution size\n",
    "    train_batch_size = 64                               # Total batch size for training\n",
    "    eval_batch_size = 64                                # Total batch size for eval\n",
    "    eval_every = 100                                    # Run prediction on validation set every so many steps (Will always run one evaluation at the end of training)\n",
    "    learning_rate = 3e-2                                # The initial learning rate for SGD\n",
    "    weight_decay = 0                                    # Weight deay if we apply some\n",
    "    num_steps = 500                                     # Total number of training epochs to perform\n",
    "    decay_type = \"cosine\"                               # How to decay the learning rate, choices=[\"cosine\", \"linear\"]\n",
    "    warmup_steps = 100                                  # Step of training to perform learning rate warmup for\n",
    "    max_grad_norm = 1.0                                 # Max gradient norm\n",
    "    local_rank = -1                                     # local_rank for distributed training on gpus\n",
    "    seed = 42                                           # random seed for initialization\n",
    "    gradient_accumulation_steps = 1                     # Number of updates steps to accumulate before performing a backward/update pass\n",
    "    fp16 = 0                                            # (action = 'store_true') Whether to use 16-bit float precision instead of 32-bit\n",
    "    fp16_opt_level = 'store_true'                       # For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']\n",
    "    loss_scale = 0                                      # Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True (0 (default value): dynamic loss scaling, Positive power of 2: static loss scaling value)\n",
    "    local_rank = -1                                     # local_rank for distributed training on gpus\n",
    "\n",
    "    device=\"cpu\"\n",
    "    n_gpu = 0\n",
    "\n",
    "    # Setup CUDA, GPU & distributed training\n",
    "    if local_rank == -1:\n",
    "        device = torch.device(\"cuda:1\" if torch.cuda.is_available() else \"cpu\")\n",
    "        n_gpu = torch.cuda.device_count()\n",
    "    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs\n",
    "        torch.cuda.set_device(local_rank)\n",
    "        device = torch.device(\"cuda:1\", local_rank)\n",
    "        torch.distributed.init_process_group(backend='nccl',\n",
    "                                             timeout=timedelta(minutes=60))\n",
    "        n_gpu = 1\n",
    "\n",
    "    # Setup logging\n",
    "    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',\n",
    "                        datefmt='%m/%d/%Y %H:%M:%S',\n",
    "                        level=logging.INFO if local_rank in [-1, 0] else logging.WARN)\n",
    "    logger.warning(\"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s\" %\n",
    "                   (local_rank, device, n_gpu, bool(local_rank != -1), fp16))\n",
    "\n",
    "    # Set seed\n",
    "    set_seed(seed, n_gpu)\n",
    "\n",
    "    # Model & Tokenizer Setup\n",
    "    model_type, img_size, pretrained_dir, device, model = setup(model_type, img_size, pretrained_dir, device, dataset)\n",
    "\n",
    "    # Training\n",
    "    train(local_rank, output_dir, name, train_batch_size, eval_batch_size, seed, n_gpu, gradient_accumulation_steps, dataset, img_size, learning_rate, weight_decay, num_steps, decay_type, warmup_steps, fp16, fp16_opt_level, device, max_grad_norm, eval_every, model)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d45403c8",
   "metadata": {},
   "source": [
    "### Running the trainer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "ee513036",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/05/2021 21:12:30 - WARNING - __main__ - Process rank: -1, device: cuda:1, n_gpu: 3, distributed training: False, 16-bits training: 0\n",
      "11/05/2021 21:12:33 - INFO - __main__ - classifier: token\n",
      "hidden_size: 768\n",
      "patches:\n",
      "  size: !!python/tuple\n",
      "  - 16\n",
      "  - 16\n",
      "representation_size: null\n",
      "transformer:\n",
      "  attention_dropout_rate: 0.0\n",
      "  dropout_rate: 0.1\n",
      "  mlp_dim: 3072\n",
      "  num_heads: 12\n",
      "  num_layers: 12\n",
      "\n",
      "--- Logging error ---\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/logging/__init__.py\", line 1081, in emit\n",
      "    msg = self.format(record)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/logging/__init__.py\", line 925, in format\n",
      "    return fmt.format(record)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/logging/__init__.py\", line 664, in format\n",
      "    record.message = record.getMessage()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/logging/__init__.py\", line 369, in getMessage\n",
      "    msg = msg % self.args\n",
      "TypeError: not all arguments converted during string formatting\n",
      "Call stack:\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/runpy.py\", line 194, in _run_module_as_main\n",
      "    return _run_code(code, main_globals, None,\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/runpy.py\", line 87, in _run_code\n",
      "    exec(code, run_globals)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/ipykernel_launcher.py\", line 16, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/traitlets/config/application.py\", line 845, in launch_instance\n",
      "    app.start()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/ipykernel/kernelapp.py\", line 612, in start\n",
      "    self.io_loop.start()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/platform/asyncio.py\", line 199, in start\n",
      "    self.asyncio_loop.run_forever()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/asyncio/base_events.py\", line 570, in run_forever\n",
      "    self._run_once()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/asyncio/base_events.py\", line 1859, in _run_once\n",
      "    handle._run()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/asyncio/events.py\", line 81, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/ioloop.py\", line 688, in <lambda>\n",
      "    lambda f: self._run_callback(functools.partial(callback, future))\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/ioloop.py\", line 741, in _run_callback\n",
      "    ret = callback()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/gen.py\", line 814, in inner\n",
      "    self.ctx_run(self.run)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/gen.py\", line 775, in run\n",
      "    yielded = self.gen.send(value)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/ipykernel/kernelbase.py\", line 365, in process_one\n",
      "    yield gen.maybe_future(dispatch(*args))\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/gen.py\", line 234, in wrapper\n",
      "    yielded = ctx_run(next, result)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/ipykernel/kernelbase.py\", line 268, in dispatch_shell\n",
      "    yield gen.maybe_future(handler(stream, idents, msg))\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/gen.py\", line 234, in wrapper\n",
      "    yielded = ctx_run(next, result)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/ipykernel/kernelbase.py\", line 543, in execute_request\n",
      "    self.do_execute(\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/gen.py\", line 234, in wrapper\n",
      "    yielded = ctx_run(next, result)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/ipykernel/ipkernel.py\", line 306, in do_execute\n",
      "    res = shell.run_cell(code, store_history=store_history, silent=silent)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/ipykernel/zmqshell.py\", line 536, in run_cell\n",
      "    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/IPython/core/interactiveshell.py\", line 2898, in run_cell\n",
      "    result = self._run_cell(\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/IPython/core/interactiveshell.py\", line 2944, in _run_cell\n",
      "    return runner(coro)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/IPython/core/async_helpers.py\", line 68, in _pseudo_sync_runner\n",
      "    coro.send(None)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/IPython/core/interactiveshell.py\", line 3169, in run_cell_async\n",
      "    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/IPython/core/interactiveshell.py\", line 3361, in run_ast_nodes\n",
      "    if (await self.run_code(code, result,  async_=asy)):\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/IPython/core/interactiveshell.py\", line 3441, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-79-263240bbee7e>\", line 1, in <module>\n",
      "    main()\n",
      "  File \"<ipython-input-78-b478d04a14d8>\", line 51, in main\n",
      "    model_type, img_size, pretrained_dir, device, model = setup(model_type, img_size, pretrained_dir, device, dataset)\n",
      "  File \"<ipython-input-75-25b0bc7a7040>\", line 42, in setup\n",
      "    logger.info(\"Training parameters %s\", model_type, img_size, pretrained_dir, device)\n",
      "Message: 'Training parameters %s'\n",
      "Arguments: ('ViT-B_16', 224, 'ViT-B_16.npz', device(type='cuda', index=1))\n",
      "11/05/2021 21:12:33 - INFO - __main__ - Total Parameter: \t85.9M\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/05/2021 21:12:35 - INFO - __main__ - ***** Running training *****\n",
      "11/05/2021 21:12:35 - INFO - __main__ -   Total optimization steps = 500\n",
      "11/05/2021 21:12:35 - INFO - __main__ -   Instantaneous batch size per GPU = 64\n",
      "11/05/2021 21:12:35 - INFO - __main__ -   Total train batch size (w. parallel, distributed & accumulation) = 64\n",
      "11/05/2021 21:12:35 - INFO - __main__ -   Gradient Accumulation steps = 1\n",
      "Training (100 / 500 Steps) (loss=3.61231):  13%|| 99/782 [01:54<10:07,  1.12it/s]11/05/2021 21:14:30 - INFO - __main__ - ***** Running Test *****\n",
      "11/05/2021 21:14:30 - INFO - __main__ -   Num steps = 157\n",
      "11/05/2021 21:14:30 - INFO - __main__ -   Batch size = 64\n",
      "\n",
      "Test... (loss=X.X):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=3.26165):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=3.26165):   1%|| 1/157 [00:00<01:56,  1.34it/s]\u001b[A\n",
      "Testing... (loss=3.31441):   1%|| 1/157 [00:00<01:56,  1.34it/s]\u001b[A\n",
      "Testing... (loss=3.31441):   1%|| 2/157 [00:00<01:08,  2.27it/s]\u001b[A\n",
      "Testing... (loss=3.36396):   1%|| 2/157 [00:01<01:08,  2.27it/s]\u001b[A\n",
      "Testing... (loss=3.36396):   2%|| 3/157 [00:01<00:52,  2.95it/s]\u001b[A\n",
      "Testing... (loss=3.29160):   2%|| 3/157 [00:01<00:52,  2.95it/s]\u001b[A\n",
      "Testing... (loss=3.29160):   3%|| 4/157 [00:01<00:44,  3.43it/s]\u001b[A\n",
      "Testing... (loss=3.32136):   3%|| 4/157 [00:01<00:44,  3.43it/s]\u001b[A\n",
      "Testing... (loss=3.32136):   3%|| 5/157 [00:01<00:40,  3.78it/s]\u001b[A\n",
      "Testing... (loss=3.41945):   3%|| 5/157 [00:01<00:40,  3.78it/s]\u001b[A\n",
      "Testing... (loss=3.41945):   4%|| 6/157 [00:01<00:37,  4.03it/s]\u001b[A\n",
      "Testing... (loss=3.46088):   4%|| 6/157 [00:02<00:37,  4.03it/s]\u001b[A\n",
      "Testing... (loss=3.46088):   4%|| 7/157 [00:02<00:35,  4.22it/s]\u001b[A\n",
      "Testing... (loss=3.37859):   4%|| 7/157 [00:02<00:35,  4.22it/s]\u001b[A\n",
      "Testing... (loss=3.37859):   5%|| 8/157 [00:02<00:34,  4.35it/s]\u001b[A\n",
      "Testing... (loss=3.42020):   5%|| 8/157 [00:02<00:34,  4.35it/s]\u001b[A\n",
      "Testing... (loss=3.42020):   6%|| 9/157 [00:02<00:33,  4.45it/s]\u001b[A\n",
      "Testing... (loss=3.25308):   6%|| 9/157 [00:02<00:33,  4.45it/s]\u001b[A\n",
      "Testing... (loss=3.25308):   6%|| 10/157 [00:02<00:32,  4.51it/s]\u001b[A\n",
      "Testing... (loss=3.29206):   6%|| 10/157 [00:02<00:32,  4.51it/s]\u001b[A\n",
      "Testing... (loss=3.29206):   7%|| 11/157 [00:02<00:32,  4.55it/s]\u001b[A\n",
      "Testing... (loss=3.22484):   7%|| 11/157 [00:03<00:32,  4.55it/s]\u001b[A\n",
      "Testing... (loss=3.22484):   8%|| 12/157 [00:03<00:31,  4.58it/s]\u001b[A\n",
      "Testing... (loss=3.25258):   8%|| 12/157 [00:03<00:31,  4.58it/s]\u001b[A\n",
      "Testing... (loss=3.25258):   8%|| 13/157 [00:03<00:31,  4.61it/s]\u001b[A\n",
      "Testing... (loss=3.40251):   8%|| 13/157 [00:03<00:31,  4.61it/s]\u001b[A\n",
      "Testing... (loss=3.40251):   9%|| 14/157 [00:03<00:30,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.47173):   9%|| 14/157 [00:03<00:30,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.47173):  10%|| 15/157 [00:03<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.28343):  10%|| 15/157 [00:03<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.28343):  10%|| 16/157 [00:03<00:30,  4.61it/s]\u001b[A\n",
      "Testing... (loss=3.33263):  10%|| 16/157 [00:04<00:30,  4.61it/s]\u001b[A\n",
      "Testing... (loss=3.33263):  11%|| 17/157 [00:04<00:30,  4.61it/s]\u001b[A\n",
      "Testing... (loss=3.33543):  11%|| 17/157 [00:04<00:30,  4.61it/s]\u001b[A\n",
      "Testing... (loss=3.33543):  11%|| 18/157 [00:04<00:29,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.35995):  11%|| 18/157 [00:04<00:29,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.35995):  12%|| 19/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.28209):  12%|| 19/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.28209):  13%|| 20/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.44792):  13%|| 20/157 [00:05<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.44792):  13%|| 21/157 [00:05<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.28297):  13%|| 21/157 [00:05<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.28297):  14%|| 22/157 [00:05<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.29235):  14%|| 22/157 [00:05<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.29235):  15%|| 23/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.48154):  15%|| 23/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.48154):  15%|| 24/157 [00:05<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.43087):  15%|| 24/157 [00:05<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.43087):  16%|| 25/157 [00:05<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.38329):  16%|| 25/157 [00:06<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.38329):  17%|| 26/157 [00:06<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.36065):  17%|| 26/157 [00:06<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.36065):  17%|| 27/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.31773):  17%|| 27/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.31773):  18%|| 28/157 [00:06<00:27,  4.67it/s]\u001b[A\n",
      "Testing... (loss=3.35351):  18%|| 28/157 [00:06<00:27,  4.67it/s]\u001b[A\n",
      "Testing... (loss=3.35351):  18%|| 29/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.20912):  18%|| 29/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.20912):  19%|| 30/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.30561):  19%|| 30/157 [00:07<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.30561):  20%|| 31/157 [00:07<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.27028):  20%|| 31/157 [00:07<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.27028):  20%|| 32/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.39672):  20%|| 32/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.39672):  21%|| 33/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.33403):  21%|| 33/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.33403):  22%|| 34/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.38490):  22%|| 34/157 [00:08<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.38490):  22%|| 35/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.33100):  22%|| 35/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.33100):  23%|| 36/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.41178):  23%|| 36/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.41178):  24%|| 37/157 [00:08<00:25,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.33385):  24%|| 37/157 [00:08<00:25,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.33385):  24%|| 38/157 [00:08<00:25,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.36041):  24%|| 38/157 [00:08<00:25,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.36041):  25%|| 39/157 [00:08<00:25,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.36822):  25%|| 39/157 [00:09<00:25,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.36822):  25%|| 40/157 [00:09<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.45415):  25%|| 40/157 [00:09<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.45415):  26%|| 41/157 [00:09<00:24,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.38790):  26%|| 41/157 [00:09<00:24,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.38790):  27%|| 42/157 [00:09<00:24,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.35942):  27%|| 42/157 [00:09<00:24,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.35942):  27%|| 43/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.36548):  27%|| 43/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.36548):  28%|| 44/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.38533):  28%|| 44/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.38533):  29%|| 45/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.34083):  29%|| 45/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.34083):  29%|| 46/157 [00:10<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.33093):  29%|| 46/157 [00:10<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.33093):  30%|| 47/157 [00:10<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.24415):  30%|| 47/157 [00:10<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.24415):  31%|| 48/157 [00:10<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.37068):  31%|| 48/157 [00:11<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.37068):  31%|| 49/157 [00:11<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.32535):  31%|| 49/157 [00:11<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.32535):  32%|| 50/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.38358):  32%|| 50/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.38358):  32%|| 51/157 [00:11<00:22,  4.67it/s]\u001b[A\n",
      "Testing... (loss=3.42192):  32%|| 51/157 [00:11<00:22,  4.67it/s]\u001b[A\n",
      "Testing... (loss=3.42192):  33%|| 52/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.31176):  33%|| 52/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.31176):  34%|| 53/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.29632):  34%|| 53/157 [00:12<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.29632):  34%|| 54/157 [00:12<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.27287):  34%|| 54/157 [00:12<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.27287):  35%|| 55/157 [00:12<00:21,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.36520):  35%|| 55/157 [00:12<00:21,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.36520):  36%|| 56/157 [00:12<00:21,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.23394):  36%|| 56/157 [00:12<00:21,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.23394):  36%|| 57/157 [00:12<00:21,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.34539):  36%|| 57/157 [00:13<00:21,  4.66it/s]\u001b[A\n",
      "Testing... (loss=3.34539):  37%|| 58/157 [00:13<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.24460):  37%|| 58/157 [00:13<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.24460):  38%|| 59/157 [00:13<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.34736):  38%|| 59/157 [00:13<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.34736):  38%|| 60/157 [00:13<00:20,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.37471):  38%|| 60/157 [00:13<00:20,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.37471):  39%|| 61/157 [00:13<00:20,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.32075):  39%|| 61/157 [00:13<00:20,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.32075):  39%|| 62/157 [00:13<00:20,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.39949):  39%|| 62/157 [00:14<00:20,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.39949):  40%|| 63/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.32524):  40%|| 63/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.32524):  41%|| 64/157 [00:14<00:20,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.34497):  41%|| 64/157 [00:14<00:20,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.34497):  41%|| 65/157 [00:14<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.33020):  41%|| 65/157 [00:14<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.33020):  42%|| 66/157 [00:14<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.33347):  42%|| 66/157 [00:14<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.33347):  43%|| 67/157 [00:14<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.31183):  43%|| 67/157 [00:15<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.31183):  43%|| 68/157 [00:15<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.23748):  43%|| 68/157 [00:15<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.23748):  44%|| 69/157 [00:15<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.19777):  44%|| 69/157 [00:15<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.19777):  45%|| 70/157 [00:15<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.36656):  45%|| 70/157 [00:15<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.36656):  45%|| 71/157 [00:15<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.30926):  45%|| 71/157 [00:16<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.30926):  46%|| 72/157 [00:16<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.36096):  46%|| 72/157 [00:16<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.36096):  46%|| 73/157 [00:16<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.43190):  46%|| 73/157 [00:16<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.43190):  47%|| 74/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.35959):  47%|| 74/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.35959):  48%|| 75/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.45696):  48%|| 75/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.45696):  48%|| 76/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.40510):  48%|| 76/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.40510):  49%|| 77/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.40298):  49%|| 77/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.40298):  50%|| 78/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.30388):  50%|| 78/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.30388):  50%|| 79/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.31404):  50%|| 79/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.31404):  51%|| 80/157 [00:17<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.28847):  51%|| 80/157 [00:17<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.28847):  52%|| 81/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.21295):  52%|| 81/157 [00:18<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.21295):  52%|| 82/157 [00:18<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.27375):  52%|| 82/157 [00:18<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.27375):  53%|| 83/157 [00:18<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.32977):  53%|| 83/157 [00:18<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.32977):  54%|| 84/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.42762):  54%|| 84/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.42762):  54%|| 85/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.30349):  54%|| 85/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.30349):  55%|| 86/157 [00:19<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.44473):  55%|| 86/157 [00:19<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.44473):  55%|| 87/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.37524):  55%|| 87/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.37524):  56%|| 88/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.33032):  56%|| 88/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.33032):  57%|| 89/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.30453):  57%|| 89/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.30453):  57%|| 90/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.33349):  57%|| 90/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.33349):  58%|| 91/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.37710):  58%|| 91/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.37710):  59%|| 92/157 [00:20<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.29829):  59%|| 92/157 [00:20<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.29829):  59%|| 93/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.36802):  59%|| 93/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.36802):  60%|| 94/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.29961):  60%|| 94/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.29961):  61%|| 95/157 [00:20<00:13,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.30453):  61%|| 95/157 [00:21<00:13,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.30453):  61%|| 96/157 [00:21<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.34695):  61%|| 96/157 [00:21<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.34695):  62%|| 97/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.35421):  62%|| 97/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.35421):  62%|| 98/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.36272):  62%|| 98/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.36272):  63%|| 99/157 [00:21<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.30554):  63%|| 99/157 [00:22<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.30554):  64%|| 100/157 [00:22<00:12,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.30960):  64%|| 100/157 [00:22<00:12,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.30960):  64%|| 101/157 [00:22<00:12,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.48345):  64%|| 101/157 [00:22<00:12,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.48345):  65%|| 102/157 [00:22<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.41161):  65%|| 102/157 [00:22<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.41161):  66%|| 103/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.28198):  66%|| 103/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.28198):  66%|| 104/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.26571):  66%|| 104/157 [00:23<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.26571):  67%|| 105/157 [00:23<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.34323):  67%|| 105/157 [00:23<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.34323):  68%|| 106/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.34453):  68%|| 106/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.34453):  68%|| 107/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.34987):  68%|| 107/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.34987):  69%|| 108/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.21626):  69%|| 108/157 [00:24<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.21626):  69%|| 109/157 [00:24<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.37207):  69%|| 109/157 [00:24<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.37207):  70%|| 110/157 [00:24<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.39631):  70%|| 110/157 [00:24<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.39631):  71%|| 111/157 [00:24<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.33209):  71%|| 111/157 [00:24<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.33209):  71%|| 112/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.30745):  71%|| 112/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.30745):  72%|| 113/157 [00:24<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.31356):  72%|| 113/157 [00:25<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.31356):  73%|| 114/157 [00:25<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.37594):  73%|| 114/157 [00:25<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.37594):  73%|| 115/157 [00:25<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.43107):  73%|| 115/157 [00:25<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.43107):  74%|| 116/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.48122):  74%|| 116/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.48122):  75%|| 117/157 [00:25<00:08,  4.61it/s]\u001b[A\n",
      "Testing... (loss=3.30506):  75%|| 117/157 [00:25<00:08,  4.61it/s]\u001b[A\n",
      "Testing... (loss=3.30506):  75%|| 118/157 [00:25<00:08,  4.60it/s]\u001b[A\n",
      "Testing... (loss=3.32078):  75%|| 118/157 [00:26<00:08,  4.60it/s]\u001b[A\n",
      "Testing... (loss=3.32078):  76%|| 119/157 [00:26<00:08,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.16228):  76%|| 119/157 [00:26<00:08,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.16228):  76%|| 120/157 [00:26<00:08,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.44460):  76%|| 120/157 [00:26<00:08,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.44460):  77%|| 121/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.34109):  77%|| 121/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.34109):  78%|| 122/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.36299):  78%|| 122/157 [00:27<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.36299):  78%|| 123/157 [00:27<00:07,  4.59it/s]\u001b[A\n",
      "Testing... (loss=3.47164):  78%|| 123/157 [00:27<00:07,  4.59it/s]\u001b[A\n",
      "Testing... (loss=3.47164):  79%|| 124/157 [00:27<00:07,  4.59it/s]\u001b[A\n",
      "Testing... (loss=3.35902):  79%|| 124/157 [00:27<00:07,  4.59it/s]\u001b[A\n",
      "Testing... (loss=3.35902):  80%|| 125/157 [00:27<00:06,  4.59it/s]\u001b[A\n",
      "Testing... (loss=3.30113):  80%|| 125/157 [00:27<00:06,  4.59it/s]\u001b[A\n",
      "Testing... (loss=3.30113):  80%|| 126/157 [00:27<00:06,  4.58it/s]\u001b[A\n",
      "Testing... (loss=3.42579):  80%|| 126/157 [00:27<00:06,  4.58it/s]\u001b[A\n",
      "Testing... (loss=3.42579):  81%|| 127/157 [00:27<00:06,  4.57it/s]\u001b[A\n",
      "Testing... (loss=3.41583):  81%|| 127/157 [00:28<00:06,  4.57it/s]\u001b[A\n",
      "Testing... (loss=3.41583):  82%|| 128/157 [00:28<00:06,  4.59it/s]\u001b[A\n",
      "Testing... (loss=3.28388):  82%|| 128/157 [00:28<00:06,  4.59it/s]\u001b[A\n",
      "Testing... (loss=3.28388):  82%|| 129/157 [00:28<00:06,  4.60it/s]\u001b[A\n",
      "Testing... (loss=3.29772):  82%|| 129/157 [00:28<00:06,  4.60it/s]\u001b[A\n",
      "Testing... (loss=3.29772):  83%|| 130/157 [00:28<00:05,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.22755):  83%|| 130/157 [00:28<00:05,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.22755):  83%|| 131/157 [00:28<00:05,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.35960):  83%|| 131/157 [00:29<00:05,  4.62it/s]\u001b[A\n",
      "Testing... (loss=3.35960):  84%|| 132/157 [00:29<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.27110):  84%|| 132/157 [00:29<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.27110):  85%|| 133/157 [00:29<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.23702):  85%|| 133/157 [00:29<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.23702):  85%|| 134/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.34753):  85%|| 134/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.34753):  86%|| 135/157 [00:29<00:04,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.31736):  86%|| 135/157 [00:29<00:04,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.31736):  87%|| 136/157 [00:29<00:04,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.17513):  87%|| 136/157 [00:30<00:04,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.17513):  87%|| 137/157 [00:30<00:04,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.43700):  87%|| 137/157 [00:30<00:04,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.43700):  88%|| 138/157 [00:30<00:04,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.32084):  88%|| 138/157 [00:30<00:04,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.32084):  89%|| 139/157 [00:30<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.40711):  89%|| 139/157 [00:30<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.40711):  89%|| 140/157 [00:30<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.32253):  89%|| 140/157 [00:30<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.32253):  90%|| 141/157 [00:30<00:03,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.32950):  90%|| 141/157 [00:31<00:03,  4.65it/s]\u001b[A\n",
      "Testing... (loss=3.32950):  90%|| 142/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.44225):  90%|| 142/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.44225):  91%|| 143/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.34675):  91%|| 143/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.34675):  92%|| 144/157 [00:31<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.37378):  92%|| 144/157 [00:31<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.37378):  92%|| 145/157 [00:31<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.28511):  92%|| 145/157 [00:32<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.28511):  93%|| 146/157 [00:32<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.27642):  93%|| 146/157 [00:32<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.27642):  94%|| 147/157 [00:32<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.40362):  94%|| 147/157 [00:32<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.40362):  94%|| 148/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.27631):  94%|| 148/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.27631):  95%|| 149/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.33860):  95%|| 149/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.33860):  96%|| 150/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.43551):  96%|| 150/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.43551):  96%|| 151/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.28520):  96%|| 151/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.28520):  97%|| 152/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.45025):  97%|| 152/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.45025):  97%|| 153/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.33185):  97%|| 153/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.33185):  98%|| 154/157 [00:33<00:00,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.36713):  98%|| 154/157 [00:33<00:00,  4.63it/s]\u001b[A\n",
      "Testing... (loss=3.36713):  99%|| 155/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.31326):  99%|| 155/157 [00:34<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.31326):  99%|| 156/157 [00:34<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=3.36171): 100%|| 157/157 [00:34<00:00,  4.57it/s]\u001b[A\n",
      "11/05/2021 21:15:04 - INFO - __main__ - \n",
      "\n",
      "11/05/2021 21:15:04 - INFO - __main__ - Test Results\n",
      "11/05/2021 21:15:04 - INFO - __main__ - Global Steps: 100\n",
      "11/05/2021 21:15:04 - INFO - __main__ - Test Loss: 3.34167\n",
      "11/05/2021 21:15:04 - INFO - __main__ - Test Accuracy: 0.70980\n",
      "11/05/2021 21:15:05 - INFO - __main__ - Saved model checkpoint to [DIR: output_cifar100]\n",
      "Training (200 / 500 Steps) (loss=1.77148):  25%|| 199/782 [04:28<08:39,  1.12it/s]  11/05/2021 21:17:03 - INFO - __main__ - ***** Running Test *****\n",
      "11/05/2021 21:17:03 - INFO - __main__ -   Num steps = 157\n",
      "11/05/2021 21:17:03 - INFO - __main__ -   Batch size = 64\n",
      "\n",
      "Test... (loss=X.X):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=1.25722):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=1.25722):   1%|| 1/157 [00:00<01:58,  1.32it/s]\u001b[A\n",
      "Testing... (loss=1.29361):   1%|| 1/157 [00:00<01:58,  1.32it/s]\u001b[A\n",
      "Testing... (loss=1.29361):   1%|| 2/157 [00:00<01:08,  2.28it/s]\u001b[A\n",
      "Testing... (loss=1.53251):   1%|| 2/157 [00:01<01:08,  2.28it/s]\u001b[A\n",
      "Testing... (loss=1.53251):   2%|| 3/157 [00:01<00:52,  2.95it/s]\u001b[A\n",
      "Testing... (loss=1.30984):   2%|| 3/157 [00:01<00:52,  2.95it/s]\u001b[A\n",
      "Testing... (loss=1.30984):   3%|| 4/157 [00:01<00:44,  3.44it/s]\u001b[A\n",
      "Testing... (loss=1.38539):   3%|| 4/157 [00:01<00:44,  3.44it/s]\u001b[A\n",
      "Testing... (loss=1.38539):   3%|| 5/157 [00:01<00:40,  3.80it/s]\u001b[A\n",
      "Testing... (loss=1.51914):   3%|| 5/157 [00:01<00:40,  3.80it/s]\u001b[A\n",
      "Testing... (loss=1.51914):   4%|| 6/157 [00:01<00:37,  4.05it/s]\u001b[A\n",
      "Testing... (loss=1.55373):   4%|| 6/157 [00:02<00:37,  4.05it/s]\u001b[A\n",
      "Testing... (loss=1.55373):   4%|| 7/157 [00:02<00:35,  4.23it/s]\u001b[A\n",
      "Testing... (loss=1.56237):   4%|| 7/157 [00:02<00:35,  4.23it/s]\u001b[A\n",
      "Testing... (loss=1.56237):   5%|| 8/157 [00:02<00:34,  4.34it/s]\u001b[A\n",
      "Testing... (loss=1.55601):   5%|| 8/157 [00:02<00:34,  4.34it/s]\u001b[A\n",
      "Testing... (loss=1.55601):   6%|| 9/157 [00:02<00:33,  4.42it/s]\u001b[A\n",
      "Testing... (loss=1.38017):   6%|| 9/157 [00:02<00:33,  4.42it/s]\u001b[A\n",
      "Testing... (loss=1.38017):   6%|| 10/157 [00:02<00:32,  4.49it/s]\u001b[A\n",
      "Testing... (loss=1.20650):   6%|| 10/157 [00:02<00:32,  4.49it/s]\u001b[A\n",
      "Testing... (loss=1.20650):   7%|| 11/157 [00:02<00:32,  4.53it/s]\u001b[A\n",
      "Testing... (loss=1.33962):   7%|| 11/157 [00:03<00:32,  4.53it/s]\u001b[A\n",
      "Testing... (loss=1.33962):   8%|| 12/157 [00:03<00:31,  4.57it/s]\u001b[A\n",
      "Testing... (loss=1.27368):   8%|| 12/157 [00:03<00:31,  4.57it/s]\u001b[A\n",
      "Testing... (loss=1.27368):   8%|| 13/157 [00:03<00:31,  4.58it/s]\u001b[A\n",
      "Testing... (loss=1.45915):   8%|| 13/157 [00:03<00:31,  4.58it/s]\u001b[A\n",
      "Testing... (loss=1.45915):   9%|| 14/157 [00:03<00:31,  4.61it/s]\u001b[A\n",
      "Testing... (loss=1.70549):   9%|| 14/157 [00:03<00:31,  4.61it/s]\u001b[A\n",
      "Testing... (loss=1.70549):  10%|| 15/157 [00:03<00:30,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.38345):  10%|| 15/157 [00:03<00:30,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.38345):  10%|| 16/157 [00:03<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.43014):  10%|| 16/157 [00:04<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.43014):  11%|| 17/157 [00:04<00:30,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.44775):  11%|| 17/157 [00:04<00:30,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.44775):  11%|| 18/157 [00:04<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.54560):  11%|| 18/157 [00:04<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.54560):  12%|| 19/157 [00:04<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.28666):  12%|| 19/157 [00:04<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.28666):  13%|| 20/157 [00:04<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.69789):  13%|| 20/157 [00:05<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.69789):  13%|| 21/157 [00:05<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.40209):  13%|| 21/157 [00:05<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.40209):  14%|| 22/157 [00:05<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.31465):  14%|| 22/157 [00:05<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.31465):  15%|| 23/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.57939):  15%|| 23/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.57939):  15%|| 24/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.57593):  15%|| 24/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.57593):  16%|| 25/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.58906):  16%|| 25/157 [00:06<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.58906):  17%|| 26/157 [00:06<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.45899):  17%|| 26/157 [00:06<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.45899):  17%|| 27/157 [00:06<00:27,  4.67it/s]\u001b[A\n",
      "Testing... (loss=1.37464):  17%|| 27/157 [00:06<00:27,  4.67it/s]\u001b[A\n",
      "Testing... (loss=1.37464):  18%|| 28/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.52368):  18%|| 28/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.52368):  18%|| 29/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.27067):  18%|| 29/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.27067):  19%|| 30/157 [00:06<00:27,  4.67it/s]\u001b[A\n",
      "Testing... (loss=1.29896):  19%|| 30/157 [00:07<00:27,  4.67it/s]\u001b[A\n",
      "Testing... (loss=1.29896):  20%|| 31/157 [00:07<00:26,  4.67it/s]\u001b[A\n",
      "Testing... (loss=1.30671):  20%|| 31/157 [00:07<00:26,  4.67it/s]\u001b[A\n",
      "Testing... (loss=1.30671):  20%|| 32/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.51324):  20%|| 32/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.51324):  21%|| 33/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.45042):  21%|| 33/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.45042):  22%|| 34/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.55803):  22%|| 34/157 [00:08<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.55803):  22%|| 35/157 [00:08<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.47133):  22%|| 35/157 [00:08<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.47133):  23%|| 36/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.49916):  23%|| 36/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.49916):  24%|| 37/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.44119):  24%|| 37/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.44119):  24%|| 38/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.44954):  24%|| 38/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.44954):  25%|| 39/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.50458):  25%|| 39/157 [00:09<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.50458):  25%|| 40/157 [00:09<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.55263):  25%|| 40/157 [00:09<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.55263):  26%|| 41/157 [00:09<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.63804):  26%|| 41/157 [00:09<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.63804):  27%|| 42/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.59264):  27%|| 42/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.59264):  27%|| 43/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.49479):  27%|| 43/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.49479):  28%|| 44/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.53284):  28%|| 44/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.53284):  29%|| 45/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.45133):  29%|| 45/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.45133):  29%|| 46/157 [00:10<00:23,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.33413):  29%|| 46/157 [00:10<00:23,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.33413):  30%|| 47/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.41209):  30%|| 47/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.41209):  31%|| 48/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.48679):  31%|| 48/157 [00:11<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.48679):  31%|| 49/157 [00:11<00:23,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.48056):  31%|| 49/157 [00:11<00:23,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.48056):  32%|| 50/157 [00:11<00:23,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.51061):  32%|| 50/157 [00:11<00:23,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.51061):  32%|| 51/157 [00:11<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.39850):  32%|| 51/157 [00:11<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.39850):  33%|| 52/157 [00:11<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.40677):  33%|| 52/157 [00:11<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.40677):  34%|| 53/157 [00:11<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.37925):  34%|| 53/157 [00:12<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.37925):  34%|| 54/157 [00:12<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.41526):  34%|| 54/157 [00:12<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.41526):  35%|| 55/157 [00:12<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.47682):  35%|| 55/157 [00:12<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.47682):  36%|| 56/157 [00:12<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.28907):  36%|| 56/157 [00:12<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.28907):  36%|| 57/157 [00:12<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.44502):  36%|| 57/157 [00:13<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.44502):  37%|| 58/157 [00:13<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.26078):  37%|| 58/157 [00:13<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.26078):  38%|| 59/157 [00:13<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.43688):  38%|| 59/157 [00:13<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.43688):  38%|| 60/157 [00:13<00:20,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.53941):  38%|| 60/157 [00:13<00:20,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.53941):  39%|| 61/157 [00:13<00:20,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.43109):  39%|| 61/157 [00:13<00:20,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.43109):  39%|| 62/157 [00:13<00:20,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.38007):  39%|| 62/157 [00:14<00:20,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.38007):  40%|| 63/157 [00:14<00:20,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.37904):  40%|| 63/157 [00:14<00:20,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.37904):  41%|| 64/157 [00:14<00:20,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.61133):  41%|| 64/157 [00:14<00:20,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.61133):  41%|| 65/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.59683):  41%|| 65/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.59683):  42%|| 66/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.31933):  42%|| 66/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.31933):  43%|| 67/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.45828):  43%|| 67/157 [00:15<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.45828):  43%|| 68/157 [00:15<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.32534):  43%|| 68/157 [00:15<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.32534):  44%|| 69/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.17416):  44%|| 69/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.17416):  45%|| 70/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.46485):  45%|| 70/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.46485):  45%|| 71/157 [00:15<00:18,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.31884):  45%|| 71/157 [00:16<00:18,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.31884):  46%|| 72/157 [00:16<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.46421):  46%|| 72/157 [00:16<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.46421):  46%|| 73/157 [00:16<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.63996):  46%|| 73/157 [00:16<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.63996):  47%|| 74/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.52323):  47%|| 74/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.52323):  48%|| 75/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.54730):  48%|| 75/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.54730):  48%|| 76/157 [00:16<00:17,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.52566):  48%|| 76/157 [00:17<00:17,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.52566):  49%|| 77/157 [00:17<00:17,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.39950):  49%|| 77/157 [00:17<00:17,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.39950):  50%|| 78/157 [00:17<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.44772):  50%|| 78/157 [00:17<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.44772):  50%|| 79/157 [00:17<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.48517):  50%|| 79/157 [00:17<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.48517):  51%|| 80/157 [00:17<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.30130):  51%|| 80/157 [00:17<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.30130):  52%|| 81/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.27023):  52%|| 81/157 [00:18<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.27023):  52%|| 82/157 [00:18<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.30414):  52%|| 82/157 [00:18<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.30414):  53%|| 83/157 [00:18<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.43790):  53%|| 83/157 [00:18<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.43790):  54%|| 84/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.47696):  54%|| 84/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.47696):  54%|| 85/157 [00:18<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.44823):  54%|| 85/157 [00:19<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.44823):  55%|| 86/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.53802):  55%|| 86/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.53802):  55%|| 87/157 [00:19<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.37140):  55%|| 87/157 [00:19<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.37140):  56%|| 88/157 [00:19<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.45602):  56%|| 88/157 [00:19<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.45602):  57%|| 89/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.37210):  57%|| 89/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.37210):  57%|| 90/157 [00:19<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.35523):  57%|| 90/157 [00:20<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.35523):  58%|| 91/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.43392):  58%|| 91/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.43392):  59%|| 92/157 [00:20<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.21567):  59%|| 92/157 [00:20<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.21567):  59%|| 93/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.42409):  59%|| 93/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.42409):  60%|| 94/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.38279):  60%|| 94/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.38279):  61%|| 95/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.36446):  61%|| 95/157 [00:21<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.36446):  61%|| 96/157 [00:21<00:13,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.44449):  61%|| 96/157 [00:21<00:13,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.44449):  62%|| 97/157 [00:21<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.38561):  62%|| 97/157 [00:21<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.38561):  62%|| 98/157 [00:21<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.44618):  62%|| 98/157 [00:21<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.44618):  63%|| 99/157 [00:21<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.35611):  63%|| 99/157 [00:22<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.35611):  64%|| 100/157 [00:22<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.34966):  64%|| 100/157 [00:22<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.34966):  64%|| 101/157 [00:22<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.56031):  64%|| 101/157 [00:22<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.56031):  65%|| 102/157 [00:22<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.55531):  65%|| 102/157 [00:22<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.55531):  66%|| 103/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.41128):  66%|| 103/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.41128):  66%|| 104/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.20549):  66%|| 104/157 [00:23<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.20549):  67%|| 105/157 [00:23<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.36549):  67%|| 105/157 [00:23<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.36549):  68%|| 106/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.41976):  68%|| 106/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.41976):  68%|| 107/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.43310):  68%|| 107/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.43310):  69%|| 108/157 [00:23<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.32794):  69%|| 108/157 [00:24<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.32794):  69%|| 109/157 [00:24<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.51054):  69%|| 109/157 [00:24<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.51054):  70%|| 110/157 [00:24<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.53274):  70%|| 110/157 [00:24<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.53274):  71%|| 111/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.24293):  71%|| 111/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.24293):  71%|| 112/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.37688):  71%|| 112/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.37688):  72%|| 113/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.30190):  72%|| 113/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.30190):  73%|| 114/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.43511):  73%|| 114/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.43511):  73%|| 115/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.43914):  73%|| 115/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.43914):  74%|| 116/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.68999):  74%|| 116/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.68999):  75%|| 117/157 [00:25<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.44548):  75%|| 117/157 [00:25<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.44548):  75%|| 118/157 [00:25<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.36034):  75%|| 118/157 [00:26<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.36034):  76%|| 119/157 [00:26<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.21282):  76%|| 119/157 [00:26<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.21282):  76%|| 120/157 [00:26<00:07,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.54979):  76%|| 120/157 [00:26<00:07,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.54979):  77%|| 121/157 [00:26<00:07,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.38468):  77%|| 121/157 [00:26<00:07,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.38468):  78%|| 122/157 [00:26<00:07,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.33537):  78%|| 122/157 [00:27<00:07,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.33537):  78%|| 123/157 [00:27<00:07,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.68827):  78%|| 123/157 [00:27<00:07,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.68827):  79%|| 124/157 [00:27<00:07,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.40875):  79%|| 124/157 [00:27<00:07,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.40875):  80%|| 125/157 [00:27<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.41537):  80%|| 125/157 [00:27<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.41537):  80%|| 126/157 [00:27<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.58812):  80%|| 126/157 [00:27<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.58812):  81%|| 127/157 [00:27<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.51628):  81%|| 127/157 [00:28<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.51628):  82%|| 128/157 [00:28<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.38714):  82%|| 128/157 [00:28<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.38714):  82%|| 129/157 [00:28<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.35214):  82%|| 129/157 [00:28<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.35214):  83%|| 130/157 [00:28<00:05,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.30489):  83%|| 130/157 [00:28<00:05,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.30489):  83%|| 131/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.50669):  83%|| 131/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.50669):  84%|| 132/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.27915):  84%|| 132/157 [00:29<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.27915):  85%|| 133/157 [00:29<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.23468):  85%|| 133/157 [00:29<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.23468):  85%|| 134/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.45616):  85%|| 134/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.45616):  86%|| 135/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.34334):  86%|| 135/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.34334):  87%|| 136/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.14904):  87%|| 136/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.14904):  87%|| 137/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.53915):  87%|| 137/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.53915):  88%|| 138/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.40353):  88%|| 138/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.40353):  89%|| 139/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.60531):  89%|| 139/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.60531):  89%|| 140/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.43228):  89%|| 140/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.43228):  90%|| 141/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.40578):  90%|| 141/157 [00:31<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.40578):  90%|| 142/157 [00:31<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.57158):  90%|| 142/157 [00:31<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.57158):  91%|| 143/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.47443):  91%|| 143/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.47443):  92%|| 144/157 [00:31<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.47720):  92%|| 144/157 [00:31<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.47720):  92%|| 145/157 [00:31<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.24928):  92%|| 145/157 [00:31<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.24928):  93%|| 146/157 [00:31<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.42171):  93%|| 146/157 [00:32<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.42171):  94%|| 147/157 [00:32<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.44894):  94%|| 147/157 [00:32<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.44894):  94%|| 148/157 [00:32<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.30806):  94%|| 148/157 [00:32<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.30806):  95%|| 149/157 [00:32<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.42411):  95%|| 149/157 [00:32<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.42411):  96%|| 150/157 [00:32<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.49591):  96%|| 150/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.49591):  96%|| 151/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.43993):  96%|| 151/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.43993):  97%|| 152/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.62049):  97%|| 152/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.62049):  97%|| 153/157 [00:33<00:00,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.50587):  97%|| 153/157 [00:33<00:00,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.50587):  98%|| 154/157 [00:33<00:00,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.33709):  98%|| 154/157 [00:33<00:00,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.33709):  99%|| 155/157 [00:33<00:00,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.36179):  99%|| 155/157 [00:34<00:00,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.36179):  99%|| 156/157 [00:34<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.27671): 100%|| 157/157 [00:34<00:00,  4.57it/s]\u001b[A\n",
      "11/05/2021 21:17:38 - INFO - __main__ - \n",
      "\n",
      "11/05/2021 21:17:38 - INFO - __main__ - Test Results\n",
      "11/05/2021 21:17:38 - INFO - __main__ - Global Steps: 200\n",
      "11/05/2021 21:17:38 - INFO - __main__ - Test Loss: 1.42926\n",
      "11/05/2021 21:17:38 - INFO - __main__ - Test Accuracy: 0.81970\n",
      "11/05/2021 21:17:39 - INFO - __main__ - Saved model checkpoint to [DIR: output_cifar100]\n",
      "Training (300 / 500 Steps) (loss=1.60135):  38%|| 299/782 [07:03<07:12,  1.12it/s]  11/05/2021 21:19:38 - INFO - __main__ - ***** Running Test *****\n",
      "11/05/2021 21:19:38 - INFO - __main__ -   Num steps = 157\n",
      "11/05/2021 21:19:38 - INFO - __main__ -   Batch size = 64\n",
      "\n",
      "Test... (loss=X.X):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.76501):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.76501):   1%|| 1/157 [00:00<01:54,  1.37it/s]\u001b[A\n",
      "Testing... (loss=0.83326):   1%|| 1/157 [00:00<01:54,  1.37it/s]\u001b[A\n",
      "Testing... (loss=0.83326):   1%|| 2/157 [00:00<01:06,  2.33it/s]\u001b[A\n",
      "Testing... (loss=1.03653):   1%|| 2/157 [00:01<01:06,  2.33it/s]\u001b[A\n",
      "Testing... (loss=1.03653):   2%|| 3/157 [00:01<00:51,  3.00it/s]\u001b[A\n",
      "Testing... (loss=0.79178):   2%|| 3/157 [00:01<00:51,  3.00it/s]\u001b[A\n",
      "Testing... (loss=0.79178):   3%|| 4/157 [00:01<00:43,  3.48it/s]\u001b[A\n",
      "Testing... (loss=0.84786):   3%|| 4/157 [00:01<00:43,  3.48it/s]\u001b[A\n",
      "Testing... (loss=0.84786):   3%|| 5/157 [00:01<00:39,  3.81it/s]\u001b[A\n",
      "Testing... (loss=0.92764):   3%|| 5/157 [00:01<00:39,  3.81it/s]\u001b[A\n",
      "Testing... (loss=0.92764):   4%|| 6/157 [00:01<00:37,  4.07it/s]\u001b[A\n",
      "Testing... (loss=0.88276):   4%|| 6/157 [00:02<00:37,  4.07it/s]\u001b[A\n",
      "Testing... (loss=0.88276):   4%|| 7/157 [00:02<00:35,  4.23it/s]\u001b[A\n",
      "Testing... (loss=0.96702):   4%|| 7/157 [00:02<00:35,  4.23it/s]\u001b[A\n",
      "Testing... (loss=0.96702):   5%|| 8/157 [00:02<00:34,  4.34it/s]\u001b[A\n",
      "Testing... (loss=0.99384):   5%|| 8/157 [00:02<00:34,  4.34it/s]\u001b[A\n",
      "Testing... (loss=0.99384):   6%|| 9/157 [00:02<00:33,  4.43it/s]\u001b[A\n",
      "Testing... (loss=0.86042):   6%|| 9/157 [00:02<00:33,  4.43it/s]\u001b[A\n",
      "Testing... (loss=0.86042):   6%|| 10/157 [00:02<00:32,  4.49it/s]\u001b[A\n",
      "Testing... (loss=0.65899):   6%|| 10/157 [00:02<00:32,  4.49it/s]\u001b[A\n",
      "Testing... (loss=0.65899):   7%|| 11/157 [00:02<00:32,  4.53it/s]\u001b[A\n",
      "Testing... (loss=0.81559):   7%|| 11/157 [00:03<00:32,  4.53it/s]\u001b[A\n",
      "Testing... (loss=0.81559):   8%|| 12/157 [00:03<00:31,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.75778):   8%|| 12/157 [00:03<00:31,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.75778):   8%|| 13/157 [00:03<00:31,  4.58it/s]\u001b[A\n",
      "Testing... (loss=1.07259):   8%|| 13/157 [00:03<00:31,  4.58it/s]\u001b[A\n",
      "Testing... (loss=1.07259):   9%|| 14/157 [00:03<00:31,  4.61it/s]\u001b[A\n",
      "Testing... (loss=1.22333):   9%|| 14/157 [00:03<00:31,  4.61it/s]\u001b[A\n",
      "Testing... (loss=1.22333):  10%|| 15/157 [00:03<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.87147):  10%|| 15/157 [00:03<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.87147):  10%|| 16/157 [00:03<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.92387):  10%|| 16/157 [00:04<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.92387):  11%|| 17/157 [00:04<00:30,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.00537):  11%|| 17/157 [00:04<00:30,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.00537):  11%|| 18/157 [00:04<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.03492):  11%|| 18/157 [00:04<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.03492):  12%|| 19/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.69957):  12%|| 19/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.69957):  13%|| 20/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.15910):  13%|| 20/157 [00:05<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.15910):  13%|| 21/157 [00:05<00:29,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.94913):  13%|| 21/157 [00:05<00:29,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.94913):  14%|| 22/157 [00:05<00:29,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.81860):  14%|| 22/157 [00:05<00:29,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.81860):  15%|| 23/157 [00:05<00:29,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.99159):  15%|| 23/157 [00:05<00:29,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.99159):  15%|| 24/157 [00:05<00:28,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.96497):  15%|| 24/157 [00:05<00:28,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.96497):  16%|| 25/157 [00:05<00:28,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.02034):  16%|| 25/157 [00:06<00:28,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.02034):  17%|| 26/157 [00:06<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.97702):  17%|| 26/157 [00:06<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.97702):  17%|| 27/157 [00:06<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.93309):  17%|| 27/157 [00:06<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.93309):  18%|| 28/157 [00:06<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.95905):  18%|| 28/157 [00:06<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.95905):  18%|| 29/157 [00:06<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.77350):  18%|| 29/157 [00:06<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.77350):  19%|| 30/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.81438):  19%|| 30/157 [00:07<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.81438):  20%|| 31/157 [00:07<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.77950):  20%|| 31/157 [00:07<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.77950):  20%|| 32/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.94524):  20%|| 32/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.94524):  21%|| 33/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.97840):  21%|| 33/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.97840):  22%|| 34/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.05067):  22%|| 34/157 [00:08<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.05067):  22%|| 35/157 [00:08<00:26,  4.67it/s]\u001b[A\n",
      "Testing... (loss=0.96452):  22%|| 35/157 [00:08<00:26,  4.67it/s]\u001b[A\n",
      "Testing... (loss=0.96452):  23%|| 36/157 [00:08<00:25,  4.67it/s]\u001b[A\n",
      "Testing... (loss=0.97830):  23%|| 36/157 [00:08<00:25,  4.67it/s]\u001b[A\n",
      "Testing... (loss=0.97830):  24%|| 37/157 [00:08<00:25,  4.67it/s]\u001b[A\n",
      "Testing... (loss=0.90964):  24%|| 37/157 [00:08<00:25,  4.67it/s]\u001b[A\n",
      "Testing... (loss=0.90964):  24%|| 38/157 [00:08<00:25,  4.68it/s]\u001b[A\n",
      "Testing... (loss=0.93596):  24%|| 38/157 [00:08<00:25,  4.68it/s]\u001b[A\n",
      "Testing... (loss=0.93596):  25%|| 39/157 [00:08<00:25,  4.67it/s]\u001b[A\n",
      "Testing... (loss=1.03440):  25%|| 39/157 [00:09<00:25,  4.67it/s]\u001b[A\n",
      "Testing... (loss=1.03440):  25%|| 40/157 [00:09<00:25,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.03764):  25%|| 40/157 [00:09<00:25,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.03764):  26%|| 41/157 [00:09<00:24,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.03135):  26%|| 41/157 [00:09<00:24,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.03135):  27%|| 42/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.01596):  27%|| 42/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.01596):  27%|| 43/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.93101):  27%|| 43/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.93101):  28%|| 44/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.02770):  28%|| 44/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.02770):  29%|| 45/157 [00:10<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.98975):  29%|| 45/157 [00:10<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.98975):  29%|| 46/157 [00:10<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.86181):  29%|| 46/157 [00:10<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.86181):  30%|| 47/157 [00:10<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.88242):  30%|| 47/157 [00:10<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.88242):  31%|| 48/157 [00:10<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.94260):  31%|| 48/157 [00:11<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.94260):  31%|| 49/157 [00:11<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.01536):  31%|| 49/157 [00:11<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=1.01536):  32%|| 50/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.95001):  32%|| 50/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.95001):  32%|| 51/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.87321):  32%|| 51/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.87321):  33%|| 52/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.86888):  33%|| 52/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.86888):  34%|| 53/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.93995):  34%|| 53/157 [00:12<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.93995):  34%|| 54/157 [00:12<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.95611):  34%|| 54/157 [00:12<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.95611):  35%|| 55/157 [00:12<00:21,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.93878):  35%|| 55/157 [00:12<00:21,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.93878):  36%|| 56/157 [00:12<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.84047):  36%|| 56/157 [00:12<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.84047):  36%|| 57/157 [00:12<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86802):  36%|| 57/157 [00:13<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86802):  37%|| 58/157 [00:13<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.73411):  37%|| 58/157 [00:13<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.73411):  38%|| 59/157 [00:13<00:21,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.85954):  38%|| 59/157 [00:13<00:21,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.85954):  38%|| 60/157 [00:13<00:21,  4.60it/s]\u001b[A\n",
      "Testing... (loss=1.00186):  38%|| 60/157 [00:13<00:21,  4.60it/s]\u001b[A\n",
      "Testing... (loss=1.00186):  39%|| 61/157 [00:13<00:20,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.85040):  39%|| 61/157 [00:13<00:20,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.85040):  39%|| 62/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.86852):  39%|| 62/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.86852):  40%|| 63/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.79505):  40%|| 63/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.79505):  41%|| 64/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.96844):  41%|| 64/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.96844):  41%|| 65/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.07723):  41%|| 65/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.07723):  42%|| 66/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79396):  42%|| 66/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79396):  43%|| 67/157 [00:14<00:19,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.90705):  43%|| 67/157 [00:15<00:19,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.90705):  43%|| 68/157 [00:15<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86537):  43%|| 68/157 [00:15<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86537):  44%|| 69/157 [00:15<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.67849):  44%|| 69/157 [00:15<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.67849):  45%|| 70/157 [00:15<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.89959):  45%|| 70/157 [00:15<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.89959):  45%|| 71/157 [00:15<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.88811):  45%|| 71/157 [00:16<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.88811):  46%|| 72/157 [00:16<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.97529):  46%|| 72/157 [00:16<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.97529):  46%|| 73/157 [00:16<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.09847):  46%|| 73/157 [00:16<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.09847):  47%|| 74/157 [00:16<00:17,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.06971):  47%|| 74/157 [00:16<00:17,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.06971):  48%|| 75/157 [00:16<00:17,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.03643):  48%|| 75/157 [00:16<00:17,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.03643):  48%|| 76/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.99470):  48%|| 76/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.99470):  49%|| 77/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.83257):  49%|| 77/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.83257):  50%|| 78/157 [00:17<00:17,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.02313):  50%|| 78/157 [00:17<00:17,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.02313):  50%|| 79/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.96470):  50%|| 79/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.96470):  51%|| 80/157 [00:17<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.77493):  51%|| 80/157 [00:17<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.77493):  52%|| 81/157 [00:17<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79407):  52%|| 81/157 [00:18<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79407):  52%|| 82/157 [00:18<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.79137):  52%|| 82/157 [00:18<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.79137):  53%|| 83/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.98156):  53%|| 83/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.98156):  54%|| 84/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.99022):  54%|| 84/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.99022):  54%|| 85/157 [00:18<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.94877):  54%|| 85/157 [00:19<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.94877):  55%|| 86/157 [00:19<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.97628):  55%|| 86/157 [00:19<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.97628):  55%|| 87/157 [00:19<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.87764):  55%|| 87/157 [00:19<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.87764):  56%|| 88/157 [00:19<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86337):  56%|| 88/157 [00:19<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86337):  57%|| 89/157 [00:19<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.87413):  57%|| 89/157 [00:19<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.87413):  57%|| 90/157 [00:19<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86066):  57%|| 90/157 [00:20<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86066):  58%|| 91/157 [00:20<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.93861):  58%|| 91/157 [00:20<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.93861):  59%|| 92/157 [00:20<00:13,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79226):  59%|| 92/157 [00:20<00:13,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79226):  59%|| 93/157 [00:20<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.90893):  59%|| 93/157 [00:20<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.90893):  60%|| 94/157 [00:20<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.88488):  60%|| 94/157 [00:20<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.88488):  61%|| 95/157 [00:20<00:13,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86062):  61%|| 95/157 [00:21<00:13,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86062):  61%|| 96/157 [00:21<00:13,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.95113):  61%|| 96/157 [00:21<00:13,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.95113):  62%|| 97/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.89246):  62%|| 97/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.89246):  62%|| 98/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.95134):  62%|| 98/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.95134):  63%|| 99/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.87942):  63%|| 99/157 [00:22<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.87942):  64%|| 100/157 [00:22<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79134):  64%|| 100/157 [00:22<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79134):  64%|| 101/157 [00:22<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.06544):  64%|| 101/157 [00:22<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.06544):  65%|| 102/157 [00:22<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.00239):  65%|| 102/157 [00:22<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.00239):  66%|| 103/157 [00:22<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.01259):  66%|| 103/157 [00:22<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=1.01259):  66%|| 104/157 [00:22<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.71335):  66%|| 104/157 [00:23<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.71335):  67%|| 105/157 [00:23<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.84406):  67%|| 105/157 [00:23<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.84406):  68%|| 106/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.87679):  68%|| 106/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.87679):  68%|| 107/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.86102):  68%|| 107/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.86102):  69%|| 108/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.84595):  69%|| 108/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.84595):  69%|| 109/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.95750):  69%|| 109/157 [00:24<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.95750):  70%|| 110/157 [00:24<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.04888):  70%|| 110/157 [00:24<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.04888):  71%|| 111/157 [00:24<00:09,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.68310):  71%|| 111/157 [00:24<00:09,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.68310):  71%|| 112/157 [00:24<00:09,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.81567):  71%|| 112/157 [00:24<00:09,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.81567):  72%|| 113/157 [00:24<00:09,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.75930):  72%|| 113/157 [00:25<00:09,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.75930):  73%|| 114/157 [00:25<00:09,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.91112):  73%|| 114/157 [00:25<00:09,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.91112):  73%|| 115/157 [00:25<00:09,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.90835):  73%|| 115/157 [00:25<00:09,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.90835):  74%|| 116/157 [00:25<00:08,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.11909):  74%|| 116/157 [00:25<00:08,  4.62it/s]\u001b[A\n",
      "Testing... (loss=1.11909):  75%|| 117/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.93541):  75%|| 117/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.93541):  75%|| 118/157 [00:25<00:08,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.91696):  75%|| 118/157 [00:26<00:08,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.91696):  76%|| 119/157 [00:26<00:08,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.76108):  76%|| 119/157 [00:26<00:08,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.76108):  76%|| 120/157 [00:26<00:08,  4.60it/s]\u001b[A\n",
      "Testing... (loss=1.00258):  76%|| 120/157 [00:26<00:08,  4.60it/s]\u001b[A\n",
      "Testing... (loss=1.00258):  77%|| 121/157 [00:26<00:07,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.85706):  77%|| 121/157 [00:26<00:07,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.85706):  78%|| 122/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.82904):  78%|| 122/157 [00:27<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.82904):  78%|| 123/157 [00:27<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.14695):  78%|| 123/157 [00:27<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.14695):  79%|| 124/157 [00:27<00:07,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.84588):  79%|| 124/157 [00:27<00:07,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.84588):  80%|| 125/157 [00:27<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.88112):  80%|| 125/157 [00:27<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.88112):  80%|| 126/157 [00:27<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.99133):  80%|| 126/157 [00:27<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.99133):  81%|| 127/157 [00:27<00:06,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.94263):  81%|| 127/157 [00:28<00:06,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.94263):  82%|| 128/157 [00:28<00:06,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.93525):  82%|| 128/157 [00:28<00:06,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.93525):  82%|| 129/157 [00:28<00:06,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.82241):  82%|| 129/157 [00:28<00:06,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.82241):  83%|| 130/157 [00:28<00:05,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.75429):  83%|| 130/157 [00:28<00:05,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.75429):  83%|| 131/157 [00:28<00:05,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.03867):  83%|| 131/157 [00:28<00:05,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.03867):  84%|| 132/157 [00:28<00:05,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.74881):  84%|| 132/157 [00:29<00:05,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.74881):  85%|| 133/157 [00:29<00:05,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.68809):  85%|| 133/157 [00:29<00:05,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.68809):  85%|| 134/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.86171):  85%|| 134/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.86171):  86%|| 135/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.82907):  86%|| 135/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.82907):  87%|| 136/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.68006):  87%|| 136/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.68006):  87%|| 137/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.97920):  87%|| 137/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.97920):  88%|| 138/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.85886):  88%|| 138/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.85886):  89%|| 139/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.05961):  89%|| 139/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.05961):  89%|| 140/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.90795):  89%|| 140/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.90795):  90%|| 141/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.93072):  90%|| 141/157 [00:31<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.93072):  90%|| 142/157 [00:31<00:03,  4.61it/s]\u001b[A\n",
      "Testing... (loss=1.02142):  90%|| 142/157 [00:31<00:03,  4.61it/s]\u001b[A\n",
      "Testing... (loss=1.02142):  91%|| 143/157 [00:31<00:03,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.96114):  91%|| 143/157 [00:31<00:03,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.96114):  92%|| 144/157 [00:31<00:02,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.95830):  92%|| 144/157 [00:31<00:02,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.95830):  92%|| 145/157 [00:31<00:02,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.74287):  92%|| 145/157 [00:31<00:02,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.74287):  93%|| 146/157 [00:31<00:02,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.98165):  93%|| 146/157 [00:32<00:02,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.98165):  94%|| 147/157 [00:32<00:02,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.86455):  94%|| 147/157 [00:32<00:02,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.86455):  94%|| 148/157 [00:32<00:01,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.80531):  94%|| 148/157 [00:32<00:01,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.80531):  95%|| 149/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.89393):  95%|| 149/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.89393):  96%|| 150/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.05620):  96%|| 150/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.05620):  96%|| 151/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.91317):  96%|| 151/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.91317):  97%|| 152/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.01404):  97%|| 152/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.01404):  97%|| 153/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.97238):  97%|| 153/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.97238):  98%|| 154/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.73429):  98%|| 154/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.73429):  99%|| 155/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.83903):  99%|| 155/157 [00:34<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.83903):  99%|| 156/157 [00:34<00:00,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.87686): 100%|| 157/157 [00:34<00:00,  4.57it/s]\u001b[A\n",
      "11/05/2021 21:20:13 - INFO - __main__ - \n",
      "\n",
      "11/05/2021 21:20:13 - INFO - __main__ - Test Results\n",
      "11/05/2021 21:20:13 - INFO - __main__ - Global Steps: 300\n",
      "11/05/2021 21:20:13 - INFO - __main__ - Test Loss: 0.91025\n",
      "11/05/2021 21:20:13 - INFO - __main__ - Test Accuracy: 0.85430\n",
      "11/05/2021 21:20:14 - INFO - __main__ - Saved model checkpoint to [DIR: output_cifar100]\n",
      "Training (400 / 500 Steps) (loss=1.48894):  51%|| 399/782 [09:37<05:42,  1.12it/s]  11/05/2021 21:22:12 - INFO - __main__ - ***** Running Test *****\n",
      "11/05/2021 21:22:12 - INFO - __main__ -   Num steps = 157\n",
      "11/05/2021 21:22:12 - INFO - __main__ -   Batch size = 64\n",
      "\n",
      "Test... (loss=X.X):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.66860):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.66860):   1%|| 1/157 [00:00<02:00,  1.29it/s]\u001b[A\n",
      "Testing... (loss=0.70803):   1%|| 1/157 [00:00<02:00,  1.29it/s]\u001b[A\n",
      "Testing... (loss=0.70803):   1%|| 2/157 [00:00<01:08,  2.25it/s]\u001b[A\n",
      "Testing... (loss=0.91716):   1%|| 2/157 [00:01<01:08,  2.25it/s]\u001b[A\n",
      "Testing... (loss=0.91716):   2%|| 3/157 [00:01<00:52,  2.95it/s]\u001b[A\n",
      "Testing... (loss=0.62581):   2%|| 3/157 [00:01<00:52,  2.95it/s]\u001b[A\n",
      "Testing... (loss=0.62581):   3%|| 4/157 [00:01<00:44,  3.44it/s]\u001b[A\n",
      "Testing... (loss=0.71621):   3%|| 4/157 [00:01<00:44,  3.44it/s]\u001b[A\n",
      "Testing... (loss=0.71621):   3%|| 5/157 [00:01<00:40,  3.79it/s]\u001b[A\n",
      "Testing... (loss=0.79989):   3%|| 5/157 [00:01<00:40,  3.79it/s]\u001b[A\n",
      "Testing... (loss=0.79989):   4%|| 6/157 [00:01<00:37,  4.05it/s]\u001b[A\n",
      "Testing... (loss=0.77005):   4%|| 6/157 [00:02<00:37,  4.05it/s]\u001b[A\n",
      "Testing... (loss=0.77005):   4%|| 7/157 [00:02<00:35,  4.24it/s]\u001b[A\n",
      "Testing... (loss=0.83358):   4%|| 7/157 [00:02<00:35,  4.24it/s]\u001b[A\n",
      "Testing... (loss=0.83358):   5%|| 8/157 [00:02<00:34,  4.37it/s]\u001b[A\n",
      "Testing... (loss=0.89910):   5%|| 8/157 [00:02<00:34,  4.37it/s]\u001b[A\n",
      "Testing... (loss=0.89910):   6%|| 9/157 [00:02<00:33,  4.46it/s]\u001b[A\n",
      "Testing... (loss=0.73594):   6%|| 9/157 [00:02<00:33,  4.46it/s]\u001b[A\n",
      "Testing... (loss=0.73594):   6%|| 10/157 [00:02<00:32,  4.52it/s]\u001b[A\n",
      "Testing... (loss=0.52965):   6%|| 10/157 [00:02<00:32,  4.52it/s]\u001b[A\n",
      "Testing... (loss=0.52965):   7%|| 11/157 [00:02<00:32,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.71716):   7%|| 11/157 [00:03<00:32,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.71716):   8%|| 12/157 [00:03<00:31,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.64376):   8%|| 12/157 [00:03<00:31,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.64376):   8%|| 13/157 [00:03<00:31,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.84393):   8%|| 13/157 [00:03<00:31,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.84393):   9%|| 14/157 [00:03<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.05919):   9%|| 14/157 [00:03<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.05919):  10%|| 15/157 [00:03<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.74482):  10%|| 15/157 [00:03<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.74482):  10%|| 16/157 [00:03<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.78356):  10%|| 16/157 [00:04<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.78356):  11%|| 17/157 [00:04<00:30,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86552):  11%|| 17/157 [00:04<00:30,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86552):  11%|| 18/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.90390):  11%|| 18/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.90390):  12%|| 19/157 [00:04<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.59314):  12%|| 19/157 [00:04<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.59314):  13%|| 20/157 [00:04<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.99594):  13%|| 20/157 [00:05<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.99594):  13%|| 21/157 [00:05<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.80597):  13%|| 21/157 [00:05<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.80597):  14%|| 22/157 [00:05<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.67216):  14%|| 22/157 [00:05<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.67216):  15%|| 23/157 [00:05<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.86468):  15%|| 23/157 [00:05<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.86468):  15%|| 24/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79964):  15%|| 24/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79964):  16%|| 25/157 [00:05<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.90547):  16%|| 25/157 [00:06<00:28,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.90547):  17%|| 26/157 [00:06<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86017):  17%|| 26/157 [00:06<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86017):  17%|| 27/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.74795):  17%|| 27/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.74795):  18%|| 28/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.85043):  18%|| 28/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.85043):  18%|| 29/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.61414):  18%|| 29/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.61414):  19%|| 30/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.65222):  19%|| 30/157 [00:07<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.65222):  20%|| 31/157 [00:07<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.68063):  20%|| 31/157 [00:07<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.68063):  20%|| 32/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.85449):  20%|| 32/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.85449):  21%|| 33/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.85917):  21%|| 33/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.85917):  22%|| 34/157 [00:07<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.89616):  22%|| 34/157 [00:08<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.89616):  22%|| 35/157 [00:08<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.83439):  22%|| 35/157 [00:08<00:26,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.83439):  23%|| 36/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.82549):  23%|| 36/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.82549):  24%|| 37/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.83316):  24%|| 37/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.83316):  24%|| 38/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.78223):  24%|| 38/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.78223):  25%|| 39/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.88314):  25%|| 39/157 [00:09<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.88314):  25%|| 40/157 [00:09<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.92851):  25%|| 40/157 [00:09<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.92851):  26%|| 41/157 [00:09<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.93842):  26%|| 41/157 [00:09<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.93842):  27%|| 42/157 [00:09<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.89615):  27%|| 42/157 [00:09<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.89615):  27%|| 43/157 [00:09<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.80227):  27%|| 43/157 [00:09<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.80227):  28%|| 44/157 [00:10<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.93059):  28%|| 44/157 [00:10<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.93059):  29%|| 45/157 [00:10<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.84730):  29%|| 45/157 [00:10<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.84730):  29%|| 46/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.75273):  29%|| 46/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.75273):  30%|| 47/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.75973):  30%|| 47/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.75973):  31%|| 48/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.78021):  31%|| 48/157 [00:11<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.78021):  31%|| 49/157 [00:11<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.93746):  31%|| 49/157 [00:11<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.93746):  32%|| 50/157 [00:11<00:23,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.80228):  32%|| 50/157 [00:11<00:23,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.80228):  32%|| 51/157 [00:11<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.70848):  32%|| 51/157 [00:11<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.70848):  33%|| 52/157 [00:11<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.68345):  33%|| 52/157 [00:11<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.68345):  34%|| 53/157 [00:11<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.75585):  34%|| 53/157 [00:12<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.75585):  34%|| 54/157 [00:12<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.84218):  34%|| 54/157 [00:12<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.84218):  35%|| 55/157 [00:12<00:22,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.81560):  35%|| 55/157 [00:12<00:22,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.81560):  36%|| 56/157 [00:12<00:21,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.68807):  36%|| 56/157 [00:12<00:21,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.68807):  36%|| 57/157 [00:12<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.81258):  36%|| 57/157 [00:13<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.81258):  37%|| 58/157 [00:13<00:21,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.59889):  37%|| 58/157 [00:13<00:21,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.59889):  38%|| 59/157 [00:13<00:21,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.70191):  38%|| 59/157 [00:13<00:21,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.70191):  38%|| 60/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.86112):  38%|| 60/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.86112):  39%|| 61/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.76189):  39%|| 61/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.76189):  39%|| 62/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.75102):  39%|| 62/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.75102):  40%|| 63/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.67252):  40%|| 63/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.67252):  41%|| 64/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.85738):  41%|| 64/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.85738):  41%|| 65/157 [00:14<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.93603):  41%|| 65/157 [00:14<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.93603):  42%|| 66/157 [00:14<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.64396):  42%|| 66/157 [00:14<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.64396):  43%|| 67/157 [00:14<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.80882):  43%|| 67/157 [00:15<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.80882):  43%|| 68/157 [00:15<00:19,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.81752):  43%|| 68/157 [00:15<00:19,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.81752):  44%|| 69/157 [00:15<00:19,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.57485):  44%|| 69/157 [00:15<00:19,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.57485):  45%|| 70/157 [00:15<00:18,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.85491):  45%|| 70/157 [00:15<00:18,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.85491):  45%|| 71/157 [00:15<00:18,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.78657):  45%|| 71/157 [00:16<00:18,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.78657):  46%|| 72/157 [00:16<00:18,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.78951):  46%|| 72/157 [00:16<00:18,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.78951):  46%|| 73/157 [00:16<00:18,  4.58it/s]\u001b[A\n",
      "Testing... (loss=1.00409):  46%|| 73/157 [00:16<00:18,  4.58it/s]\u001b[A\n",
      "Testing... (loss=1.00409):  47%|| 74/157 [00:16<00:18,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.98318):  47%|| 74/157 [00:16<00:18,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.98318):  48%|| 75/157 [00:16<00:17,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.84648):  48%|| 75/157 [00:16<00:17,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.84648):  48%|| 76/157 [00:16<00:17,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.85925):  48%|| 76/157 [00:17<00:17,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.85925):  49%|| 77/157 [00:17<00:17,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.71970):  49%|| 77/157 [00:17<00:17,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.71970):  50%|| 78/157 [00:17<00:17,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.85542):  50%|| 78/157 [00:17<00:17,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.85542):  50%|| 79/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.85599):  50%|| 79/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.85599):  51%|| 80/157 [00:17<00:16,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.65475):  51%|| 80/157 [00:17<00:16,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.65475):  52%|| 81/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.63773):  52%|| 81/157 [00:18<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.63773):  52%|| 82/157 [00:18<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.69777):  52%|| 82/157 [00:18<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.69777):  53%|| 83/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.74730):  53%|| 83/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.74730):  54%|| 84/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.79830):  54%|| 84/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.79830):  54%|| 85/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.82711):  54%|| 85/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.82711):  55%|| 86/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.88952):  55%|| 86/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.88952):  55%|| 87/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.69369):  55%|| 87/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.69369):  56%|| 88/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.76856):  56%|| 88/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.76856):  57%|| 89/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.73291):  57%|| 89/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.73291):  57%|| 90/157 [00:19<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.72204):  57%|| 90/157 [00:20<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.72204):  58%|| 91/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.85478):  58%|| 91/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.85478):  59%|| 92/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.63823):  59%|| 92/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.63823):  59%|| 93/157 [00:20<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.77237):  59%|| 93/157 [00:20<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.77237):  60%|| 94/157 [00:20<00:13,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.75554):  60%|| 94/157 [00:21<00:13,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.75554):  61%|| 95/157 [00:21<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.74034):  61%|| 95/157 [00:21<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.74034):  61%|| 96/157 [00:21<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.80424):  61%|| 96/157 [00:21<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.80424):  62%|| 97/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.76139):  62%|| 97/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.76139):  62%|| 98/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.79639):  62%|| 98/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.79639):  63%|| 99/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.74084):  63%|| 99/157 [00:22<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.74084):  64%|| 100/157 [00:22<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.65684):  64%|| 100/157 [00:22<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.65684):  64%|| 101/157 [00:22<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.94252):  64%|| 101/157 [00:22<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.94252):  65%|| 102/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.87138):  65%|| 102/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.87138):  66%|| 103/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.88737):  66%|| 103/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.88737):  66%|| 104/157 [00:22<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.57646):  66%|| 104/157 [00:23<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.57646):  67%|| 105/157 [00:23<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.69958):  67%|| 105/157 [00:23<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.69958):  68%|| 106/157 [00:23<00:10,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79335):  68%|| 106/157 [00:23<00:10,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79335):  68%|| 107/157 [00:23<00:10,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.75736):  68%|| 107/157 [00:23<00:10,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.75736):  69%|| 108/157 [00:23<00:10,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.69514):  69%|| 108/157 [00:24<00:10,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.69514):  69%|| 109/157 [00:24<00:10,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86907):  69%|| 109/157 [00:24<00:10,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.86907):  70%|| 110/157 [00:24<00:10,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.91722):  70%|| 110/157 [00:24<00:10,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.91722):  71%|| 111/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.62527):  71%|| 111/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.62527):  71%|| 112/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.69257):  71%|| 112/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.69257):  72%|| 113/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.61771):  72%|| 113/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.61771):  73%|| 114/157 [00:25<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.82493):  73%|| 114/157 [00:25<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.82493):  73%|| 115/157 [00:25<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.75569):  73%|| 115/157 [00:25<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.75569):  74%|| 116/157 [00:25<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.04030):  74%|| 116/157 [00:25<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=1.04030):  75%|| 117/157 [00:25<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.83627):  75%|| 117/157 [00:25<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.83627):  75%|| 118/157 [00:25<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.76057):  75%|| 118/157 [00:26<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.76057):  76%|| 119/157 [00:26<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.60616):  76%|| 119/157 [00:26<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.60616):  76%|| 120/157 [00:26<00:07,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.85212):  76%|| 120/157 [00:26<00:07,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.85212):  77%|| 121/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.77940):  77%|| 121/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.77940):  78%|| 122/157 [00:26<00:07,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.68342):  78%|| 122/157 [00:27<00:07,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.68342):  78%|| 123/157 [00:27<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.93643):  78%|| 123/157 [00:27<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.93643):  79%|| 124/157 [00:27<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.78239):  79%|| 124/157 [00:27<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.78239):  80%|| 125/157 [00:27<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.75864):  80%|| 125/157 [00:27<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.75864):  80%|| 126/157 [00:27<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.82576):  80%|| 126/157 [00:27<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.82576):  81%|| 127/157 [00:27<00:06,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.76568):  81%|| 127/157 [00:28<00:06,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.76568):  82%|| 128/157 [00:28<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.79799):  82%|| 128/157 [00:28<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.79799):  82%|| 129/157 [00:28<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.72566):  82%|| 129/157 [00:28<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.72566):  83%|| 130/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.63006):  83%|| 130/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.63006):  83%|| 131/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.86923):  83%|| 131/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.86923):  84%|| 132/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.60966):  84%|| 132/157 [00:29<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.60966):  85%|| 133/157 [00:29<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.59929):  85%|| 133/157 [00:29<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.59929):  85%|| 134/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.76692):  85%|| 134/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.76692):  86%|| 135/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.66113):  86%|| 135/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.66113):  87%|| 136/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.55194):  87%|| 136/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.55194):  87%|| 137/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.77552):  87%|| 137/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.77552):  88%|| 138/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.73280):  88%|| 138/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.73280):  89%|| 139/157 [00:30<00:03,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.94174):  89%|| 139/157 [00:30<00:03,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.94174):  89%|| 140/157 [00:30<00:03,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.81788):  89%|| 140/157 [00:30<00:03,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.81788):  90%|| 141/157 [00:30<00:03,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.70842):  90%|| 141/157 [00:31<00:03,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.70842):  90%|| 142/157 [00:31<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.97820):  90%|| 142/157 [00:31<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.97820):  91%|| 143/157 [00:31<00:03,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.78721):  91%|| 143/157 [00:31<00:03,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.78721):  92%|| 144/157 [00:31<00:02,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.80732):  92%|| 144/157 [00:31<00:02,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.80732):  92%|| 145/157 [00:31<00:02,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.61759):  92%|| 145/157 [00:32<00:02,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.61759):  93%|| 146/157 [00:32<00:02,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.78867):  93%|| 146/157 [00:32<00:02,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.78867):  94%|| 147/157 [00:32<00:02,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.69742):  94%|| 147/157 [00:32<00:02,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.69742):  94%|| 148/157 [00:32<00:01,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.68447):  94%|| 148/157 [00:32<00:01,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.68447):  95%|| 149/157 [00:32<00:01,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.76998):  95%|| 149/157 [00:32<00:01,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.76998):  96%|| 150/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.81868):  96%|| 150/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.81868):  96%|| 151/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.80449):  96%|| 151/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.80449):  97%|| 152/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.90671):  97%|| 152/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.90671):  97%|| 153/157 [00:33<00:00,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.87064):  97%|| 153/157 [00:33<00:00,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.87064):  98%|| 154/157 [00:33<00:00,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.67787):  98%|| 154/157 [00:33<00:00,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.67787):  99%|| 155/157 [00:33<00:00,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.68960):  99%|| 155/157 [00:34<00:00,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.68960):  99%|| 156/157 [00:34<00:00,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.73444): 100%|| 157/157 [00:34<00:00,  4.57it/s]\u001b[A\n",
      "11/05/2021 21:22:46 - INFO - __main__ - \n",
      "\n",
      "11/05/2021 21:22:46 - INFO - __main__ - Test Results\n",
      "11/05/2021 21:22:46 - INFO - __main__ - Global Steps: 400\n",
      "11/05/2021 21:22:46 - INFO - __main__ - Test Loss: 0.78003\n",
      "11/05/2021 21:22:46 - INFO - __main__ - Test Accuracy: 0.86760\n",
      "11/05/2021 21:22:48 - INFO - __main__ - Saved model checkpoint to [DIR: output_cifar100]\n",
      "Training (500 / 500 Steps) (loss=1.37319):  64%|| 499/782 [12:13<04:12,  1.12it/s]  11/05/2021 21:24:48 - INFO - __main__ - ***** Running Test *****\n",
      "11/05/2021 21:24:48 - INFO - __main__ -   Num steps = 157\n",
      "11/05/2021 21:24:48 - INFO - __main__ -   Batch size = 64\n",
      "\n",
      "Test... (loss=X.X):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.65069):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.65069):   1%|| 1/157 [00:00<01:54,  1.36it/s]\u001b[A\n",
      "Testing... (loss=0.68665):   1%|| 1/157 [00:00<01:54,  1.36it/s]\u001b[A\n",
      "Testing... (loss=0.68665):   1%|| 2/157 [00:00<01:06,  2.32it/s]\u001b[A\n",
      "Testing... (loss=0.90669):   1%|| 2/157 [00:01<01:06,  2.32it/s]\u001b[A\n",
      "Testing... (loss=0.90669):   2%|| 3/157 [00:01<00:51,  3.02it/s]\u001b[A\n",
      "Testing... (loss=0.61216):   2%|| 3/157 [00:01<00:51,  3.02it/s]\u001b[A\n",
      "Testing... (loss=0.61216):   3%|| 4/157 [00:01<00:43,  3.51it/s]\u001b[A\n",
      "Testing... (loss=0.69234):   3%|| 4/157 [00:01<00:43,  3.51it/s]\u001b[A\n",
      "Testing... (loss=0.69234):   3%|| 5/157 [00:01<00:39,  3.85it/s]\u001b[A\n",
      "Testing... (loss=0.77815):   3%|| 5/157 [00:01<00:39,  3.85it/s]\u001b[A\n",
      "Testing... (loss=0.77815):   4%|| 6/157 [00:01<00:36,  4.10it/s]\u001b[A\n",
      "Testing... (loss=0.73991):   4%|| 6/157 [00:02<00:36,  4.10it/s]\u001b[A\n",
      "Testing... (loss=0.73991):   4%|| 7/157 [00:02<00:35,  4.27it/s]\u001b[A\n",
      "Testing... (loss=0.82490):   4%|| 7/157 [00:02<00:35,  4.27it/s]\u001b[A\n",
      "Testing... (loss=0.82490):   5%|| 8/157 [00:02<00:33,  4.38it/s]\u001b[A\n",
      "Testing... (loss=0.85907):   5%|| 8/157 [00:02<00:33,  4.38it/s]\u001b[A\n",
      "Testing... (loss=0.85907):   6%|| 9/157 [00:02<00:33,  4.47it/s]\u001b[A\n",
      "Testing... (loss=0.69689):   6%|| 9/157 [00:02<00:33,  4.47it/s]\u001b[A\n",
      "Testing... (loss=0.69689):   6%|| 10/157 [00:02<00:32,  4.53it/s]\u001b[A\n",
      "Testing... (loss=0.51107):   6%|| 10/157 [00:02<00:32,  4.53it/s]\u001b[A\n",
      "Testing... (loss=0.51107):   7%|| 11/157 [00:02<00:31,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.68174):   7%|| 11/157 [00:03<00:31,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.68174):   8%|| 12/157 [00:03<00:31,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.62092):   8%|| 12/157 [00:03<00:31,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.62092):   8%|| 13/157 [00:03<00:31,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.85033):   8%|| 13/157 [00:03<00:31,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.85033):   9%|| 14/157 [00:03<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.02246):   9%|| 14/157 [00:03<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=1.02246):  10%|| 15/157 [00:03<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.70030):  10%|| 15/157 [00:03<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.70030):  10%|| 16/157 [00:03<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.75389):  10%|| 16/157 [00:04<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.75389):  11%|| 17/157 [00:04<00:30,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.82529):  11%|| 17/157 [00:04<00:30,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.82529):  11%|| 18/157 [00:04<00:30,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.85623):  11%|| 18/157 [00:04<00:30,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.85623):  12%|| 19/157 [00:04<00:30,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.57050):  12%|| 19/157 [00:04<00:30,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.57050):  13%|| 20/157 [00:04<00:29,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.96228):  13%|| 20/157 [00:05<00:29,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.96228):  13%|| 21/157 [00:05<00:29,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.79139):  13%|| 21/157 [00:05<00:29,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.79139):  14%|| 22/157 [00:05<00:29,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.65160):  14%|| 22/157 [00:05<00:29,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.65160):  15%|| 23/157 [00:05<00:29,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.84557):  15%|| 23/157 [00:05<00:29,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.84557):  15%|| 24/157 [00:05<00:28,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.77247):  15%|| 24/157 [00:05<00:28,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.77247):  16%|| 25/157 [00:05<00:28,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.87485):  16%|| 25/157 [00:06<00:28,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.87485):  17%|| 26/157 [00:06<00:28,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.85066):  17%|| 26/157 [00:06<00:28,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.85066):  17%|| 27/157 [00:06<00:28,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.71490):  17%|| 27/157 [00:06<00:28,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.71490):  18%|| 28/157 [00:06<00:27,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.82123):  18%|| 28/157 [00:06<00:27,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.82123):  18%|| 29/157 [00:06<00:27,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.58783):  18%|| 29/157 [00:06<00:27,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.58783):  19%|| 30/157 [00:06<00:27,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.62400):  19%|| 30/157 [00:07<00:27,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.62400):  20%|| 31/157 [00:07<00:27,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.64906):  20%|| 31/157 [00:07<00:27,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.64906):  20%|| 32/157 [00:07<00:27,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.80107):  20%|| 32/157 [00:07<00:27,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.80107):  21%|| 33/157 [00:07<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.85273):  21%|| 33/157 [00:07<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.85273):  22%|| 34/157 [00:07<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.85243):  22%|| 34/157 [00:08<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.85243):  22%|| 35/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.80245):  22%|| 35/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.80245):  23%|| 36/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.81190):  23%|| 36/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.81190):  24%|| 37/157 [00:08<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79693):  24%|| 37/157 [00:08<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79693):  24%|| 38/157 [00:08<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.77471):  24%|| 38/157 [00:08<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.77471):  25%|| 39/157 [00:08<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.85700):  25%|| 39/157 [00:09<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.85700):  25%|| 40/157 [00:09<00:25,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.90586):  25%|| 40/157 [00:09<00:25,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.90586):  26%|| 41/157 [00:09<00:25,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.86836):  26%|| 41/157 [00:09<00:25,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.86836):  27%|| 42/157 [00:09<00:24,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.88518):  27%|| 42/157 [00:09<00:24,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.88518):  27%|| 43/157 [00:09<00:24,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.76526):  27%|| 43/157 [00:10<00:24,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.76526):  28%|| 44/157 [00:10<00:24,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.88363):  28%|| 44/157 [00:10<00:24,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.88363):  29%|| 45/157 [00:10<00:24,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.84782):  29%|| 45/157 [00:10<00:24,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.84782):  29%|| 46/157 [00:10<00:24,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.71758):  29%|| 46/157 [00:10<00:24,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.71758):  30%|| 47/157 [00:10<00:23,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.76043):  30%|| 47/157 [00:10<00:23,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.76043):  31%|| 48/157 [00:10<00:23,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.73692):  31%|| 48/157 [00:11<00:23,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.73692):  31%|| 49/157 [00:11<00:23,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.89375):  31%|| 49/157 [00:11<00:23,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.89375):  32%|| 50/157 [00:11<00:23,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.80146):  32%|| 50/157 [00:11<00:23,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.80146):  32%|| 51/157 [00:11<00:22,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.68925):  32%|| 51/157 [00:11<00:22,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.68925):  33%|| 52/157 [00:11<00:22,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.66845):  33%|| 52/157 [00:11<00:22,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.66845):  34%|| 53/157 [00:11<00:22,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.73846):  34%|| 53/157 [00:12<00:22,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.73846):  34%|| 54/157 [00:12<00:22,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.81766):  34%|| 54/157 [00:12<00:22,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.81766):  35%|| 55/157 [00:12<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.78868):  35%|| 55/157 [00:12<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.78868):  36%|| 56/157 [00:12<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.67220):  36%|| 56/157 [00:12<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.67220):  36%|| 57/157 [00:12<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.78967):  36%|| 57/157 [00:13<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.78967):  37%|| 58/157 [00:13<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.56789):  37%|| 58/157 [00:13<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.56789):  38%|| 59/157 [00:13<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.67882):  38%|| 59/157 [00:13<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.67882):  38%|| 60/157 [00:13<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.81975):  38%|| 60/157 [00:13<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.81975):  39%|| 61/157 [00:13<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.73937):  39%|| 61/157 [00:13<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.73937):  39%|| 62/157 [00:13<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.74129):  39%|| 62/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.74129):  40%|| 63/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.65976):  40%|| 63/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.65976):  41%|| 64/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.83169):  41%|| 64/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.83169):  41%|| 65/157 [00:14<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.93063):  41%|| 65/157 [00:14<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.93063):  42%|| 66/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.61113):  42%|| 66/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.61113):  43%|| 67/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79239):  43%|| 67/157 [00:15<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.79239):  43%|| 68/157 [00:15<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.79512):  43%|| 68/157 [00:15<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.79512):  44%|| 69/157 [00:15<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.54843):  44%|| 69/157 [00:15<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.54843):  45%|| 70/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.82104):  45%|| 70/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.82104):  45%|| 71/157 [00:15<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.76593):  45%|| 71/157 [00:16<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.76593):  46%|| 72/157 [00:16<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.74218):  46%|| 72/157 [00:16<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.74218):  46%|| 73/157 [00:16<00:18,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.95858):  46%|| 73/157 [00:16<00:18,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.95858):  47%|| 74/157 [00:16<00:18,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.96255):  47%|| 74/157 [00:16<00:18,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.96255):  48%|| 75/157 [00:16<00:17,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.83829):  48%|| 75/157 [00:16<00:17,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.83829):  48%|| 76/157 [00:16<00:17,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.83975):  48%|| 76/157 [00:17<00:17,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.83975):  49%|| 77/157 [00:17<00:17,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.69687):  49%|| 77/157 [00:17<00:17,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.69687):  50%|| 78/157 [00:17<00:17,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.83875):  50%|| 78/157 [00:17<00:17,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.83875):  50%|| 79/157 [00:17<00:16,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.81132):  50%|| 79/157 [00:17<00:16,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.81132):  51%|| 80/157 [00:17<00:16,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.63520):  51%|| 80/157 [00:18<00:16,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.63520):  52%|| 81/157 [00:18<00:16,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.60995):  52%|| 81/157 [00:18<00:16,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.60995):  52%|| 82/157 [00:18<00:16,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.68732):  52%|| 82/157 [00:18<00:16,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.68732):  53%|| 83/157 [00:18<00:16,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.73455):  53%|| 83/157 [00:18<00:16,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.73455):  54%|| 84/157 [00:18<00:15,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.76696):  54%|| 84/157 [00:18<00:15,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.76696):  54%|| 85/157 [00:18<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.81251):  54%|| 85/157 [00:19<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.81251):  55%|| 86/157 [00:19<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.85566):  55%|| 86/157 [00:19<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.85566):  55%|| 87/157 [00:19<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.67838):  55%|| 87/157 [00:19<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.67838):  56%|| 88/157 [00:19<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.75256):  56%|| 88/157 [00:19<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.75256):  57%|| 89/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.71773):  57%|| 89/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.71773):  57%|| 90/157 [00:19<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.69382):  57%|| 90/157 [00:20<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.69382):  58%|| 91/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.80186):  58%|| 91/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.80186):  59%|| 92/157 [00:20<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.61722):  59%|| 92/157 [00:20<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.61722):  59%|| 93/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.75260):  59%|| 93/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.75260):  60%|| 94/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.75612):  60%|| 94/157 [00:21<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.75612):  61%|| 95/157 [00:21<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.70134):  61%|| 95/157 [00:21<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.70134):  61%|| 96/157 [00:21<00:13,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.78965):  61%|| 96/157 [00:21<00:13,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.78965):  62%|| 97/157 [00:21<00:12,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.73712):  62%|| 97/157 [00:21<00:12,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.73712):  62%|| 98/157 [00:21<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.78690):  62%|| 98/157 [00:21<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.78690):  63%|| 99/157 [00:21<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.70124):  63%|| 99/157 [00:22<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.70124):  64%|| 100/157 [00:22<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.63709):  64%|| 100/157 [00:22<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.63709):  64%|| 101/157 [00:22<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.90933):  64%|| 101/157 [00:22<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.90933):  65%|| 102/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.84458):  65%|| 102/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.84458):  66%|| 103/157 [00:22<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.87973):  66%|| 103/157 [00:22<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.87973):  66%|| 104/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.55848):  66%|| 104/157 [00:23<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.55848):  67%|| 105/157 [00:23<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.67710):  67%|| 105/157 [00:23<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.67710):  68%|| 106/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.77778):  68%|| 106/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.77778):  68%|| 107/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.73076):  68%|| 107/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.73076):  69%|| 108/157 [00:23<00:10,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.67205):  69%|| 108/157 [00:24<00:10,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.67205):  69%|| 109/157 [00:24<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.85289):  69%|| 109/157 [00:24<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.85289):  70%|| 110/157 [00:24<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.90181):  70%|| 110/157 [00:24<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.90181):  71%|| 111/157 [00:24<00:09,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.58416):  71%|| 111/157 [00:24<00:09,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.58416):  71%|| 112/157 [00:24<00:09,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.66555):  71%|| 112/157 [00:24<00:09,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.66555):  72%|| 113/157 [00:24<00:09,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.59469):  72%|| 113/157 [00:25<00:09,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.59469):  73%|| 114/157 [00:25<00:09,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.80182):  73%|| 114/157 [00:25<00:09,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.80182):  73%|| 115/157 [00:25<00:09,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.69838):  73%|| 115/157 [00:25<00:09,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.69838):  74%|| 116/157 [00:25<00:08,  4.59it/s]\u001b[A\n",
      "Testing... (loss=1.01725):  74%|| 116/157 [00:25<00:08,  4.59it/s]\u001b[A\n",
      "Testing... (loss=1.01725):  75%|| 117/157 [00:25<00:08,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.81519):  75%|| 117/157 [00:26<00:08,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.81519):  75%|| 118/157 [00:26<00:08,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.71262):  75%|| 118/157 [00:26<00:08,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.71262):  76%|| 119/157 [00:26<00:08,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.56853):  76%|| 119/157 [00:26<00:08,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.56853):  76%|| 120/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.83816):  76%|| 120/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.83816):  77%|| 121/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.75214):  77%|| 121/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.75214):  78%|| 122/157 [00:26<00:07,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.66953):  78%|| 122/157 [00:27<00:07,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.66953):  78%|| 123/157 [00:27<00:07,  4.50it/s]\u001b[A\n",
      "Testing... (loss=0.89165):  78%|| 123/157 [00:27<00:07,  4.50it/s]\u001b[A\n",
      "Testing... (loss=0.89165):  79%|| 124/157 [00:27<00:07,  4.52it/s]\u001b[A\n",
      "Testing... (loss=0.74271):  79%|| 124/157 [00:27<00:07,  4.52it/s]\u001b[A\n",
      "Testing... (loss=0.74271):  80%|| 125/157 [00:27<00:07,  4.55it/s]\u001b[A\n",
      "Testing... (loss=0.73439):  80%|| 125/157 [00:27<00:07,  4.55it/s]\u001b[A\n",
      "Testing... (loss=0.73439):  80%|| 126/157 [00:27<00:06,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.80253):  80%|| 126/157 [00:27<00:06,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.80253):  81%|| 127/157 [00:27<00:06,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.74191):  81%|| 127/157 [00:28<00:06,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.74191):  82%|| 128/157 [00:28<00:06,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.77714):  82%|| 128/157 [00:28<00:06,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.77714):  82%|| 129/157 [00:28<00:06,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.70245):  82%|| 129/157 [00:28<00:06,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.70245):  83%|| 130/157 [00:28<00:05,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.59454):  83%|| 130/157 [00:28<00:05,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.59454):  83%|| 131/157 [00:28<00:05,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.85527):  83%|| 131/157 [00:29<00:05,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.85527):  84%|| 132/157 [00:29<00:05,  4.55it/s]\u001b[A\n",
      "Testing... (loss=0.58300):  84%|| 132/157 [00:29<00:05,  4.55it/s]\u001b[A\n",
      "Testing... (loss=0.58300):  85%|| 133/157 [00:29<00:05,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.58521):  85%|| 133/157 [00:29<00:05,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.58521):  85%|| 134/157 [00:29<00:05,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.72663):  85%|| 134/157 [00:29<00:05,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.72663):  86%|| 135/157 [00:29<00:04,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.66126):  86%|| 135/157 [00:29<00:04,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.66126):  87%|| 136/157 [00:29<00:04,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.53011):  87%|| 136/157 [00:30<00:04,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.53011):  87%|| 137/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.73577):  87%|| 137/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.73577):  88%|| 138/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.71671):  88%|| 138/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.71671):  89%|| 139/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.90719):  89%|| 139/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.90719):  89%|| 140/157 [00:30<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.80331):  89%|| 140/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.80331):  90%|| 141/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.67361):  90%|| 141/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.67361):  90%|| 142/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.94237):  90%|| 142/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.94237):  91%|| 143/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.75869):  91%|| 143/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.75869):  92%|| 144/157 [00:31<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.77171):  92%|| 144/157 [00:31<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.77171):  92%|| 145/157 [00:31<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.59754):  92%|| 145/157 [00:32<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.59754):  93%|| 146/157 [00:32<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.78440):  93%|| 146/157 [00:32<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.78440):  94%|| 147/157 [00:32<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.67593):  94%|| 147/157 [00:32<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.67593):  94%|| 148/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.67865):  94%|| 148/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.67865):  95%|| 149/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.74538):  95%|| 149/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.74538):  96%|| 150/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.77896):  96%|| 150/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.77896):  96%|| 151/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.76444):  96%|| 151/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.76444):  97%|| 152/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.88179):  97%|| 152/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.88179):  97%|| 153/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.83775):  97%|| 153/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.83775):  98%|| 154/157 [00:33<00:00,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.65417):  98%|| 154/157 [00:34<00:00,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.65417):  99%|| 155/157 [00:34<00:00,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.67603):  99%|| 155/157 [00:34<00:00,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.67603):  99%|| 156/157 [00:34<00:00,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.67801): 100%|| 157/157 [00:34<00:00,  4.56it/s]\u001b[A\n",
      "11/05/2021 21:25:23 - INFO - __main__ - \n",
      "\n",
      "11/05/2021 21:25:23 - INFO - __main__ - Test Results\n",
      "11/05/2021 21:25:23 - INFO - __main__ - Global Steps: 500\n",
      "11/05/2021 21:25:23 - INFO - __main__ - Test Loss: 0.75476\n",
      "11/05/2021 21:25:23 - INFO - __main__ - Test Accuracy: 0.87280\n",
      "11/05/2021 21:25:24 - INFO - __main__ - Saved model checkpoint to [DIR: output_cifar100]\n",
      "Training (500 / 500 Steps) (loss=1.37319):  64%|| 499/782 [12:52<07:17,  1.55s/it]\n",
      "11/05/2021 21:25:27 - INFO - __main__ - Best Accuracy: \t0.872800\n",
      "11/05/2021 21:25:27 - INFO - __main__ - End Training!\n"
     ]
    }
   ],
   "source": [
    "main()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ef07f3d",
   "metadata": {},
   "source": [
    "### Testing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "5baf7c88",
   "metadata": {},
   "outputs": [],
   "source": [
    "class VisionTransformer(nn.Module):\n",
    "    def __init__(self, config, img_size=224, num_classes=21843, zero_head=False, vis=False):\n",
    "        super(VisionTransformer, self).__init__()\n",
    "        self.num_classes = num_classes\n",
    "        self.zero_head = zero_head\n",
    "        self.classifier = config.classifier\n",
    "\n",
    "        self.transformer = Transformer(config, img_size, vis)\n",
    "        self.head = Linear(config.hidden_size, num_classes)\n",
    "\n",
    "    def forward(self, x, labels=None):\n",
    "        x, attn_weights = self.transformer(x)\n",
    "        logits = self.head(x[:, 0])\n",
    "\n",
    "        if labels is not None:\n",
    "            loss_fct = CrossEntropyLoss()\n",
    "            loss = loss_fct(logits.view(-1, self.num_classes), labels.view(-1))\n",
    "            return loss\n",
    "        else:\n",
    "            return logits #, attn_weights\n",
    "\n",
    "    def load_from(self, weights):\n",
    "        with torch.no_grad():\n",
    "            if self.zero_head:\n",
    "                nn.init.zeros_(self.head.weight)\n",
    "                nn.init.zeros_(self.head.bias)\n",
    "            else:\n",
    "                self.head.weight.copy_(np2th(weights[\"head/kernel\"]).t())\n",
    "                self.head.bias.copy_(np2th(weights[\"head/bias\"]).t())\n",
    "\n",
    "            self.transformer.embeddings.patch_embeddings.weight.copy_(np2th(weights[\"embedding/kernel\"]))\n",
    "            self.transformer.embeddings.patch_embeddings.bias.copy_(np2th(weights[\"embedding/bias\"]))\n",
    "            self.transformer.embeddings.cls_token.copy_(np2th(weights[\"cls\"]))\n",
    "            self.transformer.encoder.encoder_norm.weight.copy_(np2th(weights[\"Transformer/encoder_norm/scale\"]))\n",
    "            self.transformer.encoder.encoder_norm.bias.copy_(np2th(weights[\"Transformer/encoder_norm/bias\"]))\n",
    "\n",
    "            posemb = np2th(weights[\"Transformer/posembed_input/pos_embedding\"])\n",
    "            posemb_new = self.transformer.embeddings.position_embeddings\n",
    "            if posemb.size() == posemb_new.size():\n",
    "                self.transformer.embeddings.position_embeddings.copy_(posemb)\n",
    "            else:\n",
    "                logger.info(\"load_pretrained: resized variant: %s to %s\" % (posemb.size(), posemb_new.size()))\n",
    "                ntok_new = posemb_new.size(1)\n",
    "\n",
    "                if self.classifier == \"token\":\n",
    "                    posemb_tok, posemb_grid = posemb[:, :1], posemb[0, 1:]\n",
    "                    ntok_new -= 1\n",
    "                else:\n",
    "                    posemb_tok, posemb_grid = posemb[:, :0], posemb[0]\n",
    "\n",
    "                gs_old = int(np.sqrt(len(posemb_grid)))\n",
    "                gs_new = int(np.sqrt(ntok_new))\n",
    "                print('load_pretrained: grid-size from %s to %s' % (gs_old, gs_new))\n",
    "                posemb_grid = posemb_grid.reshape(gs_old, gs_old, -1)\n",
    "\n",
    "                zoom = (gs_new / gs_old, gs_new / gs_old, 1)\n",
    "                posemb_grid = ndimage.zoom(posemb_grid, zoom, order=1)\n",
    "                posemb_grid = posemb_grid.reshape(1, gs_new * gs_new, -1)\n",
    "                posemb = np.concatenate([posemb_tok, posemb_grid], axis=1)\n",
    "                self.transformer.embeddings.position_embeddings.copy_(np2th(posemb))\n",
    "\n",
    "            for bname, block in self.transformer.encoder.named_children():\n",
    "                for uname, unit in block.named_children():\n",
    "                    unit.load_from(weights, n_block=uname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "683ccded",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VisionTransformer(\n",
       "  (transformer): Transformer(\n",
       "    (embeddings): Embeddings(\n",
       "      (patch_embeddings): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16))\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): Encoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (1): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (2): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (3): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (4): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (5): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (6): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (7): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (8): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (9): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (10): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (11): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (encoder_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (head): Linear(in_features=768, out_features=100, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = VisionTransformer(config, img_size=224, zero_head=True, num_classes=100)\n",
    "checkpoint = torch.load(\"output_cifar100/cifar100_500_checkpoint.bin\") \n",
    "\n",
    "model.load_state_dict(checkpoint) \n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "a6677bb8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VisionTransformer(\n",
       "  (transformer): Transformer(\n",
       "    (embeddings): Embeddings(\n",
       "      (patch_embeddings): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16))\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): Encoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (1): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (2): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (3): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (4): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (5): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (6): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (7): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (8): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (9): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (10): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (11): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (encoder_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (head): Linear(in_features=768, out_features=100, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "56ba59cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "\n",
    "image_dir = '0005.png'\n",
    "\n",
    "input_image = Image.open(image_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "da6b271b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAIXElEQVR4nF2WW49cVxGFV9Xe+5y+d09P22MnsTNx7FwUomDiAEIRUgAJBC+Ef4KExL/hhTdAiMs7IAIoIQECuTmOEzuZxGPPpe99+py9d1XxMBEg1lO9VH3SqpJqkZnhv/pPTQYQYKL1ahHX8w/+9Mqff/Pz69/+7tUbLwYO7cGgGAyKbhdMABSgszb8v8jMPmeoGSAOBCJJ1eHhp39/89PX3zi5eWt6fFjl9Wy58NvULgtiPxhNBpceGV25svf0Fy7duH7+2lWiYKpmRkQEOkMR0ecAAtTATHExu/fG3z597fU7r716961/6Ww2DIUPyIPiNPPDqoMitcoOzav5ZnMicUm+u3/18kvfvPrSN7/49W8U/U5WZSYGfQ4zMzGBMbOe3Hrz1s9/cfLH1xcH905Xq+l2k1SGnV7Bom1dbeRyv/fk04+e7+2sDg4Ojw5OtqtFsvlaThtOvZ3rL//g5R/9cPfaE6ISiAxgYn9mjSc6eeuN93/1E3nnZi+tV26TbNVwqpQ0ZZBaSC1PFbbl+T6HEtue5J6f5bBpELIzlc3svV/+Wqr6+z/+0UPPPC1mIAbAAHtym/sHH//1tyM93O2q58ZR7akmbaLE46a6XVd3N3FmOC7qxWAz7c7mg23VE+owk3hGQTTq9i7vTJp3P3r1pz9bHR2DGWZnAEDy0c2/lHrYGTkwekX7XK+71+v1QC0VkqZqmjuz+p1FfVfomPO81ayGafLU+ML+qN3mMlAgdIoSsDLnk7ffu/3Kayx6tmcPYDs7loP3dkTiUtuuVw47xFSt445bZWdONAa3SHRUJUzt6CihxXFZhcLLYi1JHHPLsxORlKKt7ejk49/95dqN57v7DwPwMKxuvtc9uE+IZdMK435qCxXF4Gjd56kFC6yWxQquvWubnb52Wy2PYVt2TcyrpdQZjhxLllQbHLb10c3377397hP7FwHysliv/vmPwWxqllxngH7Hs3ZXtNudzMtT1lVpUngZORKyDqxzMm9lbbc7Zlo1sVEXcyYvwo00qwLkYxln80/+/uZjL301dPt+e3qITz5BlRy5UBYSgzVZ11Zmv9sdBbNtXRfWJEBy7lGAWaO22SZlbFWYQssFAbJllW1DzGUrSGf+wYebo6PRY32fPj0sD2eWRQtvyGQV1caVtNl3W22QdjvDXqpny1lUCcpiRkASU9KiYDK1jDqrEYGsyVVuVq7Tr6enm+OT0WOP+/zZg7zelilx8Gm9ZXIePmhICGQoWy01jy3a3luiZJJVQUZM5JwP7LI1kEAEI4WSZmsay01TrbbzGQCvizUltSqTz855mAkpEbP3ZShNYwZDlFU9IzODoZKJ4EGsBBgRGYiJyYwNDLOmTtttvV4D8LKpXW1OHCSbbMzIQtDCMdugPypSrJsYjVgssE+qxhDAGTyI1IQAIiMYAAWfHX5ucr2N2xqAd+TI2LsyxyquN+x8FN9kCt4Tcyt0HEJFrmRPZEJOJDsDMZlqVtPgyDtSM4VzDkYMZc3e4MEAvPNOHcfGyAhM6rXRtcI1mSWJpKw5W6680ygGZlEhIhgaUXFkCjAbsqnCjMCs6sHsqRgMAHi0i6jqU7aY2XtxClGIEAE5xk2lOWmuzDIRcs5KUAUMoohm7IiNmIgJBPPsiB2DqFW0xjsAGOORli2KWVNjmlSSqcFAqpBcsDmIqRAxMZsZgYjJTJnJOTYzVVE1ImJimBHAREV/2JucA8B+77wN++KQLao0qamhollzAnMgZjUFwExMxM4RExODzgRVNbOzX8tETOyIGTTYu9Ce7ALG5fmJG47UcdJoUOeZHLuyFB+Mgxpn0ZSzqrH3xIAZEzkiMiOj4LwZ6PO57NkVLrAvhpcv+3aribUvdsc2mQgclGOM5kmDVwICJImIgqGAEYupsjGRN3LkoKpiSnDOiyRWhhkIDi76snjoYl1tsmN242F47ol6MCh816OwRrHNVEUXBWZCFlVAlFVjzmQonWeQiQI4M8SpBTARiMHMkQyTYf+Riyens+l0xmAef/mL9OTV1O6JK428iOaYpI5mRsQQNUM2ySqsYDVWYxATAYCoVxTkyIgpsCu2RTn+yvN1v//JZ/fv33vgAZQX9rrPP7t8/5azSjiSsaZsZKrIKZHBTA2mps55y2ZEmWBkICJDMEpmjpm5SK7d3X+8/+wX7symLpkPjgFo4fsvPGfXruROV9mi5CxIWbJklWTM6hwMzoxUWRSqzpEnKgBPQmRGBKbMqHbHnRvPr5SXRyerxWy9XHgAZtq6enn3e9+6v5jTzXWK8+jVCZwYgQDyHITglVwmYTgzqKqBYEJWIyduiRV1e9R++plmZ1RNZxycloEd+bN01zjuvPhCf7p4MF/qnVrSKpt5saooqVXoeqmOfCh81FjCtjHnnBnqQ0qUmRoqUjnkx5/M586t18tOu9stemXZ6nZ7HgDMXMRK0PvajcV29fGvVri99E1j0Xauf6ns9U7ffrsz7CTTOF0OntqP9+dtQwouhqKaV7M7H22Kkq7s20O7TlObud3vDibj0c7OcDTyZwmVVKlJcxO9/lSh33nwSj++f2t1ePx4k30171x5pv/oxXt372ytNb727M6NXYhMV0tr4sNF58Pf/8F6/fajl9rndyfn98YXLnT3zg13xoNev9vrkZlJlqy5qZrlZnU6O559fHB8+87p3YPjj+5U01ObLtocervD9fFR3mxkZxgm51Ql5ZzNxhcudsa7F/b39y49Mrl0cXBu0h4OQrdThNI7H0Kg/8R3zdKkuK7W25Pp/MHxYjpbHp8sVou82tSbzTbXqCNMzIXgi3a73e32Rrs7w8lkfGFvtHe+OxqWvY4rC3LF/8b3fwMyLXZMzPBN7AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<PIL.PngImagePlugin.PngImageFile image mode=RGB size=32x32 at 0x7FC71DF58D00>"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "a0626c18",
   "metadata": {},
   "outputs": [],
   "source": [
    "transform = transforms.Compose([\n",
    "    transforms.Resize((224, 224)),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),\n",
    "])\n",
    "\n",
    "x = transform(input_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "2a001f65",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 224, 224])"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "d6f7ff78",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = x.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "25aa7795",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0], device='cuda:1')"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits = model(x.unsqueeze(0))\n",
    "\n",
    "preds = torch.argmax(logits, dim=-1)\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "0b85dd20",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 7.8992e+00,  4.6909e-01,  5.0083e-02, -7.1851e-01, -6.4249e-01,\n",
       "         -3.7722e-01,  8.6283e-02,  2.3017e-01, -2.8317e-01,  4.6696e-01,\n",
       "          9.9816e-01,  9.3340e-02, -4.5178e-01, -3.7654e-01, -8.7495e-02,\n",
       "         -8.0172e-01,  6.3487e-01, -2.3182e-01, -2.1290e-01, -2.5988e-01,\n",
       "         -8.8064e-02, -5.8691e-01,  2.7822e-01, -5.9778e-01,  3.9020e-01,\n",
       "         -1.5187e-01,  4.4171e-01, -5.8582e-01,  7.5145e-01, -5.2795e-01,\n",
       "         -1.9603e-01, -4.7016e-01, -1.4039e-01, -4.4834e-01, -7.8456e-01,\n",
       "          2.4838e-01, -4.2332e-01, -3.7657e-01, -6.3094e-01,  2.3180e-01,\n",
       "          3.6288e-01,  9.0661e-02, -6.5950e-01, -3.1426e-01, -2.9390e-01,\n",
       "          7.1401e-01,  2.6698e-02,  2.4943e-01, -4.5388e-01, -7.5234e-01,\n",
       "         -4.3814e-01, -6.1359e-02, -5.1932e-01,  1.8496e+00,  5.0920e-01,\n",
       "         -8.9061e-01, -2.3098e-01,  3.1055e+00, -5.0354e-01, -5.1488e-01,\n",
       "         -7.4981e-01,  7.3391e-01,  7.2163e-01, -5.8690e-01, -6.2303e-01,\n",
       "         -6.0522e-01, -7.9643e-01, -1.5570e-02, -6.1442e-01, -1.7153e-01,\n",
       "          1.1032e+00, -4.0617e-01, -9.0171e-01, -5.9499e-02, -6.3074e-01,\n",
       "         -4.1081e-01, -1.9709e-01, -1.1973e-01, -1.1013e-01, -1.0455e-01,\n",
       "         -5.8953e-01, -2.8286e-01,  1.2999e-01,  2.2888e+00,  4.7378e-03,\n",
       "         -3.7203e-01,  1.0030e-01, -1.0376e-01, -6.0852e-01, -2.8759e-01,\n",
       "         -5.2587e-01,  1.9997e-01,  1.4718e+00, -1.8190e-01,  9.9548e-04,\n",
       "         -2.3756e-01, -3.3400e-01, -5.9347e-01,  3.3547e-01,  3.6693e-02]],\n",
       "       device='cuda:1', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d588516",
   "metadata": {},
   "source": [
    "### ART Attack "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "4d6bf45e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/05/2021 21:47:54 - INFO - art.estimators.classification.pytorch - Inferred 1 hidden layers on PyTorch classifier.\n"
     ]
    }
   ],
   "source": [
    "from art.estimators.classification import PyTorchClassifier\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# Create the ART classifier\n",
    "\n",
    "classifier = PyTorchClassifier(\n",
    "    model=model,\n",
    "    loss=criterion,\n",
    "    input_shape=(3, 224, 224),\n",
    "    nb_classes=100,\n",
    "    device_type='cuda:0'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "8fea3ceb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 7.89917612e+00,  4.69092488e-01,  5.00827096e-02,\n",
       "        -7.18508303e-01, -6.42487347e-01, -3.77223313e-01,\n",
       "         8.62825960e-02,  2.30170950e-01, -2.83168972e-01,\n",
       "         4.66958195e-01,  9.98158514e-01,  9.33395326e-02,\n",
       "        -4.51776594e-01, -3.76543254e-01, -8.74946415e-02,\n",
       "        -8.01720321e-01,  6.34873986e-01, -2.31822580e-01,\n",
       "        -2.12903559e-01, -2.59884596e-01, -8.80644470e-02,\n",
       "        -5.86911798e-01,  2.78219730e-01, -5.97784162e-01,\n",
       "         3.90199095e-01, -1.51874021e-01,  4.41707015e-01,\n",
       "        -5.85820675e-01,  7.51449049e-01, -5.27953088e-01,\n",
       "        -1.96030915e-01, -4.70155746e-01, -1.40386432e-01,\n",
       "        -4.48342979e-01, -7.84562171e-01,  2.48379648e-01,\n",
       "        -4.23319548e-01, -3.76574636e-01, -6.30944729e-01,\n",
       "         2.31804639e-01,  3.62877011e-01,  9.06610563e-02,\n",
       "        -6.59502745e-01, -3.14258665e-01, -2.93902874e-01,\n",
       "         7.14007378e-01,  2.66979113e-02,  2.49432296e-01,\n",
       "        -4.53883708e-01, -7.52340078e-01, -4.38137025e-01,\n",
       "        -6.13587648e-02, -5.19321024e-01,  1.84962225e+00,\n",
       "         5.09203672e-01, -8.90610576e-01, -2.30978414e-01,\n",
       "         3.10548949e+00, -5.03543019e-01, -5.14881909e-01,\n",
       "        -7.49805033e-01,  7.33913183e-01,  7.21625507e-01,\n",
       "        -5.86901486e-01, -6.23031318e-01, -6.05217934e-01,\n",
       "        -7.96428919e-01, -1.55698899e-02, -6.14419281e-01,\n",
       "        -1.71525240e-01,  1.10318232e+00, -4.06172663e-01,\n",
       "        -9.01711285e-01, -5.94989844e-02, -6.30743980e-01,\n",
       "        -4.10805553e-01, -1.97091401e-01, -1.19726956e-01,\n",
       "        -1.10134110e-01, -1.04550406e-01, -5.89534163e-01,\n",
       "        -2.82858104e-01,  1.29994497e-01,  2.28883362e+00,\n",
       "         4.73784655e-03, -3.72029990e-01,  1.00297593e-01,\n",
       "        -1.03764787e-01, -6.08517110e-01, -2.87588984e-01,\n",
       "        -5.25871933e-01,  1.99965581e-01,  1.47181034e+00,\n",
       "        -1.81901067e-01,  9.95480455e-04, -2.37559140e-01,\n",
       "        -3.34004283e-01, -5.93467355e-01,  3.35474938e-01,\n",
       "         3.66933942e-02]], dtype=float32)"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.predict(x.unsqueeze(0).cpu())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "1042bf97",
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax_activation(inputs): \n",
    "    inputs = inputs.tolist()\n",
    "    exp_values = np.exp(inputs - np.max(inputs)) \n",
    "    # Normalize \n",
    "    probabilities = exp_values / np.sum(exp_values)\n",
    "\n",
    "    return probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "8392e757",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted label: [0], confidence: 95.34%\n"
     ]
    }
   ],
   "source": [
    "predection = classifier.predict(x.unsqueeze(0).cpu())\n",
    "\n",
    "probability = np.max(softmax_activation(predection), axis=1)[0]\n",
    "probability = round(probability*100, 2) \n",
    "\n",
    "label = np.argmax(predection, axis=1) \n",
    "\n",
    "print(\"Predicted label: {}, confidence: {}%\".format(label, probability))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7738ab81",
   "metadata": {},
   "source": [
    "### Torch Attack "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "922e0a47",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchattacks\n",
    "\n",
    "atk = torchattacks.PGD(model, eps=0.01, alpha=2/255, steps=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "ee4add34",
   "metadata": {},
   "outputs": [],
   "source": [
    "adv_images = atk(x.unsqueeze(0), torch.tensor([1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "7420ca0f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 3, 224, 224])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adv_images.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "28a57ba2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3], device='cuda:0')"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits = model(adv_images)\n",
    "\n",
    "preds = torch.argmax(logits, dim=-1)\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b76c89c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}