ViT - Cifar 10 Training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0fb6b65b",
   "metadata": {},
   "source": [
    "## Vision Transformer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1d398c3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: http://ftp.daumkakao.com/pypi/simple\r\n",
      "Requirement already satisfied: ml_collections in /home/ubuntu/miniconda3/lib/python3.8/site-packages (0.1.0)\r\n",
      "Requirement already satisfied: six in /home/ubuntu/miniconda3/lib/python3.8/site-packages (from ml_collections) (1.16.0)\r\n",
      "Requirement already satisfied: PyYAML in /home/ubuntu/miniconda3/lib/python3.8/site-packages (from ml_collections) (5.4.1)\r\n",
      "Requirement already satisfied: contextlib2 in /home/ubuntu/miniconda3/lib/python3.8/site-packages (from ml_collections) (21.6.0)\r\n",
      "Requirement already satisfied: absl-py in /home/ubuntu/miniconda3/lib/python3.8/site-packages (from ml_collections) (0.12.0)\r\n"
     ]
    }
   ],
   "source": [
    "!pip install ml_collections"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aaf10b6b",
   "metadata": {},
   "source": [
    "### Imporing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0a89f65f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, copy, logging, math, random, time, typing, io, cv2\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import keras\n",
    "import keras.backend as K\n",
    "from keras.datasets import cifar10, cifar100\n",
    "\n",
    "# import albumentations as albu\n",
    "from sklearn.model_selection import train_test_split, StratifiedShuffleSplit\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from scipy import ndimage\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import CrossEntropyLoss, Dropout, Softmax, Linear, Conv2d, LayerNorm\n",
    "from torch.nn.modules.utils import _pair\n",
    "import torchvision\n",
    "from torchvision import transforms, datasets\n",
    "import torch.distributed as dist\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "from torch.utils.data import DataLoader, RandomSampler, DistributedSampler, SequentialSampler\n",
    "from torch.optim.lr_scheduler import LambdaLR\n",
    "import torchvision.models as models\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "from torchvision import utils\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from matplotlib.colors import LogNorm \n",
    "\n",
    "from urllib.request import urlretrieve\n",
    "from PIL import Image\n",
    "from os.path import join as pjoin\n",
    "\n",
    "from tqdm import tqdm\n",
    "from datetime import timedelta\n",
    "from __future__ import absolute_import, division, print_function \n",
    "import ml_collections"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34536021",
   "metadata": {},
   "source": [
    "### Download Pre-trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "27cfadd8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2021-11-05 17:55:46--  https://storage.googleapis.com/vit_models/imagenet21k/ViT-B_16.npz\n",
      "Resolving storage.googleapis.com (storage.googleapis.com)... 172.217.174.112, 216.58.220.144, 216.58.220.112, ...\n",
      "Connecting to storage.googleapis.com (storage.googleapis.com)|172.217.174.112|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 412815506 (394M) [application/octet-stream]\n",
      "Saving to: ‘ViT-B_16.npz.1’\n",
      "\n",
      "ViT-B_16.npz.1      100%[===================>] 393.69M  22.0MB/s    in 19s     \n",
      "\n",
      "2021-11-05 17:56:07 (20.5 MB/s) - ‘ViT-B_16.npz.1’ saved [412815506/412815506]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://storage.googleapis.com/vit_models/imagenet21k/ViT-B_16.npz"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "154336ea",
   "metadata": {},
   "source": [
    "### Check GPU Access"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "634fc358",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU:  Tesla V100-SXM2-32GB\n"
     ]
    }
   ],
   "source": [
    "if torch.cuda.is_available():    \n",
    "    device = torch.device(\"cuda:1\")\n",
    "    print('GPU: ', torch.cuda.get_device_name(0))\n",
    "\n",
    "else:\n",
    "    device = torch.device(\"cpu\")\n",
    "    print('No GPU available')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9a9432a",
   "metadata": {},
   "source": [
    "### Configurations "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8e431563",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_testing():\n",
    "    '''\n",
    "    Returns a minimal configuration for testing\n",
    "    '''\n",
    "    config = ml_collections.ConfigDict()\n",
    "    config.patches = ml_collections.ConfigDict({'size': (16, 16)})\n",
    "    config.hidden_size = 1\n",
    "    config.transformer = ml_collections.ConfigDict()\n",
    "    config.transformer.mlp_dim = 1\n",
    "    config.transformer.num_heads = 1\n",
    "    config.transformer.num_layers = 1\n",
    "    config.transformer.attention_dropout_rate = 0.0\n",
    "    config.transformer.dropout_rate = 0.1\n",
    "    config.classifier = 'token'\n",
    "    config.representation_size = None\n",
    "    return config\n",
    "\n",
    "def get_b16_config():\n",
    "    '''\n",
    "    Returns the ViT-B/16 configuration\n",
    "    '''\n",
    "    config = ml_collections.ConfigDict()\n",
    "    config.patches = ml_collections.ConfigDict({'size': (16, 16)})\n",
    "    config.hidden_size = 768\n",
    "    config.transformer = ml_collections.ConfigDict()\n",
    "    config.transformer.mlp_dim = 3072\n",
    "    config.transformer.num_heads = 12\n",
    "    config.transformer.num_layers = 12\n",
    "    config.transformer.attention_dropout_rate = 0.0\n",
    "    config.transformer.dropout_rate = 0.1\n",
    "    config.classifier = 'token'\n",
    "    config.representation_size = None\n",
    "    return config\n",
    "\n",
    "def get_b32_config():\n",
    "    '''\n",
    "    Returns the ViT-B/32 configuration\n",
    "    '''\n",
    "    config = get_b16_config()\n",
    "    config.patches.size = (32, 32)\n",
    "    return config\n",
    "\n",
    "def get_l16_config():\n",
    "    '''\n",
    "    Returns the ViT-L/16 configuration\n",
    "    '''\n",
    "    config = ml_collections.ConfigDict()\n",
    "    config.patches = ml_collections.ConfigDict({'size': (16, 16)})\n",
    "    config.hidden_size = 1024\n",
    "    config.transformer = ml_collections.ConfigDict()\n",
    "    config.transformer.mlp_dim = 4096\n",
    "    config.transformer.num_heads = 16\n",
    "    config.transformer.num_layers = 24\n",
    "    config.transformer.attention_dropout_rate = 0.0\n",
    "    config.transformer.dropout_rate = 0.1\n",
    "    config.classifier = 'token'\n",
    "    config.representation_size = None\n",
    "    return config\n",
    "\n",
    "def get_l32_config():\n",
    "    '''\n",
    "    Returns the ViT-L/32 configuration\n",
    "    '''\n",
    "    config = get_l16_config()\n",
    "    config.patches.size = (32, 32)\n",
    "    return config\n",
    "\n",
    "def get_h14_config():\n",
    "    '''\n",
    "    Returns the ViT-L/16 configuration\n",
    "    '''\n",
    "    config = ml_collections.ConfigDict()\n",
    "    config.patches = ml_collections.ConfigDict({'size': (14, 14)})\n",
    "    config.hidden_size = 1280\n",
    "    config.transformer = ml_collections.ConfigDict()\n",
    "    config.transformer.mlp_dim = 5120\n",
    "    config.transformer.num_heads = 16\n",
    "    config.transformer.num_layers = 32\n",
    "    config.transformer.attention_dropout_rate = 0.0\n",
    "    config.transformer.dropout_rate = 0.1\n",
    "    config.classifier = 'token'\n",
    "    config.representation_size = None\n",
    "    return config"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ec951b3",
   "metadata": {},
   "source": [
    "### Model Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5d0c68cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "ATTENTION_Q = \"MultiHeadDotProductAttention_1/query\"\n",
    "ATTENTION_K = \"MultiHeadDotProductAttention_1/key\"\n",
    "ATTENTION_V = \"MultiHeadDotProductAttention_1/value\"\n",
    "ATTENTION_OUT = \"MultiHeadDotProductAttention_1/out\"\n",
    "FC_0 = \"MlpBlock_3/Dense_0\"\n",
    "FC_1 = \"MlpBlock_3/Dense_1\"\n",
    "ATTENTION_NORM = \"LayerNorm_0\"\n",
    "MLP_NORM = \"LayerNorm_2\"\n",
    "\n",
    "\n",
    "def np2th(weights):\n",
    "    \"\"\"Possibly convert HWIO to OIHW.\"\"\"\n",
    "    if weights.ndim == 4:\n",
    "        weights = weights.transpose([3, 2, 0, 1])\n",
    "    return torch.from_numpy(weights)\n",
    "\n",
    "def swish(x):\n",
    "    return x * torch.sigmoid(x)\n",
    "\n",
    "ACT2FN = {\"gelu\": torch.nn.functional.gelu, \"relu\": torch.nn.functional.relu, \"swish\": swish}\n",
    "\n",
    "\n",
    "class Attention(nn.Module):\n",
    "    def __init__(self, config, vis):\n",
    "        super(Attention, self).__init__()\n",
    "        self.vis = vis\n",
    "        self.num_attention_heads = config.transformer[\"num_heads\"]\n",
    "        self.attention_head_size = int(config.hidden_size / self.num_attention_heads)\n",
    "        self.all_head_size = self.num_attention_heads * self.attention_head_size\n",
    "\n",
    "        self.query = Linear(config.hidden_size, self.all_head_size)\n",
    "        self.key = Linear(config.hidden_size, self.all_head_size)\n",
    "        self.value = Linear(config.hidden_size, self.all_head_size)\n",
    "\n",
    "        self.out = Linear(config.hidden_size, config.hidden_size)\n",
    "        self.attn_dropout = Dropout(config.transformer[\"attention_dropout_rate\"])\n",
    "        self.proj_dropout = Dropout(config.transformer[\"attention_dropout_rate\"])\n",
    "\n",
    "        self.softmax = Softmax(dim=-1)\n",
    "\n",
    "    def transpose_for_scores(self, x):\n",
    "        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)\n",
    "        x = x.view(*new_x_shape)\n",
    "        return x.permute(0, 2, 1, 3)\n",
    "\n",
    "    def forward(self, hidden_states):\n",
    "        mixed_query_layer = self.query(hidden_states)\n",
    "        mixed_key_layer = self.key(hidden_states)\n",
    "        mixed_value_layer = self.value(hidden_states)\n",
    "\n",
    "        query_layer = self.transpose_for_scores(mixed_query_layer)\n",
    "        key_layer = self.transpose_for_scores(mixed_key_layer)\n",
    "        value_layer = self.transpose_for_scores(mixed_value_layer)\n",
    "\n",
    "        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))\n",
    "        attention_scores = attention_scores / math.sqrt(self.attention_head_size)\n",
    "        attention_probs = self.softmax(attention_scores)\n",
    "        weights = attention_probs if self.vis else None\n",
    "        attention_probs = self.attn_dropout(attention_probs)\n",
    "\n",
    "        context_layer = torch.matmul(attention_probs, value_layer)\n",
    "        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()\n",
    "        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)\n",
    "        context_layer = context_layer.view(*new_context_layer_shape)\n",
    "        attention_output = self.out(context_layer)\n",
    "        attention_output = self.proj_dropout(attention_output)\n",
    "        return attention_output, weights\n",
    "\n",
    "\n",
    "class Mlp(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super(Mlp, self).__init__()\n",
    "        self.fc1 = Linear(config.hidden_size, config.transformer[\"mlp_dim\"])\n",
    "        self.fc2 = Linear(config.transformer[\"mlp_dim\"], config.hidden_size)\n",
    "        self.act_fn = ACT2FN[\"gelu\"]\n",
    "        self.dropout = Dropout(config.transformer[\"dropout_rate\"])\n",
    "\n",
    "        self._init_weights()\n",
    "\n",
    "    def _init_weights(self):\n",
    "        nn.init.xavier_uniform_(self.fc1.weight)\n",
    "        nn.init.xavier_uniform_(self.fc2.weight)\n",
    "        nn.init.normal_(self.fc1.bias, std=1e-6)\n",
    "        nn.init.normal_(self.fc2.bias, std=1e-6)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.fc1(x)\n",
    "        x = self.act_fn(x)\n",
    "        x = self.dropout(x)\n",
    "        x = self.fc2(x)\n",
    "        x = self.dropout(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "class Embeddings(nn.Module):\n",
    "    '''\n",
    "    Construct the embeddings from patch, position embeddings\n",
    "    '''\n",
    "    def __init__(self, config, img_size, in_channels=3):\n",
    "        super(Embeddings, self).__init__()\n",
    "        img_size = _pair(img_size)\n",
    "        patch_size = _pair(config.patches[\"size\"])\n",
    "        n_patches = (img_size[0]//patch_size[0]) * (img_size[1]//patch_size[1])\n",
    "\n",
    "        self.patch_embeddings = Conv2d(in_channels=in_channels,\n",
    "                                       out_channels=config.hidden_size,\n",
    "                                       kernel_size=patch_size,\n",
    "                                       stride=patch_size)\n",
    "        self.position_embeddings = nn.Parameter(torch.zeros(1, n_patches+1, config.hidden_size))\n",
    "        self.cls_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size))\n",
    "\n",
    "        self.dropout = Dropout(config.transformer[\"dropout_rate\"])\n",
    "\n",
    "    def forward(self, x):\n",
    "        B = x.shape[0]\n",
    "        cls_tokens = self.cls_token.expand(B, -1, -1)\n",
    "\n",
    "        x = self.patch_embeddings(x)\n",
    "        x = x.flatten(2)\n",
    "        x = x.transpose(-1, -2)\n",
    "        x = torch.cat((cls_tokens, x), dim=1)\n",
    "\n",
    "        embeddings = x + self.position_embeddings\n",
    "        embeddings = self.dropout(embeddings)\n",
    "        return embeddings\n",
    "\n",
    "\n",
    "class Block(nn.Module):\n",
    "    def __init__(self, config, vis):\n",
    "        super(Block, self).__init__()\n",
    "        self.hidden_size = config.hidden_size\n",
    "        self.attention_norm = LayerNorm(config.hidden_size, eps=1e-6)\n",
    "        self.ffn_norm = LayerNorm(config.hidden_size, eps=1e-6)\n",
    "        self.ffn = Mlp(config)\n",
    "        self.attn = Attention(config, vis)\n",
    "\n",
    "    def forward(self, x):\n",
    "        h = x\n",
    "        x = self.attention_norm(x)\n",
    "        x, weights = self.attn(x)\n",
    "        x = x + h\n",
    "\n",
    "        h = x\n",
    "        x = self.ffn_norm(x)\n",
    "        x = self.ffn(x)\n",
    "        x = x + h\n",
    "        return x, weights\n",
    "\n",
    "    def load_from(self, weights, n_block):\n",
    "        ROOT = f\"Transformer/encoderblock_{n_block}\"\n",
    "        with torch.no_grad():\n",
    "            query_weight = np2th(weights[pjoin(ROOT, ATTENTION_Q, \"kernel\")]).view(self.hidden_size, self.hidden_size).t()\n",
    "            key_weight = np2th(weights[pjoin(ROOT, ATTENTION_K, \"kernel\")]).view(self.hidden_size, self.hidden_size).t()\n",
    "            value_weight = np2th(weights[pjoin(ROOT, ATTENTION_V, \"kernel\")]).view(self.hidden_size, self.hidden_size).t()\n",
    "            out_weight = np2th(weights[pjoin(ROOT, ATTENTION_OUT, \"kernel\")]).view(self.hidden_size, self.hidden_size).t()\n",
    "\n",
    "            query_bias = np2th(weights[pjoin(ROOT, ATTENTION_Q, \"bias\")]).view(-1)\n",
    "            key_bias = np2th(weights[pjoin(ROOT, ATTENTION_K, \"bias\")]).view(-1)\n",
    "            value_bias = np2th(weights[pjoin(ROOT, ATTENTION_V, \"bias\")]).view(-1)\n",
    "            out_bias = np2th(weights[pjoin(ROOT, ATTENTION_OUT, \"bias\")]).view(-1)\n",
    "\n",
    "            self.attn.query.weight.copy_(query_weight)\n",
    "            self.attn.key.weight.copy_(key_weight)\n",
    "            self.attn.value.weight.copy_(value_weight)\n",
    "            self.attn.out.weight.copy_(out_weight)\n",
    "            self.attn.query.bias.copy_(query_bias)\n",
    "            self.attn.key.bias.copy_(key_bias)\n",
    "            self.attn.value.bias.copy_(value_bias)\n",
    "            self.attn.out.bias.copy_(out_bias)\n",
    "\n",
    "            mlp_weight_0 = np2th(weights[pjoin(ROOT, FC_0, \"kernel\")]).t()\n",
    "            mlp_weight_1 = np2th(weights[pjoin(ROOT, FC_1, \"kernel\")]).t()\n",
    "            mlp_bias_0 = np2th(weights[pjoin(ROOT, FC_0, \"bias\")]).t()\n",
    "            mlp_bias_1 = np2th(weights[pjoin(ROOT, FC_1, \"bias\")]).t()\n",
    "\n",
    "            self.ffn.fc1.weight.copy_(mlp_weight_0)\n",
    "            self.ffn.fc2.weight.copy_(mlp_weight_1)\n",
    "            self.ffn.fc1.bias.copy_(mlp_bias_0)\n",
    "            self.ffn.fc2.bias.copy_(mlp_bias_1)\n",
    "\n",
    "            self.attention_norm.weight.copy_(np2th(weights[pjoin(ROOT, ATTENTION_NORM, \"scale\")]))\n",
    "            self.attention_norm.bias.copy_(np2th(weights[pjoin(ROOT, ATTENTION_NORM, \"bias\")]))\n",
    "            self.ffn_norm.weight.copy_(np2th(weights[pjoin(ROOT, MLP_NORM, \"scale\")]))\n",
    "            self.ffn_norm.bias.copy_(np2th(weights[pjoin(ROOT, MLP_NORM, \"bias\")]))\n",
    "\n",
    "\n",
    "class Encoder(nn.Module):\n",
    "    def __init__(self, config, vis):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.vis = vis\n",
    "        self.layer = nn.ModuleList()\n",
    "        self.encoder_norm = LayerNorm(config.hidden_size, eps=1e-6)\n",
    "        for _ in range(config.transformer[\"num_layers\"]):\n",
    "            layer = Block(config, vis)\n",
    "            self.layer.append(copy.deepcopy(layer))\n",
    "\n",
    "    def forward(self, hidden_states):\n",
    "        attn_weights = []\n",
    "        for layer_block in self.layer:\n",
    "            hidden_states, weights = layer_block(hidden_states)\n",
    "            if self.vis:\n",
    "                attn_weights.append(weights)\n",
    "        encoded = self.encoder_norm(hidden_states)\n",
    "        return encoded, attn_weights\n",
    "\n",
    "\n",
    "class Transformer(nn.Module):\n",
    "    def __init__(self, config, img_size, vis):\n",
    "        super(Transformer, self).__init__()\n",
    "        self.embeddings = Embeddings(config, img_size=img_size)\n",
    "        self.encoder = Encoder(config, vis)\n",
    "\n",
    "    def forward(self, input_ids):\n",
    "        embedding_output = self.embeddings(input_ids)\n",
    "        encoded, attn_weights = self.encoder(embedding_output)\n",
    "        return encoded, attn_weights\n",
    "\n",
    "\n",
    "class VisionTransformer(nn.Module):\n",
    "    def __init__(self, config, img_size=224, num_classes=21843, zero_head=False, vis=False):\n",
    "        super(VisionTransformer, self).__init__()\n",
    "        self.num_classes = num_classes\n",
    "        self.zero_head = zero_head\n",
    "        self.classifier = config.classifier\n",
    "\n",
    "        self.transformer = Transformer(config, img_size, vis)\n",
    "        self.head = Linear(config.hidden_size, num_classes)\n",
    "\n",
    "    def forward(self, x, labels=None):\n",
    "        x, attn_weights = self.transformer(x)\n",
    "        logits = self.head(x[:, 0])\n",
    "\n",
    "        if labels is not None:\n",
    "            loss_fct = CrossEntropyLoss()\n",
    "            loss = loss_fct(logits.view(-1, self.num_classes), labels.view(-1))\n",
    "            return loss\n",
    "        else:\n",
    "            return logits, attn_weights\n",
    "\n",
    "    def load_from(self, weights):\n",
    "        with torch.no_grad():\n",
    "            if self.zero_head:\n",
    "                nn.init.zeros_(self.head.weight)\n",
    "                nn.init.zeros_(self.head.bias)\n",
    "            else:\n",
    "                self.head.weight.copy_(np2th(weights[\"head/kernel\"]).t())\n",
    "                self.head.bias.copy_(np2th(weights[\"head/bias\"]).t())\n",
    "\n",
    "            self.transformer.embeddings.patch_embeddings.weight.copy_(np2th(weights[\"embedding/kernel\"]))\n",
    "            self.transformer.embeddings.patch_embeddings.bias.copy_(np2th(weights[\"embedding/bias\"]))\n",
    "            self.transformer.embeddings.cls_token.copy_(np2th(weights[\"cls\"]))\n",
    "            self.transformer.encoder.encoder_norm.weight.copy_(np2th(weights[\"Transformer/encoder_norm/scale\"]))\n",
    "            self.transformer.encoder.encoder_norm.bias.copy_(np2th(weights[\"Transformer/encoder_norm/bias\"]))\n",
    "\n",
    "            posemb = np2th(weights[\"Transformer/posembed_input/pos_embedding\"])\n",
    "            posemb_new = self.transformer.embeddings.position_embeddings\n",
    "            if posemb.size() == posemb_new.size():\n",
    "                self.transformer.embeddings.position_embeddings.copy_(posemb)\n",
    "            else:\n",
    "                logger.info(\"load_pretrained: resized variant: %s to %s\" % (posemb.size(), posemb_new.size()))\n",
    "                ntok_new = posemb_new.size(1)\n",
    "\n",
    "                if self.classifier == \"token\":\n",
    "                    posemb_tok, posemb_grid = posemb[:, :1], posemb[0, 1:]\n",
    "                    ntok_new -= 1\n",
    "                else:\n",
    "                    posemb_tok, posemb_grid = posemb[:, :0], posemb[0]\n",
    "\n",
    "                gs_old = int(np.sqrt(len(posemb_grid)))\n",
    "                gs_new = int(np.sqrt(ntok_new))\n",
    "                print('load_pretrained: grid-size from %s to %s' % (gs_old, gs_new))\n",
    "                posemb_grid = posemb_grid.reshape(gs_old, gs_old, -1)\n",
    "\n",
    "                zoom = (gs_new / gs_old, gs_new / gs_old, 1)\n",
    "                posemb_grid = ndimage.zoom(posemb_grid, zoom, order=1)\n",
    "                posemb_grid = posemb_grid.reshape(1, gs_new * gs_new, -1)\n",
    "                posemb = np.concatenate([posemb_tok, posemb_grid], axis=1)\n",
    "                self.transformer.embeddings.position_embeddings.copy_(np2th(posemb))\n",
    "\n",
    "            for bname, block in self.transformer.encoder.named_children():\n",
    "                for uname, unit in block.named_children():\n",
    "                    unit.load_from(weights, n_block=uname)\n",
    "\n",
    "\n",
    "CONFIGS = {\n",
    "    'ViT-B_16': get_b16_config(),\n",
    "    'ViT-B_32': get_b32_config(),\n",
    "    'ViT-L_16': get_l16_config(),\n",
    "    'ViT-L_32': get_l32_config(),\n",
    "    'ViT-H_14': get_h14_config(),\n",
    "    'testing': get_testing(),\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04cfeab1",
   "metadata": {},
   "source": [
    "### Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "15530c24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VisionTransformer(\n",
       "  (transformer): Transformer(\n",
       "    (embeddings): Embeddings(\n",
       "      (patch_embeddings): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16))\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): Encoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (1): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (2): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (3): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (4): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (5): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (6): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (7): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (8): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (9): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (10): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (11): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (encoder_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (head): Linear(in_features=768, out_features=21843, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = CONFIGS[\"ViT-B_16\"]\n",
    "model = VisionTransformer(config, img_size=224, num_classes=21843, zero_head=False, vis=True)\n",
    "model.load_from(np.load(\"ViT-B_16.npz\"))\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca533f29",
   "metadata": {},
   "source": [
    "### Utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "eef2ba1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_world_size():\n",
    "    if not dist.is_available():\n",
    "        return 1\n",
    "    if not dist.is_initialized():\n",
    "        return 1\n",
    "    return dist.get_world_size()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68c859fd",
   "metadata": {},
   "source": [
    "### Dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7c8128aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_loader(local_rank, img_size, dataset, train_batch_size, eval_batch_size):\n",
    "    if local_rank not in [-1, 0]:\n",
    "        torch.distributed.barrier()\n",
    "\n",
    "    transform_train = transforms.Compose([\n",
    "        transforms.RandomResizedCrop((img_size, img_size), scale=(0.05, 1.0)),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),\n",
    "    ])\n",
    "    transform_test = transforms.Compose([\n",
    "        transforms.Resize((img_size, img_size)),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),\n",
    "    ])\n",
    "\n",
    "    if dataset == \"cifar10\":\n",
    "        trainset = datasets.CIFAR10(root=\"./data\",\n",
    "                                    train=True,\n",
    "                                    download=True,\n",
    "                                    transform=transform_train)\n",
    "        testset = datasets.CIFAR10(root=\"./data\",\n",
    "                                   train=False,\n",
    "                                   download=True,\n",
    "                                   transform=transform_test) if local_rank in [-1, 0] else None\n",
    "\n",
    "    else:\n",
    "        trainset = datasets.CIFAR100(root=\"./data\",\n",
    "                                     train=True,\n",
    "                                     download=True,\n",
    "                                     transform=transform_train)\n",
    "        testset = datasets.CIFAR100(root=\"./data\",\n",
    "                                    train=False,\n",
    "                                    download=True,\n",
    "                                    transform=transform_test) if local_rank in [-1, 0] else None\n",
    "    if local_rank == 0:\n",
    "        torch.distributed.barrier()\n",
    "\n",
    "    train_sampler = RandomSampler(trainset) if local_rank == -1 else DistributedSampler(trainset)\n",
    "    test_sampler = SequentialSampler(testset)\n",
    "    train_loader = DataLoader(trainset,\n",
    "                              sampler=train_sampler,\n",
    "                              batch_size=train_batch_size,\n",
    "                              num_workers=4,\n",
    "                              pin_memory=True)\n",
    "    test_loader = DataLoader(testset,\n",
    "                             sampler=test_sampler,\n",
    "                             batch_size=eval_batch_size,\n",
    "                             num_workers=4,\n",
    "                             pin_memory=True) if testset is not None else None\n",
    "\n",
    "    return train_loader, test_loader"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1dd266e6",
   "metadata": {},
   "source": [
    "### Scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4c71f996",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConstantLRSchedule(LambdaLR):\n",
    "    \"\"\" Constant learning rate schedule.\n",
    "    \"\"\"\n",
    "    def __init__(self, optimizer, last_epoch=-1):\n",
    "        super(ConstantLRSchedule, self).__init__(optimizer, lambda _: 1.0, last_epoch=last_epoch)\n",
    "\n",
    "\n",
    "class WarmupConstantSchedule(LambdaLR):\n",
    "    \"\"\" Linear warmup and then constant.\n",
    "        Linearly increases learning rate schedule from 0 to 1 over `warmup_steps` training steps.\n",
    "        Keeps learning rate schedule equal to 1. after warmup_steps.\n",
    "    \"\"\"\n",
    "    def __init__(self, optimizer, warmup_steps, last_epoch=-1):\n",
    "        self.warmup_steps = warmup_steps\n",
    "        super(WarmupConstantSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)\n",
    "\n",
    "    def lr_lambda(self, step):\n",
    "        if step < self.warmup_steps:\n",
    "            return float(step) / float(max(1.0, self.warmup_steps))\n",
    "        return 1.\n",
    "\n",
    "\n",
    "class WarmupLinearSchedule(LambdaLR):\n",
    "    \"\"\" Linear warmup and then linear decay.\n",
    "        Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.\n",
    "        Linearly decreases learning rate from 1. to 0. over remaining `t_total - warmup_steps` steps.\n",
    "    \"\"\"\n",
    "    def __init__(self, optimizer, warmup_steps, t_total, last_epoch=-1):\n",
    "        self.warmup_steps = warmup_steps\n",
    "        self.t_total = t_total\n",
    "        super(WarmupLinearSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)\n",
    "\n",
    "    def lr_lambda(self, step):\n",
    "        if step < self.warmup_steps:\n",
    "            return float(step) / float(max(1, self.warmup_steps))\n",
    "        return max(0.0, float(self.t_total - step) / float(max(1.0, self.t_total - self.warmup_steps)))\n",
    "\n",
    "\n",
    "class WarmupCosineSchedule(LambdaLR):\n",
    "    \"\"\" Linear warmup and then cosine decay.\n",
    "        Linearly increases learning rate from 0 to 1 over `warmup_steps` training steps.\n",
    "        Decreases learning rate from 1. to 0. over remaining `t_total - warmup_steps` steps following a cosine curve.\n",
    "        If `cycles` (default=0.5) is different from default, learning rate follows cosine function after warmup.\n",
    "    \"\"\"\n",
    "    def __init__(self, optimizer, warmup_steps, t_total, cycles=.5, last_epoch=-1):\n",
    "        self.warmup_steps = warmup_steps\n",
    "        self.t_total = t_total\n",
    "        self.cycles = cycles\n",
    "        super(WarmupCosineSchedule, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)\n",
    "\n",
    "    def lr_lambda(self, step):\n",
    "        if step < self.warmup_steps:\n",
    "            return float(step) / float(max(1.0, self.warmup_steps))\n",
    "        # progress after warmup\n",
    "        progress = float(step - self.warmup_steps) / float(max(1, self.t_total - self.warmup_steps))\n",
    "        return max(0.0, 0.5 * (1. + math.cos(math.pi * float(self.cycles) * 2.0 * progress)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f812376",
   "metadata": {},
   "source": [
    "### Training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b7c81fa0",
   "metadata": {},
   "outputs": [],
   "source": [
    "class AverageMeter(object):\n",
    "    \"\"\"Computes and stores the average and current value\"\"\"\n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "\n",
    "    def reset(self):\n",
    "        self.val = 0\n",
    "        self.avg = 0\n",
    "        self.sum = 0\n",
    "        self.count = 0\n",
    "\n",
    "    def update(self, val, n=1):\n",
    "        self.val = val\n",
    "        self.sum += val * n\n",
    "        self.count += n\n",
    "        self.avg = self.sum / self.count\n",
    "\n",
    "\n",
    "def simple_accuracy(preds, labels):\n",
    "    return (preds == labels).mean()\n",
    "\n",
    "\n",
    "def save_model(output_dir, name, model):\n",
    "    model_to_save = model.module if hasattr(model, 'module') else model\n",
    "    model_checkpoint = os.path.join(output_dir, \"%s_checkpoint.bin\" % name)\n",
    "    torch.save(model_to_save.state_dict(), model_checkpoint)\n",
    "    logger.info(\"Saved model checkpoint to [DIR: %s]\", output_dir)\n",
    "\n",
    "\n",
    "def setup(model_type, img_size, pretrained_dir, device, dataset):\n",
    "    # Prepare model\n",
    "    config = CONFIGS[model_type]\n",
    "\n",
    "    num_classes = 10 if dataset == \"cifar10\" else 100\n",
    "\n",
    "    model = VisionTransformer(config, img_size, zero_head=True, num_classes=num_classes)\n",
    "    model.load_from(np.load(pretrained_dir))\n",
    "    model.to(device)\n",
    "    num_params = count_parameters(model)\n",
    "\n",
    "    logger.info(\"{}\".format(config))\n",
    "    logger.info(\"Training parameters %s\", model_type, img_size, pretrained_dir, device)\n",
    "    logger.info(\"Total Parameter: \\t%2.1fM\" % num_params)\n",
    "    return model_type, img_size, pretrained_dir, device, model\n",
    "\n",
    "\n",
    "def count_parameters(model):\n",
    "    params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "    return params/1000000\n",
    "\n",
    "\n",
    "def set_seed(seed, n_gpu):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    if n_gpu > 0:\n",
    "        torch.cuda.manual_seed_all(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c2392c0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def valid(eval_batch_size, local_rank, device, model, writer, test_loader, global_step):\n",
    "    # Validation!\n",
    "    eval_losses = AverageMeter()\n",
    "\n",
    "    logger.info(\"***** Running Test *****\")\n",
    "    logger.info(\"  Num steps = %d\", len(test_loader))\n",
    "    logger.info(\"  Batch size = %d\", eval_batch_size)\n",
    "\n",
    "    model.eval()\n",
    "    all_preds, all_label = [], []\n",
    "    epoch_iterator = tqdm(test_loader,\n",
    "                          desc=\"Test... (loss=X.X)\",\n",
    "                          bar_format=\"{l_bar}{r_bar}\",\n",
    "                          dynamic_ncols=True,\n",
    "                          disable=local_rank not in [-1, 0])\n",
    "    loss_fct = torch.nn.CrossEntropyLoss()\n",
    "    for step, batch in enumerate(epoch_iterator):\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        x, y = batch\n",
    "        with torch.no_grad():\n",
    "            logits, attn_weights = model(x)\n",
    "\n",
    "            eval_loss = loss_fct(logits, y)\n",
    "            eval_losses.update(eval_loss.item())\n",
    "\n",
    "            preds = torch.argmax(logits, dim=-1)\n",
    "\n",
    "        if len(all_preds) == 0:\n",
    "            all_preds.append(preds.detach().cpu().numpy())\n",
    "            all_label.append(y.detach().cpu().numpy())\n",
    "        else:\n",
    "            all_preds[0] = np.append(\n",
    "                all_preds[0], preds.detach().cpu().numpy(), axis=0\n",
    "            )\n",
    "            all_label[0] = np.append(\n",
    "                all_label[0], y.detach().cpu().numpy(), axis=0\n",
    "            )\n",
    "        epoch_iterator.set_description(\"Testing... (loss=%2.5f)\" % eval_losses.val)\n",
    "\n",
    "    all_preds, all_label = all_preds[0], all_label[0]\n",
    "    accuracy = simple_accuracy(all_preds, all_label)\n",
    "\n",
    "    logger.info(\"\\n\")\n",
    "    logger.info(\"Test Results\")\n",
    "    logger.info(\"Global Steps: %d\" % global_step)\n",
    "    logger.info(\"Test Loss: %2.5f\" % eval_losses.avg)\n",
    "    logger.info(\"Test Accuracy: %2.5f\" % accuracy)\n",
    "\n",
    "    writer.add_scalar(\"test/accuracy\", scalar_value=accuracy, global_step=global_step)\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f1b826e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(local_rank, output_dir, name, train_batch_size, eval_batch_size, seed, n_gpu, gradient_accumulation_steps, dataset, img_size, learning_rate, weight_decay, num_steps, decay_type, warmup_steps, fp16, fp16_opt_level, device, max_grad_norm, eval_every, model):\n",
    "    \"\"\" Train the model \"\"\"\n",
    "    if local_rank in [-1, 0]:\n",
    "        os.makedirs(output_dir, exist_ok=True)\n",
    "        writer = SummaryWriter(log_dir=os.path.join(\"logs\", name))\n",
    "\n",
    "    train_batch_size = train_batch_size // gradient_accumulation_steps\n",
    "\n",
    "    # Prepare dataset\n",
    "    train_loader, test_loader = get_loader(local_rank, img_size, dataset, train_batch_size, eval_batch_size)\n",
    "\n",
    "    # Prepare optimizer and scheduler\n",
    "    optimizer = torch.optim.SGD(model.parameters(),\n",
    "                                lr=learning_rate,\n",
    "                                momentum=0.9,\n",
    "                                weight_decay=weight_decay)\n",
    "    t_total = num_steps\n",
    "    if decay_type == \"cosine\":\n",
    "        scheduler = WarmupCosineSchedule(optimizer, warmup_steps=warmup_steps, t_total=t_total)\n",
    "    else:\n",
    "        scheduler = WarmupLinearSchedule(optimizer, warmup_steps=warmup_steps, t_total=t_total)\n",
    "\n",
    "    if fp16:\n",
    "        model, optimizer = amp.initialize(models=model,\n",
    "                                          optimizers=optimizer,\n",
    "                                          opt_level=fp16_opt_level)\n",
    "        amp._amp_state.loss_scalers[0]._loss_scale = 2**20\n",
    "\n",
    "    # Distributed training\n",
    "    if local_rank != -1:\n",
    "        model = DDP(model, message_size=250000000, gradient_predivide_factor=get_world_size())\n",
    "\n",
    "    # Train!\n",
    "    logger.info(\"***** Running training *****\")\n",
    "    logger.info(\"  Total optimization steps = %d\", num_steps)\n",
    "    logger.info(\"  Instantaneous batch size per GPU = %d\", train_batch_size)\n",
    "    logger.info(\"  Total train batch size (w. parallel, distributed & accumulation) = %d\",\n",
    "                train_batch_size * gradient_accumulation_steps * (\n",
    "                    torch.distributed.get_world_size() if local_rank != -1 else 1))\n",
    "    logger.info(\"  Gradient Accumulation steps = %d\", gradient_accumulation_steps)\n",
    "\n",
    "    model.zero_grad()\n",
    "    set_seed(seed, n_gpu)  # Added here for reproducibility (even between python 2 and 3)\n",
    "    losses = AverageMeter()\n",
    "    global_step, best_acc = 0, 0\n",
    "    train_accs = []\n",
    "    test_accs = []\n",
    "\n",
    "    while True:\n",
    "        model.train()\n",
    "        epoch_iterator = tqdm(train_loader,\n",
    "                              desc=\"Training (X / X Steps) (loss=X.X)\",\n",
    "                              bar_format=\"{l_bar}{r_bar}\",\n",
    "                              dynamic_ncols=True,\n",
    "                              disable=local_rank not in [-1, 0])\n",
    "        for step, batch in enumerate(epoch_iterator):\n",
    "            batch = tuple(t.to(device) for t in batch)\n",
    "            x, y = batch\n",
    "            loss = model(x, y)\n",
    "\n",
    "            if gradient_accumulation_steps > 1:\n",
    "                loss = loss / gradient_accumulation_steps\n",
    "            if fp16:\n",
    "                with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "                    scaled_loss.backward()\n",
    "            else:\n",
    "                loss.backward()\n",
    "\n",
    "            if (step + 1) % gradient_accumulation_steps == 0:\n",
    "                losses.update(loss.item()*gradient_accumulation_steps)\n",
    "                if fp16:\n",
    "                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), max_grad_norm)\n",
    "                else:\n",
    "                    torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)\n",
    "                scheduler.step()\n",
    "                optimizer.step()\n",
    "                optimizer.zero_grad()\n",
    "                global_step += 1\n",
    "\n",
    "                epoch_iterator.set_description(\n",
    "                    \"Training (%d / %d Steps) (loss=%2.5f)\" % (global_step, t_total, losses.val)\n",
    "                )\n",
    "                if local_rank in [-1, 0]:\n",
    "                    writer.add_scalar(\"train/loss\", scalar_value=losses.val, global_step=global_step)\n",
    "                    writer.add_scalar(\"train/lr\", scalar_value=scheduler.get_lr()[0], global_step=global_step)\n",
    "                if global_step % eval_every == 0 and local_rank in [-1, 0]:\n",
    "                    accuracy = valid(eval_batch_size, local_rank, device, model, writer, test_loader, global_step)\n",
    "                    test_accs.append(accuracy)\n",
    "                    if best_acc < accuracy:\n",
    "                        save_model(output_dir, name, model)\n",
    "                        best_acc = accuracy\n",
    "                    model.train()\n",
    "\n",
    "                if global_step % 5 == 0:\n",
    "                    torch.save({\n",
    "                        'epoch': global_step,\n",
    "                        'model': model,\n",
    "                        'optimizer': optimizer,\n",
    "                        'scheduler': scheduler,\n",
    "                        'test_acc': test_accs\n",
    "                    }, './checkpoint.pt')\n",
    "\n",
    "                if global_step % t_total == 0:\n",
    "                    break\n",
    "        losses.reset()\n",
    "        if global_step % t_total == 0:\n",
    "            break\n",
    "\n",
    "    if local_rank in [-1, 0]:\n",
    "        writer.close()\n",
    "    logger.info(\"Best Accuracy: \\t%f\" % best_acc)\n",
    "    logger.info(\"End Training!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e98d6eee",
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    # Required parameters\n",
    "    name = \"cifar10_500\"                               # Name of this run. Used for monitoring\n",
    "    dataset = \"cifar10\"                                # Which downstream task, choices=[\"cifar10\", \"cifar100\"]\n",
    "    model_type = \"ViT-B_16\"                             # Which variant to use\n",
    "    pretrained_dir = \"ViT-B_16.npz\"                     # Where to search for pretrained ViT models\n",
    "    output_dir = \"output_cifar10\"                      # The output directory where checkpoints will be written\n",
    "    img_size = 224  #224                                 # Resolution size\n",
    "    train_batch_size = 64                               # Total batch size for training\n",
    "    eval_batch_size = 64                                # Total batch size for eval\n",
    "    eval_every = 100                                    # Run prediction on validation set every so many steps (Will always run one evaluation at the end of training)\n",
    "    learning_rate = 3e-2                                # The initial learning rate for SGD\n",
    "    weight_decay = 0                                    # Weight deay if we apply some\n",
    "    num_steps = 500                                     # Total number of training epochs to perform\n",
    "    decay_type = \"cosine\"                               # How to decay the learning rate, choices=[\"cosine\", \"linear\"]\n",
    "    warmup_steps = 100                                  # Step of training to perform learning rate warmup for\n",
    "    max_grad_norm = 1.0                                 # Max gradient norm\n",
    "    local_rank = -1                                     # local_rank for distributed training on gpus\n",
    "    seed = 42                                           # random seed for initialization\n",
    "    gradient_accumulation_steps = 1                     # Number of updates steps to accumulate before performing a backward/update pass\n",
    "    fp16 = 0                                            # (action = 'store_true') Whether to use 16-bit float precision instead of 32-bit\n",
    "    fp16_opt_level = 'store_true'                       # For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']\n",
    "    loss_scale = 0                                      # Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True (0 (default value): dynamic loss scaling, Positive power of 2: static loss scaling value)\n",
    "    local_rank = -1                                     # local_rank for distributed training on gpus\n",
    "\n",
    "    device=\"cpu\"\n",
    "    n_gpu = 0\n",
    "\n",
    "    # Setup CUDA, GPU & distributed training\n",
    "    if local_rank == -1:\n",
    "        device = torch.device(\"cuda:1\" if torch.cuda.is_available() else \"cpu\")\n",
    "        n_gpu = torch.cuda.device_count()\n",
    "    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs\n",
    "        torch.cuda.set_device(local_rank)\n",
    "        device = torch.device(\"cuda:1\", local_rank)\n",
    "        torch.distributed.init_process_group(backend='nccl',\n",
    "                                             timeout=timedelta(minutes=60))\n",
    "        n_gpu = 1\n",
    "\n",
    "    # Setup logging\n",
    "    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',\n",
    "                        datefmt='%m/%d/%Y %H:%M:%S',\n",
    "                        level=logging.INFO if local_rank in [-1, 0] else logging.WARN)\n",
    "    logger.warning(\"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s\" %\n",
    "                   (local_rank, device, n_gpu, bool(local_rank != -1), fp16))\n",
    "\n",
    "    # Set seed\n",
    "    set_seed(seed, n_gpu)\n",
    "\n",
    "    # Model & Tokenizer Setup\n",
    "    model_type, img_size, pretrained_dir, device, model = setup(model_type, img_size, pretrained_dir, device, dataset)\n",
    "\n",
    "    # Training\n",
    "    train(local_rank, output_dir, name, train_batch_size, eval_batch_size, seed, n_gpu, gradient_accumulation_steps, dataset, img_size, learning_rate, weight_decay, num_steps, decay_type, warmup_steps, fp16, fp16_opt_level, device, max_grad_norm, eval_every, model)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0045df69",
   "metadata": {},
   "source": [
    "### Running the trainer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2b5b50e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/05/2021 18:08:37 - WARNING - __main__ - Process rank: -1, device: cuda:1, n_gpu: 3, distributed training: False, 16-bits training: 0\n",
      "11/05/2021 18:08:40 - INFO - __main__ - classifier: token\n",
      "hidden_size: 768\n",
      "patches:\n",
      "  size: !!python/tuple\n",
      "  - 16\n",
      "  - 16\n",
      "representation_size: null\n",
      "transformer:\n",
      "  attention_dropout_rate: 0.0\n",
      "  dropout_rate: 0.1\n",
      "  mlp_dim: 3072\n",
      "  num_heads: 12\n",
      "  num_layers: 12\n",
      "\n",
      "--- Logging error ---\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/logging/__init__.py\", line 1081, in emit\n",
      "    msg = self.format(record)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/logging/__init__.py\", line 925, in format\n",
      "    return fmt.format(record)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/logging/__init__.py\", line 664, in format\n",
      "    record.message = record.getMessage()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/logging/__init__.py\", line 369, in getMessage\n",
      "    msg = msg % self.args\n",
      "TypeError: not all arguments converted during string formatting\n",
      "Call stack:\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/runpy.py\", line 194, in _run_module_as_main\n",
      "    return _run_code(code, main_globals, None,\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/runpy.py\", line 87, in _run_code\n",
      "    exec(code, run_globals)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/ipykernel_launcher.py\", line 16, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/traitlets/config/application.py\", line 845, in launch_instance\n",
      "    app.start()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/ipykernel/kernelapp.py\", line 612, in start\n",
      "    self.io_loop.start()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/platform/asyncio.py\", line 199, in start\n",
      "    self.asyncio_loop.run_forever()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/asyncio/base_events.py\", line 570, in run_forever\n",
      "    self._run_once()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/asyncio/base_events.py\", line 1859, in _run_once\n",
      "    handle._run()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/asyncio/events.py\", line 81, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/ioloop.py\", line 688, in <lambda>\n",
      "    lambda f: self._run_callback(functools.partial(callback, future))\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/ioloop.py\", line 741, in _run_callback\n",
      "    ret = callback()\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/gen.py\", line 814, in inner\n",
      "    self.ctx_run(self.run)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/gen.py\", line 775, in run\n",
      "    yielded = self.gen.send(value)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/ipykernel/kernelbase.py\", line 365, in process_one\n",
      "    yield gen.maybe_future(dispatch(*args))\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/gen.py\", line 234, in wrapper\n",
      "    yielded = ctx_run(next, result)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/ipykernel/kernelbase.py\", line 268, in dispatch_shell\n",
      "    yield gen.maybe_future(handler(stream, idents, msg))\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/gen.py\", line 234, in wrapper\n",
      "    yielded = ctx_run(next, result)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/ipykernel/kernelbase.py\", line 543, in execute_request\n",
      "    self.do_execute(\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/tornado/gen.py\", line 234, in wrapper\n",
      "    yielded = ctx_run(next, result)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/ipykernel/ipkernel.py\", line 306, in do_execute\n",
      "    res = shell.run_cell(code, store_history=store_history, silent=silent)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/ipykernel/zmqshell.py\", line 536, in run_cell\n",
      "    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/IPython/core/interactiveshell.py\", line 2898, in run_cell\n",
      "    result = self._run_cell(\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/IPython/core/interactiveshell.py\", line 2944, in _run_cell\n",
      "    return runner(coro)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/IPython/core/async_helpers.py\", line 68, in _pseudo_sync_runner\n",
      "    coro.send(None)\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/IPython/core/interactiveshell.py\", line 3169, in run_cell_async\n",
      "    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/IPython/core/interactiveshell.py\", line 3361, in run_ast_nodes\n",
      "    if (await self.run_code(code, result,  async_=asy)):\n",
      "  File \"/home/ubuntu/miniconda3/lib/python3.8/site-packages/IPython/core/interactiveshell.py\", line 3441, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-13-263240bbee7e>\", line 1, in <module>\n",
      "    main()\n",
      "  File \"<ipython-input-12-8a06637ad683>\", line 51, in main\n",
      "    model_type, img_size, pretrained_dir, device, model = setup(model_type, img_size, pretrained_dir, device, dataset)\n",
      "  File \"<ipython-input-9-25b0bc7a7040>\", line 42, in setup\n",
      "    logger.info(\"Training parameters %s\", model_type, img_size, pretrained_dir, device)\n",
      "Message: 'Training parameters %s'\n",
      "Arguments: ('ViT-B_16', 224, 'ViT-B_16.npz', device(type='cuda', index=1))\n",
      "11/05/2021 18:08:40 - INFO - __main__ - Total Parameter: \t85.8M\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/05/2021 18:08:42 - INFO - __main__ - ***** Running training *****\n",
      "11/05/2021 18:08:42 - INFO - __main__ -   Total optimization steps = 500\n",
      "11/05/2021 18:08:42 - INFO - __main__ -   Instantaneous batch size per GPU = 64\n",
      "11/05/2021 18:08:42 - INFO - __main__ -   Total train batch size (w. parallel, distributed & accumulation) = 64\n",
      "11/05/2021 18:08:42 - INFO - __main__ -   Gradient Accumulation steps = 1\n",
      "Training (X / X Steps) (loss=X.X):   0%|| 0/782 [00:00<?, ?it/s]/home/ubuntu/miniconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:131: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  warnings.warn(\"Detected call of `lr_scheduler.step()` before `optimizer.step()`. \"\n",
      "Training (1 / 500 Steps) (loss=2.30258):   0%|| 0/782 [00:01<?, ?it/s]/home/ubuntu/miniconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:247: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n",
      "Training (100 / 500 Steps) (loss=0.42397):  13%|| 99/782 [01:55<10:05,  1.13it/s]11/05/2021 18:10:38 - INFO - __main__ - ***** Running Test *****\n",
      "11/05/2021 18:10:38 - INFO - __main__ -   Num steps = 157\n",
      "11/05/2021 18:10:38 - INFO - __main__ -   Batch size = 64\n",
      "\n",
      "Test... (loss=X.X):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.26138):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.26138):   1%|| 1/157 [00:00<01:59,  1.30it/s]\u001b[A\n",
      "Testing... (loss=0.17797):   1%|| 1/157 [00:01<01:59,  1.30it/s]\u001b[A\n",
      "Testing... (loss=0.17797):   1%|| 2/157 [00:01<01:11,  2.17it/s]\u001b[A\n",
      "Testing... (loss=0.23106):   1%|| 2/157 [00:01<01:11,  2.17it/s]\u001b[A\n",
      "Testing... (loss=0.23106):   2%|| 3/157 [00:01<00:53,  2.86it/s]\u001b[A\n",
      "Testing... (loss=0.30254):   2%|| 3/157 [00:01<00:53,  2.86it/s]\u001b[A\n",
      "Testing... (loss=0.30254):   3%|| 4/157 [00:01<00:45,  3.35it/s]\u001b[A\n",
      "Testing... (loss=0.19157):   3%|| 4/157 [00:01<00:45,  3.35it/s]\u001b[A\n",
      "Testing... (loss=0.19157):   3%|| 5/157 [00:01<00:41,  3.64it/s]\u001b[A\n",
      "Testing... (loss=0.12414):   3%|| 5/157 [00:01<00:41,  3.64it/s]\u001b[A\n",
      "Testing... (loss=0.12414):   4%|| 6/157 [00:01<00:38,  3.90it/s]\u001b[A\n",
      "Testing... (loss=0.23097):   4%|| 6/157 [00:02<00:38,  3.90it/s]\u001b[A\n",
      "Testing... (loss=0.23097):   4%|| 7/157 [00:02<00:36,  4.12it/s]\u001b[A\n",
      "Testing... (loss=0.13086):   4%|| 7/157 [00:02<00:36,  4.12it/s]\u001b[A\n",
      "Testing... (loss=0.13086):   5%|| 8/157 [00:02<00:34,  4.28it/s]\u001b[A\n",
      "Testing... (loss=0.16253):   5%|| 8/157 [00:02<00:34,  4.28it/s]\u001b[A\n",
      "Testing... (loss=0.16253):   6%|| 9/157 [00:02<00:33,  4.37it/s]\u001b[A\n",
      "Testing... (loss=0.16418):   6%|| 9/157 [00:02<00:33,  4.37it/s]\u001b[A\n",
      "Testing... (loss=0.16418):   6%|| 10/157 [00:02<00:33,  4.44it/s]\u001b[A\n",
      "Testing... (loss=0.20388):   6%|| 10/157 [00:02<00:33,  4.44it/s]\u001b[A\n",
      "Testing... (loss=0.20388):   7%|| 11/157 [00:02<00:32,  4.49it/s]\u001b[A\n",
      "Testing... (loss=0.15530):   7%|| 11/157 [00:03<00:32,  4.49it/s]\u001b[A\n",
      "Testing... (loss=0.15530):   8%|| 12/157 [00:03<00:32,  4.50it/s]\u001b[A\n",
      "Testing... (loss=0.19793):   8%|| 12/157 [00:03<00:32,  4.50it/s]\u001b[A\n",
      "Testing... (loss=0.19793):   8%|| 13/157 [00:03<00:31,  4.55it/s]\u001b[A\n",
      "Testing... (loss=0.19651):   8%|| 13/157 [00:03<00:31,  4.55it/s]\u001b[A\n",
      "Testing... (loss=0.19651):   9%|| 14/157 [00:03<00:31,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.21874):   9%|| 14/157 [00:03<00:31,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.21874):  10%|| 15/157 [00:03<00:31,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.19984):  10%|| 15/157 [00:04<00:31,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.19984):  10%|| 16/157 [00:04<00:30,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.13522):  10%|| 16/157 [00:04<00:30,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.13522):  11%|| 17/157 [00:04<00:30,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.21399):  11%|| 17/157 [00:04<00:30,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.21399):  11%|| 18/157 [00:04<00:30,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.14286):  11%|| 18/157 [00:04<00:30,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.14286):  12%|| 19/157 [00:04<00:30,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.16865):  12%|| 19/157 [00:04<00:30,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.16865):  13%|| 20/157 [00:04<00:29,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.14375):  13%|| 20/157 [00:05<00:29,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.14375):  13%|| 21/157 [00:05<00:29,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.28831):  13%|| 21/157 [00:05<00:29,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.28831):  14%|| 22/157 [00:05<00:29,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.09903):  14%|| 22/157 [00:05<00:29,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.09903):  15%|| 23/157 [00:05<00:29,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.13454):  15%|| 23/157 [00:05<00:29,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.13454):  15%|| 24/157 [00:05<00:28,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.15219):  15%|| 24/157 [00:06<00:28,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.15219):  16%|| 25/157 [00:06<00:28,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.20160):  16%|| 25/157 [00:06<00:28,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.20160):  17%|| 26/157 [00:06<00:28,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.12353):  17%|| 26/157 [00:06<00:28,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.12353):  17%|| 27/157 [00:06<00:28,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.19090):  17%|| 27/157 [00:06<00:28,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.19090):  18%|| 28/157 [00:06<00:27,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.14481):  18%|| 28/157 [00:06<00:27,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.14481):  18%|| 29/157 [00:06<00:27,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.20099):  18%|| 29/157 [00:07<00:27,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.20099):  19%|| 30/157 [00:07<00:27,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.23516):  19%|| 30/157 [00:07<00:27,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.23516):  20%|| 31/157 [00:07<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.25993):  20%|| 31/157 [00:07<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.25993):  20%|| 32/157 [00:07<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.18919):  20%|| 32/157 [00:07<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.18919):  21%|| 33/157 [00:07<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.25720):  21%|| 33/157 [00:07<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.25720):  22%|| 34/157 [00:07<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.22142):  22%|| 34/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.22142):  22%|| 35/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.22291):  22%|| 35/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.22291):  23%|| 36/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.19634):  23%|| 36/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.19634):  24%|| 37/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.26741):  24%|| 37/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.26741):  24%|| 38/157 [00:08<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.26432):  24%|| 38/157 [00:09<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.26432):  25%|| 39/157 [00:09<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.20660):  25%|| 39/157 [00:09<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.20660):  25%|| 40/157 [00:09<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.17829):  25%|| 40/157 [00:09<00:25,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.17829):  26%|| 41/157 [00:09<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.17527):  26%|| 41/157 [00:09<00:24,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.17527):  27%|| 42/157 [00:09<00:24,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.18754):  27%|| 42/157 [00:09<00:24,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.18754):  27%|| 43/157 [00:09<00:24,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.21242):  27%|| 43/157 [00:10<00:24,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.21242):  28%|| 44/157 [00:10<00:24,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.20435):  28%|| 44/157 [00:10<00:24,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.20435):  29%|| 45/157 [00:10<00:24,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.15720):  29%|| 45/157 [00:10<00:24,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.15720):  29%|| 46/157 [00:10<00:24,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.19891):  29%|| 46/157 [00:10<00:24,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.19891):  30%|| 47/157 [00:10<00:23,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.16641):  30%|| 47/157 [00:10<00:23,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.16641):  31%|| 48/157 [00:10<00:23,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.27770):  31%|| 48/157 [00:11<00:23,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.27770):  31%|| 49/157 [00:11<00:23,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.21005):  31%|| 49/157 [00:11<00:23,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.21005):  32%|| 50/157 [00:11<00:23,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.19330):  32%|| 50/157 [00:11<00:23,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.19330):  32%|| 51/157 [00:11<00:22,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.26715):  32%|| 51/157 [00:11<00:22,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.26715):  33%|| 52/157 [00:11<00:22,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15077):  33%|| 52/157 [00:12<00:22,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15077):  34%|| 53/157 [00:12<00:22,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.22313):  34%|| 53/157 [00:12<00:22,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.22313):  34%|| 54/157 [00:12<00:22,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.15048):  34%|| 54/157 [00:12<00:22,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.15048):  35%|| 55/157 [00:12<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.21858):  35%|| 55/157 [00:12<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.21858):  36%|| 56/157 [00:12<00:21,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.18108):  36%|| 56/157 [00:12<00:21,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.18108):  36%|| 57/157 [00:12<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.18961):  36%|| 57/157 [00:13<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.18961):  37%|| 58/157 [00:13<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.25559):  37%|| 58/157 [00:13<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.25559):  38%|| 59/157 [00:13<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19205):  38%|| 59/157 [00:13<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19205):  38%|| 60/157 [00:13<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.16202):  38%|| 60/157 [00:13<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.16202):  39%|| 61/157 [00:13<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.16247):  39%|| 61/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.16247):  39%|| 62/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19654):  39%|| 62/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19654):  40%|| 63/157 [00:14<00:20,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.23504):  40%|| 63/157 [00:14<00:20,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.23504):  41%|| 64/157 [00:14<00:20,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.21400):  41%|| 64/157 [00:14<00:20,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.21400):  41%|| 65/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.13169):  41%|| 65/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.13169):  42%|| 66/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17705):  42%|| 66/157 [00:15<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17705):  43%|| 67/157 [00:15<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.13178):  43%|| 67/157 [00:15<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.13178):  43%|| 68/157 [00:15<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.23994):  43%|| 68/157 [00:15<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.23994):  44%|| 69/157 [00:15<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.20233):  44%|| 69/157 [00:15<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.20233):  45%|| 70/157 [00:15<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.27659):  45%|| 70/157 [00:15<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.27659):  45%|| 71/157 [00:15<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.22673):  45%|| 71/157 [00:16<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.22673):  46%|| 72/157 [00:16<00:18,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.18592):  46%|| 72/157 [00:16<00:18,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.18592):  46%|| 73/157 [00:16<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.16084):  46%|| 73/157 [00:16<00:18,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.16084):  47%|| 74/157 [00:16<00:17,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.29475):  47%|| 74/157 [00:16<00:17,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.29475):  48%|| 75/157 [00:16<00:17,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.14257):  48%|| 75/157 [00:17<00:17,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.14257):  48%|| 76/157 [00:17<00:17,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.16755):  48%|| 76/157 [00:17<00:17,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.16755):  49%|| 77/157 [00:17<00:17,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.32155):  49%|| 77/157 [00:17<00:17,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.32155):  50%|| 78/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.20319):  50%|| 78/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.20319):  50%|| 79/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.15021):  50%|| 79/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.15021):  51%|| 80/157 [00:17<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.18406):  51%|| 80/157 [00:18<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.18406):  52%|| 81/157 [00:18<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.18838):  52%|| 81/157 [00:18<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.18838):  52%|| 82/157 [00:18<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.18874):  52%|| 82/157 [00:18<00:16,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.18874):  53%|| 83/157 [00:18<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.14159):  53%|| 83/157 [00:18<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.14159):  54%|| 84/157 [00:18<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17865):  54%|| 84/157 [00:18<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17865):  54%|| 85/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19889):  54%|| 85/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19889):  55%|| 86/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14500):  55%|| 86/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14500):  55%|| 87/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.18690):  55%|| 87/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.18690):  56%|| 88/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.28730):  56%|| 88/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.28730):  57%|| 89/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.15418):  57%|| 89/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.15418):  57%|| 90/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14047):  57%|| 90/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14047):  58%|| 91/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.21620):  58%|| 91/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.21620):  59%|| 92/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14873):  59%|| 92/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14873):  59%|| 93/157 [00:20<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.16052):  59%|| 93/157 [00:20<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.16052):  60%|| 94/157 [00:20<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.18167):  60%|| 94/157 [00:21<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.18167):  61%|| 95/157 [00:21<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.15606):  61%|| 95/157 [00:21<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.15606):  61%|| 96/157 [00:21<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.24284):  61%|| 96/157 [00:21<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.24284):  62%|| 97/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.24057):  62%|| 97/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.24057):  62%|| 98/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10847):  62%|| 98/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10847):  63%|| 99/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.18292):  63%|| 99/157 [00:22<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.18292):  64%|| 100/157 [00:22<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.16400):  64%|| 100/157 [00:22<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.16400):  64%|| 101/157 [00:22<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.16928):  64%|| 101/157 [00:22<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.16928):  65%|| 102/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.20742):  65%|| 102/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.20742):  66%|| 103/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19104):  66%|| 103/157 [00:23<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19104):  66%|| 104/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.26593):  66%|| 104/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.26593):  67%|| 105/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.21441):  67%|| 105/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.21441):  68%|| 106/157 [00:23<00:11,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.17622):  68%|| 106/157 [00:23<00:11,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.17622):  68%|| 107/157 [00:23<00:10,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.16273):  68%|| 107/157 [00:23<00:10,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.16273):  69%|| 108/157 [00:23<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.21866):  69%|| 108/157 [00:24<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.21866):  69%|| 109/157 [00:24<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.20176):  69%|| 109/157 [00:24<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.20176):  70%|| 110/157 [00:24<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.24944):  70%|| 110/157 [00:24<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.24944):  71%|| 111/157 [00:24<00:09,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.21467):  71%|| 111/157 [00:24<00:09,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.21467):  71%|| 112/157 [00:24<00:09,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.16391):  71%|| 112/157 [00:24<00:09,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.16391):  72%|| 113/157 [00:24<00:09,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17082):  72%|| 113/157 [00:25<00:09,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17082):  73%|| 114/157 [00:25<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.17859):  73%|| 114/157 [00:25<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.17859):  73%|| 115/157 [00:25<00:09,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17918):  73%|| 115/157 [00:25<00:09,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17918):  74%|| 116/157 [00:25<00:08,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17569):  74%|| 116/157 [00:25<00:08,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17569):  75%|| 117/157 [00:25<00:08,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.21246):  75%|| 117/157 [00:26<00:08,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.21246):  75%|| 118/157 [00:26<00:08,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.18150):  75%|| 118/157 [00:26<00:08,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.18150):  76%|| 119/157 [00:26<00:08,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.21830):  76%|| 119/157 [00:26<00:08,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.21830):  76%|| 120/157 [00:26<00:07,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.16145):  76%|| 120/157 [00:26<00:07,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.16145):  77%|| 121/157 [00:26<00:07,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.23726):  77%|| 121/157 [00:26<00:07,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.23726):  78%|| 122/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.17235):  78%|| 122/157 [00:27<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.17235):  78%|| 123/157 [00:27<00:07,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.18250):  78%|| 123/157 [00:27<00:07,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.18250):  79%|| 124/157 [00:27<00:07,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.18629):  79%|| 124/157 [00:27<00:07,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.18629):  80%|| 125/157 [00:27<00:06,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.16673):  80%|| 125/157 [00:27<00:06,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.16673):  80%|| 126/157 [00:27<00:06,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.18437):  80%|| 126/157 [00:28<00:06,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.18437):  81%|| 127/157 [00:28<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.15773):  81%|| 127/157 [00:28<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.15773):  82%|| 128/157 [00:28<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.17545):  82%|| 128/157 [00:28<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.17545):  82%|| 129/157 [00:28<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.22461):  82%|| 129/157 [00:28<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.22461):  83%|| 130/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.22157):  83%|| 130/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.22157):  83%|| 131/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.25709):  83%|| 131/157 [00:29<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.25709):  84%|| 132/157 [00:29<00:05,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.24372):  84%|| 132/157 [00:29<00:05,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.24372):  85%|| 133/157 [00:29<00:05,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.21099):  85%|| 133/157 [00:29<00:05,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.21099):  85%|| 134/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.19701):  85%|| 134/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.19701):  86%|| 135/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.17704):  86%|| 135/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.17704):  87%|| 136/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.25649):  87%|| 136/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.25649):  87%|| 137/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.14285):  87%|| 137/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.14285):  88%|| 138/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.21943):  88%|| 138/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.21943):  89%|| 139/157 [00:30<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.16474):  89%|| 139/157 [00:30<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.16474):  89%|| 140/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.19332):  89%|| 140/157 [00:31<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.19332):  90%|| 141/157 [00:31<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.18398):  90%|| 141/157 [00:31<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.18398):  90%|| 142/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.15840):  90%|| 142/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.15840):  91%|| 143/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13063):  91%|| 143/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13063):  92%|| 144/157 [00:31<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.31756):  92%|| 144/157 [00:31<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.31756):  92%|| 145/157 [00:31<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.18717):  92%|| 145/157 [00:32<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.18717):  93%|| 146/157 [00:32<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.18535):  93%|| 146/157 [00:32<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.18535):  94%|| 147/157 [00:32<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.16052):  94%|| 147/157 [00:32<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.16052):  94%|| 148/157 [00:32<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.23964):  94%|| 148/157 [00:32<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.23964):  95%|| 149/157 [00:32<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.22902):  95%|| 149/157 [00:32<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.22902):  96%|| 150/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.17094):  96%|| 150/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.17094):  96%|| 151/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13126):  96%|| 151/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13126):  97%|| 152/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.22997):  97%|| 152/157 [00:33<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.22997):  97%|| 153/157 [00:33<00:00,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.23577):  97%|| 153/157 [00:33<00:00,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.23577):  98%|| 154/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19816):  98%|| 154/157 [00:34<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19816):  99%|| 155/157 [00:34<00:00,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.15871):  99%|| 155/157 [00:34<00:00,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.15871):  99%|| 156/157 [00:34<00:00,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.20203): 100%|| 157/157 [00:34<00:00,  4.56it/s]\u001b[A\n",
      "11/05/2021 18:11:12 - INFO - __main__ - \n",
      "\n",
      "11/05/2021 18:11:12 - INFO - __main__ - Test Results\n",
      "11/05/2021 18:11:12 - INFO - __main__ - Global Steps: 100\n",
      "11/05/2021 18:11:12 - INFO - __main__ - Test Loss: 0.19480\n",
      "11/05/2021 18:11:12 - INFO - __main__ - Test Accuracy: 0.96970\n",
      "11/05/2021 18:11:13 - INFO - __main__ - Saved model checkpoint to [DIR: output_cifar10]\n",
      "Training (200 / 500 Steps) (loss=0.34867):  25%|| 199/782 [04:29<08:41,  1.12it/s]  11/05/2021 18:13:12 - INFO - __main__ - ***** Running Test *****\n",
      "11/05/2021 18:13:12 - INFO - __main__ -   Num steps = 157\n",
      "11/05/2021 18:13:12 - INFO - __main__ -   Batch size = 64\n",
      "\n",
      "Test... (loss=X.X):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.18466):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.18466):   1%|| 1/157 [00:00<01:36,  1.62it/s]\u001b[A\n",
      "Testing... (loss=0.10184):   1%|| 1/157 [00:00<01:36,  1.62it/s]\u001b[A\n",
      "Testing... (loss=0.10184):   1%|| 2/157 [00:00<00:58,  2.63it/s]\u001b[A\n",
      "Testing... (loss=0.11773):   1%|| 2/157 [00:01<00:58,  2.63it/s]\u001b[A\n",
      "Testing... (loss=0.11773):   2%|| 3/157 [00:01<00:47,  3.27it/s]\u001b[A\n",
      "Testing... (loss=0.21484):   2%|| 3/157 [00:01<00:47,  3.27it/s]\u001b[A\n",
      "Testing... (loss=0.21484):   3%|| 4/157 [00:01<00:41,  3.71it/s]\u001b[A\n",
      "Testing... (loss=0.14818):   3%|| 4/157 [00:01<00:41,  3.71it/s]\u001b[A\n",
      "Testing... (loss=0.14818):   3%|| 5/157 [00:01<00:37,  4.00it/s]\u001b[A\n",
      "Testing... (loss=0.05908):   3%|| 5/157 [00:01<00:37,  4.00it/s]\u001b[A\n",
      "Testing... (loss=0.05908):   4%|| 6/157 [00:01<00:35,  4.21it/s]\u001b[A\n",
      "Testing... (loss=0.16010):   4%|| 6/157 [00:01<00:35,  4.21it/s]\u001b[A\n",
      "Testing... (loss=0.16010):   4%|| 7/157 [00:01<00:34,  4.34it/s]\u001b[A\n",
      "Testing... (loss=0.07640):   4%|| 7/157 [00:02<00:34,  4.34it/s]\u001b[A\n",
      "Testing... (loss=0.07640):   5%|| 8/157 [00:02<00:33,  4.44it/s]\u001b[A\n",
      "Testing... (loss=0.05918):   5%|| 8/157 [00:02<00:33,  4.44it/s]\u001b[A\n",
      "Testing... (loss=0.05918):   6%|| 9/157 [00:02<00:32,  4.50it/s]\u001b[A\n",
      "Testing... (loss=0.07765):   6%|| 9/157 [00:02<00:32,  4.50it/s]\u001b[A\n",
      "Testing... (loss=0.07765):   6%|| 10/157 [00:02<00:32,  4.55it/s]\u001b[A\n",
      "Testing... (loss=0.10545):   6%|| 10/157 [00:02<00:32,  4.55it/s]\u001b[A\n",
      "Testing... (loss=0.10545):   7%|| 11/157 [00:02<00:31,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.10350):   7%|| 11/157 [00:02<00:31,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.10350):   8%|| 12/157 [00:02<00:31,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.11282):   8%|| 12/157 [00:03<00:31,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.11282):   8%|| 13/157 [00:03<00:31,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.14300):   8%|| 13/157 [00:03<00:31,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.14300):   9%|| 14/157 [00:03<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07317):   9%|| 14/157 [00:03<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07317):  10%|| 15/157 [00:03<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08731):  10%|| 15/157 [00:03<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08731):  10%|| 16/157 [00:03<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05521):  10%|| 16/157 [00:04<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05521):  11%|| 17/157 [00:04<00:30,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.11118):  11%|| 17/157 [00:04<00:30,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.11118):  11%|| 18/157 [00:04<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.13387):  11%|| 18/157 [00:04<00:29,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.13387):  12%|| 19/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.12231):  12%|| 19/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.12231):  13%|| 20/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06276):  13%|| 20/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06276):  13%|| 21/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09853):  13%|| 21/157 [00:05<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09853):  14%|| 22/157 [00:05<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09442):  14%|| 22/157 [00:05<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09442):  15%|| 23/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09925):  15%|| 23/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09925):  15%|| 24/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06221):  15%|| 24/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06221):  16%|| 25/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.08747):  16%|| 25/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.08747):  17%|| 26/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.07447):  17%|| 26/157 [00:06<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.07447):  17%|| 27/157 [00:06<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13701):  17%|| 27/157 [00:06<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13701):  18%|| 28/157 [00:06<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06439):  18%|| 28/157 [00:06<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06439):  18%|| 29/157 [00:06<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13071):  18%|| 29/157 [00:06<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13071):  19%|| 30/157 [00:06<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.20882):  19%|| 30/157 [00:07<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.20882):  20%|| 31/157 [00:07<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.10585):  20%|| 31/157 [00:07<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.10585):  20%|| 32/157 [00:07<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.11511):  20%|| 32/157 [00:07<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.11511):  21%|| 33/157 [00:07<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13955):  21%|| 33/157 [00:07<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13955):  22%|| 34/157 [00:07<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07269):  22%|| 34/157 [00:07<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07269):  22%|| 35/157 [00:07<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10970):  22%|| 35/157 [00:08<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10970):  23%|| 36/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.10442):  23%|| 36/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.10442):  24%|| 37/157 [00:08<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.21446):  24%|| 37/157 [00:08<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.21446):  24%|| 38/157 [00:08<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.16145):  24%|| 38/157 [00:08<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.16145):  25%|| 39/157 [00:08<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.20385):  25%|| 39/157 [00:08<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.20385):  25%|| 40/157 [00:09<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.10784):  25%|| 40/157 [00:09<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.10784):  26%|| 41/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.08907):  26%|| 41/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.08907):  27%|| 42/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09252):  27%|| 42/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09252):  27%|| 43/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.14500):  27%|| 43/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.14500):  28%|| 44/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17536):  28%|| 44/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17536):  29%|| 45/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09262):  29%|| 45/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09262):  29%|| 46/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.14073):  29%|| 46/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.14073):  30%|| 47/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.10541):  30%|| 47/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.10541):  31%|| 48/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.18789):  31%|| 48/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.18789):  31%|| 49/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.15661):  31%|| 49/157 [00:11<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.15661):  32%|| 50/157 [00:11<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.12533):  32%|| 50/157 [00:11<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.12533):  32%|| 51/157 [00:11<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.07166):  32%|| 51/157 [00:11<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.07166):  33%|| 52/157 [00:11<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.07280):  33%|| 52/157 [00:11<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.07280):  34%|| 53/157 [00:11<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17034):  34%|| 53/157 [00:12<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17034):  34%|| 54/157 [00:12<00:22,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10573):  34%|| 54/157 [00:12<00:22,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10573):  35%|| 55/157 [00:12<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17724):  35%|| 55/157 [00:12<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17724):  36%|| 56/157 [00:12<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.07345):  36%|| 56/157 [00:12<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.07345):  36%|| 57/157 [00:12<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10588):  36%|| 57/157 [00:12<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10588):  37%|| 58/157 [00:12<00:21,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15414):  37%|| 58/157 [00:13<00:21,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15414):  38%|| 59/157 [00:13<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11599):  38%|| 59/157 [00:13<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11599):  38%|| 60/157 [00:13<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06776):  38%|| 60/157 [00:13<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06776):  39%|| 61/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05968):  39%|| 61/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05968):  39%|| 62/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10290):  39%|| 62/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10290):  40%|| 63/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.25554):  40%|| 63/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.25554):  41%|| 64/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07875):  41%|| 64/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07875):  41%|| 65/157 [00:14<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07151):  41%|| 65/157 [00:14<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07151):  42%|| 66/157 [00:14<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14022):  42%|| 66/157 [00:14<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14022):  43%|| 67/157 [00:14<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06563):  43%|| 67/157 [00:15<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06563):  43%|| 68/157 [00:15<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06503):  43%|| 68/157 [00:15<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06503):  44%|| 69/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.11357):  44%|| 69/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.11357):  45%|| 70/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.13543):  45%|| 70/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.13543):  45%|| 71/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.14337):  45%|| 71/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.14337):  46%|| 72/157 [00:15<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.24321):  46%|| 72/157 [00:16<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.24321):  46%|| 73/157 [00:16<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05979):  46%|| 73/157 [00:16<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05979):  47%|| 74/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.20477):  47%|| 74/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.20477):  48%|| 75/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05326):  48%|| 75/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05326):  48%|| 76/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11851):  48%|| 76/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11851):  49%|| 77/157 [00:16<00:17,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.33712):  49%|| 77/157 [00:17<00:17,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.33712):  50%|| 78/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.12478):  50%|| 78/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.12478):  50%|| 79/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05234):  50%|| 79/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05234):  51%|| 80/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.12529):  51%|| 80/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.12529):  52%|| 81/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09179):  52%|| 81/157 [00:18<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09179):  52%|| 82/157 [00:18<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09250):  52%|| 82/157 [00:18<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09250):  53%|| 83/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05106):  53%|| 83/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05106):  54%|| 84/157 [00:18<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12137):  54%|| 84/157 [00:18<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12137):  54%|| 85/157 [00:18<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.13653):  54%|| 85/157 [00:18<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.13653):  55%|| 86/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11112):  55%|| 86/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11112):  55%|| 87/157 [00:19<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08009):  55%|| 87/157 [00:19<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08009):  56%|| 88/157 [00:19<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10828):  56%|| 88/157 [00:19<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10828):  57%|| 89/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07843):  57%|| 89/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07843):  57%|| 90/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08225):  57%|| 90/157 [00:19<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08225):  58%|| 91/157 [00:19<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.11982):  58%|| 91/157 [00:20<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.11982):  59%|| 92/157 [00:20<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05678):  59%|| 92/157 [00:20<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05678):  59%|| 93/157 [00:20<00:13,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.13567):  59%|| 93/157 [00:20<00:13,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.13567):  60%|| 94/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10820):  60%|| 94/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10820):  61%|| 95/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06714):  61%|| 95/157 [00:21<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06714):  61%|| 96/157 [00:21<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13806):  61%|| 96/157 [00:21<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13806):  62%|| 97/157 [00:21<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15881):  62%|| 97/157 [00:21<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15881):  62%|| 98/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05260):  62%|| 98/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05260):  63%|| 99/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.13676):  63%|| 99/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.13676):  64%|| 100/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09833):  64%|| 100/157 [00:22<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09833):  64%|| 101/157 [00:22<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05286):  64%|| 101/157 [00:22<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05286):  65%|| 102/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09957):  65%|| 102/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09957):  66%|| 103/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08929):  66%|| 103/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08929):  66%|| 104/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.20837):  66%|| 104/157 [00:23<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.20837):  67%|| 105/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.13455):  67%|| 105/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.13455):  68%|| 106/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10607):  68%|| 106/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10607):  68%|| 107/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06528):  68%|| 107/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06528):  69%|| 108/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14750):  69%|| 108/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14750):  69%|| 109/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.15449):  69%|| 109/157 [00:24<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.15449):  70%|| 110/157 [00:24<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.22978):  70%|| 110/157 [00:24<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.22978):  71%|| 111/157 [00:24<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11888):  71%|| 111/157 [00:24<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11888):  71%|| 112/157 [00:24<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09764):  71%|| 112/157 [00:24<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09764):  72%|| 113/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05001):  72%|| 113/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05001):  73%|| 114/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.09576):  73%|| 114/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.09576):  73%|| 115/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12811):  73%|| 115/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12811):  74%|| 116/157 [00:25<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09219):  74%|| 116/157 [00:25<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09219):  75%|| 117/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.25436):  75%|| 117/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.25436):  75%|| 118/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12153):  75%|| 118/157 [00:26<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12153):  76%|| 119/157 [00:26<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10766):  76%|| 119/157 [00:26<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10766):  76%|| 120/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07979):  76%|| 120/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07979):  77%|| 121/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.14753):  77%|| 121/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.14753):  78%|| 122/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.14584):  78%|| 122/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.14584):  78%|| 123/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04924):  78%|| 123/157 [00:27<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04924):  79%|| 124/157 [00:27<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12942):  79%|| 124/157 [00:27<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12942):  80%|| 125/157 [00:27<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13011):  80%|| 125/157 [00:27<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13011):  80%|| 126/157 [00:27<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08658):  80%|| 126/157 [00:27<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08658):  81%|| 127/157 [00:27<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.11432):  81%|| 127/157 [00:27<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.11432):  82%|| 128/157 [00:27<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10084):  82%|| 128/157 [00:28<00:06,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10084):  82%|| 129/157 [00:28<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.17765):  82%|| 129/157 [00:28<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.17765):  83%|| 130/157 [00:28<00:05,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14287):  83%|| 130/157 [00:28<00:05,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14287):  83%|| 131/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.17505):  83%|| 131/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.17505):  84%|| 132/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12889):  84%|| 132/157 [00:29<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12889):  85%|| 133/157 [00:29<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.14855):  85%|| 133/157 [00:29<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.14855):  85%|| 134/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15367):  85%|| 134/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15367):  86%|| 135/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05829):  86%|| 135/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05829):  87%|| 136/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15798):  87%|| 136/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15798):  87%|| 137/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08205):  87%|| 137/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08205):  88%|| 138/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.14971):  88%|| 138/157 [00:30<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.14971):  89%|| 139/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10009):  89%|| 139/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10009):  89%|| 140/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10386):  89%|| 140/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10386):  90%|| 141/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07392):  90%|| 141/157 [00:31<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07392):  90%|| 142/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09810):  90%|| 142/157 [00:31<00:03,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09810):  91%|| 143/157 [00:31<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07694):  91%|| 143/157 [00:31<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07694):  92%|| 144/157 [00:31<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19389):  92%|| 144/157 [00:31<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19389):  92%|| 145/157 [00:31<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19512):  92%|| 145/157 [00:31<00:02,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19512):  93%|| 146/157 [00:31<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08378):  93%|| 146/157 [00:32<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08378):  94%|| 147/157 [00:32<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10822):  94%|| 147/157 [00:32<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10822):  94%|| 148/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.18180):  94%|| 148/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.18180):  95%|| 149/157 [00:32<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13706):  95%|| 149/157 [00:32<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13706):  96%|| 150/157 [00:32<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11453):  96%|| 150/157 [00:32<00:01,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11453):  96%|| 151/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10537):  96%|| 151/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10537):  97%|| 152/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.09726):  97%|| 152/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.09726):  97%|| 153/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19606):  97%|| 153/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.19606):  98%|| 154/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08561):  98%|| 154/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08561):  99%|| 155/157 [00:33<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08039):  99%|| 155/157 [00:34<00:00,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08039):  99%|| 156/157 [00:34<00:00,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.07672): 100%|| 157/157 [00:34<00:00,  4.60it/s]\u001b[A\n",
      "11/05/2021 18:13:46 - INFO - __main__ - \n",
      "\n",
      "11/05/2021 18:13:46 - INFO - __main__ - Test Results\n",
      "11/05/2021 18:13:46 - INFO - __main__ - Global Steps: 200\n",
      "11/05/2021 18:13:46 - INFO - __main__ - Test Loss: 0.11800\n",
      "11/05/2021 18:13:46 - INFO - __main__ - Test Accuracy: 0.97290\n",
      "11/05/2021 18:13:47 - INFO - __main__ - Saved model checkpoint to [DIR: output_cifar10]\n",
      "Training (300 / 500 Steps) (loss=0.38152):  38%|| 299/782 [07:02<07:07,  1.13it/s]  11/05/2021 18:15:44 - INFO - __main__ - ***** Running Test *****\n",
      "11/05/2021 18:15:44 - INFO - __main__ -   Num steps = 157\n",
      "11/05/2021 18:15:44 - INFO - __main__ -   Batch size = 64\n",
      "\n",
      "Test... (loss=X.X):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.19512):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.19512):   1%|| 1/157 [00:00<01:42,  1.52it/s]\u001b[A\n",
      "Testing... (loss=0.08851):   1%|| 1/157 [00:00<01:42,  1.52it/s]\u001b[A\n",
      "Testing... (loss=0.08851):   1%|| 2/157 [00:00<01:01,  2.51it/s]\u001b[A\n",
      "Testing... (loss=0.09176):   1%|| 2/157 [00:01<01:01,  2.51it/s]\u001b[A\n",
      "Testing... (loss=0.09176):   2%|| 3/157 [00:01<00:48,  3.17it/s]\u001b[A\n",
      "Testing... (loss=0.17458):   2%|| 3/157 [00:01<00:48,  3.17it/s]\u001b[A\n",
      "Testing... (loss=0.17458):   3%|| 4/157 [00:01<00:42,  3.61it/s]\u001b[A\n",
      "Testing... (loss=0.13474):   3%|| 4/157 [00:01<00:42,  3.61it/s]\u001b[A\n",
      "Testing... (loss=0.13474):   3%|| 5/157 [00:01<00:38,  3.92it/s]\u001b[A\n",
      "Testing... (loss=0.03328):   3%|| 5/157 [00:01<00:38,  3.92it/s]\u001b[A\n",
      "Testing... (loss=0.03328):   4%|| 6/157 [00:01<00:36,  4.15it/s]\u001b[A\n",
      "Testing... (loss=0.11795):   4%|| 6/157 [00:01<00:36,  4.15it/s]\u001b[A\n",
      "Testing... (loss=0.11795):   4%|| 7/157 [00:01<00:34,  4.30it/s]\u001b[A\n",
      "Testing... (loss=0.04631):   4%|| 7/157 [00:02<00:34,  4.30it/s]\u001b[A\n",
      "Testing... (loss=0.04631):   5%|| 8/157 [00:02<00:33,  4.40it/s]\u001b[A\n",
      "Testing... (loss=0.04105):   5%|| 8/157 [00:02<00:33,  4.40it/s]\u001b[A\n",
      "Testing... (loss=0.04105):   6%|| 9/157 [00:02<00:33,  4.48it/s]\u001b[A\n",
      "Testing... (loss=0.04455):   6%|| 9/157 [00:02<00:33,  4.48it/s]\u001b[A\n",
      "Testing... (loss=0.04455):   6%|| 10/157 [00:02<00:32,  4.53it/s]\u001b[A\n",
      "Testing... (loss=0.08076):   6%|| 10/157 [00:02<00:32,  4.53it/s]\u001b[A\n",
      "Testing... (loss=0.08076):   7%|| 11/157 [00:02<00:31,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.08598):   7%|| 11/157 [00:03<00:31,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.08598):   8%|| 12/157 [00:03<00:31,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.08395):   8%|| 12/157 [00:03<00:31,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.08395):   8%|| 13/157 [00:03<00:31,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.13402):   8%|| 13/157 [00:03<00:31,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.13402):   9%|| 14/157 [00:03<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04461):   9%|| 14/157 [00:03<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04461):  10%|| 15/157 [00:03<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06181):  10%|| 15/157 [00:03<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06181):  10%|| 16/157 [00:03<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03035):  10%|| 16/157 [00:04<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03035):  11%|| 17/157 [00:04<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07869):  11%|| 17/157 [00:04<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07869):  11%|| 18/157 [00:04<00:29,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10461):  11%|| 18/157 [00:04<00:29,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10461):  12%|| 19/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09821):  12%|| 19/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09821):  13%|| 20/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.04429):  13%|| 20/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.04429):  13%|| 21/157 [00:04<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.12701):  13%|| 21/157 [00:05<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.12701):  14%|| 22/157 [00:05<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05584):  14%|| 22/157 [00:05<00:29,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05584):  15%|| 23/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.03909):  15%|| 23/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.03909):  15%|| 24/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05066):  15%|| 24/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05066):  16%|| 25/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.03632):  16%|| 25/157 [00:06<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.03632):  17%|| 26/157 [00:06<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06259):  17%|| 26/157 [00:06<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06259):  17%|| 27/157 [00:06<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.11941):  17%|| 27/157 [00:06<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.11941):  18%|| 28/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.03690):  18%|| 28/157 [00:06<00:27,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.03690):  18%|| 29/157 [00:06<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.07059):  18%|| 29/157 [00:06<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.07059):  19%|| 30/157 [00:06<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.12912):  19%|| 30/157 [00:07<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.12912):  20%|| 31/157 [00:07<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05955):  20%|| 31/157 [00:07<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05955):  20%|| 32/157 [00:07<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05316):  20%|| 32/157 [00:07<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05316):  21%|| 33/157 [00:07<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.13884):  21%|| 33/157 [00:07<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.13884):  22%|| 34/157 [00:07<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.08956):  22%|| 34/157 [00:07<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.08956):  22%|| 35/157 [00:07<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09760):  22%|| 35/157 [00:08<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09760):  23%|| 36/157 [00:08<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11130):  23%|| 36/157 [00:08<00:26,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11130):  24%|| 37/157 [00:08<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17481):  24%|| 37/157 [00:08<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17481):  24%|| 38/157 [00:08<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09979):  24%|| 38/157 [00:08<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09979):  25%|| 39/157 [00:08<00:25,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13624):  25%|| 39/157 [00:09<00:25,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13624):  25%|| 40/157 [00:09<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06271):  25%|| 40/157 [00:09<00:25,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06271):  26%|| 41/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06453):  26%|| 41/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06453):  27%|| 42/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06632):  27%|| 42/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06632):  27%|| 43/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.16241):  27%|| 43/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.16241):  28%|| 44/157 [00:09<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.12717):  28%|| 44/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.12717):  29%|| 45/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.11186):  29%|| 45/157 [00:10<00:24,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.11186):  29%|| 46/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.10554):  29%|| 46/157 [00:10<00:23,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.10554):  30%|| 47/157 [00:10<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.08421):  30%|| 47/157 [00:10<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.08421):  31%|| 48/157 [00:10<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.12071):  31%|| 48/157 [00:10<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.12071):  31%|| 49/157 [00:10<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.08565):  31%|| 49/157 [00:11<00:23,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.08565):  32%|| 50/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.08472):  32%|| 50/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.08472):  32%|| 51/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.09842):  32%|| 51/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.09842):  33%|| 52/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.08878):  33%|| 52/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.08878):  34%|| 53/157 [00:11<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.13585):  34%|| 53/157 [00:12<00:22,  4.66it/s]\u001b[A\n",
      "Testing... (loss=0.13585):  34%|| 54/157 [00:12<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05058):  34%|| 54/157 [00:12<00:22,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05058):  35%|| 55/157 [00:12<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.11043):  35%|| 55/157 [00:12<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.11043):  36%|| 56/157 [00:12<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04485):  36%|| 56/157 [00:12<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04485):  36%|| 57/157 [00:12<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06947):  36%|| 57/157 [00:12<00:21,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06947):  37%|| 58/157 [00:12<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.13243):  37%|| 58/157 [00:13<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.13243):  38%|| 59/157 [00:13<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.07246):  38%|| 59/157 [00:13<00:21,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.07246):  38%|| 60/157 [00:13<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04534):  38%|| 60/157 [00:13<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04534):  39%|| 61/157 [00:13<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04865):  39%|| 61/157 [00:13<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04865):  39%|| 62/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08838):  39%|| 62/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08838):  40%|| 63/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.13206):  40%|| 63/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.13206):  41%|| 64/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07921):  41%|| 64/157 [00:14<00:20,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07921):  41%|| 65/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.04786):  41%|| 65/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.04786):  42%|| 66/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06831):  42%|| 66/157 [00:14<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06831):  43%|| 67/157 [00:14<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04033):  43%|| 67/157 [00:15<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04033):  43%|| 68/157 [00:15<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09399):  43%|| 68/157 [00:15<00:19,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.09399):  44%|| 69/157 [00:15<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10091):  44%|| 69/157 [00:15<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10091):  45%|| 70/157 [00:15<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.12896):  45%|| 70/157 [00:15<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.12896):  45%|| 71/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10313):  45%|| 71/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10313):  46%|| 72/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15177):  46%|| 72/157 [00:16<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15177):  46%|| 73/157 [00:16<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05367):  46%|| 73/157 [00:16<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05367):  47%|| 74/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.20929):  47%|| 74/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.20929):  48%|| 75/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.02990):  48%|| 75/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.02990):  48%|| 76/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09922):  48%|| 76/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09922):  49%|| 77/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.24169):  49%|| 77/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.24169):  50%|| 78/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08354):  50%|| 78/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08354):  50%|| 79/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03080):  50%|| 79/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03080):  51%|| 80/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05009):  51%|| 80/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05009):  52%|| 81/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10642):  52%|| 81/157 [00:18<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10642):  52%|| 82/157 [00:18<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05105):  52%|| 82/157 [00:18<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05105):  53%|| 83/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04457):  53%|| 83/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04457):  54%|| 84/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08018):  54%|| 84/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08018):  54%|| 85/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07809):  54%|| 85/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07809):  55%|| 86/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05067):  55%|| 86/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05067):  55%|| 87/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08214):  55%|| 87/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08214):  56%|| 88/157 [00:19<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.11475):  56%|| 88/157 [00:19<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.11475):  57%|| 89/157 [00:19<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.04767):  57%|| 89/157 [00:19<00:14,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.04767):  57%|| 90/157 [00:19<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.03124):  57%|| 90/157 [00:20<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.03124):  58%|| 91/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11016):  58%|| 91/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11016):  59%|| 92/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03607):  59%|| 92/157 [00:20<00:14,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03607):  59%|| 93/157 [00:20<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07857):  59%|| 93/157 [00:20<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07857):  60%|| 94/157 [00:20<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09727):  60%|| 94/157 [00:20<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09727):  61%|| 95/157 [00:20<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06304):  61%|| 95/157 [00:21<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06304):  61%|| 96/157 [00:21<00:13,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.11995):  61%|| 96/157 [00:21<00:13,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.11995):  62%|| 97/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17033):  62%|| 97/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.17033):  62%|| 98/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.03147):  62%|| 98/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.03147):  63%|| 99/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.08268):  63%|| 99/157 [00:21<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.08268):  64%|| 100/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07711):  64%|| 100/157 [00:22<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07711):  64%|| 101/157 [00:22<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05488):  64%|| 101/157 [00:22<00:12,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05488):  65%|| 102/157 [00:22<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.10073):  65%|| 102/157 [00:22<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.10073):  66%|| 103/157 [00:22<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05378):  66%|| 103/157 [00:22<00:11,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05378):  66%|| 104/157 [00:22<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13358):  66%|| 104/157 [00:23<00:11,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.13358):  67%|| 105/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.13801):  67%|| 105/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.13801):  68%|| 106/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07470):  68%|| 106/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07470):  68%|| 107/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06262):  68%|| 107/157 [00:23<00:10,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06262):  69%|| 108/157 [00:23<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.14841):  69%|| 108/157 [00:23<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.14841):  69%|| 109/157 [00:23<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10879):  69%|| 109/157 [00:24<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10879):  70%|| 110/157 [00:24<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.17370):  70%|| 110/157 [00:24<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.17370):  71%|| 111/157 [00:24<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10739):  71%|| 111/157 [00:24<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10739):  71%|| 112/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10042):  71%|| 112/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10042):  72%|| 113/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04686):  72%|| 113/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04686):  73%|| 114/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12348):  73%|| 114/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12348):  73%|| 115/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12086):  73%|| 115/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12086):  74%|| 116/157 [00:25<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11144):  74%|| 116/157 [00:25<00:08,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11144):  75%|| 117/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.23808):  75%|| 117/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.23808):  75%|| 118/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08810):  75%|| 118/157 [00:26<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08810):  76%|| 119/157 [00:26<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05417):  76%|| 119/157 [00:26<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05417):  76%|| 120/157 [00:26<00:08,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.13988):  76%|| 120/157 [00:26<00:08,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.13988):  77%|| 121/157 [00:26<00:07,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.12819):  77%|| 121/157 [00:26<00:07,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.12819):  78%|| 122/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.14669):  78%|| 122/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.14669):  78%|| 123/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04243):  78%|| 123/157 [00:27<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04243):  79%|| 124/157 [00:27<00:07,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10725):  79%|| 124/157 [00:27<00:07,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10725):  80%|| 125/157 [00:27<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09127):  80%|| 125/157 [00:27<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09127):  80%|| 126/157 [00:27<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03515):  80%|| 126/157 [00:27<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03515):  81%|| 127/157 [00:27<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08421):  81%|| 127/157 [00:28<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08421):  82%|| 128/157 [00:28<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09008):  82%|| 128/157 [00:28<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09008):  82%|| 129/157 [00:28<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14453):  82%|| 129/157 [00:28<00:06,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14453):  83%|| 130/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10191):  83%|| 130/157 [00:28<00:05,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10191):  83%|| 131/157 [00:28<00:05,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11681):  83%|| 131/157 [00:28<00:05,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11681):  84%|| 132/157 [00:28<00:05,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08089):  84%|| 132/157 [00:29<00:05,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08089):  85%|| 133/157 [00:29<00:05,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08021):  85%|| 133/157 [00:29<00:05,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08021):  85%|| 134/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10953):  85%|| 134/157 [00:29<00:04,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10953):  86%|| 135/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03264):  86%|| 135/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03264):  87%|| 136/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.14195):  87%|| 136/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.14195):  87%|| 137/157 [00:29<00:04,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.06376):  87%|| 137/157 [00:30<00:04,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.06376):  88%|| 138/157 [00:30<00:04,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.15354):  88%|| 138/157 [00:30<00:04,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.15354):  89%|| 139/157 [00:30<00:03,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.05008):  89%|| 139/157 [00:30<00:03,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.05008):  89%|| 140/157 [00:30<00:03,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.05824):  89%|| 140/157 [00:30<00:03,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.05824):  90%|| 141/157 [00:30<00:03,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.04945):  90%|| 141/157 [00:31<00:03,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.04945):  90%|| 142/157 [00:31<00:03,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.05494):  90%|| 142/157 [00:31<00:03,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.05494):  91%|| 143/157 [00:31<00:03,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.04212):  91%|| 143/157 [00:31<00:03,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.04212):  92%|| 144/157 [00:31<00:02,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.15541):  92%|| 144/157 [00:31<00:02,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.15541):  92%|| 145/157 [00:31<00:02,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.09141):  92%|| 145/157 [00:31<00:02,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.09141):  93%|| 146/157 [00:31<00:02,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.04623):  93%|| 146/157 [00:32<00:02,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.04623):  94%|| 147/157 [00:32<00:02,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.04433):  94%|| 147/157 [00:32<00:02,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.04433):  94%|| 148/157 [00:32<00:01,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.15153):  94%|| 148/157 [00:32<00:01,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.15153):  95%|| 149/157 [00:32<00:01,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.13822):  95%|| 149/157 [00:32<00:01,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.13822):  96%|| 150/157 [00:32<00:01,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.07572):  96%|| 150/157 [00:32<00:01,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.07572):  96%|| 151/157 [00:32<00:01,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.07271):  96%|| 151/157 [00:33<00:01,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.07271):  97%|| 152/157 [00:33<00:01,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.09465):  97%|| 152/157 [00:33<00:01,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.09465):  97%|| 153/157 [00:33<00:00,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.16501):  97%|| 153/157 [00:33<00:00,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.16501):  98%|| 154/157 [00:33<00:00,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.08991):  98%|| 154/157 [00:33<00:00,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.08991):  99%|| 155/157 [00:33<00:00,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.07278):  99%|| 155/157 [00:34<00:00,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.07278):  99%|| 156/157 [00:34<00:00,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03655): 100%|| 157/157 [00:34<00:00,  4.59it/s]\u001b[A\n",
      "11/05/2021 18:16:18 - INFO - __main__ - \n",
      "\n",
      "11/05/2021 18:16:18 - INFO - __main__ - Test Results\n",
      "11/05/2021 18:16:18 - INFO - __main__ - Global Steps: 300\n",
      "11/05/2021 18:16:18 - INFO - __main__ - Test Loss: 0.09098\n",
      "11/05/2021 18:16:18 - INFO - __main__ - Test Accuracy: 0.97580\n",
      "11/05/2021 18:16:19 - INFO - __main__ - Saved model checkpoint to [DIR: output_cifar10]\n",
      "Training (400 / 500 Steps) (loss=0.24288):  51%|| 399/782 [09:34<05:44,  1.11it/s]  11/05/2021 18:18:16 - INFO - __main__ - ***** Running Test *****\n",
      "11/05/2021 18:18:16 - INFO - __main__ -   Num steps = 157\n",
      "11/05/2021 18:18:16 - INFO - __main__ -   Batch size = 64\n",
      "\n",
      "Test... (loss=X.X):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.11855):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.11855):   1%|| 1/157 [00:00<01:41,  1.53it/s]\u001b[A\n",
      "Testing... (loss=0.06530):   1%|| 1/157 [00:00<01:41,  1.53it/s]\u001b[A\n",
      "Testing... (loss=0.06530):   1%|| 2/157 [00:00<01:01,  2.52it/s]\u001b[A\n",
      "Testing... (loss=0.07911):   1%|| 2/157 [00:01<01:01,  2.52it/s]\u001b[A\n",
      "Testing... (loss=0.07911):   2%|| 3/157 [00:01<00:48,  3.19it/s]\u001b[A\n",
      "Testing... (loss=0.10708):   2%|| 3/157 [00:01<00:48,  3.19it/s]\u001b[A\n",
      "Testing... (loss=0.10708):   3%|| 4/157 [00:01<00:41,  3.64it/s]\u001b[A\n",
      "Testing... (loss=0.13633):   3%|| 4/157 [00:01<00:41,  3.64it/s]\u001b[A\n",
      "Testing... (loss=0.13633):   3%|| 5/157 [00:01<00:38,  3.96it/s]\u001b[A\n",
      "Testing... (loss=0.03112):   3%|| 5/157 [00:01<00:38,  3.96it/s]\u001b[A\n",
      "Testing... (loss=0.03112):   4%|| 6/157 [00:01<00:36,  4.17it/s]\u001b[A\n",
      "Testing... (loss=0.10560):   4%|| 6/157 [00:01<00:36,  4.17it/s]\u001b[A\n",
      "Testing... (loss=0.10560):   4%|| 7/157 [00:01<00:34,  4.31it/s]\u001b[A\n",
      "Testing... (loss=0.03113):   4%|| 7/157 [00:02<00:34,  4.31it/s]\u001b[A\n",
      "Testing... (loss=0.03113):   5%|| 8/157 [00:02<00:33,  4.41it/s]\u001b[A\n",
      "Testing... (loss=0.03232):   5%|| 8/157 [00:02<00:33,  4.41it/s]\u001b[A\n",
      "Testing... (loss=0.03232):   6%|| 9/157 [00:02<00:32,  4.49it/s]\u001b[A\n",
      "Testing... (loss=0.05744):   6%|| 9/157 [00:02<00:32,  4.49it/s]\u001b[A\n",
      "Testing... (loss=0.05744):   6%|| 10/157 [00:02<00:32,  4.54it/s]\u001b[A\n",
      "Testing... (loss=0.05392):   6%|| 10/157 [00:02<00:32,  4.54it/s]\u001b[A\n",
      "Testing... (loss=0.05392):   7%|| 11/157 [00:02<00:31,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.07374):   7%|| 11/157 [00:03<00:31,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.07374):   8%|| 12/157 [00:03<00:31,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.09794):   8%|| 12/157 [00:03<00:31,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.09794):   8%|| 13/157 [00:03<00:31,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.13456):   8%|| 13/157 [00:03<00:31,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.13456):   9%|| 14/157 [00:03<00:30,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.05424):   9%|| 14/157 [00:03<00:30,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.05424):  10%|| 15/157 [00:03<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07059):  10%|| 15/157 [00:03<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07059):  10%|| 16/157 [00:03<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.02486):  10%|| 16/157 [00:04<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.02486):  11%|| 17/157 [00:04<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05869):  11%|| 17/157 [00:04<00:30,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05869):  11%|| 18/157 [00:04<00:30,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.07835):  11%|| 18/157 [00:04<00:30,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.07835):  12%|| 19/157 [00:04<00:29,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08903):  12%|| 19/157 [00:04<00:29,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08903):  13%|| 20/157 [00:04<00:29,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04119):  13%|| 20/157 [00:04<00:29,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04119):  13%|| 21/157 [00:04<00:29,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07839):  13%|| 21/157 [00:05<00:29,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07839):  14%|| 22/157 [00:05<00:29,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04674):  14%|| 22/157 [00:05<00:29,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04674):  15%|| 23/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05327):  15%|| 23/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05327):  15%|| 24/157 [00:05<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03329):  15%|| 24/157 [00:05<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03329):  16%|| 25/157 [00:05<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03251):  16%|| 25/157 [00:06<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03251):  17%|| 26/157 [00:06<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04013):  17%|| 26/157 [00:06<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04013):  17%|| 27/157 [00:06<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09494):  17%|| 27/157 [00:06<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09494):  18%|| 28/157 [00:06<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.02785):  18%|| 28/157 [00:06<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.02785):  18%|| 29/157 [00:06<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07048):  18%|| 29/157 [00:06<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07048):  19%|| 30/157 [00:06<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09056):  19%|| 30/157 [00:07<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09056):  20%|| 31/157 [00:07<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04235):  20%|| 31/157 [00:07<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04235):  20%|| 32/157 [00:07<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06027):  20%|| 32/157 [00:07<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06027):  21%|| 33/157 [00:07<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.13741):  21%|| 33/157 [00:07<00:26,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.13741):  22%|| 34/157 [00:07<00:26,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.07843):  22%|| 34/157 [00:07<00:26,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.07843):  22%|| 35/157 [00:07<00:26,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.05597):  22%|| 35/157 [00:08<00:26,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.05597):  23%|| 36/157 [00:08<00:26,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.07010):  23%|| 36/157 [00:08<00:26,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.07010):  24%|| 37/157 [00:08<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.17630):  24%|| 37/157 [00:08<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.17630):  24%|| 38/157 [00:08<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.11803):  24%|| 38/157 [00:08<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.11803):  25%|| 39/157 [00:08<00:25,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.05240):  25%|| 39/157 [00:09<00:25,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.05240):  25%|| 40/157 [00:09<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06458):  25%|| 40/157 [00:09<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06458):  26%|| 41/157 [00:09<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04341):  26%|| 41/157 [00:09<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04341):  27%|| 42/157 [00:09<00:24,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06402):  27%|| 42/157 [00:09<00:24,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.06402):  27%|| 43/157 [00:09<00:24,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.11304):  27%|| 43/157 [00:09<00:24,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.11304):  28%|| 44/157 [00:09<00:24,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.12644):  28%|| 44/157 [00:10<00:24,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.12644):  29%|| 45/157 [00:10<00:24,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.06334):  29%|| 45/157 [00:10<00:24,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.06334):  29%|| 46/157 [00:10<00:24,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.06942):  29%|| 46/157 [00:10<00:24,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.06942):  30%|| 47/157 [00:10<00:24,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.07480):  30%|| 47/157 [00:10<00:24,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.07480):  31%|| 48/157 [00:10<00:23,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.10585):  31%|| 48/157 [00:11<00:23,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.10585):  31%|| 49/157 [00:11<00:23,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.08167):  31%|| 49/157 [00:11<00:23,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.08167):  32%|| 50/157 [00:11<00:23,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.06200):  32%|| 50/157 [00:11<00:23,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.06200):  32%|| 51/157 [00:11<00:23,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.11107):  32%|| 51/157 [00:11<00:23,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.11107):  33%|| 52/157 [00:11<00:22,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.07830):  33%|| 52/157 [00:11<00:22,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.07830):  34%|| 53/157 [00:11<00:22,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.15811):  34%|| 53/157 [00:12<00:22,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.15811):  34%|| 54/157 [00:12<00:22,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.05391):  34%|| 54/157 [00:12<00:22,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.05391):  35%|| 55/157 [00:12<00:22,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.08715):  35%|| 55/157 [00:12<00:22,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.08715):  36%|| 56/157 [00:12<00:22,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.03822):  36%|| 56/157 [00:12<00:22,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.03822):  36%|| 57/157 [00:12<00:21,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.06822):  36%|| 57/157 [00:12<00:21,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.06822):  37%|| 58/157 [00:12<00:21,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.07103):  37%|| 58/157 [00:13<00:21,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.07103):  38%|| 59/157 [00:13<00:21,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.04356):  38%|| 59/157 [00:13<00:21,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.04356):  38%|| 60/157 [00:13<00:21,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.03519):  38%|| 60/157 [00:13<00:21,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.03519):  39%|| 61/157 [00:13<00:20,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.02747):  39%|| 61/157 [00:13<00:20,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.02747):  39%|| 62/157 [00:13<00:20,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.05588):  39%|| 62/157 [00:14<00:20,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.05588):  40%|| 63/157 [00:14<00:20,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.18710):  40%|| 63/157 [00:14<00:20,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.18710):  41%|| 64/157 [00:14<00:20,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.10225):  41%|| 64/157 [00:14<00:20,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.10225):  41%|| 65/157 [00:14<00:19,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.03711):  41%|| 65/157 [00:14<00:19,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.03711):  42%|| 66/157 [00:14<00:19,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.05536):  42%|| 66/157 [00:14<00:19,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.05536):  43%|| 67/157 [00:14<00:19,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.03837):  43%|| 67/157 [00:15<00:19,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.03837):  43%|| 68/157 [00:15<00:19,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.09646):  43%|| 68/157 [00:15<00:19,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.09646):  44%|| 69/157 [00:15<00:19,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.10538):  44%|| 69/157 [00:15<00:19,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.10538):  45%|| 70/157 [00:15<00:18,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.09922):  45%|| 70/157 [00:15<00:18,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.09922):  45%|| 71/157 [00:15<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.09086):  45%|| 71/157 [00:16<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.09086):  46%|| 72/157 [00:16<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12755):  46%|| 72/157 [00:16<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.12755):  46%|| 73/157 [00:16<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04118):  46%|| 73/157 [00:16<00:18,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04118):  47%|| 74/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.23146):  47%|| 74/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.23146):  48%|| 75/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03643):  48%|| 75/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03643):  48%|| 76/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09378):  48%|| 76/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09378):  49%|| 77/157 [00:17<00:17,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.29222):  49%|| 77/157 [00:17<00:17,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.29222):  50%|| 78/157 [00:17<00:17,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.09112):  50%|| 78/157 [00:17<00:17,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.09112):  50%|| 79/157 [00:17<00:16,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.02537):  50%|| 79/157 [00:17<00:16,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.02537):  51%|| 80/157 [00:17<00:16,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05870):  51%|| 80/157 [00:17<00:16,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05870):  52%|| 81/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05247):  52%|| 81/157 [00:18<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05247):  52%|| 82/157 [00:18<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08252):  52%|| 82/157 [00:18<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08252):  53%|| 83/157 [00:18<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.02201):  53%|| 83/157 [00:18<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.02201):  54%|| 84/157 [00:18<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.09427):  54%|| 84/157 [00:18<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.09427):  54%|| 85/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07954):  54%|| 85/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07954):  55%|| 86/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05719):  55%|| 86/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05719):  55%|| 87/157 [00:19<00:15,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.05902):  55%|| 87/157 [00:19<00:15,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.05902):  56%|| 88/157 [00:19<00:14,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.09070):  56%|| 88/157 [00:19<00:14,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.09070):  57%|| 89/157 [00:19<00:14,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03223):  57%|| 89/157 [00:19<00:14,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03223):  57%|| 90/157 [00:19<00:14,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.04272):  57%|| 90/157 [00:20<00:14,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.04272):  58%|| 91/157 [00:20<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06871):  58%|| 91/157 [00:20<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06871):  59%|| 92/157 [00:20<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.02496):  59%|| 92/157 [00:20<00:14,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.02496):  59%|| 93/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04041):  59%|| 93/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04041):  60%|| 94/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06490):  60%|| 94/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06490):  61%|| 95/157 [00:20<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05728):  61%|| 95/157 [00:21<00:13,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05728):  61%|| 96/157 [00:21<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.16982):  61%|| 96/157 [00:21<00:13,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.16982):  62%|| 97/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14992):  62%|| 97/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.14992):  62%|| 98/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.02091):  62%|| 98/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.02091):  63%|| 99/157 [00:21<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08126):  63%|| 99/157 [00:22<00:12,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08126):  64%|| 100/157 [00:22<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07488):  64%|| 100/157 [00:22<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07488):  64%|| 101/157 [00:22<00:12,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.02791):  64%|| 101/157 [00:22<00:12,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.02791):  65%|| 102/157 [00:22<00:11,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08651):  65%|| 102/157 [00:22<00:11,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08651):  66%|| 103/157 [00:22<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08908):  66%|| 103/157 [00:22<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08908):  66%|| 104/157 [00:22<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.11394):  66%|| 104/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.11394):  67%|| 105/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10569):  67%|| 105/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10569):  68%|| 106/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05300):  68%|| 106/157 [00:23<00:11,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05300):  68%|| 107/157 [00:23<00:10,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.04390):  68%|| 107/157 [00:23<00:10,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.04390):  69%|| 108/157 [00:23<00:10,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.12449):  69%|| 108/157 [00:24<00:10,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.12449):  69%|| 109/157 [00:24<00:10,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08313):  69%|| 109/157 [00:24<00:10,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08313):  70%|| 110/157 [00:24<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15189):  70%|| 110/157 [00:24<00:10,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15189):  71%|| 111/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08224):  71%|| 111/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08224):  71%|| 112/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05795):  71%|| 112/157 [00:24<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05795):  72%|| 113/157 [00:24<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04905):  72%|| 113/157 [00:25<00:09,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04905):  73%|| 114/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07816):  73%|| 114/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07816):  73%|| 115/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08343):  73%|| 115/157 [00:25<00:09,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.08343):  74%|| 116/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06427):  74%|| 116/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06427):  75%|| 117/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.27161):  75%|| 117/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.27161):  75%|| 118/157 [00:25<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10395):  75%|| 118/157 [00:26<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10395):  76%|| 119/157 [00:26<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06162):  76%|| 119/157 [00:26<00:08,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06162):  76%|| 120/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.09562):  76%|| 120/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.09562):  77%|| 121/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10295):  77%|| 121/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10295):  78%|| 122/157 [00:26<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10087):  78%|| 122/157 [00:27<00:07,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10087):  78%|| 123/157 [00:27<00:07,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03934):  78%|| 123/157 [00:27<00:07,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03934):  79%|| 124/157 [00:27<00:07,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.11144):  79%|| 124/157 [00:27<00:07,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.11144):  80%|| 125/157 [00:27<00:06,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.08622):  80%|| 125/157 [00:27<00:06,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.08622):  80%|| 126/157 [00:27<00:06,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.03564):  80%|| 126/157 [00:27<00:06,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.03564):  81%|| 127/157 [00:27<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08017):  81%|| 127/157 [00:28<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08017):  82%|| 128/157 [00:28<00:06,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.04133):  82%|| 128/157 [00:28<00:06,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.04133):  82%|| 129/157 [00:28<00:06,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.14139):  82%|| 129/157 [00:28<00:06,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.14139):  83%|| 130/157 [00:28<00:05,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.06150):  83%|| 130/157 [00:28<00:05,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.06150):  83%|| 131/157 [00:28<00:05,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08302):  83%|| 131/157 [00:28<00:05,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08302):  84%|| 132/157 [00:28<00:05,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08868):  84%|| 132/157 [00:29<00:05,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08868):  85%|| 133/157 [00:29<00:05,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.05801):  85%|| 133/157 [00:29<00:05,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.05801):  85%|| 134/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.07661):  85%|| 134/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.07661):  86%|| 135/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.02719):  86%|| 135/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.02719):  87%|| 136/157 [00:29<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.12299):  87%|| 136/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.12299):  87%|| 137/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03394):  87%|| 137/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03394):  88%|| 138/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.15204):  88%|| 138/157 [00:30<00:04,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.15204):  89%|| 139/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.03364):  89%|| 139/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.03364):  89%|| 140/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06830):  89%|| 140/157 [00:30<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06830):  90%|| 141/157 [00:30<00:03,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03930):  90%|| 141/157 [00:31<00:03,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03930):  90%|| 142/157 [00:31<00:03,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.04692):  90%|| 142/157 [00:31<00:03,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.04692):  91%|| 143/157 [00:31<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05166):  91%|| 143/157 [00:31<00:03,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05166):  92%|| 144/157 [00:31<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15265):  92%|| 144/157 [00:31<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.15265):  92%|| 145/157 [00:31<00:02,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08270):  92%|| 145/157 [00:32<00:02,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08270):  93%|| 146/157 [00:32<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.03549):  93%|| 146/157 [00:32<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.03549):  94%|| 147/157 [00:32<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06279):  94%|| 147/157 [00:32<00:02,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06279):  94%|| 148/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.09848):  94%|| 148/157 [00:32<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.09848):  95%|| 149/157 [00:32<00:01,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.13231):  95%|| 149/157 [00:32<00:01,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.13231):  96%|| 150/157 [00:32<00:01,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.07991):  96%|| 150/157 [00:33<00:01,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.07991):  96%|| 151/157 [00:33<00:01,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08742):  96%|| 151/157 [00:33<00:01,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08742):  97%|| 152/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07570):  97%|| 152/157 [00:33<00:01,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07570):  97%|| 153/157 [00:33<00:00,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10923):  97%|| 153/157 [00:33<00:00,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10923):  98%|| 154/157 [00:33<00:00,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06882):  98%|| 154/157 [00:33<00:00,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06882):  99%|| 155/157 [00:33<00:00,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05046):  99%|| 155/157 [00:34<00:00,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05046):  99%|| 156/157 [00:34<00:00,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.03776): 100%|| 157/157 [00:34<00:00,  4.57it/s]\u001b[A\n",
      "11/05/2021 18:18:51 - INFO - __main__ - \n",
      "\n",
      "11/05/2021 18:18:51 - INFO - __main__ - Test Results\n",
      "11/05/2021 18:18:51 - INFO - __main__ - Global Steps: 400\n",
      "11/05/2021 18:18:51 - INFO - __main__ - Test Loss: 0.07871\n",
      "11/05/2021 18:18:51 - INFO - __main__ - Test Accuracy: 0.97820\n",
      "11/05/2021 18:18:52 - INFO - __main__ - Saved model checkpoint to [DIR: output_cifar10]\n",
      "Training (500 / 500 Steps) (loss=0.27852):  64%|| 499/782 [12:09<04:12,  1.12it/s]  11/05/2021 18:20:51 - INFO - __main__ - ***** Running Test *****\n",
      "11/05/2021 18:20:51 - INFO - __main__ -   Num steps = 157\n",
      "11/05/2021 18:20:51 - INFO - __main__ -   Batch size = 64\n",
      "\n",
      "Test... (loss=X.X):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.11033):   0%|| 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Testing... (loss=0.11033):   1%|| 1/157 [00:00<01:48,  1.44it/s]\u001b[A\n",
      "Testing... (loss=0.06576):   1%|| 1/157 [00:00<01:48,  1.44it/s]\u001b[A\n",
      "Testing... (loss=0.06576):   1%|| 2/157 [00:00<01:04,  2.40it/s]\u001b[A\n",
      "Testing... (loss=0.06514):   1%|| 2/157 [00:01<01:04,  2.40it/s]\u001b[A\n",
      "Testing... (loss=0.06514):   2%|| 3/157 [00:01<00:50,  3.07it/s]\u001b[A\n",
      "Testing... (loss=0.12430):   2%|| 3/157 [00:01<00:50,  3.07it/s]\u001b[A\n",
      "Testing... (loss=0.12430):   3%|| 4/157 [00:01<00:43,  3.54it/s]\u001b[A\n",
      "Testing... (loss=0.11675):   3%|| 4/157 [00:01<00:43,  3.54it/s]\u001b[A\n",
      "Testing... (loss=0.11675):   3%|| 5/157 [00:01<00:39,  3.86it/s]\u001b[A\n",
      "Testing... (loss=0.02599):   3%|| 5/157 [00:01<00:39,  3.86it/s]\u001b[A\n",
      "Testing... (loss=0.02599):   4%|| 6/157 [00:01<00:37,  4.07it/s]\u001b[A\n",
      "Testing... (loss=0.09104):   4%|| 6/157 [00:02<00:37,  4.07it/s]\u001b[A\n",
      "Testing... (loss=0.09104):   4%|| 7/157 [00:02<00:35,  4.21it/s]\u001b[A\n",
      "Testing... (loss=0.02756):   4%|| 7/157 [00:02<00:35,  4.21it/s]\u001b[A\n",
      "Testing... (loss=0.02756):   5%|| 8/157 [00:02<00:34,  4.31it/s]\u001b[A\n",
      "Testing... (loss=0.02935):   5%|| 8/157 [00:02<00:34,  4.31it/s]\u001b[A\n",
      "Testing... (loss=0.02935):   6%|| 9/157 [00:02<00:33,  4.39it/s]\u001b[A\n",
      "Testing... (loss=0.05412):   6%|| 9/157 [00:02<00:33,  4.39it/s]\u001b[A\n",
      "Testing... (loss=0.05412):   6%|| 10/157 [00:02<00:32,  4.46it/s]\u001b[A\n",
      "Testing... (loss=0.04786):   6%|| 10/157 [00:02<00:32,  4.46it/s]\u001b[A\n",
      "Testing... (loss=0.04786):   7%|| 11/157 [00:02<00:32,  4.52it/s]\u001b[A\n",
      "Testing... (loss=0.05506):   7%|| 11/157 [00:03<00:32,  4.52it/s]\u001b[A\n",
      "Testing... (loss=0.05506):   8%|| 12/157 [00:03<00:31,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.09883):   8%|| 12/157 [00:03<00:31,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.09883):   8%|| 13/157 [00:03<00:31,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.11005):   8%|| 13/157 [00:03<00:31,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.11005):   9%|| 14/157 [00:03<00:31,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.04667):   9%|| 14/157 [00:03<00:31,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.04667):  10%|| 15/157 [00:03<00:30,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.06018):  10%|| 15/157 [00:03<00:30,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.06018):  10%|| 16/157 [00:03<00:30,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.02268):  10%|| 16/157 [00:04<00:30,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.02268):  11%|| 17/157 [00:04<00:30,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.05474):  11%|| 17/157 [00:04<00:30,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.05474):  11%|| 18/157 [00:04<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06777):  11%|| 18/157 [00:04<00:30,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.06777):  12%|| 19/157 [00:04<00:29,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07672):  12%|| 19/157 [00:04<00:29,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07672):  13%|| 20/157 [00:04<00:29,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03768):  13%|| 20/157 [00:05<00:29,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03768):  13%|| 21/157 [00:05<00:29,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07279):  13%|| 21/157 [00:05<00:29,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07279):  14%|| 22/157 [00:05<00:29,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.03755):  14%|| 22/157 [00:05<00:29,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.03755):  15%|| 23/157 [00:05<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04950):  15%|| 23/157 [00:05<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04950):  15%|| 24/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.03512):  15%|| 24/157 [00:05<00:28,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.03512):  16%|| 25/157 [00:05<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.02973):  16%|| 25/157 [00:06<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.02973):  17%|| 26/157 [00:06<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03238):  17%|| 26/157 [00:06<00:28,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03238):  17%|| 27/157 [00:06<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08744):  17%|| 27/157 [00:06<00:27,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08744):  18%|| 28/157 [00:06<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.02433):  18%|| 28/157 [00:06<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.02433):  18%|| 29/157 [00:06<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06393):  18%|| 29/157 [00:06<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.06393):  19%|| 30/157 [00:06<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.10050):  19%|| 30/157 [00:07<00:27,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.10050):  20%|| 31/157 [00:07<00:27,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.03793):  20%|| 31/157 [00:07<00:27,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.03793):  20%|| 32/157 [00:07<00:27,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04788):  20%|| 32/157 [00:07<00:27,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04788):  21%|| 33/157 [00:07<00:26,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.14327):  21%|| 33/157 [00:07<00:26,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.14327):  22%|| 34/157 [00:07<00:26,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.05172):  22%|| 34/157 [00:08<00:26,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.05172):  22%|| 35/157 [00:08<00:26,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.05308):  22%|| 35/157 [00:08<00:26,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.05308):  23%|| 36/157 [00:08<00:26,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.06605):  23%|| 36/157 [00:08<00:26,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.06605):  24%|| 37/157 [00:08<00:25,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.16311):  24%|| 37/157 [00:08<00:25,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.16311):  24%|| 38/157 [00:08<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.11524):  24%|| 38/157 [00:08<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.11524):  25%|| 39/157 [00:08<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04334):  25%|| 39/157 [00:09<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04334):  25%|| 40/157 [00:09<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05446):  25%|| 40/157 [00:09<00:25,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05446):  26%|| 41/157 [00:09<00:25,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03589):  26%|| 41/157 [00:09<00:25,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.03589):  27%|| 42/157 [00:09<00:24,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05935):  27%|| 42/157 [00:09<00:24,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.05935):  27%|| 43/157 [00:09<00:24,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11343):  27%|| 43/157 [00:10<00:24,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.11343):  28%|| 44/157 [00:10<00:24,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.12386):  28%|| 44/157 [00:10<00:24,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.12386):  29%|| 45/157 [00:10<00:24,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.05849):  29%|| 45/157 [00:10<00:24,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.05849):  29%|| 46/157 [00:10<00:24,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.07030):  29%|| 46/157 [00:10<00:24,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.07030):  30%|| 47/157 [00:10<00:23,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07277):  30%|| 47/157 [00:10<00:23,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07277):  31%|| 48/157 [00:10<00:23,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10474):  31%|| 48/157 [00:11<00:23,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10474):  31%|| 49/157 [00:11<00:23,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07264):  31%|| 49/157 [00:11<00:23,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07264):  32%|| 50/157 [00:11<00:23,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05504):  32%|| 50/157 [00:11<00:23,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05504):  32%|| 51/157 [00:11<00:22,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10645):  32%|| 51/157 [00:11<00:22,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.10645):  33%|| 52/157 [00:11<00:22,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.06602):  33%|| 52/157 [00:11<00:22,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.06602):  34%|| 53/157 [00:11<00:22,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.14543):  34%|| 53/157 [00:12<00:22,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.14543):  34%|| 54/157 [00:12<00:22,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.05145):  34%|| 54/157 [00:12<00:22,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.05145):  35%|| 55/157 [00:12<00:22,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.08885):  35%|| 55/157 [00:12<00:22,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.08885):  36%|| 56/157 [00:12<00:21,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.03379):  36%|| 56/157 [00:12<00:21,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.03379):  36%|| 57/157 [00:12<00:21,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.06667):  36%|| 57/157 [00:13<00:21,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.06667):  37%|| 58/157 [00:13<00:21,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.06082):  37%|| 58/157 [00:13<00:21,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.06082):  38%|| 59/157 [00:13<00:21,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03896):  38%|| 59/157 [00:13<00:21,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03896):  38%|| 60/157 [00:13<00:20,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.02978):  38%|| 60/157 [00:13<00:20,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.02978):  39%|| 61/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.02997):  39%|| 61/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.02997):  39%|| 62/157 [00:13<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05825):  39%|| 62/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05825):  40%|| 63/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.16921):  40%|| 63/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.16921):  41%|| 64/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.11285):  41%|| 64/157 [00:14<00:20,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.11285):  41%|| 65/157 [00:14<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.03054):  41%|| 65/157 [00:14<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.03054):  42%|| 66/157 [00:14<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04719):  42%|| 66/157 [00:14<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04719):  43%|| 67/157 [00:14<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04048):  43%|| 67/157 [00:15<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.04048):  43%|| 68/157 [00:15<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10036):  43%|| 68/157 [00:15<00:19,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.10036):  44%|| 69/157 [00:15<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.09454):  44%|| 69/157 [00:15<00:19,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.09454):  45%|| 70/157 [00:15<00:18,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08924):  45%|| 70/157 [00:15<00:18,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08924):  45%|| 71/157 [00:15<00:18,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08205):  45%|| 71/157 [00:16<00:18,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08205):  46%|| 72/157 [00:16<00:18,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.12792):  46%|| 72/157 [00:16<00:18,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.12792):  46%|| 73/157 [00:16<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04076):  46%|| 73/157 [00:16<00:18,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04076):  47%|| 74/157 [00:16<00:17,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.23779):  47%|| 74/157 [00:16<00:17,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.23779):  48%|| 75/157 [00:16<00:17,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.03487):  48%|| 75/157 [00:16<00:17,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.03487):  48%|| 76/157 [00:16<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09004):  48%|| 76/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09004):  49%|| 77/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.27016):  49%|| 77/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.27016):  50%|| 78/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09953):  50%|| 78/157 [00:17<00:17,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.09953):  50%|| 79/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.02360):  50%|| 79/157 [00:17<00:16,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.02360):  51%|| 80/157 [00:17<00:16,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05941):  51%|| 80/157 [00:18<00:16,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05941):  52%|| 81/157 [00:18<00:16,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04759):  52%|| 81/157 [00:18<00:16,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04759):  52%|| 82/157 [00:18<00:16,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07311):  52%|| 82/157 [00:18<00:16,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07311):  53%|| 83/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.02046):  53%|| 83/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.02046):  54%|| 84/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08996):  54%|| 84/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.08996):  54%|| 85/157 [00:18<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07417):  54%|| 85/157 [00:19<00:15,  4.64it/s]\u001b[A\n",
      "Testing... (loss=0.07417):  55%|| 86/157 [00:19<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05935):  55%|| 86/157 [00:19<00:15,  4.65it/s]\u001b[A\n",
      "Testing... (loss=0.05935):  55%|| 87/157 [00:19<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05259):  55%|| 87/157 [00:19<00:15,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.05259):  56%|| 88/157 [00:19<00:14,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08428):  56%|| 88/157 [00:19<00:14,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08428):  57%|| 89/157 [00:19<00:14,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.02804):  57%|| 89/157 [00:19<00:14,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.02804):  57%|| 90/157 [00:19<00:14,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.04026):  57%|| 90/157 [00:20<00:14,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.04026):  58%|| 91/157 [00:20<00:14,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.06089):  58%|| 91/157 [00:20<00:14,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.06089):  59%|| 92/157 [00:20<00:14,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.02331):  59%|| 92/157 [00:20<00:14,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.02331):  59%|| 93/157 [00:20<00:13,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03761):  59%|| 93/157 [00:20<00:13,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03761):  60%|| 94/157 [00:20<00:13,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.06229):  60%|| 94/157 [00:21<00:13,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.06229):  61%|| 95/157 [00:21<00:13,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.05133):  61%|| 95/157 [00:21<00:13,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.05133):  61%|| 96/157 [00:21<00:13,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.16017):  61%|| 96/157 [00:21<00:13,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.16017):  62%|| 97/157 [00:21<00:13,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.14095):  62%|| 97/157 [00:21<00:13,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.14095):  62%|| 98/157 [00:21<00:12,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.01965):  62%|| 98/157 [00:21<00:12,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.01965):  63%|| 99/157 [00:21<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07457):  63%|| 99/157 [00:22<00:12,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.07457):  64%|| 100/157 [00:22<00:12,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.06642):  64%|| 100/157 [00:22<00:12,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.06642):  64%|| 101/157 [00:22<00:12,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.02193):  64%|| 101/157 [00:22<00:12,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.02193):  65%|| 102/157 [00:22<00:11,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.07800):  65%|| 102/157 [00:22<00:11,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.07800):  66%|| 103/157 [00:22<00:11,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.09639):  66%|| 103/157 [00:22<00:11,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.09639):  66%|| 104/157 [00:22<00:11,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.11372):  66%|| 104/157 [00:23<00:11,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.11372):  67%|| 105/157 [00:23<00:11,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.10530):  67%|| 105/157 [00:23<00:11,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.10530):  68%|| 106/157 [00:23<00:11,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.04806):  68%|| 106/157 [00:23<00:11,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.04806):  68%|| 107/157 [00:23<00:10,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.04152):  68%|| 107/157 [00:23<00:10,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.04152):  69%|| 108/157 [00:23<00:10,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.10856):  69%|| 108/157 [00:24<00:10,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.10856):  69%|| 109/157 [00:24<00:10,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.08376):  69%|| 109/157 [00:24<00:10,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.08376):  70%|| 110/157 [00:24<00:10,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.14656):  70%|| 110/157 [00:24<00:10,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.14656):  71%|| 111/157 [00:24<00:09,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.08295):  71%|| 111/157 [00:24<00:09,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.08295):  71%|| 112/157 [00:24<00:09,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.05925):  71%|| 112/157 [00:24<00:09,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.05925):  72%|| 113/157 [00:24<00:09,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.04566):  72%|| 113/157 [00:25<00:09,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.04566):  73%|| 114/157 [00:25<00:09,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.07166):  73%|| 114/157 [00:25<00:09,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.07166):  73%|| 115/157 [00:25<00:09,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.08096):  73%|| 115/157 [00:25<00:09,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.08096):  74%|| 116/157 [00:25<00:08,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.05988):  74%|| 116/157 [00:25<00:08,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.05988):  75%|| 117/157 [00:25<00:08,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.26679):  75%|| 117/157 [00:26<00:08,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.26679):  75%|| 118/157 [00:26<00:08,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.10077):  75%|| 118/157 [00:26<00:08,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.10077):  76%|| 119/157 [00:26<00:08,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.06180):  76%|| 119/157 [00:26<00:08,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.06180):  76%|| 120/157 [00:26<00:08,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08433):  76%|| 120/157 [00:26<00:08,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.08433):  77%|| 121/157 [00:26<00:07,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.09871):  77%|| 121/157 [00:26<00:07,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.09871):  78%|| 122/157 [00:26<00:07,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.08509):  78%|| 122/157 [00:27<00:07,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.08509):  78%|| 123/157 [00:27<00:07,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.03396):  78%|| 123/157 [00:27<00:07,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.03396):  79%|| 124/157 [00:27<00:07,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.10336):  79%|| 124/157 [00:27<00:07,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.10336):  80%|| 125/157 [00:27<00:06,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.09280):  80%|| 125/157 [00:27<00:06,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.09280):  80%|| 126/157 [00:27<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03112):  80%|| 126/157 [00:27<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03112):  81%|| 127/157 [00:27<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.06670):  81%|| 127/157 [00:28<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.06670):  82%|| 128/157 [00:28<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03868):  82%|| 128/157 [00:28<00:06,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.03868):  82%|| 129/157 [00:28<00:06,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.12515):  82%|| 129/157 [00:28<00:06,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.12515):  83%|| 130/157 [00:28<00:05,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.06260):  83%|| 130/157 [00:28<00:05,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.06260):  83%|| 131/157 [00:28<00:05,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.07482):  83%|| 131/157 [00:29<00:05,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.07482):  84%|| 132/157 [00:29<00:05,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.08329):  84%|| 132/157 [00:29<00:05,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.08329):  85%|| 133/157 [00:29<00:05,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.05461):  85%|| 133/157 [00:29<00:05,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.05461):  85%|| 134/157 [00:29<00:05,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.06673):  85%|| 134/157 [00:29<00:05,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.06673):  86%|| 135/157 [00:29<00:04,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.02600):  86%|| 135/157 [00:29<00:04,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.02600):  87%|| 136/157 [00:29<00:04,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.11565):  87%|| 136/157 [00:30<00:04,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.11565):  87%|| 137/157 [00:30<00:04,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.03338):  87%|| 137/157 [00:30<00:04,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.03338):  88%|| 138/157 [00:30<00:04,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.14425):  88%|| 138/157 [00:30<00:04,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.14425):  89%|| 139/157 [00:30<00:03,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.03088):  89%|| 139/157 [00:30<00:03,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.03088):  89%|| 140/157 [00:30<00:03,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.05289):  89%|| 140/157 [00:31<00:03,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.05289):  90%|| 141/157 [00:31<00:03,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.04604):  90%|| 141/157 [00:31<00:03,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.04604):  90%|| 142/157 [00:31<00:03,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.03893):  90%|| 142/157 [00:31<00:03,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.03893):  91%|| 143/157 [00:31<00:03,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.03984):  91%|| 143/157 [00:31<00:03,  4.59it/s]\u001b[A\n",
      "Testing... (loss=0.03984):  92%|| 144/157 [00:31<00:02,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.14757):  92%|| 144/157 [00:31<00:02,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.14757):  92%|| 145/157 [00:31<00:02,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.08480):  92%|| 145/157 [00:32<00:02,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.08480):  93%|| 146/157 [00:32<00:02,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.03403):  93%|| 146/157 [00:32<00:02,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.03403):  94%|| 147/157 [00:32<00:02,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.06115):  94%|| 147/157 [00:32<00:02,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.06115):  94%|| 148/157 [00:32<00:01,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.08558):  94%|| 148/157 [00:32<00:01,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.08558):  95%|| 149/157 [00:32<00:01,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.12539):  95%|| 149/157 [00:33<00:01,  4.57it/s]\u001b[A\n",
      "Testing... (loss=0.12539):  96%|| 150/157 [00:33<00:01,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.07364):  96%|| 150/157 [00:33<00:01,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.07364):  96%|| 151/157 [00:33<00:01,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.08205):  96%|| 151/157 [00:33<00:01,  4.56it/s]\u001b[A\n",
      "Testing... (loss=0.08205):  97%|| 152/157 [00:33<00:01,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.07612):  97%|| 152/157 [00:33<00:01,  4.58it/s]\u001b[A\n",
      "Testing... (loss=0.07612):  97%|| 153/157 [00:33<00:00,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.10025):  97%|| 153/157 [00:33<00:00,  4.60it/s]\u001b[A\n",
      "Testing... (loss=0.10025):  98%|| 154/157 [00:33<00:00,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.06551):  98%|| 154/157 [00:34<00:00,  4.61it/s]\u001b[A\n",
      "Testing... (loss=0.06551):  99%|| 155/157 [00:34<00:00,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.05086):  99%|| 155/157 [00:34<00:00,  4.62it/s]\u001b[A\n",
      "Testing... (loss=0.05086):  99%|| 156/157 [00:34<00:00,  4.63it/s]\u001b[A\n",
      "Testing... (loss=0.04197): 100%|| 157/157 [00:34<00:00,  4.56it/s]\u001b[A\n",
      "11/05/2021 18:21:26 - INFO - __main__ - \n",
      "\n",
      "11/05/2021 18:21:26 - INFO - __main__ - Test Results\n",
      "11/05/2021 18:21:26 - INFO - __main__ - Global Steps: 500\n",
      "11/05/2021 18:21:26 - INFO - __main__ - Test Loss: 0.07383\n",
      "11/05/2021 18:21:26 - INFO - __main__ - Test Accuracy: 0.97990\n",
      "11/05/2021 18:21:27 - INFO - __main__ - Saved model checkpoint to [DIR: output_cifar10]\n",
      "Training (500 / 500 Steps) (loss=0.27852):  64%|| 499/782 [12:47<07:15,  1.54s/it]\n",
      "11/05/2021 18:21:30 - INFO - __main__ - Best Accuracy: \t0.979900\n",
      "11/05/2021 18:21:30 - INFO - __main__ - End Training!\n"
     ]
    }
   ],
   "source": [
    "main()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "419facb1",
   "metadata": {},
   "source": [
    "### Testing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "66f45206",
   "metadata": {},
   "outputs": [],
   "source": [
    "class VisionTransformer(nn.Module):\n",
    "    def __init__(self, config, img_size=224, num_classes=21843, zero_head=False, vis=False):\n",
    "        super(VisionTransformer, self).__init__()\n",
    "        self.num_classes = num_classes\n",
    "        self.zero_head = zero_head\n",
    "        self.classifier = config.classifier\n",
    "\n",
    "        self.transformer = Transformer(config, img_size, vis)\n",
    "        self.head = Linear(config.hidden_size, num_classes)\n",
    "\n",
    "    def forward(self, x, labels=None):\n",
    "        x, attn_weights = self.transformer(x)\n",
    "        logits = self.head(x[:, 0])\n",
    "\n",
    "        if labels is not None:\n",
    "            loss_fct = CrossEntropyLoss()\n",
    "            loss = loss_fct(logits.view(-1, self.num_classes), labels.view(-1))\n",
    "            return loss\n",
    "        else:\n",
    "            return logits #, attn_weights\n",
    "\n",
    "    def load_from(self, weights):\n",
    "        with torch.no_grad():\n",
    "            if self.zero_head:\n",
    "                nn.init.zeros_(self.head.weight)\n",
    "                nn.init.zeros_(self.head.bias)\n",
    "            else:\n",
    "                self.head.weight.copy_(np2th(weights[\"head/kernel\"]).t())\n",
    "                self.head.bias.copy_(np2th(weights[\"head/bias\"]).t())\n",
    "\n",
    "            self.transformer.embeddings.patch_embeddings.weight.copy_(np2th(weights[\"embedding/kernel\"]))\n",
    "            self.transformer.embeddings.patch_embeddings.bias.copy_(np2th(weights[\"embedding/bias\"]))\n",
    "            self.transformer.embeddings.cls_token.copy_(np2th(weights[\"cls\"]))\n",
    "            self.transformer.encoder.encoder_norm.weight.copy_(np2th(weights[\"Transformer/encoder_norm/scale\"]))\n",
    "            self.transformer.encoder.encoder_norm.bias.copy_(np2th(weights[\"Transformer/encoder_norm/bias\"]))\n",
    "\n",
    "            posemb = np2th(weights[\"Transformer/posembed_input/pos_embedding\"])\n",
    "            posemb_new = self.transformer.embeddings.position_embeddings\n",
    "            if posemb.size() == posemb_new.size():\n",
    "                self.transformer.embeddings.position_embeddings.copy_(posemb)\n",
    "            else:\n",
    "                logger.info(\"load_pretrained: resized variant: %s to %s\" % (posemb.size(), posemb_new.size()))\n",
    "                ntok_new = posemb_new.size(1)\n",
    "\n",
    "                if self.classifier == \"token\":\n",
    "                    posemb_tok, posemb_grid = posemb[:, :1], posemb[0, 1:]\n",
    "                    ntok_new -= 1\n",
    "                else:\n",
    "                    posemb_tok, posemb_grid = posemb[:, :0], posemb[0]\n",
    "\n",
    "                gs_old = int(np.sqrt(len(posemb_grid)))\n",
    "                gs_new = int(np.sqrt(ntok_new))\n",
    "                print('load_pretrained: grid-size from %s to %s' % (gs_old, gs_new))\n",
    "                posemb_grid = posemb_grid.reshape(gs_old, gs_old, -1)\n",
    "\n",
    "                zoom = (gs_new / gs_old, gs_new / gs_old, 1)\n",
    "                posemb_grid = ndimage.zoom(posemb_grid, zoom, order=1)\n",
    "                posemb_grid = posemb_grid.reshape(1, gs_new * gs_new, -1)\n",
    "                posemb = np.concatenate([posemb_tok, posemb_grid], axis=1)\n",
    "                self.transformer.embeddings.position_embeddings.copy_(np2th(posemb))\n",
    "\n",
    "            for bname, block in self.transformer.encoder.named_children():\n",
    "                for uname, unit in block.named_children():\n",
    "                    unit.load_from(weights, n_block=uname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "8520c5c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VisionTransformer(\n",
       "  (transformer): Transformer(\n",
       "    (embeddings): Embeddings(\n",
       "      (patch_embeddings): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16))\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): Encoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (1): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (2): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (3): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (4): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (5): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (6): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (7): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (8): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (9): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (10): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (11): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (encoder_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (head): Linear(in_features=768, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = VisionTransformer(config, img_size=224, zero_head=True, num_classes=10)\n",
    "checkpoint = torch.load(\"output_cifar10/cifar10_500_checkpoint.bin\") \n",
    "\n",
    "model.load_state_dict(checkpoint) \n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "9c3fe0b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VisionTransformer(\n",
       "  (transformer): Transformer(\n",
       "    (embeddings): Embeddings(\n",
       "      (patch_embeddings): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16))\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): Encoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (1): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (2): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (3): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (4): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (5): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (6): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (7): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (8): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (9): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (10): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "        (11): Block(\n",
       "          (attention_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "          (ffn): Mlp(\n",
       "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (attn): Attention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (proj_dropout): Dropout(p=0.0, inplace=False)\n",
       "            (softmax): Softmax(dim=-1)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (encoder_norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (head): Linear(in_features=768, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "a3a4e101",
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "\n",
    "image_dir = 'automobile_ex2.png'\n",
    "\n",
    "input_image = Image.open(image_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "fd45b9a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAJLElEQVR4nDWWyW6k53mF3/eb/qlmzqRIij1QsgQFLcMwnGSTnRbJyoAXvgFfjy8hydbwwl4k8MpDIltA245akntQq2XOLFYVa/qnb36zaPsKnvMc4AAH//0/fqpo/oN/+tf+xoOf/fw/r8bnXKki63TzwhO1znDOBWK/08uS5L1Hp6apn718WltHto1WO8BiNDjaPz3aPUzyri+G8/nsV//1s/33HguZv3r2lUiSREYplUoSxTkyjpwzH+xq3VrvtHeEUGTZ4eFBLx9sbO6nQulg7xaL1WJaV68Crcezs+n9169f92Ten4V0WjZ385sX/3dHQZl5KxABGQOAGIOxRreNIoqCSQHIyZvWedftpT7YVVkBiNa0Idh+rwsikF8cDcysbBl4wfzCrOqSXJC9AaC3icrXAIIxxiJjDBGBKMTofbBEjGKEt0LAVCKLXjcayaWcL28ur74modahPhzxg74YdvreBe2gcuZ4Z1B5sShbbWKwxoVKIDLGGCIjiiF4gsg4E5xzhhGIc8ZlKmQik4RQIAdCYgh125ahWglpbrUBZU3QPsw1SuuAAja2bSliwjodgQgMkDEERER4q4KIDBkiAhOMc6VypTIXPDJyLpZrbQAswlcXZtDdYcmg1+ncz2dLU1Y303q57BaDg42Hn/zbj6bzUvAQGEdgkdAjA0SC4GwIQaCUgnwkAooEnKGQDDlF9MBsrKv18v0Pf/Dg5DtV6R+fPP7zl0+/OXvlKpqUq35WHO0+LIJ4dXsrrNZpBwEiADnvQghpwkMkACKi4F0I5F0kYMgQAEP0gaILIXocdHa62YjappC5CIx07GSDo+PHOzt7ea8nlbi+vWCSCyBCACIAYAAYI8VIAAiARAAAyAQCB+BE6KN1PhiDRnPvk+AUgPTBR+eZZxxl0R91N7ZUkgSKxjkhuWCAfwcgRQyeAhEnRGQxIhIAAeOckABASJHlXc+KXuzl6YZ3PHhYLhdOGxFF651n4CIwwcfj8d3dncC3sQkhgrOeIhABvLUhjsgBWQh2Oh1LTKtqnRfd09N/QNWdLlbauOvrm6uLM93OMFiKzDnvOIRIyHlZV1XbCCWkd66pqkL5EOLfm6EYKHICwk6nsL59+eqrbjZoF8v5fEE82z96/PSPn08mVw+O39VNc3n+sq6Wu9t7abcIHCl6QFiu11XTCCLyPn75xZfDfYuMh0jGWAKIEI3VnHEumLN10RuuV7PF7eTNtzcfPfnH//ndp7/79NdCuFQCBHLOGmsuby7TIt/e3eEAZVWdXZwbo0UTbQRxP56ez+2qbgJ5BsH7yNMky+T9/f3kxfX3n3zoq4mK4mKpKd0yEZ99/mk1+UZKtd7c1Nrc3V3JRAiVa1smSdGWegUrberuIBUmBiQgjK1utLOAyBB7vf6j9x8R2JubcZrkw+GQkWuD+nZ80x1tnF9dXV9dsghBu3pdZ2ne1i0XnV53kGWZQNFWbS8tpFJSCeFcEOB91CEABU8EUuWj0Y5u4vXNtWkw6RXPXlwEb21IjUv3djp/evWXpgkpzzqdXMi8P9g8Pf0g72QopHdetxoIvA9V1cSIwvmInHwMwbs8zQe7Ww9OHnnH7iZ3B3uP93ffb9tAQtrgXRBbstfq+XI+6/X63Zw9fvSIsVTKZHvnoKyWzbp+uxtnfVXVurWpKoQPyJERyV5n8MF3nnzw0YcAyV9evD7ubI5GwzTpOidE2jfBz5erq/OL+9nZwcG2eGczTUyeqdlMY2uSFLU2AmVVVzFGbz3koGRSWc8iYYxqa+v43aPTd/aOh/3dyaSua+BJD2ThMQfea1u6ncxev3l+ff3CtKtBvyi6JIvFqr4BiFxgIC8kN8Z88eyLl89fcsb39w+kTJrGCC7SEAxBbjRK2bm8nJ2dT4vB9mB7gMiqOnirb2/OrqYXbXvXNrd1XVlnk8wkubdGQZQ++OidZHy9Wgcfgvez6cx5e35xWTorIs/Pz2dP//DbJx9//Punz3cOT/PRZtvY+maBTIYYr28uVstrBd7X66aqgJCr1EdczWMMmSUIEUJkxvuyaY9OTmKMrTGry4uyLJErsa7M3WT+5q9/Pb++uJuufvjjn3z8z/s6AjioyhWBvb296Obi/nby589+73x7+OAk6/d8EOj7FCCGELwPMQbj2rYdDod5UQghXbAELPogdK0H/e4nn/zL9d3VkaHD4x0lIKAUadFFcXn2Epxxlf/st599/fy5CxqketDtuEA8YrAOiMgHBmCdXy6WnHPO+Gq5Ag5Spd5FMb4ZL+fjbkc9fHjY29w7ebDHJSeVqjTTzi5nd4L86+ffXH57vj3aXZTzxf3SahcIIDBvQrleTiaTEIMUsqobYEKozBHsHextFV2tndBt+/LFc+8qG/3H3/v+d598jzHmGGhdXp+9rhbTPMU3r17mWXb63vvXk5uyXUji3oTx7WQ6vSvrFRDlRZF3OqcffTcv8o2NzeFwsLM1YoyvVrV4+OD4f3/zq/v7sXOexT+mQkjZI5GX2tzdjpWkg4PN+8X0YGe36HX6YVRfzC+/Obsd35vSqU66vXs4GAw3NjZUlqNSSaKKouh1Op1cAKD3TFxdXezt7xOF5Xw5vr7971/+AkDaAN3B6Pj4Yb41Ws7nRtdJLh35NEuasj5bvxmMdk8/eG+wv5uNegwZ5zzJ88AF51wliUiVkMCQSRXE6zfnGzv7m7t70/FtVS5i8FYba8NoYzjoZtG2dVVS01qjmeTNvCmK7u7e/vY7h4PRVtobEuN1XTdN23oQaZFniqJASCIAAUMFYmtnV+um1Q1KIdPMtFpIFBKrqvryi8+5EBiJI5vN5zwvFvPl/tHJ3sE72WDAstwRtFWttWaMhxCDNrrRIYZUyVyAkFJlHeFdcC54HwgZcgGMu+CMbo21PgSiyIDJJLlfLBzy46N3D49PkrzD09RFtK0RTPT7QyEE4xxQEJD33rStbhtXloRr4VxgyJRKkSMXgvOEs5oLlSExxgFIMB4jjJTa3Ns7PjpJiz4x4QkCUZqqRArOOSK+ZfztrvW6ZFttbd0YgcgZi0IIwRRzClEKkeTdAEBSSillopTgivc6xWCQJjlwFYFBhExKiQyJBBfIGEJEipwxIbiSCUiWJGmShP8Hc6PRYWXJgAwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<PIL.PngImagePlugin.PngImageFile image mode=RGB size=32x32 at 0x7FC7906094F0>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "f845d51c",
   "metadata": {},
   "outputs": [],
   "source": [
    "transform = transforms.Compose([\n",
    "    transforms.Resize((224, 224)),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),\n",
    "])\n",
    "\n",
    "x = transform(input_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "a59c4ee4",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = x.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "e62f7323",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 224, 224])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "33261ab5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1], device='cuda:1')"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits = model(x.unsqueeze(0))\n",
    "\n",
    "preds = torch.argmax(logits, dim=-1)\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "5984109a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.3349,  6.6046, -1.1946, -1.1217, -1.7440, -1.1695, -1.7322, -0.8939,\n",
       "         -0.1608,  1.7470]], device='cuda:1', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13b3ecef",
   "metadata": {},
   "source": [
    "### ART Attack "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "f736258a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11/05/2021 21:01:13 - INFO - art.config - set ART_DATA_PATH to /home/ubuntu/.art/data\n",
      "11/05/2021 21:01:14 - INFO - art.estimators.classification.pytorch - Inferred 1 hidden layers on PyTorch classifier.\n"
     ]
    }
   ],
   "source": [
    "from art.estimators.classification import PyTorchClassifier\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# Create the ART classifier\n",
    "\n",
    "classifier = PyTorchClassifier(\n",
    "    model=model,\n",
    "    loss=criterion,\n",
    "    input_shape=(3, 224, 224),\n",
    "    nb_classes=1000,\n",
    "    device_type='cuda:0'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "c5efbf4d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.33490533,  6.6046057 , -1.1946402 , -1.1217006 , -1.7439765 ,\n",
       "        -1.1694728 , -1.7322187 , -0.8938796 , -0.16083284,  1.7470232 ]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.predict(x.unsqueeze(0).cpu())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "837548d6",
   "metadata": {},
   "source": [
    "### Torch Attack "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "9ae5e0c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchattacks\n",
    "\n",
    "atk = torchattacks.PGD(model, eps=0.01, alpha=2/255, steps=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "4a308cae",
   "metadata": {},
   "outputs": [],
   "source": [
    "adv_images = atk(x.unsqueeze(0), torch.tensor([1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "39a3dbc7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 3, 224, 224])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adv_images.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "93aeaa31",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3], device='cuda:0')"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits = model(adv_images)\n",
    "\n",
    "preds = torch.argmax(logits, dim=-1)\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d60623f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}