d6ab1d3a-92f5-4847-8f3b-69faabc89c67.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
import contextlib
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.nn.attention.flex_attention import BlockMask, flex_attention #KoszarskyB

# -----------------------------------------------------------------------------
# Muon optimizer

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        ns_steps: The number of Newton-Schulz iteration steps to use.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5):
        self.world_size = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ['RANK'])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, ns_steps=ns_steps)
        params = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                'params': [p for p in params if p.numel() == size],
                'update_buffer': [
                    torch.empty(size, device='cuda', dtype=torch.bfloat16)
                    for _ in range(self.world_size)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):

        for group in self.param_groups:

            lr = group['lr']
            momentum = group['momentum']
            nesterov = group['nesterov']
            ns_steps = group['ns_steps']
            update_buffers = group['update_buffer']
            # generate weight updates in distributed fashion
            params = group['params']
            assert len(params) % self.world_size == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.world_size]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p]
                if 'momentum_buffer' not in state:
                    state['momentum_buffer'] = torch.zeros_like(g)
                buf = state['momentum_buffer']
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_via_newtonschulz5(g, steps=ns_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.world_size]
            update_prev()

# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, num_heads):
        super().__init__()
        assert dim % num_heads == 0
        self.num_heads = num_heads
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5]))
        self.rotary = Rotary(dim // num_heads) # dim // num_heads = head_dim
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x, vi, block_mask):
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q = self.c_q(x).view(B, T, self.num_heads, -1)
        k = self.c_k(x).view(B, T, self.num_heads, -1)
        v = self.c_v(x).view(B, T, self.num_heads, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @KoszarskyB & @Grad62304977
        q, k = norm(q), norm(k) # QK norm @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask, enable_gqa=True)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x):
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.model_dim, config.num_heads)
        self.mlp = MLP(config.model_dim)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x, vi, x0, block_mask):
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

class ValueEmbedding(nn.Module):
    def __init__(self, config: "GPTConfig"):
        super().__init__()
        self.__setattr__
        self.embed = nn.ModuleList([
            nn.Embedding(config.vocab_size, config.model_dim)
            for _ in range(6)
        ])

    def forward(self, inputs) -> "list[torch.Tensor]":
        ve = [emb(inputs) for emb in self.embed]
        ve += reversed(ve)
        return ve


# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    num_layers : int = 12
    num_heads : int = 6 # head dim 128 suggested by @Grad62304977
    model_dim : int = 768

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.num_layers = config.num_layers

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.num_layers // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.num_layers - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.embed = nn.Embedding(config.vocab_size, config.model_dim)
        self.blocks = nn.ModuleList([Block(config) for _ in range(config.num_layers)])
        # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
        # U-net structure on token value embeddings by @leloykun
        self.value_embeds = ValueEmbedding(config)
        self.lm_head = CastedLinear(config.model_dim, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(
        self,
        inputs: torch.Tensor,
        targets: torch.Tensor,
        sliding_window_num_blocks: torch.Tensor,
    ):
        BLOCK_SIZE = 128
        assert inputs.ndim == 1
        docs = (inputs == 50256).cumsum(0)
        docs_low = docs.view(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.view(-1, BLOCK_SIZE)[:, -1].contiguous()

        def document_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            return causal_mask & document_mask

        def dense_to_ordered(dense_mask: torch.Tensor):
            num_blocks = dense_mask.sum(dim=-1, dtype=torch.int32)
            indices = dense_mask.argsort(dim=-1, descending=True, stable=True).to(torch.int32)
            return num_blocks[None, None].contiguous(), indices[None, None].contiguous()

        def create_doc_swc_block_mask(sliding_window_num_blocks: torch.Tensor):
            kv_idx = block_idx = torch.arange(512, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_bm = q_idx >= kv_idx
            causal_full_bm = q_idx > kv_idx
            window_bm = q_idx - kv_idx < sliding_window_num_blocks
            window_full_bm = window_bm
            # document_bm = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            document_bm = (docs_low[:, None] <= docs_high) & (docs_low <= docs_high[:, None])
            document_full_bm = (docs_low[:, None] == docs_high) & (docs_low == docs_high[:, None])
            nonzero_bm = causal_bm & window_bm & document_bm
            full_bm  = causal_full_bm & window_full_bm & document_full_bm
            kv_num_blocks, kv_indices = dense_to_ordered(nonzero_bm ^ full_bm)
            full_kv_num_blocks, full_kv_indices = dense_to_ordered(full_bm)
            return BlockMask.from_kv_blocks(
                kv_num_blocks,
                kv_indices,
                full_kv_num_blocks,
                full_kv_indices,
                BLOCK_SIZE=BLOCK_SIZE,
                mask_mod=document_causal,
            )

        block_mask = create_doc_swc_block_mask(sliding_window_num_blocks)

        # forward the GPT model itself
        x = self.embed(inputs[None]) # token embeddings of shape (b, t, model_dim)
        x = norm(x) # @Grad62304977
        x0 = x
        ve = self.value_embeds(inputs)
        ve_enc, ve_dec = ve[:self.num_encoder_layers], ve[self.num_encoder_layers:]

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.blocks[i](x, ve_enc[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.blocks[self.num_encoder_layers + i](x, ve_dec[i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = 30 * torch.tanh(logits / 30) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(path: Path, num_tokens):
    with path.open("rb", buffering=0) as f:
        tokens = torch.empty(num_tokens, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * num_tokens, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, seq_len, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.seq_len = seq_len

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.files_num_tokens = [_peek_data_shard(file) for file in self.files]
        assert min(self.files_num_tokens) >= num_processes * seq_len + 1
        self.total_num_tokens = sum(self.files_num_tokens)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.seq_len
        self.tokens = _load_data_shard(self.files[self.current_shard], self.files_num_tokens[self.current_shard])

    def next_batch(self):
        batch_size = self.seq_len * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.seq_len+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        inputs = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        targets = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return inputs, targets

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
assert torch.cuda.is_available()
device = torch.device(f"cuda:{ddp_local_rank}")
torch.cuda.set_device(device)
print(f"using device: {device}")
dist.init_process_group(backend='nccl', device_id=device)
dist.barrier()
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = uuid.uuid4()
    logdir = Path("logs") / f"{run_id}"
    logdir.mkdir(exist_ok=True)
    logfile = Path("logs") / f"{run_id}.txt"
    print(logfile.stem)
    # create the log file
    with logfile.open("w") as f:
        # begin the log by printing this file (the Python code)
        print(code, file=f)
        print("=" * 100, file=f)
def print0(s, logonly=False):
    if master_process:
        with logfile.open("a") as f:
            if not logonly:
                print(s)
            print(s, file=f)
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running python {sys.version}")
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# calculate the number of steps to take in the val loop.
assert args.val_tokens % (args.sequence_length * ddp_world_size) == 0
val_steps = args.val_tokens // (args.sequence_length * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size

# load tokens
train_loader = DistributedDataLoader(args.input_bin, args.sequence_length, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, args.sequence_length, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.total_num_tokens} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.total_num_tokens} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
inputs_train, targets_train = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, num_layers=12, num_heads=6, model_dim=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank], broadcast_buffers=False, gradient_as_bucket_view=True)
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
embed_params = [*raw_model.embed.parameters(), *raw_model.value_embeds.parameters()]
optimizer1 = torch.optim.Adam(embed_params, lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.blocks.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_num_blocks = torch.tensor(1, dtype=torch.int32, device="cuda")
sw_num_blocks_prev = 1
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Linearly increase the sliding window size over training in chunks of 64 from 64 -> 1792. By @fernbear.bsky.social
    frac_done = step / args.num_iterations # training progress
    sw_num_blocks = int(((1 - frac_done) * 64 + frac_done * 1792 + 64) // 128)
    if sw_num_blocks != sw_num_blocks_prev:
        sliding_window_num_blocks.copy_(sw_num_blocks, non_blocking=True)
        sw_num_blocks_prev = sw_num_blocks

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                inputs_val, targets_val = val_loader.next_batch()
                val_loss += model(inputs_val, targets_val, sliding_window_num_blocks)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    for i in range(1, train_accumulation_steps + 1):
        with contextlib.ExitStack() as stack:
            if i < train_accumulation_steps: # there's no need to sync gradients every accumulation step
                stack.enter_context(model.no_sync())
            if step >= 5:
                stack.enter_context(torch.compiler.set_stance(skip_guard_eval_unsafe=True))
            model(inputs_train, targets_train, sliding_window_num_blocks).backward()
            inputs_train, targets_train = train_loader.next_batch()
    if train_accumulation_steps != 1:
        for p in model.parameters():
            p.grad /= train_accumulation_steps
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

print0(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()

====================================================================================================
Running python 3.10.12 (main, Sep 11 2024, 15:47:36) [GCC 11.4.0]
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Wed Dec 11 09:35:14 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.183.06             Driver Version: 535.183.06   CUDA Version: 12.2     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:19:00.0 Off |                    0 |
| N/A   38C    P0             125W / 700W |   7084MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:3B:00.0 Off |                    0 |
| N/A   30C    P0             116W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:4C:00.0 Off |                    0 |
| N/A   28C    P0             112W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:5D:00.0 Off |                    0 |
| N/A   36C    P0             114W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:9B:00.0 Off |                    0 |
| N/A   38C    P0             119W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:BB:00.0 Off |                    0 |
| N/A   30C    P0             117W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:CB:00.0 Off |                    0 |
| N/A   36C    P0             119W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:DB:00.0 Off |                    0 |
| N/A   30C    P0             118W / 700W |   3211MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 1000000000 across 10 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:29104ms step_avg:nanms
step:2/1480 train_time:29208ms step_avg:nanms
step:3/1480 train_time:29331ms step_avg:nanms
step:4/1480 train_time:29472ms step_avg:nanms
step:5/1480 train_time:29616ms step_avg:nanms
step:6/1480 train_time:29756ms step_avg:nanms
step:7/1480 train_time:29903ms step_avg:nanms
step:8/1480 train_time:30039ms step_avg:nanms
step:9/1480 train_time:30182ms step_avg:nanms
step:10/1480 train_time:30328ms step_avg:nanms
step:11/1480 train_time:143ms step_avg:nanms
step:12/1480 train_time:284ms step_avg:nanms
step:13/1480 train_time:427ms step_avg:142.35ms
step:14/1480 train_time:570ms step_avg:142.39ms
step:15/1480 train_time:710ms step_avg:142.06ms
step:16/1480 train_time:852ms step_avg:142.04ms
step:17/1480 train_time:995ms step_avg:142.21ms
step:18/1480 train_time:1140ms step_avg:142.45ms
step:19/1480 train_time:1283ms step_avg:142.56ms
step:20/1480 train_time:1427ms step_avg:142.66ms
step:21/1480 train_time:1569ms step_avg:142.67ms
step:22/1480 train_time:1711ms step_avg:142.55ms
step:23/1480 train_time:1854ms step_avg:142.58ms
step:24/1480 train_time:1995ms step_avg:142.52ms
step:25/1480 train_time:2139ms step_avg:142.60ms
step:26/1480 train_time:2282ms step_avg:142.63ms
step:27/1480 train_time:2425ms step_avg:142.66ms
step:28/1480 train_time:2569ms step_avg:142.71ms
step:29/1480 train_time:2710ms step_avg:142.61ms
step:30/1480 train_time:2852ms step_avg:142.60ms
step:31/1480 train_time:2996ms step_avg:142.67ms
step:32/1480 train_time:3141ms step_avg:142.76ms
step:33/1480 train_time:3285ms step_avg:142.82ms
step:34/1480 train_time:3427ms step_avg:142.80ms
step:35/1480 train_time:3570ms step_avg:142.80ms
step:36/1480 train_time:3712ms step_avg:142.76ms
step:37/1480 train_time:3853ms step_avg:142.71ms
step:38/1480 train_time:3997ms step_avg:142.76ms
step:39/1480 train_time:4142ms step_avg:142.81ms
step:40/1480 train_time:4285ms step_avg:142.83ms
step:41/1480 train_time:4427ms step_avg:142.81ms
step:42/1480 train_time:4568ms step_avg:142.76ms
step:43/1480 train_time:4710ms step_avg:142.72ms
step:44/1480 train_time:4852ms step_avg:142.71ms
step:45/1480 train_time:4995ms step_avg:142.71ms
step:46/1480 train_time:5139ms step_avg:142.76ms
step:47/1480 train_time:5282ms step_avg:142.75ms
step:48/1480 train_time:5425ms step_avg:142.77ms
step:49/1480 train_time:5568ms step_avg:142.76ms
step:50/1480 train_time:5710ms step_avg:142.76ms
step:51/1480 train_time:5852ms step_avg:142.73ms
step:52/1480 train_time:5996ms step_avg:142.76ms
step:53/1480 train_time:6140ms step_avg:142.78ms
step:54/1480 train_time:6284ms step_avg:142.81ms
step:55/1480 train_time:6426ms step_avg:142.80ms
step:56/1480 train_time:6568ms step_avg:142.78ms
step:57/1480 train_time:6710ms step_avg:142.76ms
step:58/1480 train_time:6850ms step_avg:142.71ms
step:59/1480 train_time:6993ms step_avg:142.72ms
step:60/1480 train_time:7137ms step_avg:142.74ms
step:61/1480 train_time:7280ms step_avg:142.75ms
step:62/1480 train_time:7424ms step_avg:142.77ms
step:63/1480 train_time:7566ms step_avg:142.75ms
step:64/1480 train_time:7709ms step_avg:142.76ms
step:65/1480 train_time:7850ms step_avg:142.72ms
step:66/1480 train_time:7993ms step_avg:142.73ms
step:67/1480 train_time:8137ms step_avg:142.75ms
step:68/1480 train_time:8279ms step_avg:142.75ms
step:69/1480 train_time:8424ms step_avg:142.77ms
step:70/1480 train_time:8566ms step_avg:142.77ms
step:71/1480 train_time:8710ms step_avg:142.78ms
step:72/1480 train_time:8852ms step_avg:142.78ms
step:73/1480 train_time:8994ms step_avg:142.76ms
step:74/1480 train_time:9137ms step_avg:142.76ms
step:75/1480 train_time:9281ms step_avg:142.78ms
step:76/1480 train_time:9424ms step_avg:142.79ms
step:77/1480 train_time:9567ms step_avg:142.78ms
step:78/1480 train_time:9708ms step_avg:142.76ms
step:79/1480 train_time:10224ms step_avg:148.18ms
step:80/1480 train_time:10751ms step_avg:153.59ms
step:81/1480 train_time:10849ms step_avg:152.80ms
step:82/1480 train_time:10990ms step_avg:152.63ms
step:83/1480 train_time:11131ms step_avg:152.48ms
step:84/1480 train_time:11273ms step_avg:152.34ms
step:85/1480 train_time:11416ms step_avg:152.22ms
step:86/1480 train_time:11559ms step_avg:152.09ms
step:87/1480 train_time:11702ms step_avg:151.97ms
step:88/1480 train_time:11846ms step_avg:151.87ms
step:89/1480 train_time:11987ms step_avg:151.74ms
step:90/1480 train_time:12130ms step_avg:151.62ms
step:91/1480 train_time:12272ms step_avg:151.51ms
step:92/1480 train_time:12415ms step_avg:151.40ms
step:93/1480 train_time:12558ms step_avg:151.30ms
step:94/1480 train_time:12700ms step_avg:151.19ms
step:95/1480 train_time:12845ms step_avg:151.12ms
step:96/1480 train_time:12987ms step_avg:151.01ms
step:97/1480 train_time:13129ms step_avg:150.91ms
step:98/1480 train_time:13666ms step_avg:155.29ms
step:99/1480 train_time:13761ms step_avg:154.62ms
step:100/1480 train_time:13904ms step_avg:154.49ms
step:101/1480 train_time:14050ms step_avg:154.39ms
step:102/1480 train_time:14187ms step_avg:154.21ms
step:103/1480 train_time:14329ms step_avg:154.07ms
step:104/1480 train_time:14470ms step_avg:153.93ms
step:105/1480 train_time:14614ms step_avg:153.83ms
step:106/1480 train_time:14756ms step_avg:153.71ms
step:107/1480 train_time:14900ms step_avg:153.61ms
step:108/1480 train_time:15045ms step_avg:153.52ms
step:109/1480 train_time:15187ms step_avg:153.40ms
step:110/1480 train_time:15330ms step_avg:153.30ms
step:111/1480 train_time:15473ms step_avg:153.19ms
step:112/1480 train_time:15618ms step_avg:153.12ms
step:113/1480 train_time:15767ms step_avg:153.07ms
step:114/1480 train_time:15911ms step_avg:152.99ms
step:115/1480 train_time:16056ms step_avg:152.92ms
step:116/1480 train_time:16204ms step_avg:152.87ms
step:117/1480 train_time:16349ms step_avg:152.80ms
step:118/1480 train_time:16494ms step_avg:152.72ms
step:119/1480 train_time:16641ms step_avg:152.67ms
step:120/1480 train_time:16787ms step_avg:152.61ms
step:121/1480 train_time:16931ms step_avg:152.53ms
step:122/1480 train_time:17076ms step_avg:152.47ms
step:123/1480 train_time:17224ms step_avg:152.42ms
step:124/1480 train_time:17369ms step_avg:152.36ms
step:125/1480 train_time:17514ms step_avg:152.30ms
step:125/1480 val_loss:4.4042 train_time:17579ms step_avg:152.86ms
step:126/1480 train_time:17678ms step_avg:152.40ms
step:127/1480 train_time:17816ms step_avg:152.27ms
step:128/1480 train_time:17963ms step_avg:152.23ms
step:129/1480 train_time:18109ms step_avg:152.18ms
step:130/1480 train_time:18254ms step_avg:152.11ms
step:131/1480 train_time:18399ms step_avg:152.06ms
step:132/1480 train_time:18545ms step_avg:152.01ms
step:133/1480 train_time:18692ms step_avg:151.96ms
step:134/1480 train_time:18837ms step_avg:151.92ms
step:135/1480 train_time:18986ms step_avg:151.89ms
step:136/1480 train_time:19131ms step_avg:151.83ms
step:137/1480 train_time:19277ms step_avg:151.78ms
step:138/1480 train_time:19424ms step_avg:151.75ms
step:139/1480 train_time:19570ms step_avg:151.71ms
step:140/1480 train_time:19715ms step_avg:151.65ms
step:141/1480 train_time:19861ms step_avg:151.61ms
step:142/1480 train_time:20009ms step_avg:151.58ms
step:143/1480 train_time:20153ms step_avg:151.53ms
step:144/1480 train_time:20299ms step_avg:151.48ms
step:145/1480 train_time:20446ms step_avg:151.45ms
step:146/1480 train_time:20591ms step_avg:151.41ms
step:147/1480 train_time:20735ms step_avg:151.35ms
step:148/1480 train_time:20882ms step_avg:151.32ms
step:149/1480 train_time:21028ms step_avg:151.28ms
step:150/1480 train_time:21175ms step_avg:151.25ms
step:151/1480 train_time:21321ms step_avg:151.22ms
step:152/1480 train_time:21468ms step_avg:151.18ms
step:153/1480 train_time:21613ms step_avg:151.14ms
step:154/1480 train_time:21758ms step_avg:151.10ms
step:155/1480 train_time:21905ms step_avg:151.07ms
step:156/1480 train_time:22051ms step_avg:151.04ms
step:157/1480 train_time:22197ms step_avg:151.00ms
step:158/1480 train_time:22344ms step_avg:150.97ms
step:159/1480 train_time:22491ms step_avg:150.95ms
step:160/1480 train_time:22635ms step_avg:150.90ms
step:161/1480 train_time:22781ms step_avg:150.87ms
step:162/1480 train_time:22927ms step_avg:150.83ms
step:163/1480 train_time:23073ms step_avg:150.80ms
step:164/1480 train_time:23219ms step_avg:150.77ms
step:165/1480 train_time:23366ms step_avg:150.75ms
step:166/1480 train_time:23512ms step_avg:150.72ms
step:167/1480 train_time:23657ms step_avg:150.68ms
step:168/1480 train_time:23803ms step_avg:150.65ms
step:169/1480 train_time:23949ms step_avg:150.62ms
step:170/1480 train_time:24094ms step_avg:150.59ms
step:171/1480 train_time:24239ms step_avg:150.55ms
step:172/1480 train_time:24387ms step_avg:150.54ms
step:173/1480 train_time:24532ms step_avg:150.50ms
step:174/1480 train_time:24678ms step_avg:150.47ms
step:175/1480 train_time:24824ms step_avg:150.45ms
step:176/1480 train_time:24970ms step_avg:150.42ms
step:177/1480 train_time:25115ms step_avg:150.39ms
step:178/1480 train_time:25262ms step_avg:150.37ms
step:179/1480 train_time:25409ms step_avg:150.35ms
step:180/1480 train_time:25939ms step_avg:152.58ms
step:181/1480 train_time:26046ms step_avg:152.32ms
step:182/1480 train_time:26192ms step_avg:152.28ms
step:183/1480 train_time:26336ms step_avg:152.23ms
step:184/1480 train_time:26482ms step_avg:152.19ms
step:185/1480 train_time:26627ms step_avg:152.16ms
step:186/1480 train_time:26773ms step_avg:152.12ms
step:187/1480 train_time:26919ms step_avg:152.08ms
step:188/1480 train_time:27068ms step_avg:152.07ms
step:189/1480 train_time:27231ms step_avg:152.13ms
step:190/1480 train_time:27361ms step_avg:152.01ms
step:191/1480 train_time:27508ms step_avg:151.98ms
step:192/1480 train_time:27654ms step_avg:151.95ms
step:193/1480 train_time:27799ms step_avg:151.91ms
step:194/1480 train_time:27945ms step_avg:151.88ms
step:195/1480 train_time:28092ms step_avg:151.85ms
step:196/1480 train_time:28237ms step_avg:151.81ms
step:197/1480 train_time:28384ms step_avg:151.79ms
step:198/1480 train_time:28531ms step_avg:151.76ms
step:199/1480 train_time:28676ms step_avg:151.72ms
step:200/1480 train_time:28823ms step_avg:151.70ms
step:201/1480 train_time:28970ms step_avg:151.67ms
step:202/1480 train_time:29114ms step_avg:151.64ms
step:203/1480 train_time:29261ms step_avg:151.61ms
step:204/1480 train_time:29408ms step_avg:151.59ms
step:205/1480 train_time:29555ms step_avg:151.56ms
step:206/1480 train_time:29701ms step_avg:151.54ms
step:207/1480 train_time:29848ms step_avg:151.51ms
step:208/1480 train_time:29992ms step_avg:151.48ms
step:209/1480 train_time:30137ms step_avg:151.44ms
step:210/1480 train_time:30285ms step_avg:151.42ms
step:211/1480 train_time:30431ms step_avg:151.40ms
step:212/1480 train_time:30578ms step_avg:151.38ms
step:213/1480 train_time:30725ms step_avg:151.35ms
step:214/1480 train_time:30871ms step_avg:151.33ms
step:215/1480 train_time:31015ms step_avg:151.29ms
step:216/1480 train_time:31161ms step_avg:151.27ms
step:217/1480 train_time:31308ms step_avg:151.25ms
step:218/1480 train_time:31453ms step_avg:151.22ms
step:219/1480 train_time:31598ms step_avg:151.19ms
step:220/1480 train_time:31745ms step_avg:151.17ms
step:221/1480 train_time:32337ms step_avg:153.26ms
step:222/1480 train_time:32443ms step_avg:153.03ms
step:223/1480 train_time:32591ms step_avg:153.01ms
step:224/1480 train_time:32738ms step_avg:152.98ms
step:225/1480 train_time:32887ms step_avg:152.96ms
step:226/1480 train_time:33034ms step_avg:152.94ms
step:227/1480 train_time:33182ms step_avg:152.91ms
step:228/1480 train_time:33331ms step_avg:152.89ms
step:229/1480 train_time:33480ms step_avg:152.87ms
step:230/1480 train_time:33629ms step_avg:152.86ms
step:231/1480 train_time:33778ms step_avg:152.84ms
step:232/1480 train_time:33927ms step_avg:152.82ms
step:233/1480 train_time:34075ms step_avg:152.80ms
step:234/1480 train_time:34224ms step_avg:152.79ms
step:235/1480 train_time:34373ms step_avg:152.77ms
step:236/1480 train_time:34522ms step_avg:152.75ms
step:237/1480 train_time:34671ms step_avg:152.74ms
step:238/1480 train_time:34820ms step_avg:152.72ms
step:239/1480 train_time:34969ms step_avg:152.70ms
step:240/1480 train_time:35117ms step_avg:152.68ms
step:241/1480 train_time:35267ms step_avg:152.67ms
step:242/1480 train_time:35415ms step_avg:152.65ms
step:243/1480 train_time:35563ms step_avg:152.63ms
step:244/1480 train_time:35712ms step_avg:152.62ms
step:245/1480 train_time:35861ms step_avg:152.60ms
step:246/1480 train_time:36011ms step_avg:152.59ms
step:247/1480 train_time:36159ms step_avg:152.57ms
step:248/1480 train_time:36309ms step_avg:152.56ms
step:249/1480 train_time:36456ms step_avg:152.54ms
step:250/1480 train_time:36606ms step_avg:152.53ms
step:250/1480 val_loss:3.9928 train_time:36673ms step_avg:152.80ms
step:251/1480 train_time:36769ms step_avg:152.57ms
step:252/1480 train_time:36913ms step_avg:152.53ms
step:253/1480 train_time:37062ms step_avg:152.52ms
step:254/1480 train_time:37210ms step_avg:152.50ms
step:255/1480 train_time:37359ms step_avg:152.48ms
step:256/1480 train_time:37507ms step_avg:152.47ms
step:257/1480 train_time:37656ms step_avg:152.45ms
step:258/1480 train_time:37804ms step_avg:152.44ms
step:259/1480 train_time:37954ms step_avg:152.42ms
step:260/1480 train_time:38102ms step_avg:152.41ms
step:261/1480 train_time:38252ms step_avg:152.40ms
step:262/1480 train_time:38400ms step_avg:152.38ms
step:263/1480 train_time:38549ms step_avg:152.37ms
step:264/1480 train_time:38697ms step_avg:152.35ms
step:265/1480 train_time:38844ms step_avg:152.33ms
step:266/1480 train_time:38993ms step_avg:152.32ms
step:267/1480 train_time:39141ms step_avg:152.30ms
step:268/1480 train_time:39290ms step_avg:152.29ms
step:269/1480 train_time:39438ms step_avg:152.27ms
step:270/1480 train_time:39586ms step_avg:152.25ms
step:271/1480 train_time:39736ms step_avg:152.24ms
step:272/1480 train_time:39884ms step_avg:152.23ms
step:273/1480 train_time:40034ms step_avg:152.22ms
step:274/1480 train_time:40182ms step_avg:152.20ms
step:275/1480 train_time:40331ms step_avg:152.19ms
step:276/1480 train_time:40479ms step_avg:152.18ms
step:277/1480 train_time:40627ms step_avg:152.16ms
step:278/1480 train_time:40777ms step_avg:152.15ms
step:279/1480 train_time:40924ms step_avg:152.13ms
step:280/1480 train_time:41074ms step_avg:152.13ms
step:281/1480 train_time:41221ms step_avg:152.11ms
step:282/1480 train_time:41370ms step_avg:152.10ms
step:283/1480 train_time:41519ms step_avg:152.08ms
step:284/1480 train_time:41667ms step_avg:152.07ms
step:285/1480 train_time:41817ms step_avg:152.06ms
step:286/1480 train_time:41964ms step_avg:152.04ms
step:287/1480 train_time:42113ms step_avg:152.03ms
step:288/1480 train_time:42260ms step_avg:152.01ms
step:289/1480 train_time:42408ms step_avg:152.00ms
step:290/1480 train_time:42556ms step_avg:151.99ms
step:291/1480 train_time:42704ms step_avg:151.97ms
step:292/1480 train_time:42852ms step_avg:151.96ms
step:293/1480 train_time:43000ms step_avg:151.94ms
step:294/1480 train_time:43150ms step_avg:151.94ms
step:295/1480 train_time:43298ms step_avg:151.92ms
step:296/1480 train_time:43447ms step_avg:151.91ms
step:297/1480 train_time:43596ms step_avg:151.90ms
step:298/1480 train_time:43743ms step_avg:151.88ms
step:299/1480 train_time:43892ms step_avg:151.88ms
step:300/1480 train_time:44040ms step_avg:151.86ms
step:301/1480 train_time:44189ms step_avg:151.85ms
step:302/1480 train_time:44338ms step_avg:151.84ms
step:303/1480 train_time:44486ms step_avg:151.83ms
step:304/1480 train_time:44635ms step_avg:151.82ms
step:305/1480 train_time:44783ms step_avg:151.81ms
step:306/1480 train_time:44933ms step_avg:151.80ms
step:307/1480 train_time:45080ms step_avg:151.78ms
step:308/1480 train_time:45229ms step_avg:151.78ms
step:309/1480 train_time:45378ms step_avg:151.77ms
step:310/1480 train_time:45526ms step_avg:151.75ms
step:311/1480 train_time:45676ms step_avg:151.75ms
step:312/1480 train_time:45823ms step_avg:151.73ms
step:313/1480 train_time:45972ms step_avg:151.72ms
step:314/1480 train_time:46120ms step_avg:151.71ms
step:315/1480 train_time:46268ms step_avg:151.70ms
step:316/1480 train_time:46418ms step_avg:151.69ms
step:317/1480 train_time:46566ms step_avg:151.68ms
step:318/1480 train_time:46715ms step_avg:151.67ms
step:319/1480 train_time:46863ms step_avg:151.66ms
step:320/1480 train_time:47012ms step_avg:151.65ms
step:321/1480 train_time:47160ms step_avg:151.64ms
step:322/1480 train_time:47308ms step_avg:151.63ms
step:323/1480 train_time:47457ms step_avg:151.62ms
step:324/1480 train_time:47605ms step_avg:151.61ms
step:325/1480 train_time:47754ms step_avg:151.60ms
step:326/1480 train_time:47902ms step_avg:151.59ms
step:327/1480 train_time:48050ms step_avg:151.58ms
step:328/1480 train_time:48198ms step_avg:151.57ms
step:329/1480 train_time:48346ms step_avg:151.56ms
step:330/1480 train_time:48496ms step_avg:151.55ms
step:331/1480 train_time:48646ms step_avg:151.55ms
step:332/1480 train_time:48798ms step_avg:151.55ms
step:333/1480 train_time:48948ms step_avg:151.54ms
step:334/1480 train_time:49099ms step_avg:151.54ms
step:335/1480 train_time:49248ms step_avg:151.53ms
step:336/1480 train_time:49399ms step_avg:151.53ms
step:337/1480 train_time:49551ms step_avg:151.53ms
step:338/1480 train_time:49701ms step_avg:151.53ms
step:339/1480 train_time:49852ms step_avg:151.53ms
step:340/1480 train_time:50002ms step_avg:151.52ms
step:341/1480 train_time:50153ms step_avg:151.52ms
step:342/1480 train_time:50304ms step_avg:151.52ms
step:343/1480 train_time:50455ms step_avg:151.52ms
step:344/1480 train_time:50608ms step_avg:151.52ms
step:345/1480 train_time:50760ms step_avg:151.52ms
step:346/1480 train_time:50911ms step_avg:151.52ms
step:347/1480 train_time:51062ms step_avg:151.52ms
step:348/1480 train_time:51214ms step_avg:151.52ms
step:349/1480 train_time:51364ms step_avg:151.52ms
step:350/1480 train_time:51516ms step_avg:151.52ms
step:351/1480 train_time:51666ms step_avg:151.51ms
step:352/1480 train_time:51818ms step_avg:151.51ms
step:353/1480 train_time:51968ms step_avg:151.51ms
step:354/1480 train_time:52119ms step_avg:151.51ms
step:355/1480 train_time:52269ms step_avg:151.51ms
step:356/1480 train_time:52420ms step_avg:151.50ms
step:357/1480 train_time:52572ms step_avg:151.50ms
step:358/1480 train_time:52722ms step_avg:151.50ms
step:359/1480 train_time:52874ms step_avg:151.50ms
step:360/1480 train_time:53024ms step_avg:151.50ms
step:361/1480 train_time:53176ms step_avg:151.50ms
step:362/1480 train_time:53326ms step_avg:151.50ms
step:363/1480 train_time:53478ms step_avg:151.49ms
step:364/1480 train_time:53628ms step_avg:151.49ms
step:365/1480 train_time:53779ms step_avg:151.49ms
step:366/1480 train_time:53930ms step_avg:151.49ms
step:367/1480 train_time:54080ms step_avg:151.48ms
step:368/1480 train_time:54231ms step_avg:151.48ms
step:369/1480 train_time:54383ms step_avg:151.48ms
step:370/1480 train_time:54535ms step_avg:151.49ms
step:371/1480 train_time:54686ms step_avg:151.48ms
step:372/1480 train_time:54837ms step_avg:151.48ms
step:373/1480 train_time:54988ms step_avg:151.48ms
step:374/1480 train_time:55138ms step_avg:151.48ms
step:375/1480 train_time:55289ms step_avg:151.48ms
step:375/1480 val_loss:3.8109 train_time:55358ms step_avg:151.66ms
step:376/1480 train_time:55453ms step_avg:151.51ms
step:377/1480 train_time:55599ms step_avg:151.50ms
step:378/1480 train_time:55751ms step_avg:151.50ms
step:379/1480 train_time:55915ms step_avg:151.53ms
step:380/1480 train_time:56052ms step_avg:151.49ms
step:381/1480 train_time:56201ms step_avg:151.49ms
step:382/1480 train_time:56353ms step_avg:151.49ms
step:383/1480 train_time:56504ms step_avg:151.49ms
step:384/1480 train_time:56656ms step_avg:151.49ms
step:385/1480 train_time:56808ms step_avg:151.49ms
step:386/1480 train_time:56957ms step_avg:151.48ms
step:387/1480 train_time:57109ms step_avg:151.48ms
step:388/1480 train_time:57259ms step_avg:151.48ms
step:389/1480 train_time:57410ms step_avg:151.48ms
step:390/1480 train_time:57560ms step_avg:151.47ms
step:391/1480 train_time:57712ms step_avg:151.47ms
step:392/1480 train_time:57862ms step_avg:151.47ms
step:393/1480 train_time:58013ms step_avg:151.47ms
step:394/1480 train_time:58164ms step_avg:151.47ms
step:395/1480 train_time:58315ms step_avg:151.47ms
step:396/1480 train_time:58466ms step_avg:151.47ms
step:397/1480 train_time:58617ms step_avg:151.46ms
step:398/1480 train_time:58768ms step_avg:151.46ms
step:399/1480 train_time:58918ms step_avg:151.46ms
step:400/1480 train_time:59070ms step_avg:151.46ms
step:401/1480 train_time:59222ms step_avg:151.46ms
step:402/1480 train_time:59373ms step_avg:151.46ms
step:403/1480 train_time:59524ms step_avg:151.46ms
step:404/1480 train_time:59676ms step_avg:151.46ms
step:405/1480 train_time:59827ms step_avg:151.46ms
step:406/1480 train_time:59978ms step_avg:151.46ms
step:407/1480 train_time:60130ms step_avg:151.46ms
step:408/1480 train_time:60280ms step_avg:151.46ms
step:409/1480 train_time:60432ms step_avg:151.46ms
step:410/1480 train_time:60581ms step_avg:151.45ms
step:411/1480 train_time:60733ms step_avg:151.45ms
step:412/1480 train_time:60884ms step_avg:151.45ms
step:413/1480 train_time:61035ms step_avg:151.45ms
step:414/1480 train_time:61186ms step_avg:151.45ms
step:415/1480 train_time:61336ms step_avg:151.45ms
step:416/1480 train_time:61487ms step_avg:151.45ms
step:417/1480 train_time:61637ms step_avg:151.44ms
step:418/1480 train_time:61788ms step_avg:151.44ms
step:419/1480 train_time:61938ms step_avg:151.44ms
step:420/1480 train_time:62090ms step_avg:151.44ms
step:421/1480 train_time:62239ms step_avg:151.43ms
step:422/1480 train_time:62391ms step_avg:151.43ms
step:423/1480 train_time:62541ms step_avg:151.43ms
step:424/1480 train_time:62693ms step_avg:151.43ms
step:425/1480 train_time:62843ms step_avg:151.43ms
step:426/1480 train_time:62994ms step_avg:151.43ms
step:427/1480 train_time:63145ms step_avg:151.43ms
step:428/1480 train_time:63295ms step_avg:151.42ms
step:429/1480 train_time:63446ms step_avg:151.42ms
step:430/1480 train_time:63597ms step_avg:151.42ms
step:431/1480 train_time:63749ms step_avg:151.42ms
step:432/1480 train_time:63899ms step_avg:151.42ms
step:433/1480 train_time:64051ms step_avg:151.42ms
step:434/1480 train_time:64202ms step_avg:151.42ms
step:435/1480 train_time:64353ms step_avg:151.42ms
step:436/1480 train_time:64505ms step_avg:151.42ms
step:437/1480 train_time:64656ms step_avg:151.42ms
step:438/1480 train_time:64809ms step_avg:151.42ms
step:439/1480 train_time:64960ms step_avg:151.42ms
step:440/1480 train_time:65112ms step_avg:151.42ms
step:441/1480 train_time:65265ms step_avg:151.43ms
step:442/1480 train_time:65417ms step_avg:151.43ms
step:443/1480 train_time:65570ms step_avg:151.43ms
step:444/1480 train_time:65722ms step_avg:151.43ms
step:445/1480 train_time:65875ms step_avg:151.44ms
step:446/1480 train_time:66029ms step_avg:151.44ms
step:447/1480 train_time:66184ms step_avg:151.45ms
step:448/1480 train_time:66336ms step_avg:151.45ms
step:449/1480 train_time:66490ms step_avg:151.46ms
step:450/1480 train_time:66642ms step_avg:151.46ms
step:451/1480 train_time:66795ms step_avg:151.46ms
step:452/1480 train_time:66948ms step_avg:151.47ms
step:453/1480 train_time:67102ms step_avg:151.47ms
step:454/1480 train_time:67255ms step_avg:151.48ms
step:455/1480 train_time:67409ms step_avg:151.48ms
step:456/1480 train_time:67561ms step_avg:151.48ms
step:457/1480 train_time:67714ms step_avg:151.49ms
step:458/1480 train_time:67867ms step_avg:151.49ms
step:459/1480 train_time:68020ms step_avg:151.49ms
step:460/1480 train_time:68172ms step_avg:151.49ms
step:461/1480 train_time:68326ms step_avg:151.50ms
step:462/1480 train_time:68479ms step_avg:151.50ms
step:463/1480 train_time:68633ms step_avg:151.51ms
step:464/1480 train_time:68786ms step_avg:151.51ms
step:465/1480 train_time:68938ms step_avg:151.51ms
step:466/1480 train_time:69091ms step_avg:151.52ms
step:467/1480 train_time:69243ms step_avg:151.52ms
step:468/1480 train_time:69397ms step_avg:151.52ms
step:469/1480 train_time:69550ms step_avg:151.53ms
step:470/1480 train_time:69703ms step_avg:151.53ms
step:471/1480 train_time:69856ms step_avg:151.53ms
step:472/1480 train_time:70009ms step_avg:151.54ms
step:473/1480 train_time:70162ms step_avg:151.54ms
step:474/1480 train_time:70315ms step_avg:151.54ms
step:475/1480 train_time:70467ms step_avg:151.54ms
step:476/1480 train_time:70620ms step_avg:151.55ms
step:477/1480 train_time:70773ms step_avg:151.55ms
step:478/1480 train_time:70927ms step_avg:151.55ms
step:479/1480 train_time:71080ms step_avg:151.56ms
step:480/1480 train_time:71233ms step_avg:151.56ms
step:481/1480 train_time:71386ms step_avg:151.56ms
step:482/1480 train_time:71538ms step_avg:151.56ms
step:483/1480 train_time:71691ms step_avg:151.57ms
step:484/1480 train_time:71844ms step_avg:151.57ms
step:485/1480 train_time:71997ms step_avg:151.57ms
step:486/1480 train_time:72151ms step_avg:151.58ms
step:487/1480 train_time:72304ms step_avg:151.58ms
step:488/1480 train_time:72456ms step_avg:151.58ms
step:489/1480 train_time:72610ms step_avg:151.59ms
step:490/1480 train_time:72762ms step_avg:151.59ms
step:491/1480 train_time:72914ms step_avg:151.59ms
step:492/1480 train_time:73066ms step_avg:151.59ms
step:493/1480 train_time:73219ms step_avg:151.59ms
step:494/1480 train_time:73372ms step_avg:151.60ms
step:495/1480 train_time:73527ms step_avg:151.60ms
step:496/1480 train_time:73681ms step_avg:151.61ms
step:497/1480 train_time:73834ms step_avg:151.61ms
step:498/1480 train_time:73986ms step_avg:151.61ms
step:499/1480 train_time:74139ms step_avg:151.61ms
step:500/1480 train_time:74292ms step_avg:151.62ms
step:500/1480 val_loss:3.6869 train_time:74360ms step_avg:151.75ms
step:501/1480 train_time:74451ms step_avg:151.63ms
step:502/1480 train_time:74603ms step_avg:151.63ms
step:503/1480 train_time:74756ms step_avg:151.64ms
step:504/1480 train_time:74909ms step_avg:151.64ms
step:505/1480 train_time:75061ms step_avg:151.64ms
step:506/1480 train_time:75213ms step_avg:151.64ms
step:507/1480 train_time:75366ms step_avg:151.64ms
step:508/1480 train_time:75519ms step_avg:151.64ms
step:509/1480 train_time:75673ms step_avg:151.65ms
step:510/1480 train_time:75827ms step_avg:151.65ms
step:511/1480 train_time:75980ms step_avg:151.66ms
step:512/1480 train_time:76132ms step_avg:151.66ms
step:513/1480 train_time:76285ms step_avg:151.66ms
step:514/1480 train_time:76436ms step_avg:151.66ms
step:515/1480 train_time:76590ms step_avg:151.66ms
step:516/1480 train_time:76744ms step_avg:151.67ms
step:517/1480 train_time:76898ms step_avg:151.67ms
step:518/1480 train_time:77051ms step_avg:151.68ms
step:519/1480 train_time:77204ms step_avg:151.68ms
step:520/1480 train_time:77357ms step_avg:151.68ms
step:521/1480 train_time:77510ms step_avg:151.68ms
step:522/1480 train_time:77663ms step_avg:151.69ms
step:523/1480 train_time:77816ms step_avg:151.69ms
step:524/1480 train_time:77969ms step_avg:151.69ms
step:525/1480 train_time:78123ms step_avg:151.69ms
step:526/1480 train_time:78276ms step_avg:151.70ms
step:527/1480 train_time:78429ms step_avg:151.70ms
step:528/1480 train_time:78581ms step_avg:151.70ms
step:529/1480 train_time:78734ms step_avg:151.70ms
step:530/1480 train_time:78888ms step_avg:151.71ms
step:531/1480 train_time:79040ms step_avg:151.71ms
step:532/1480 train_time:79193ms step_avg:151.71ms
step:533/1480 train_time:79347ms step_avg:151.71ms
step:534/1480 train_time:79499ms step_avg:151.72ms
step:535/1480 train_time:79652ms step_avg:151.72ms
step:536/1480 train_time:79805ms step_avg:151.72ms
step:537/1480 train_time:79957ms step_avg:151.72ms
step:538/1480 train_time:80110ms step_avg:151.72ms
step:539/1480 train_time:80263ms step_avg:151.73ms
step:540/1480 train_time:80417ms step_avg:151.73ms
step:541/1480 train_time:80570ms step_avg:151.73ms
step:542/1480 train_time:80723ms step_avg:151.74ms
step:543/1480 train_time:80876ms step_avg:151.74ms
step:544/1480 train_time:81028ms step_avg:151.74ms
step:545/1480 train_time:81181ms step_avg:151.74ms
step:546/1480 train_time:81334ms step_avg:151.74ms
step:547/1480 train_time:81487ms step_avg:151.75ms
step:548/1480 train_time:81639ms step_avg:151.75ms
step:549/1480 train_time:81792ms step_avg:151.75ms
step:550/1480 train_time:81947ms step_avg:151.75ms
step:551/1480 train_time:82102ms step_avg:151.76ms
step:552/1480 train_time:82257ms step_avg:151.77ms
step:553/1480 train_time:82412ms step_avg:151.77ms
step:554/1480 train_time:82566ms step_avg:151.78ms
step:555/1480 train_time:82720ms step_avg:151.78ms
step:556/1480 train_time:82875ms step_avg:151.79ms
step:557/1480 train_time:83030ms step_avg:151.79ms
step:558/1480 train_time:83185ms step_avg:151.80ms
step:559/1480 train_time:83339ms step_avg:151.80ms
step:560/1480 train_time:83494ms step_avg:151.81ms
step:561/1480 train_time:83649ms step_avg:151.81ms
step:562/1480 train_time:83804ms step_avg:151.82ms
step:563/1480 train_time:83959ms step_avg:151.83ms
step:564/1480 train_time:84113ms step_avg:151.83ms
step:565/1480 train_time:84267ms step_avg:151.83ms
step:566/1480 train_time:84423ms step_avg:151.84ms
step:567/1480 train_time:84578ms step_avg:151.85ms
step:568/1480 train_time:84732ms step_avg:151.85ms
step:569/1480 train_time:84899ms step_avg:151.88ms
step:570/1480 train_time:85041ms step_avg:151.86ms
step:571/1480 train_time:85196ms step_avg:151.87ms
step:572/1480 train_time:85352ms step_avg:151.87ms
step:573/1480 train_time:85506ms step_avg:151.88ms
step:574/1480 train_time:85662ms step_avg:151.88ms
step:575/1480 train_time:85818ms step_avg:151.89ms
step:576/1480 train_time:85973ms step_avg:151.90ms
step:577/1480 train_time:86127ms step_avg:151.90ms
step:578/1480 train_time:86281ms step_avg:151.90ms
step:579/1480 train_time:86436ms step_avg:151.91ms
step:580/1480 train_time:86591ms step_avg:151.91ms
step:581/1480 train_time:86745ms step_avg:151.92ms
step:582/1480 train_time:86899ms step_avg:151.92ms
step:583/1480 train_time:87053ms step_avg:151.93ms
step:584/1480 train_time:87209ms step_avg:151.93ms
step:585/1480 train_time:87362ms step_avg:151.93ms
step:586/1480 train_time:87518ms step_avg:151.94ms
step:587/1480 train_time:87673ms step_avg:151.95ms
step:588/1480 train_time:87828ms step_avg:151.95ms
step:589/1480 train_time:87983ms step_avg:151.96ms
step:590/1480 train_time:88139ms step_avg:151.96ms
step:591/1480 train_time:88293ms step_avg:151.97ms
step:592/1480 train_time:88448ms step_avg:151.97ms
step:593/1480 train_time:88604ms step_avg:151.98ms
step:594/1480 train_time:88760ms step_avg:151.99ms
step:595/1480 train_time:88916ms step_avg:151.99ms
step:596/1480 train_time:89073ms step_avg:152.00ms
step:597/1480 train_time:89228ms step_avg:152.01ms
step:598/1480 train_time:89381ms step_avg:152.01ms
step:599/1480 train_time:89535ms step_avg:152.01ms
step:600/1480 train_time:89691ms step_avg:152.02ms
step:601/1480 train_time:89847ms step_avg:152.02ms
step:602/1480 train_time:90001ms step_avg:152.03ms
step:603/1480 train_time:90156ms step_avg:152.03ms
step:604/1480 train_time:90311ms step_avg:152.04ms
step:605/1480 train_time:90465ms step_avg:152.04ms
step:606/1480 train_time:90620ms step_avg:152.05ms
step:607/1480 train_time:90776ms step_avg:152.05ms
step:608/1480 train_time:90931ms step_avg:152.06ms
step:609/1480 train_time:91086ms step_avg:152.06ms
step:610/1480 train_time:91239ms step_avg:152.07ms
step:611/1480 train_time:91394ms step_avg:152.07ms
step:612/1480 train_time:91549ms step_avg:152.08ms
step:613/1480 train_time:91705ms step_avg:152.08ms
step:614/1480 train_time:91860ms step_avg:152.09ms
step:615/1480 train_time:92015ms step_avg:152.09ms
step:616/1480 train_time:92168ms step_avg:152.09ms
step:617/1480 train_time:92324ms step_avg:152.10ms
step:618/1480 train_time:92479ms step_avg:152.10ms
step:619/1480 train_time:92633ms step_avg:152.11ms
step:620/1480 train_time:92789ms step_avg:152.11ms
step:621/1480 train_time:92944ms step_avg:152.12ms
step:622/1480 train_time:93099ms step_avg:152.12ms
step:623/1480 train_time:93254ms step_avg:152.13ms
step:624/1480 train_time:93409ms step_avg:152.13ms
step:625/1480 train_time:93563ms step_avg:152.13ms
step:625/1480 val_loss:3.6089 train_time:93634ms step_avg:152.25ms
step:626/1480 train_time:93725ms step_avg:152.15ms
step:627/1480 train_time:93879ms step_avg:152.15ms
step:628/1480 train_time:94034ms step_avg:152.16ms
step:629/1480 train_time:94188ms step_avg:152.16ms
step:630/1480 train_time:94342ms step_avg:152.16ms
step:631/1480 train_time:94496ms step_avg:152.17ms
step:632/1480 train_time:94650ms step_avg:152.17ms
step:633/1480 train_time:94805ms step_avg:152.17ms
step:634/1480 train_time:94959ms step_avg:152.18ms
step:635/1480 train_time:95114ms step_avg:152.18ms
step:636/1480 train_time:95268ms step_avg:152.19ms
step:637/1480 train_time:95423ms step_avg:152.19ms
step:638/1480 train_time:95578ms step_avg:152.19ms
step:639/1480 train_time:95732ms step_avg:152.20ms
step:640/1480 train_time:95887ms step_avg:152.20ms
step:641/1480 train_time:96042ms step_avg:152.21ms
step:642/1480 train_time:96196ms step_avg:152.21ms
step:643/1480 train_time:96351ms step_avg:152.21ms
step:644/1480 train_time:96505ms step_avg:152.22ms
step:645/1480 train_time:96660ms step_avg:152.22ms
step:646/1480 train_time:96815ms step_avg:152.23ms
step:647/1480 train_time:96970ms step_avg:152.23ms
step:648/1480 train_time:97125ms step_avg:152.23ms
step:649/1480 train_time:97280ms step_avg:152.24ms
step:650/1480 train_time:97436ms step_avg:152.24ms
step:651/1480 train_time:97591ms step_avg:152.25ms
step:652/1480 train_time:97746ms step_avg:152.25ms
step:653/1480 train_time:97902ms step_avg:152.26ms
step:654/1480 train_time:98056ms step_avg:152.26ms
step:655/1480 train_time:98211ms step_avg:152.26ms
step:656/1480 train_time:98365ms step_avg:152.27ms
step:657/1480 train_time:98520ms step_avg:152.27ms
step:658/1480 train_time:98675ms step_avg:152.28ms
step:659/1480 train_time:98830ms step_avg:152.28ms
step:660/1480 train_time:98988ms step_avg:152.29ms
step:661/1480 train_time:99144ms step_avg:152.29ms
step:662/1480 train_time:99300ms step_avg:152.30ms
step:663/1480 train_time:99455ms step_avg:152.30ms
step:664/1480 train_time:99612ms step_avg:152.31ms
step:665/1480 train_time:99770ms step_avg:152.32ms
step:666/1480 train_time:99926ms step_avg:152.33ms
step:667/1480 train_time:100082ms step_avg:152.33ms
step:668/1480 train_time:100238ms step_avg:152.34ms
step:669/1480 train_time:100396ms step_avg:152.35ms
step:670/1480 train_time:100552ms step_avg:152.35ms
step:671/1480 train_time:100708ms step_avg:152.36ms
step:672/1480 train_time:100864ms step_avg:152.36ms
step:673/1480 train_time:101019ms step_avg:152.37ms
step:674/1480 train_time:101176ms step_avg:152.37ms
step:675/1480 train_time:101332ms step_avg:152.38ms
step:676/1480 train_time:101490ms step_avg:152.39ms
step:677/1480 train_time:101648ms step_avg:152.40ms
step:678/1480 train_time:101804ms step_avg:152.40ms
step:679/1480 train_time:101959ms step_avg:152.41ms
step:680/1480 train_time:102116ms step_avg:152.41ms
step:681/1480 train_time:102272ms step_avg:152.42ms
step:682/1480 train_time:102428ms step_avg:152.42ms
step:683/1480 train_time:102584ms step_avg:152.43ms
step:684/1480 train_time:102742ms step_avg:152.44ms
step:685/1480 train_time:102900ms step_avg:152.44ms
step:686/1480 train_time:103056ms step_avg:152.45ms
step:687/1480 train_time:103212ms step_avg:152.46ms
step:688/1480 train_time:103370ms step_avg:152.46ms
step:689/1480 train_time:103528ms step_avg:152.47ms
step:690/1480 train_time:103686ms step_avg:152.48ms
step:691/1480 train_time:103843ms step_avg:152.49ms
step:692/1480 train_time:103999ms step_avg:152.49ms
step:693/1480 train_time:104155ms step_avg:152.50ms
step:694/1480 train_time:104312ms step_avg:152.50ms
step:695/1480 train_time:104470ms step_avg:152.51ms
step:696/1480 train_time:104625ms step_avg:152.51ms
step:697/1480 train_time:104782ms step_avg:152.52ms
step:698/1480 train_time:104938ms step_avg:152.53ms
step:699/1480 train_time:105095ms step_avg:152.53ms
step:700/1480 train_time:105251ms step_avg:152.54ms
step:701/1480 train_time:105407ms step_avg:152.54ms
step:702/1480 train_time:105562ms step_avg:152.55ms
step:703/1480 train_time:105719ms step_avg:152.55ms
step:704/1480 train_time:105875ms step_avg:152.56ms
step:705/1480 train_time:106031ms step_avg:152.56ms
step:706/1480 train_time:106189ms step_avg:152.57ms
step:707/1480 train_time:106346ms step_avg:152.58ms
step:708/1480 train_time:106500ms step_avg:152.58ms
step:709/1480 train_time:106656ms step_avg:152.58ms
step:710/1480 train_time:106812ms step_avg:152.59ms
step:711/1480 train_time:106968ms step_avg:152.59ms
step:712/1480 train_time:107124ms step_avg:152.60ms
step:713/1480 train_time:107281ms step_avg:152.60ms
step:714/1480 train_time:107437ms step_avg:152.61ms
step:715/1480 train_time:107593ms step_avg:152.61ms
step:716/1480 train_time:107749ms step_avg:152.62ms
step:717/1480 train_time:107905ms step_avg:152.62ms
step:718/1480 train_time:108059ms step_avg:152.63ms
step:719/1480 train_time:108216ms step_avg:152.63ms
step:720/1480 train_time:108373ms step_avg:152.64ms
step:721/1480 train_time:108532ms step_avg:152.65ms
step:722/1480 train_time:108689ms step_avg:152.65ms
step:723/1480 train_time:108845ms step_avg:152.66ms
step:724/1480 train_time:109002ms step_avg:152.66ms
step:725/1480 train_time:109157ms step_avg:152.67ms
step:726/1480 train_time:109315ms step_avg:152.67ms
step:727/1480 train_time:109472ms step_avg:152.68ms
step:728/1480 train_time:109628ms step_avg:152.69ms
step:729/1480 train_time:109786ms step_avg:152.69ms
step:730/1480 train_time:109944ms step_avg:152.70ms
step:731/1480 train_time:110101ms step_avg:152.71ms
step:732/1480 train_time:110256ms step_avg:152.71ms
step:733/1480 train_time:110414ms step_avg:152.72ms
step:734/1480 train_time:110572ms step_avg:152.72ms
step:735/1480 train_time:110728ms step_avg:152.73ms
step:736/1480 train_time:110884ms step_avg:152.73ms
step:737/1480 train_time:111039ms step_avg:152.74ms
step:738/1480 train_time:111195ms step_avg:152.74ms
step:739/1480 train_time:111352ms step_avg:152.75ms
step:740/1480 train_time:111510ms step_avg:152.75ms
step:741/1480 train_time:111669ms step_avg:152.76ms
step:742/1480 train_time:111825ms step_avg:152.77ms
step:743/1480 train_time:111981ms step_avg:152.77ms
step:744/1480 train_time:112137ms step_avg:152.77ms
step:745/1480 train_time:112295ms step_avg:152.78ms
step:746/1480 train_time:112451ms step_avg:152.79ms
step:747/1480 train_time:112607ms step_avg:152.79ms
step:748/1480 train_time:112765ms step_avg:152.80ms
step:749/1480 train_time:112921ms step_avg:152.80ms
step:750/1480 train_time:113077ms step_avg:152.81ms
step:750/1480 val_loss:3.5532 train_time:113147ms step_avg:152.90ms
step:751/1480 train_time:113239ms step_avg:152.82ms
step:752/1480 train_time:113395ms step_avg:152.82ms
step:753/1480 train_time:113551ms step_avg:152.83ms
step:754/1480 train_time:113706ms step_avg:152.83ms
step:755/1480 train_time:113863ms step_avg:152.84ms
step:756/1480 train_time:114018ms step_avg:152.84ms
step:757/1480 train_time:114176ms step_avg:152.85ms
step:758/1480 train_time:114333ms step_avg:152.85ms
step:759/1480 train_time:114503ms step_avg:152.87ms
step:760/1480 train_time:114647ms step_avg:152.86ms
step:761/1480 train_time:114803ms step_avg:152.87ms
step:762/1480 train_time:114960ms step_avg:152.87ms
step:763/1480 train_time:115116ms step_avg:152.88ms
step:764/1480 train_time:115274ms step_avg:152.88ms
step:765/1480 train_time:115431ms step_avg:152.89ms
step:766/1480 train_time:115589ms step_avg:152.90ms
step:767/1480 train_time:115747ms step_avg:152.90ms
step:768/1480 train_time:115903ms step_avg:152.91ms
step:769/1480 train_time:116061ms step_avg:152.91ms
step:770/1480 train_time:116218ms step_avg:152.92ms
step:771/1480 train_time:116375ms step_avg:152.92ms
step:772/1480 train_time:116534ms step_avg:152.93ms
step:773/1480 train_time:116691ms step_avg:152.94ms
step:774/1480 train_time:116849ms step_avg:152.94ms
step:775/1480 train_time:117006ms step_avg:152.95ms
step:776/1480 train_time:117164ms step_avg:152.96ms
step:777/1480 train_time:117323ms step_avg:152.96ms
step:778/1480 train_time:117481ms step_avg:152.97ms
step:779/1480 train_time:117637ms step_avg:152.97ms
step:780/1480 train_time:117796ms step_avg:152.98ms
step:781/1480 train_time:117953ms step_avg:152.99ms
step:782/1480 train_time:118110ms step_avg:152.99ms
step:783/1480 train_time:118267ms step_avg:153.00ms
step:784/1480 train_time:118424ms step_avg:153.00ms
step:785/1480 train_time:118581ms step_avg:153.01ms
step:786/1480 train_time:118739ms step_avg:153.01ms
step:787/1480 train_time:118899ms step_avg:153.02ms
step:788/1480 train_time:119057ms step_avg:153.03ms
step:789/1480 train_time:119214ms step_avg:153.03ms
step:790/1480 train_time:119373ms step_avg:153.04ms
step:791/1480 train_time:119532ms step_avg:153.05ms
step:792/1480 train_time:119690ms step_avg:153.06ms
step:793/1480 train_time:119847ms step_avg:153.06ms
step:794/1480 train_time:120005ms step_avg:153.07ms
step:795/1480 train_time:120164ms step_avg:153.08ms
step:796/1480 train_time:120324ms step_avg:153.08ms
step:797/1480 train_time:120482ms step_avg:153.09ms
step:798/1480 train_time:120640ms step_avg:153.10ms
step:799/1480 train_time:120802ms step_avg:153.11ms
step:800/1480 train_time:120960ms step_avg:153.11ms
step:801/1480 train_time:121117ms step_avg:153.12ms
step:802/1480 train_time:121279ms step_avg:153.13ms
step:803/1480 train_time:121437ms step_avg:153.14ms
step:804/1480 train_time:121595ms step_avg:153.14ms
step:805/1480 train_time:121754ms step_avg:153.15ms
step:806/1480 train_time:121911ms step_avg:153.15ms
step:807/1480 train_time:122069ms step_avg:153.16ms
step:808/1480 train_time:122227ms step_avg:153.17ms
step:809/1480 train_time:122386ms step_avg:153.17ms
step:810/1480 train_time:122543ms step_avg:153.18ms
step:811/1480 train_time:122700ms step_avg:153.18ms
step:812/1480 train_time:122857ms step_avg:153.19ms
step:813/1480 train_time:123014ms step_avg:153.19ms
step:814/1480 train_time:123172ms step_avg:153.20ms
step:815/1480 train_time:123329ms step_avg:153.20ms
step:816/1480 train_time:123490ms step_avg:153.21ms
step:817/1480 train_time:123648ms step_avg:153.22ms
step:818/1480 train_time:123805ms step_avg:153.22ms
step:819/1480 train_time:123964ms step_avg:153.23ms
step:820/1480 train_time:124122ms step_avg:153.24ms
step:821/1480 train_time:124279ms step_avg:153.24ms
step:822/1480 train_time:124436ms step_avg:153.25ms
step:823/1480 train_time:124596ms step_avg:153.25ms
step:824/1480 train_time:124754ms step_avg:153.26ms
step:825/1480 train_time:124913ms step_avg:153.27ms
step:826/1480 train_time:125074ms step_avg:153.28ms
step:827/1480 train_time:125233ms step_avg:153.28ms
step:828/1480 train_time:125391ms step_avg:153.29ms
step:829/1480 train_time:125549ms step_avg:153.30ms
step:830/1480 train_time:125707ms step_avg:153.30ms
step:831/1480 train_time:125865ms step_avg:153.31ms
step:832/1480 train_time:126023ms step_avg:153.31ms
step:833/1480 train_time:126181ms step_avg:153.32ms
step:834/1480 train_time:126339ms step_avg:153.32ms
step:835/1480 train_time:126498ms step_avg:153.33ms
step:836/1480 train_time:126656ms step_avg:153.34ms
step:837/1480 train_time:126813ms step_avg:153.34ms
step:838/1480 train_time:126971ms step_avg:153.35ms
step:839/1480 train_time:127128ms step_avg:153.35ms
step:840/1480 train_time:127286ms step_avg:153.36ms
step:841/1480 train_time:127442ms step_avg:153.36ms
step:842/1480 train_time:127602ms step_avg:153.37ms
step:843/1480 train_time:127758ms step_avg:153.37ms
step:844/1480 train_time:127914ms step_avg:153.37ms
step:845/1480 train_time:128071ms step_avg:153.38ms
step:846/1480 train_time:128230ms step_avg:153.38ms
step:847/1480 train_time:128389ms step_avg:153.39ms
step:848/1480 train_time:128546ms step_avg:153.40ms
step:849/1480 train_time:128704ms step_avg:153.40ms
step:850/1480 train_time:128862ms step_avg:153.41ms
step:851/1480 train_time:129020ms step_avg:153.41ms
step:852/1480 train_time:129178ms step_avg:153.42ms
step:853/1480 train_time:129336ms step_avg:153.42ms
step:854/1480 train_time:129496ms step_avg:153.43ms
step:855/1480 train_time:129654ms step_avg:153.44ms
step:856/1480 train_time:129811ms step_avg:153.44ms
step:857/1480 train_time:129969ms step_avg:153.45ms
step:858/1480 train_time:130128ms step_avg:153.45ms
step:859/1480 train_time:130284ms step_avg:153.46ms
step:860/1480 train_time:130441ms step_avg:153.46ms
step:861/1480 train_time:130601ms step_avg:153.47ms
step:862/1480 train_time:130764ms step_avg:153.48ms
step:863/1480 train_time:130923ms step_avg:153.48ms
step:864/1480 train_time:131081ms step_avg:153.49ms
step:865/1480 train_time:131239ms step_avg:153.50ms
step:866/1480 train_time:131398ms step_avg:153.50ms
step:867/1480 train_time:131559ms step_avg:153.51ms
step:868/1480 train_time:131716ms step_avg:153.51ms
step:869/1480 train_time:131875ms step_avg:153.52ms
step:870/1480 train_time:132033ms step_avg:153.53ms
step:871/1480 train_time:132190ms step_avg:153.53ms
step:872/1480 train_time:132347ms step_avg:153.53ms
step:873/1480 train_time:132504ms step_avg:153.54ms
step:874/1480 train_time:132664ms step_avg:153.55ms
step:875/1480 train_time:132823ms step_avg:153.55ms
step:875/1480 val_loss:3.5087 train_time:132896ms step_avg:153.64ms
step:876/1480 train_time:132986ms step_avg:153.56ms
step:877/1480 train_time:133141ms step_avg:153.57ms
step:878/1480 train_time:133299ms step_avg:153.57ms
step:879/1480 train_time:133457ms step_avg:153.58ms
step:880/1480 train_time:133614ms step_avg:153.58ms
step:881/1480 train_time:133772ms step_avg:153.58ms
step:882/1480 train_time:133931ms step_avg:153.59ms
step:883/1480 train_time:134090ms step_avg:153.60ms
step:884/1480 train_time:134253ms step_avg:153.61ms
step:885/1480 train_time:134413ms step_avg:153.62ms
step:886/1480 train_time:134575ms step_avg:153.62ms
step:887/1480 train_time:134735ms step_avg:153.63ms
step:888/1480 train_time:134899ms step_avg:153.64ms
step:889/1480 train_time:135060ms step_avg:153.65ms
step:890/1480 train_time:135216ms step_avg:153.65ms
step:891/1480 train_time:135375ms step_avg:153.66ms
step:892/1480 train_time:135535ms step_avg:153.67ms
step:893/1480 train_time:135694ms step_avg:153.67ms
step:894/1480 train_time:135853ms step_avg:153.68ms
step:895/1480 train_time:136015ms step_avg:153.69ms
step:896/1480 train_time:136172ms step_avg:153.69ms
step:897/1480 train_time:136334ms step_avg:153.70ms
step:898/1480 train_time:136495ms step_avg:153.71ms
step:899/1480 train_time:136653ms step_avg:153.72ms
step:900/1480 train_time:136812ms step_avg:153.72ms
step:901/1480 train_time:136974ms step_avg:153.73ms
step:902/1480 train_time:137132ms step_avg:153.74ms
step:903/1480 train_time:137294ms step_avg:153.74ms
step:904/1480 train_time:137454ms step_avg:153.75ms
step:905/1480 train_time:137612ms step_avg:153.76ms
step:906/1480 train_time:137772ms step_avg:153.76ms
step:907/1480 train_time:137935ms step_avg:153.77ms
step:908/1480 train_time:138092ms step_avg:153.78ms
step:909/1480 train_time:138253ms step_avg:153.79ms
step:910/1480 train_time:138417ms step_avg:153.80ms
step:911/1480 train_time:138576ms step_avg:153.80ms
step:912/1480 train_time:138735ms step_avg:153.81ms
step:913/1480 train_time:138896ms step_avg:153.82ms
step:914/1480 train_time:139056ms step_avg:153.82ms
step:915/1480 train_time:139216ms step_avg:153.83ms
step:916/1480 train_time:139375ms step_avg:153.84ms
step:917/1480 train_time:139533ms step_avg:153.84ms
step:918/1480 train_time:139695ms step_avg:153.85ms
step:919/1480 train_time:139857ms step_avg:153.86ms
step:920/1480 train_time:140016ms step_avg:153.86ms
step:921/1480 train_time:140177ms step_avg:153.87ms
step:922/1480 train_time:140338ms step_avg:153.88ms
step:923/1480 train_time:140495ms step_avg:153.88ms
step:924/1480 train_time:140655ms step_avg:153.89ms
step:925/1480 train_time:140814ms step_avg:153.89ms
step:926/1480 train_time:140973ms step_avg:153.90ms
step:927/1480 train_time:141130ms step_avg:153.90ms
step:928/1480 train_time:141290ms step_avg:153.91ms
step:929/1480 train_time:141452ms step_avg:153.92ms
step:930/1480 train_time:141613ms step_avg:153.93ms
step:931/1480 train_time:141771ms step_avg:153.93ms
step:932/1480 train_time:141931ms step_avg:153.94ms
step:933/1480 train_time:142090ms step_avg:153.94ms
step:934/1480 train_time:142248ms step_avg:153.95ms
step:935/1480 train_time:142410ms step_avg:153.96ms
step:936/1480 train_time:142570ms step_avg:153.96ms
step:937/1480 train_time:142731ms step_avg:153.97ms
step:938/1480 train_time:142889ms step_avg:153.98ms
step:939/1480 train_time:143052ms step_avg:153.99ms
step:940/1480 train_time:143214ms step_avg:153.99ms
step:941/1480 train_time:143373ms step_avg:154.00ms
step:942/1480 train_time:143532ms step_avg:154.00ms
step:943/1480 train_time:143694ms step_avg:154.01ms
step:944/1480 train_time:143858ms step_avg:154.02ms
step:945/1480 train_time:144015ms step_avg:154.03ms
step:946/1480 train_time:144178ms step_avg:154.04ms
step:947/1480 train_time:144338ms step_avg:154.04ms
step:948/1480 train_time:144497ms step_avg:154.05ms
step:949/1480 train_time:144668ms step_avg:154.07ms
step:950/1480 train_time:144815ms step_avg:154.06ms
step:951/1480 train_time:144977ms step_avg:154.07ms
step:952/1480 train_time:145135ms step_avg:154.07ms
step:953/1480 train_time:145295ms step_avg:154.08ms
step:954/1480 train_time:145459ms step_avg:154.09ms
step:955/1480 train_time:145616ms step_avg:154.09ms
step:956/1480 train_time:145775ms step_avg:154.10ms
step:957/1480 train_time:145937ms step_avg:154.10ms
step:958/1480 train_time:146099ms step_avg:154.11ms
step:959/1480 train_time:146257ms step_avg:154.12ms
step:960/1480 train_time:146416ms step_avg:154.12ms
step:961/1480 train_time:146575ms step_avg:154.13ms
step:962/1480 train_time:146735ms step_avg:154.13ms
step:963/1480 train_time:146895ms step_avg:154.14ms
step:964/1480 train_time:147056ms step_avg:154.15ms
step:965/1480 train_time:147215ms step_avg:154.15ms
step:966/1480 train_time:147374ms step_avg:154.16ms
step:967/1480 train_time:147533ms step_avg:154.16ms
step:968/1480 train_time:147692ms step_avg:154.17ms
step:969/1480 train_time:147852ms step_avg:154.17ms
step:970/1480 train_time:148011ms step_avg:154.18ms
step:971/1480 train_time:148172ms step_avg:154.19ms
step:972/1480 train_time:148330ms step_avg:154.19ms
step:973/1480 train_time:148488ms step_avg:154.19ms
step:974/1480 train_time:148650ms step_avg:154.20ms
step:975/1480 train_time:148811ms step_avg:154.21ms
step:976/1480 train_time:148971ms step_avg:154.21ms
step:977/1480 train_time:149132ms step_avg:154.22ms
step:978/1480 train_time:149292ms step_avg:154.23ms
step:979/1480 train_time:149452ms step_avg:154.23ms
step:980/1480 train_time:149612ms step_avg:154.24ms
step:981/1480 train_time:149774ms step_avg:154.25ms
step:982/1480 train_time:149932ms step_avg:154.25ms
step:983/1480 train_time:150092ms step_avg:154.26ms
step:984/1480 train_time:150251ms step_avg:154.26ms
step:985/1480 train_time:150413ms step_avg:154.27ms
step:986/1480 train_time:150573ms step_avg:154.28ms
step:987/1480 train_time:150731ms step_avg:154.28ms
step:988/1480 train_time:150892ms step_avg:154.29ms
step:989/1480 train_time:151052ms step_avg:154.29ms
step:990/1480 train_time:151214ms step_avg:154.30ms
step:991/1480 train_time:151375ms step_avg:154.31ms
step:992/1480 train_time:151539ms step_avg:154.32ms
step:993/1480 train_time:151708ms step_avg:154.33ms
step:994/1480 train_time:151868ms step_avg:154.34ms
step:995/1480 train_time:152028ms step_avg:154.34ms
step:996/1480 train_time:152186ms step_avg:154.35ms
step:997/1480 train_time:152347ms step_avg:154.35ms
step:998/1480 train_time:152508ms step_avg:154.36ms
step:999/1480 train_time:152668ms step_avg:154.37ms
step:1000/1480 train_time:152829ms step_avg:154.37ms
step:1000/1480 val_loss:3.4441 train_time:152903ms step_avg:154.45ms
step:1001/1480 train_time:152998ms step_avg:154.39ms
step:1002/1480 train_time:153155ms step_avg:154.39ms
step:1003/1480 train_time:153317ms step_avg:154.40ms
step:1004/1480 train_time:153478ms step_avg:154.40ms
step:1005/1480 train_time:153639ms step_avg:154.41ms
step:1006/1480 train_time:153798ms step_avg:154.42ms
step:1007/1480 train_time:153958ms step_avg:154.42ms
step:1008/1480 train_time:154118ms step_avg:154.43ms
step:1009/1480 train_time:154282ms step_avg:154.44ms
step:1010/1480 train_time:154440ms step_avg:154.44ms
step:1011/1480 train_time:154601ms step_avg:154.45ms
step:1012/1480 train_time:154759ms step_avg:154.45ms
step:1013/1480 train_time:154920ms step_avg:154.46ms
step:1014/1480 train_time:155081ms step_avg:154.46ms
step:1015/1480 train_time:155243ms step_avg:154.47ms
step:1016/1480 train_time:155402ms step_avg:154.48ms
step:1017/1480 train_time:155563ms step_avg:154.48ms
step:1018/1480 train_time:155724ms step_avg:154.49ms
step:1019/1480 train_time:155886ms step_avg:154.50ms
step:1020/1480 train_time:156046ms step_avg:154.50ms
step:1021/1480 train_time:156206ms step_avg:154.51ms
step:1022/1480 train_time:156367ms step_avg:154.51ms
step:1023/1480 train_time:156531ms step_avg:154.52ms
step:1024/1480 train_time:156691ms step_avg:154.53ms
step:1025/1480 train_time:156852ms step_avg:154.53ms
step:1026/1480 train_time:157014ms step_avg:154.54ms
step:1027/1480 train_time:157173ms step_avg:154.55ms
step:1028/1480 train_time:157337ms step_avg:154.55ms
step:1029/1480 train_time:157500ms step_avg:154.56ms
step:1030/1480 train_time:157660ms step_avg:154.57ms
step:1031/1480 train_time:157819ms step_avg:154.57ms
step:1032/1480 train_time:157981ms step_avg:154.58ms
step:1033/1480 train_time:158140ms step_avg:154.58ms
step:1034/1480 train_time:158299ms step_avg:154.59ms
step:1035/1480 train_time:158459ms step_avg:154.59ms
step:1036/1480 train_time:158618ms step_avg:154.60ms
step:1037/1480 train_time:158778ms step_avg:154.60ms
step:1038/1480 train_time:158938ms step_avg:154.61ms
step:1039/1480 train_time:159099ms step_avg:154.61ms
step:1040/1480 train_time:159258ms step_avg:154.62ms
step:1041/1480 train_time:159419ms step_avg:154.63ms
step:1042/1480 train_time:159577ms step_avg:154.63ms
step:1043/1480 train_time:159737ms step_avg:154.63ms
step:1044/1480 train_time:159897ms step_avg:154.64ms
step:1045/1480 train_time:160058ms step_avg:154.65ms
step:1046/1480 train_time:160218ms step_avg:154.65ms
step:1047/1480 train_time:160377ms step_avg:154.65ms
step:1048/1480 train_time:160538ms step_avg:154.66ms
step:1049/1480 train_time:160698ms step_avg:154.67ms
step:1050/1480 train_time:160859ms step_avg:154.67ms
step:1051/1480 train_time:161020ms step_avg:154.68ms
step:1052/1480 train_time:161179ms step_avg:154.68ms
step:1053/1480 train_time:161340ms step_avg:154.69ms
step:1054/1480 train_time:161500ms step_avg:154.69ms
step:1055/1480 train_time:161659ms step_avg:154.70ms
step:1056/1480 train_time:161817ms step_avg:154.70ms
step:1057/1480 train_time:161977ms step_avg:154.71ms
step:1058/1480 train_time:162138ms step_avg:154.71ms
step:1059/1480 train_time:162301ms step_avg:154.72ms
step:1060/1480 train_time:162463ms step_avg:154.73ms
step:1061/1480 train_time:162620ms step_avg:154.73ms
step:1062/1480 train_time:162779ms step_avg:154.73ms
step:1063/1480 train_time:162938ms step_avg:154.74ms
step:1064/1480 train_time:163096ms step_avg:154.74ms
step:1065/1480 train_time:163257ms step_avg:154.75ms
step:1066/1480 train_time:163418ms step_avg:154.75ms
step:1067/1480 train_time:163579ms step_avg:154.76ms
step:1068/1480 train_time:163739ms step_avg:154.76ms
step:1069/1480 train_time:163901ms step_avg:154.77ms
step:1070/1480 train_time:164061ms step_avg:154.77ms
step:1071/1480 train_time:164227ms step_avg:154.78ms
step:1072/1480 train_time:164386ms step_avg:154.79ms
step:1073/1480 train_time:164544ms step_avg:154.79ms
step:1074/1480 train_time:164705ms step_avg:154.80ms
step:1075/1480 train_time:164866ms step_avg:154.80ms
step:1076/1480 train_time:165025ms step_avg:154.81ms
step:1077/1480 train_time:165185ms step_avg:154.81ms
step:1078/1480 train_time:165350ms step_avg:154.82ms
step:1079/1480 train_time:165515ms step_avg:154.83ms
step:1080/1480 train_time:165676ms step_avg:154.84ms
step:1081/1480 train_time:165837ms step_avg:154.84ms
step:1082/1480 train_time:165996ms step_avg:154.85ms
step:1083/1480 train_time:166156ms step_avg:154.85ms
step:1084/1480 train_time:166315ms step_avg:154.86ms
step:1085/1480 train_time:166475ms step_avg:154.86ms
step:1086/1480 train_time:166637ms step_avg:154.87ms
step:1087/1480 train_time:166798ms step_avg:154.87ms
step:1088/1480 train_time:166957ms step_avg:154.88ms
step:1089/1480 train_time:167120ms step_avg:154.88ms
step:1090/1480 train_time:167282ms step_avg:154.89ms
step:1091/1480 train_time:167442ms step_avg:154.90ms
step:1092/1480 train_time:167604ms step_avg:154.90ms
step:1093/1480 train_time:167765ms step_avg:154.91ms
step:1094/1480 train_time:167925ms step_avg:154.91ms
step:1095/1480 train_time:168084ms step_avg:154.92ms
step:1096/1480 train_time:168247ms step_avg:154.92ms
step:1097/1480 train_time:168410ms step_avg:154.93ms
step:1098/1480 train_time:168573ms step_avg:154.94ms
step:1099/1480 train_time:168735ms step_avg:154.95ms
step:1100/1480 train_time:168898ms step_avg:154.95ms
step:1101/1480 train_time:169060ms step_avg:154.96ms
step:1102/1480 train_time:169222ms step_avg:154.97ms
step:1103/1480 train_time:169391ms step_avg:154.98ms
step:1104/1480 train_time:169554ms step_avg:154.98ms
step:1105/1480 train_time:169716ms step_avg:154.99ms
step:1106/1480 train_time:169877ms step_avg:155.00ms
step:1107/1480 train_time:170039ms step_avg:155.00ms
step:1108/1480 train_time:170198ms step_avg:155.01ms
step:1109/1480 train_time:170358ms step_avg:155.01ms
step:1110/1480 train_time:170518ms step_avg:155.02ms
step:1111/1480 train_time:170678ms step_avg:155.02ms
step:1112/1480 train_time:170840ms step_avg:155.03ms
step:1113/1480 train_time:171007ms step_avg:155.04ms
step:1114/1480 train_time:171169ms step_avg:155.04ms
step:1115/1480 train_time:171332ms step_avg:155.05ms
step:1116/1480 train_time:171494ms step_avg:155.06ms
step:1117/1480 train_time:171657ms step_avg:155.07ms
step:1118/1480 train_time:171822ms step_avg:155.07ms
step:1119/1480 train_time:171981ms step_avg:155.08ms
step:1120/1480 train_time:172142ms step_avg:155.08ms
step:1121/1480 train_time:172306ms step_avg:155.09ms
step:1122/1480 train_time:172467ms step_avg:155.10ms
step:1123/1480 train_time:172629ms step_avg:155.10ms
step:1124/1480 train_time:172793ms step_avg:155.11ms
step:1125/1480 train_time:172956ms step_avg:155.12ms
step:1125/1480 val_loss:3.3877 train_time:173031ms step_avg:155.18ms
step:1126/1480 train_time:173121ms step_avg:155.13ms
step:1127/1480 train_time:173282ms step_avg:155.13ms
step:1128/1480 train_time:173445ms step_avg:155.14ms
step:1129/1480 train_time:173609ms step_avg:155.15ms
step:1130/1480 train_time:173770ms step_avg:155.15ms
step:1131/1480 train_time:173939ms step_avg:155.16ms
step:1132/1480 train_time:174099ms step_avg:155.17ms
step:1133/1480 train_time:174264ms step_avg:155.18ms
step:1134/1480 train_time:174428ms step_avg:155.18ms
step:1135/1480 train_time:174588ms step_avg:155.19ms
step:1136/1480 train_time:174748ms step_avg:155.19ms
step:1137/1480 train_time:174908ms step_avg:155.20ms
step:1138/1480 train_time:175072ms step_avg:155.21ms
step:1139/1480 train_time:175244ms step_avg:155.22ms
step:1140/1480 train_time:175392ms step_avg:155.21ms
step:1141/1480 train_time:175558ms step_avg:155.22ms
step:1142/1480 train_time:175719ms step_avg:155.23ms
step:1143/1480 train_time:175884ms step_avg:155.24ms
step:1144/1480 train_time:176046ms step_avg:155.24ms
step:1145/1480 train_time:176204ms step_avg:155.25ms
step:1146/1480 train_time:176368ms step_avg:155.25ms
step:1147/1480 train_time:176528ms step_avg:155.26ms
step:1148/1480 train_time:176689ms step_avg:155.26ms
step:1149/1480 train_time:176852ms step_avg:155.27ms
step:1150/1480 train_time:177012ms step_avg:155.27ms
step:1151/1480 train_time:177179ms step_avg:155.28ms
step:1152/1480 train_time:177343ms step_avg:155.29ms
step:1153/1480 train_time:177509ms step_avg:155.30ms
step:1154/1480 train_time:177669ms step_avg:155.31ms
step:1155/1480 train_time:177831ms step_avg:155.31ms
step:1156/1480 train_time:177998ms step_avg:155.32ms
step:1157/1480 train_time:178163ms step_avg:155.33ms
step:1158/1480 train_time:178324ms step_avg:155.33ms
step:1159/1480 train_time:178486ms step_avg:155.34ms
step:1160/1480 train_time:178646ms step_avg:155.34ms
step:1161/1480 train_time:178807ms step_avg:155.35ms
step:1162/1480 train_time:178970ms step_avg:155.36ms
step:1163/1480 train_time:179130ms step_avg:155.36ms
step:1164/1480 train_time:179291ms step_avg:155.36ms
step:1165/1480 train_time:179449ms step_avg:155.37ms
step:1166/1480 train_time:179612ms step_avg:155.37ms
step:1167/1480 train_time:179771ms step_avg:155.38ms
step:1168/1480 train_time:179932ms step_avg:155.38ms
step:1169/1480 train_time:180097ms step_avg:155.39ms
step:1170/1480 train_time:180259ms step_avg:155.40ms
step:1171/1480 train_time:180421ms step_avg:155.40ms
step:1172/1480 train_time:180582ms step_avg:155.41ms
step:1173/1480 train_time:180745ms step_avg:155.41ms
step:1174/1480 train_time:180912ms step_avg:155.42ms
step:1175/1480 train_time:181075ms step_avg:155.43ms
step:1176/1480 train_time:181240ms step_avg:155.44ms
step:1177/1480 train_time:181407ms step_avg:155.45ms
step:1178/1480 train_time:181567ms step_avg:155.45ms
step:1179/1480 train_time:181726ms step_avg:155.45ms
step:1180/1480 train_time:181894ms step_avg:155.46ms
step:1181/1480 train_time:182058ms step_avg:155.47ms
step:1182/1480 train_time:182218ms step_avg:155.48ms
step:1183/1480 train_time:182382ms step_avg:155.48ms
step:1184/1480 train_time:182544ms step_avg:155.49ms
step:1185/1480 train_time:182708ms step_avg:155.50ms
step:1186/1480 train_time:182870ms step_avg:155.50ms
step:1187/1480 train_time:183042ms step_avg:155.52ms
step:1188/1480 train_time:183203ms step_avg:155.52ms
step:1189/1480 train_time:183367ms step_avg:155.53ms
step:1190/1480 train_time:183528ms step_avg:155.53ms
step:1191/1480 train_time:183693ms step_avg:155.54ms
step:1192/1480 train_time:183852ms step_avg:155.54ms
step:1193/1480 train_time:184013ms step_avg:155.55ms
step:1194/1480 train_time:184174ms step_avg:155.55ms
step:1195/1480 train_time:184337ms step_avg:155.56ms
step:1196/1480 train_time:184507ms step_avg:155.57ms
step:1197/1480 train_time:184669ms step_avg:155.58ms
step:1198/1480 train_time:184837ms step_avg:155.59ms
step:1199/1480 train_time:185000ms step_avg:155.59ms
step:1200/1480 train_time:185162ms step_avg:155.60ms
step:1201/1480 train_time:185322ms step_avg:155.60ms
step:1202/1480 train_time:185493ms step_avg:155.61ms
step:1203/1480 train_time:185659ms step_avg:155.62ms
step:1204/1480 train_time:185824ms step_avg:155.63ms
step:1205/1480 train_time:185986ms step_avg:155.64ms
step:1206/1480 train_time:186146ms step_avg:155.64ms
step:1207/1480 train_time:186307ms step_avg:155.65ms
step:1208/1480 train_time:186468ms step_avg:155.65ms
step:1209/1480 train_time:186630ms step_avg:155.65ms
step:1210/1480 train_time:186796ms step_avg:155.66ms
step:1211/1480 train_time:186961ms step_avg:155.67ms
step:1212/1480 train_time:187124ms step_avg:155.68ms
step:1213/1480 train_time:187289ms step_avg:155.68ms
step:1214/1480 train_time:187454ms step_avg:155.69ms
step:1215/1480 train_time:187618ms step_avg:155.70ms
step:1216/1480 train_time:187779ms step_avg:155.70ms
step:1217/1480 train_time:187944ms step_avg:155.71ms
step:1218/1480 train_time:188104ms step_avg:155.72ms
step:1219/1480 train_time:188271ms step_avg:155.72ms
step:1220/1480 train_time:188431ms step_avg:155.73ms
step:1221/1480 train_time:188592ms step_avg:155.73ms
step:1222/1480 train_time:188754ms step_avg:155.74ms
step:1223/1480 train_time:188916ms step_avg:155.74ms
step:1224/1480 train_time:189084ms step_avg:155.75ms
step:1225/1480 train_time:189249ms step_avg:155.76ms
step:1226/1480 train_time:189413ms step_avg:155.77ms
step:1227/1480 train_time:189576ms step_avg:155.77ms
step:1228/1480 train_time:189740ms step_avg:155.78ms
step:1229/1480 train_time:189902ms step_avg:155.79ms
step:1230/1480 train_time:190071ms step_avg:155.80ms
step:1231/1480 train_time:190237ms step_avg:155.80ms
step:1232/1480 train_time:190403ms step_avg:155.81ms
step:1233/1480 train_time:190565ms step_avg:155.82ms
step:1234/1480 train_time:190725ms step_avg:155.82ms
step:1235/1480 train_time:190890ms step_avg:155.83ms
step:1236/1480 train_time:191051ms step_avg:155.83ms
step:1237/1480 train_time:191211ms step_avg:155.84ms
step:1238/1480 train_time:191386ms step_avg:155.85ms
step:1239/1480 train_time:191548ms step_avg:155.86ms
step:1240/1480 train_time:191711ms step_avg:155.86ms
step:1241/1480 train_time:191876ms step_avg:155.87ms
step:1242/1480 train_time:192039ms step_avg:155.88ms
step:1243/1480 train_time:192204ms step_avg:155.88ms
step:1244/1480 train_time:192365ms step_avg:155.89ms
step:1245/1480 train_time:192526ms step_avg:155.89ms
step:1246/1480 train_time:192688ms step_avg:155.90ms
step:1247/1480 train_time:192850ms step_avg:155.90ms
step:1248/1480 train_time:193010ms step_avg:155.90ms
step:1249/1480 train_time:193171ms step_avg:155.91ms
step:1250/1480 train_time:193331ms step_avg:155.91ms
step:1250/1480 val_loss:3.3382 train_time:193407ms step_avg:155.97ms
step:1251/1480 train_time:193501ms step_avg:155.92ms
step:1252/1480 train_time:193665ms step_avg:155.93ms
step:1253/1480 train_time:193827ms step_avg:155.94ms
step:1254/1480 train_time:193989ms step_avg:155.94ms
step:1255/1480 train_time:194159ms step_avg:155.95ms
step:1256/1480 train_time:194325ms step_avg:155.96ms
step:1257/1480 train_time:194488ms step_avg:155.96ms
step:1258/1480 train_time:194653ms step_avg:155.97ms
step:1259/1480 train_time:194816ms step_avg:155.98ms
step:1260/1480 train_time:194975ms step_avg:155.98ms
step:1261/1480 train_time:195137ms step_avg:155.98ms
step:1262/1480 train_time:195306ms step_avg:155.99ms
step:1263/1480 train_time:195471ms step_avg:156.00ms
step:1264/1480 train_time:195630ms step_avg:156.00ms
step:1265/1480 train_time:195790ms step_avg:156.01ms
step:1266/1480 train_time:195951ms step_avg:156.01ms
step:1267/1480 train_time:196112ms step_avg:156.02ms
step:1268/1480 train_time:196274ms step_avg:156.02ms
step:1269/1480 train_time:196441ms step_avg:156.03ms
step:1270/1480 train_time:196604ms step_avg:156.03ms
step:1271/1480 train_time:196766ms step_avg:156.04ms
step:1272/1480 train_time:196928ms step_avg:156.04ms
step:1273/1480 train_time:197092ms step_avg:156.05ms
step:1274/1480 train_time:197257ms step_avg:156.06ms
step:1275/1480 train_time:197416ms step_avg:156.06ms
step:1276/1480 train_time:197575ms step_avg:156.06ms
step:1277/1480 train_time:197737ms step_avg:156.07ms
step:1278/1480 train_time:197898ms step_avg:156.07ms
step:1279/1480 train_time:198061ms step_avg:156.08ms
step:1280/1480 train_time:198229ms step_avg:156.09ms
step:1281/1480 train_time:198390ms step_avg:156.09ms
step:1282/1480 train_time:198549ms step_avg:156.09ms
step:1283/1480 train_time:198711ms step_avg:156.10ms
step:1284/1480 train_time:198873ms step_avg:156.10ms
step:1285/1480 train_time:199034ms step_avg:156.11ms
step:1286/1480 train_time:199197ms step_avg:156.11ms
step:1287/1480 train_time:199360ms step_avg:156.12ms
step:1288/1480 train_time:199524ms step_avg:156.12ms
step:1289/1480 train_time:199695ms step_avg:156.13ms
step:1290/1480 train_time:199864ms step_avg:156.14ms
step:1291/1480 train_time:200030ms step_avg:156.15ms
step:1292/1480 train_time:200192ms step_avg:156.16ms
step:1293/1480 train_time:200359ms step_avg:156.16ms
step:1294/1480 train_time:200523ms step_avg:156.17ms
step:1295/1480 train_time:200687ms step_avg:156.18ms
step:1296/1480 train_time:200851ms step_avg:156.18ms
step:1297/1480 train_time:201013ms step_avg:156.19ms
step:1298/1480 train_time:201175ms step_avg:156.19ms
step:1299/1480 train_time:201337ms step_avg:156.20ms
step:1300/1480 train_time:201497ms step_avg:156.20ms
step:1301/1480 train_time:201661ms step_avg:156.21ms
step:1302/1480 train_time:201826ms step_avg:156.21ms
step:1303/1480 train_time:201993ms step_avg:156.22ms
step:1304/1480 train_time:202157ms step_avg:156.23ms
step:1305/1480 train_time:202318ms step_avg:156.23ms
step:1306/1480 train_time:202485ms step_avg:156.24ms
step:1307/1480 train_time:202647ms step_avg:156.24ms
step:1308/1480 train_time:202808ms step_avg:156.25ms
step:1309/1480 train_time:202973ms step_avg:156.25ms
step:1310/1480 train_time:203135ms step_avg:156.26ms
step:1311/1480 train_time:203296ms step_avg:156.26ms
step:1312/1480 train_time:203462ms step_avg:156.27ms
step:1313/1480 train_time:203626ms step_avg:156.28ms
step:1314/1480 train_time:203790ms step_avg:156.28ms
step:1315/1480 train_time:203952ms step_avg:156.29ms
step:1316/1480 train_time:204111ms step_avg:156.29ms
step:1317/1480 train_time:204272ms step_avg:156.29ms
step:1318/1480 train_time:204439ms step_avg:156.30ms
step:1319/1480 train_time:204606ms step_avg:156.31ms
step:1320/1480 train_time:204774ms step_avg:156.32ms
step:1321/1480 train_time:204936ms step_avg:156.32ms
step:1322/1480 train_time:205108ms step_avg:156.33ms
step:1323/1480 train_time:205271ms step_avg:156.34ms
step:1324/1480 train_time:205434ms step_avg:156.34ms
step:1325/1480 train_time:205604ms step_avg:156.35ms
step:1326/1480 train_time:205768ms step_avg:156.36ms
step:1327/1480 train_time:205931ms step_avg:156.36ms
step:1328/1480 train_time:206092ms step_avg:156.37ms
step:1329/1480 train_time:206276ms step_avg:156.39ms
step:1330/1480 train_time:206438ms step_avg:156.39ms
step:1331/1480 train_time:206602ms step_avg:156.40ms
step:1332/1480 train_time:206765ms step_avg:156.40ms
step:1333/1480 train_time:206931ms step_avg:156.41ms
step:1334/1480 train_time:207095ms step_avg:156.42ms
step:1335/1480 train_time:207254ms step_avg:156.42ms
step:1336/1480 train_time:207425ms step_avg:156.43ms
step:1337/1480 train_time:207591ms step_avg:156.44ms
step:1338/1480 train_time:207753ms step_avg:156.44ms
step:1339/1480 train_time:207918ms step_avg:156.45ms
step:1340/1480 train_time:208083ms step_avg:156.45ms
step:1341/1480 train_time:208245ms step_avg:156.46ms
step:1342/1480 train_time:208410ms step_avg:156.46ms
step:1343/1480 train_time:208571ms step_avg:156.47ms
step:1344/1480 train_time:208734ms step_avg:156.47ms
step:1345/1480 train_time:208904ms step_avg:156.48ms
step:1346/1480 train_time:209066ms step_avg:156.49ms
step:1347/1480 train_time:209229ms step_avg:156.49ms
step:1348/1480 train_time:209391ms step_avg:156.50ms
step:1349/1480 train_time:209552ms step_avg:156.50ms
step:1350/1480 train_time:209719ms step_avg:156.51ms
step:1351/1480 train_time:209883ms step_avg:156.51ms
step:1352/1480 train_time:210046ms step_avg:156.52ms
step:1353/1480 train_time:210212ms step_avg:156.52ms
step:1354/1480 train_time:210374ms step_avg:156.53ms
step:1355/1480 train_time:210535ms step_avg:156.53ms
step:1356/1480 train_time:210700ms step_avg:156.54ms
step:1357/1480 train_time:210866ms step_avg:156.54ms
step:1358/1480 train_time:211030ms step_avg:156.55ms
step:1359/1480 train_time:211195ms step_avg:156.56ms
step:1360/1480 train_time:211362ms step_avg:156.56ms
step:1361/1480 train_time:211532ms step_avg:156.57ms
step:1362/1480 train_time:211696ms step_avg:156.58ms
step:1363/1480 train_time:211863ms step_avg:156.59ms
step:1364/1480 train_time:212026ms step_avg:156.59ms
step:1365/1480 train_time:212186ms step_avg:156.59ms
step:1366/1480 train_time:212349ms step_avg:156.60ms
step:1367/1480 train_time:212512ms step_avg:156.60ms
step:1368/1480 train_time:212676ms step_avg:156.61ms
step:1369/1480 train_time:212848ms step_avg:156.62ms
step:1370/1480 train_time:213014ms step_avg:156.63ms
step:1371/1480 train_time:213176ms step_avg:156.63ms
step:1372/1480 train_time:213344ms step_avg:156.64ms
step:1373/1480 train_time:213506ms step_avg:156.64ms
step:1374/1480 train_time:213671ms step_avg:156.65ms
step:1375/1480 train_time:213833ms step_avg:156.65ms
step:1375/1480 val_loss:3.2997 train_time:213907ms step_avg:156.71ms
step:1376/1480 train_time:213999ms step_avg:156.66ms
step:1377/1480 train_time:214164ms step_avg:156.67ms
step:1378/1480 train_time:214326ms step_avg:156.67ms
step:1379/1480 train_time:214491ms step_avg:156.68ms
step:1380/1480 train_time:214654ms step_avg:156.68ms
step:1381/1480 train_time:214821ms step_avg:156.69ms
step:1382/1480 train_time:214986ms step_avg:156.70ms
step:1383/1480 train_time:215150ms step_avg:156.70ms
step:1384/1480 train_time:215315ms step_avg:156.71ms
step:1385/1480 train_time:215474ms step_avg:156.71ms
step:1386/1480 train_time:215636ms step_avg:156.71ms
step:1387/1480 train_time:215802ms step_avg:156.72ms
step:1388/1480 train_time:215963ms step_avg:156.72ms
step:1389/1480 train_time:216129ms step_avg:156.73ms
step:1390/1480 train_time:216291ms step_avg:156.73ms
step:1391/1480 train_time:216452ms step_avg:156.74ms
step:1392/1480 train_time:216614ms step_avg:156.74ms
step:1393/1480 train_time:216776ms step_avg:156.74ms
step:1394/1480 train_time:216941ms step_avg:156.75ms
step:1395/1480 train_time:217104ms step_avg:156.75ms
step:1396/1480 train_time:217268ms step_avg:156.76ms
step:1397/1480 train_time:217429ms step_avg:156.76ms
step:1398/1480 train_time:217589ms step_avg:156.76ms
step:1399/1480 train_time:217752ms step_avg:156.77ms
step:1400/1480 train_time:217918ms step_avg:156.78ms
step:1401/1480 train_time:218079ms step_avg:156.78ms
step:1402/1480 train_time:218244ms step_avg:156.78ms
step:1403/1480 train_time:218410ms step_avg:156.79ms
step:1404/1480 train_time:218573ms step_avg:156.80ms
step:1405/1480 train_time:218737ms step_avg:156.80ms
step:1406/1480 train_time:218902ms step_avg:156.81ms
step:1407/1480 train_time:219064ms step_avg:156.81ms
step:1408/1480 train_time:219227ms step_avg:156.82ms
step:1409/1480 train_time:219399ms step_avg:156.83ms
step:1410/1480 train_time:219563ms step_avg:156.83ms
step:1411/1480 train_time:219725ms step_avg:156.83ms
step:1412/1480 train_time:219889ms step_avg:156.84ms
step:1413/1480 train_time:220051ms step_avg:156.84ms
step:1414/1480 train_time:220214ms step_avg:156.85ms
step:1415/1480 train_time:220379ms step_avg:156.85ms
step:1416/1480 train_time:220554ms step_avg:156.87ms
step:1417/1480 train_time:220719ms step_avg:156.87ms
step:1418/1480 train_time:220884ms step_avg:156.88ms
step:1419/1480 train_time:221050ms step_avg:156.88ms
step:1420/1480 train_time:221213ms step_avg:156.89ms
step:1421/1480 train_time:221377ms step_avg:156.89ms
step:1422/1480 train_time:221543ms step_avg:156.90ms
step:1423/1480 train_time:221706ms step_avg:156.90ms
step:1424/1480 train_time:221872ms step_avg:156.91ms
step:1425/1480 train_time:222041ms step_avg:156.92ms
step:1426/1480 train_time:222206ms step_avg:156.93ms
step:1427/1480 train_time:222373ms step_avg:156.93ms
step:1428/1480 train_time:222534ms step_avg:156.94ms
step:1429/1480 train_time:222694ms step_avg:156.94ms
step:1430/1480 train_time:222860ms step_avg:156.94ms
step:1431/1480 train_time:223027ms step_avg:156.95ms
step:1432/1480 train_time:223194ms step_avg:156.96ms
step:1433/1480 train_time:223363ms step_avg:156.97ms
step:1434/1480 train_time:223532ms step_avg:156.97ms
step:1435/1480 train_time:223697ms step_avg:156.98ms
step:1436/1480 train_time:223864ms step_avg:156.99ms
step:1437/1480 train_time:224028ms step_avg:156.99ms
step:1438/1480 train_time:224190ms step_avg:157.00ms
step:1439/1480 train_time:224356ms step_avg:157.00ms
step:1440/1480 train_time:224519ms step_avg:157.01ms
step:1441/1480 train_time:224683ms step_avg:157.01ms
step:1442/1480 train_time:224850ms step_avg:157.02ms
step:1443/1480 train_time:225023ms step_avg:157.03ms
step:1444/1480 train_time:225187ms step_avg:157.03ms
step:1445/1480 train_time:225350ms step_avg:157.04ms
step:1446/1480 train_time:225515ms step_avg:157.04ms
step:1447/1480 train_time:225684ms step_avg:157.05ms
step:1448/1480 train_time:225848ms step_avg:157.06ms
step:1449/1480 train_time:226011ms step_avg:157.06ms
step:1450/1480 train_time:226174ms step_avg:157.07ms
step:1451/1480 train_time:226337ms step_avg:157.07ms
step:1452/1480 train_time:226504ms step_avg:157.08ms
step:1453/1480 train_time:226667ms step_avg:157.08ms
step:1454/1480 train_time:226830ms step_avg:157.08ms
step:1455/1480 train_time:226997ms step_avg:157.09ms
step:1456/1480 train_time:227162ms step_avg:157.10ms
step:1457/1480 train_time:227325ms step_avg:157.10ms
step:1458/1480 train_time:227490ms step_avg:157.11ms
step:1459/1480 train_time:227654ms step_avg:157.11ms
step:1460/1480 train_time:227818ms step_avg:157.12ms
step:1461/1480 train_time:227982ms step_avg:157.12ms
step:1462/1480 train_time:228147ms step_avg:157.13ms
step:1463/1480 train_time:228312ms step_avg:157.13ms
step:1464/1480 train_time:228475ms step_avg:157.14ms
step:1465/1480 train_time:228640ms step_avg:157.14ms
step:1466/1480 train_time:228804ms step_avg:157.15ms
step:1467/1480 train_time:228971ms step_avg:157.15ms
step:1468/1480 train_time:229133ms step_avg:157.16ms
step:1469/1480 train_time:229295ms step_avg:157.16ms
step:1470/1480 train_time:229464ms step_avg:157.17ms
step:1471/1480 train_time:229636ms step_avg:157.18ms
step:1472/1480 train_time:229807ms step_avg:157.19ms
step:1473/1480 train_time:229970ms step_avg:157.19ms
step:1474/1480 train_time:230136ms step_avg:157.20ms
step:1475/1480 train_time:230305ms step_avg:157.21ms
step:1476/1480 train_time:230469ms step_avg:157.21ms
step:1477/1480 train_time:230637ms step_avg:157.22ms
step:1478/1480 train_time:230807ms step_avg:157.23ms
step:1479/1480 train_time:230974ms step_avg:157.23ms
step:1480/1480 train_time:231137ms step_avg:157.24ms
step:1480/1480 val_loss:3.2812 train_time:231214ms step_avg:157.29ms
peak memory consumption: 34239 MiB