[08/29 22:23:34] libai INFO: Rank of current process: 0. World size: 8
[08/29 22:23:34] libai INFO: Command line arguments: Namespace(config_file='configs/swinv2_imagenet.py', eval_only=False, fast_dev_run=False, opts=[], resume=False)
[08/29 22:23:34] libai INFO: Contents of args.config_file=configs/swinv2_imagenet.py:
from libai.config import LazyCall
from .common.models.swinv2.swinv2_tiny_patch4_window8_256 import model
from .common.models.graph import graph
from .common.train import train
from .common.optim import optim
from .common.data.imagenet import dataloader

from flowvision import transforms
from flowvision.data import Mixup
from flowvision.loss.cross_entropy import SoftTargetCrossEntropy
from flowvision.transforms import InterpolationMode
from flowvision.transforms.functional import str_to_interp_mode
from flowvision.data.constants import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
)
from flowvision.data.auto_augment import rand_augment_transform
from flowvision.data.random_erasing import RandomErasing


# Refine data path to imagenet
dataloader.train.dataset[0].root = "/data/ImageNet/extract"
dataloader.test[0].dataset.root = "/data/ImageNet/extract"

# Add Mixup Func
dataloader.train.mixup_func = LazyCall(Mixup)(
    mixup_alpha=0.8,
    cutmix_alpha=1.0,
    prob=1.0,
    switch_prob=0.5,
    mode="batch",
    num_classes=1000,
)

dataloader.train.dataset[0].transform = LazyCall(transforms.Compose)(
    transforms=[
        LazyCall(transforms.RandomResizedCrop)(
            size=256,
            scale=(0.08, 1.0),
            ratio=(3.0 / 4.0, 4.0 / 3.0),
            interpolation=InterpolationMode.BICUBIC,
        ),
        LazyCall(transforms.RandomHorizontalFlip)(p=0.5),
        LazyCall(rand_augment_transform)(
            config_str="rand-m9-mstd0.5-inc1",
            hparams=dict(
                translate_const=int(256 * 0.45),
                img_mean=tuple([min(255, round(255 * x)) for x in IMAGENET_DEFAULT_MEAN]),
                interpolation=str_to_interp_mode("bicubic"),
            ),
        ),
        LazyCall(transforms.ToTensor)(),
        LazyCall(transforms.Normalize)(
            mean=IMAGENET_DEFAULT_MEAN,
            std=IMAGENET_DEFAULT_STD,
        ),
        LazyCall(RandomErasing)(
            probability=0.25,
            mode="pixel",
            max_count=1,
            num_splits=0,
            device="cpu",
        ),
    ]
)
dataloader.test[0].dataset.transform = LazyCall(transforms.Compose)(
    transforms=[
        LazyCall(transforms.Resize)(
            size=256,
            interpolation=InterpolationMode.BICUBIC,
        ),
        LazyCall(transforms.CenterCrop)(
            size=256,
        ),
        LazyCall(transforms.ToTensor)(),
        LazyCall(transforms.Normalize)(
            mean=IMAGENET_DEFAULT_MEAN,
            std=IMAGENET_DEFAULT_STD,
        ),
    ]
)


# Refine model cfg for vit training on imagenet
model.cfg.num_classes = 1000
model.cfg.loss_func = SoftTargetCrossEntropy()

# Refine optimizer cfg for vit model
optim.lr = 1e-3  # The pytorch version is 1024 as the total batch size, 1e-3 as the learning rate
optim.eps = 1e-8
optim.weight_decay = 0.05


def check_keywords_in_name(name, keywords=()):
    isin = False
    for keyword in keywords:
        if keyword in name:
            isin = True
    return isin


def set_weight_decay(model, skip_list=(), skip_keywords=()):
    has_decay = []
    no_decay = []

    for name, param in model.named_parameters():
        if not param.requires_grad:
            continue  # frozen weights
        if (
            len(param.shape) == 1
            or name.endswith(".bias")
            or (name in skip_list)
            or check_keywords_in_name(name, skip_keywords)
        ):
            no_decay.append(param)
        else:
            has_decay.append(param)
    return [{"params": has_decay}, {"params": no_decay, "weight_decay": 0.0}]


optim.params = LazyCall(set_weight_decay)(
    model=model,
    skip_list=("absolute_pos_embed"),
    skip_keywords=("cpb_mlp", "logit_scale", "relative_position_bias_table"),
)

# Refine train cfg for vit model
train.train_micro_batch_size = 128
train.test_micro_batch_size = 128
train.train_epoch = 300
train.warmup_ratio = 20 / 300
train.eval_period = 1000
train.log_period = 100
graph.enabled = False
train.rdma_enabled = True
# Scheduler
train.scheduler.warmup_factor = 0.001
train.scheduler.alpha = 0.01
train.scheduler.warmup_method = "linear"
train.output_dir = "./output_20220829"
# Set fp16 ON
train.amp.enabled = True

[08/29 22:23:34] lb.config.lazy WARNING: The config contains objects that cannot serialize to a valid yaml. ./output_20220829/config.yaml is human-readable but cannot be loaded.
[08/29 22:23:34] lb.config.lazy WARNING: Config is saved using cloudpickle at ./output_20220829/config.yaml.pkl.
[08/29 22:23:34] libai INFO: Full config saved to ./output_20220829/config.yaml
[08/29 22:23:34] lb.engine.default INFO: > compiling dataset index builder ...
[08/29 22:23:34] lb.engine.default INFO: >>> done with dataset index builder. Compilation time: 0.070 seconds
[08/29 22:23:34] lb.engine.default INFO: >>> done with compiling. Compilation time: 0.071 seconds
[08/29 22:23:34] lb.engine.default INFO: Prepare training, validating, testing set
[08/29 22:23:38] lb.engine.default INFO: Prepare testing set
[08/29 22:23:38] lb.engine.default INFO: Auto-scaling the config to train.train_iter=375342, train.warmup_iter=25023
[08/29 22:23:47] lb.engine.default INFO: Model:
SwinTransformerV2(
  (patch_embed): PatchEmbed(
    (proj): Conv2d(3, 96, kernel_size=(4, 4), stride=(4, 4))
    (norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
  )
  (pos_drop): Dropout(p=0.0, inplace=False)
  (layers): ModuleList(
    (0): BasicLayer(
      (blocks): ModuleList(
        (0): SwinTransformerBlock(
          (norm1): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
          (attn): WindowAttention(
            (cpb_mlp): Sequential(
              (0): Linear1D(in_features=2, out_features=512, bias=True, parallel=data)
              (1): ReLU(inplace=True)
              (2): Linear1D(in_features=512, out_features=3, bias=False, parallel=data)
            )
            (qkv): Linear1D(in_features=96, out_features=288, bias=False, parallel=data)
            (attn_drop): Dropout(p=0.0, inplace=False)
            (proj): Linear1D(in_features=96, out_features=96, bias=True, parallel=data)
            (proj_drop): Dropout(p=0.0, inplace=False)
            (softmax): Softmax(dim=-1)
          )
          (drop_path): Identity()
          (norm2): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
          (mlp): MLP(
            bias_gelu_fusion=True, bias_dropout_fusion=True, dropout=0.0
            (dense_h_to_4h): Linear1D(in_features=96, out_features=384, bias=True, parallel=col)
            (dense_4h_to_h): Linear1D(in_features=384, out_features=96, bias=True, parallel=row)
          )
        )
        (1): SwinTransformerBlock(
          (norm1): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
          (attn): WindowAttention(
            (cpb_mlp): Sequential(
              (0): Linear1D(in_features=2, out_features=512, bias=True, parallel=data)
              (1): ReLU(inplace=True)
              (2): Linear1D(in_features=512, out_features=3, bias=False, parallel=data)
            )
            (qkv): Linear1D(in_features=96, out_features=288, bias=False, parallel=data)
            (attn_drop): Dropout(p=0.0, inplace=False)
            (proj): Linear1D(in_features=96, out_features=96, bias=True, parallel=data)
            (proj_drop): Dropout(p=0.0, inplace=False)
            (softmax): Softmax(dim=-1)
          )
          (drop_path): DropPath()
          (norm2): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
          (mlp): MLP(
            bias_gelu_fusion=True, bias_dropout_fusion=True, dropout=0.0
            (dense_h_to_4h): Linear1D(in_features=96, out_features=384, bias=True, parallel=col)
            (dense_4h_to_h): Linear1D(in_features=384, out_features=96, bias=True, parallel=row)
          )
        )
      )
      (downsample): PatchMerging(
        (reduction): Linear1D(in_features=384, out_features=192, bias=False, parallel=data)
        (norm): LayerNorm((192,), eps=1e-05, elementwise_affine=True)
      )
    )
    (1): BasicLayer(
      (blocks): ModuleList(
        (0): SwinTransformerBlock(
          (norm1): LayerNorm((192,), eps=1e-05, elementwise_affine=True)
          (attn): WindowAttention(
            (cpb_mlp): Sequential(
              (0): Linear1D(in_features=2, out_features=512, bias=True, parallel=data)
              (1): ReLU(inplace=True)
              (2): Linear1D(in_features=512, out_features=6, bias=False, parallel=data)
            )
            (qkv): Linear1D(in_features=192, out_features=576, bias=False, parallel=data)
            (attn_drop): Dropout(p=0.0, inplace=False)
            (proj): Linear1D(in_features=192, out_features=192, bias=True, parallel=data)
            (proj_drop): Dropout(p=0.0, inplace=False)
            (softmax): Softmax(dim=-1)
          )
          (drop_path): DropPath()
          (norm2): LayerNorm((192,), eps=1e-05, elementwise_affine=True)
          (mlp): MLP(
            bias_gelu_fusion=True, bias_dropout_fusion=True, dropout=0.0
            (dense_h_to_4h): Linear1D(in_features=192, out_features=768, bias=True, parallel=col)
            (dense_4h_to_h): Linear1D(in_features=768, out_features=192, bias=True, parallel=row)
          )
        )
        (1): SwinTransformerBlock(
          (norm1): LayerNorm((192,), eps=1e-05, elementwise_affine=True)
          (attn): WindowAttention(
            (cpb_mlp): Sequential(
              (0): Linear1D(in_features=2, out_features=512, bias=True, parallel=data)
              (1): ReLU(inplace=True)
              (2): Linear1D(in_features=512, out_features=6, bias=False, parallel=data)
            )
            (qkv): Linear1D(in_features=192, out_features=576, bias=False, parallel=data)
            (attn_drop): Dropout(p=0.0, inplace=False)
            (proj): Linear1D(in_features=192, out_features=192, bias=True, parallel=data)
            (proj_drop): Dropout(p=0.0, inplace=False)
            (softmax): Softmax(dim=-1)
          )
          (drop_path): DropPath()
          (norm2): LayerNorm((192,), eps=1e-05, elementwise_affine=True)
          (mlp): MLP(
            bias_gelu_fusion=True, bias_dropout_fusion=True, dropout=0.0
            (dense_h_to_4h): Linear1D(in_features=192, out_features=768, bias=True, parallel=col)
            (dense_4h_to_h): Linear1D(in_features=768, out_features=192, bias=True, parallel=row)
          )
        )
      )
      (downsample): PatchMerging(
        (reduction): Linear1D(in_features=768, out_features=384, bias=False, parallel=data)
        (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
      )
    )
    (2): BasicLayer(
      (blocks): ModuleList(
        (0): SwinTransformerBlock(
          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
          (attn): WindowAttention(
            (cpb_mlp): Sequential(
              (0): Linear1D(in_features=2, out_features=512, bias=True, parallel=data)
              (1): ReLU(inplace=True)
              (2): Linear1D(in_features=512, out_features=12, bias=False, parallel=data)
            )
            (qkv): Linear1D(in_features=384, out_features=1152, bias=False, parallel=data)
            (attn_drop): Dropout(p=0.0, inplace=False)
            (proj): Linear1D(in_features=384, out_features=384, bias=True, parallel=data)
            (proj_drop): Dropout(p=0.0, inplace=False)
            (softmax): Softmax(dim=-1)
          )
          (drop_path): DropPath()
          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
          (mlp): MLP(
            bias_gelu_fusion=True, bias_dropout_fusion=True, dropout=0.0
            (dense_h_to_4h): Linear1D(in_features=384, out_features=1536, bias=True, parallel=col)
            (dense_4h_to_h): Linear1D(in_features=1536, out_features=384, bias=True, parallel=row)
          )
        )
        (1): SwinTransformerBlock(
          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
          (attn): WindowAttention(
            (cpb_mlp): Sequential(
              (0): Linear1D(in_features=2, out_features=512, bias=True, parallel=data)
              (1): ReLU(inplace=True)
              (2): Linear1D(in_features=512, out_features=12, bias=False, parallel=data)
            )
            (qkv): Linear1D(in_features=384, out_features=1152, bias=False, parallel=data)
            (attn_drop): Dropout(p=0.0, inplace=False)
            (proj): Linear1D(in_features=384, out_features=384, bias=True, parallel=data)
            (proj_drop): Dropout(p=0.0, inplace=False)
            (softmax): Softmax(dim=-1)
          )
          (drop_path): DropPath()
          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
          (mlp): MLP(
            bias_gelu_fusion=True, bias_dropout_fusion=True, dropout=0.0
            (dense_h_to_4h): Linear1D(in_features=384, out_features=1536, bias=True, parallel=col)
            (dense_4h_to_h): Linear1D(in_features=1536, out_features=384, bias=True, parallel=row)
          )
        )
        (2): SwinTransformerBlock(
          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
          (attn): WindowAttention(
            (cpb_mlp): Sequential(
              (0): Linear1D(in_features=2, out_features=512, bias=True, parallel=data)
              (1): ReLU(inplace=True)
              (2): Linear1D(in_features=512, out_features=12, bias=False, parallel=data)
            )
            (qkv): Linear1D(in_features=384, out_features=1152, bias=False, parallel=data)
            (attn_drop): Dropout(p=0.0, inplace=False)
            (proj): Linear1D(in_features=384, out_features=384, bias=True, parallel=data)
            (proj_drop): Dropout(p=0.0, inplace=False)
            (softmax): Softmax(dim=-1)
          )
          (drop_path): DropPath()
          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
          (mlp): MLP(
            bias_gelu_fusion=True, bias_dropout_fusion=True, dropout=0.0
            (dense_h_to_4h): Linear1D(in_features=384, out_features=1536, bias=True, parallel=col)
            (dense_4h_to_h): Linear1D(in_features=1536, out_features=384, bias=True, parallel=row)
          )
        )
        (3): SwinTransformerBlock(
          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
          (attn): WindowAttention(
            (cpb_mlp): Sequential(
              (0): Linear1D(in_features=2, out_features=512, bias=True, parallel=data)
              (1): ReLU(inplace=True)
              (2): Linear1D(in_features=512, out_features=12, bias=False, parallel=data)
            )
            (qkv): Linear1D(in_features=384, out_features=1152, bias=False, parallel=data)
            (attn_drop): Dropout(p=0.0, inplace=False)
            (proj): Linear1D(in_features=384, out_features=384, bias=True, parallel=data)
            (proj_drop): Dropout(p=0.0, inplace=False)
            (softmax): Softmax(dim=-1)
          )
          (drop_path): DropPath()
          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
          (mlp): MLP(
            bias_gelu_fusion=True, bias_dropout_fusion=True, dropout=0.0
            (dense_h_to_4h): Linear1D(in_features=384, out_features=1536, bias=True, parallel=col)
            (dense_4h_to_h): Linear1D(in_features=1536, out_features=384, bias=True, parallel=row)
          )
        )
        (4): SwinTransformerBlock(
          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
          (attn): WindowAttention(
            (cpb_mlp): Sequential(
              (0): Linear1D(in_features=2, out_features=512, bias=True, parallel=data)
              (1): ReLU(inplace=True)
              (2): Linear1D(in_features=512, out_features=12, bias=False, parallel=data)
            )
            (qkv): Linear1D(in_features=384, out_features=1152, bias=False, parallel=data)
            (attn_drop): Dropout(p=0.0, inplace=False)
            (proj): Linear1D(in_features=384, out_features=384, bias=True, parallel=data)
            (proj_drop): Dropout(p=0.0, inplace=False)
            (softmax): Softmax(dim=-1)
          )
          (drop_path): DropPath()
          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
          (mlp): MLP(
            bias_gelu_fusion=True, bias_dropout_fusion=True, dropout=0.0
            (dense_h_to_4h): Linear1D(in_features=384, out_features=1536, bias=True, parallel=col)
            (dense_4h_to_h): Linear1D(in_features=1536, out_features=384, bias=True, parallel=row)
          )
        )
        (5): SwinTransformerBlock(
          (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
          (attn): WindowAttention(
            (cpb_mlp): Sequential(
              (0): Linear1D(in_features=2, out_features=512, bias=True, parallel=data)
              (1): ReLU(inplace=True)
              (2): Linear1D(in_features=512, out_features=12, bias=False, parallel=data)
            )
            (qkv): Linear1D(in_features=384, out_features=1152, bias=False, parallel=data)
            (attn_drop): Dropout(p=0.0, inplace=False)
            (proj): Linear1D(in_features=384, out_features=384, bias=True, parallel=data)
            (proj_drop): Dropout(p=0.0, inplace=False)
            (softmax): Softmax(dim=-1)
          )
          (drop_path): DropPath()
          (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
          (mlp): MLP(
            bias_gelu_fusion=True, bias_dropout_fusion=True, dropout=0.0
            (dense_h_to_4h): Linear1D(in_features=384, out_features=1536, bias=True, parallel=col)
            (dense_4h_to_h): Linear1D(in_features=1536, out_features=384, bias=True, parallel=row)
          )
        )
      )
      (downsample): PatchMerging(
        (reduction): Linear1D(in_features=1536, out_features=768, bias=False, parallel=data)
        (norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
      )
    )
    (3): BasicLayer(
      (blocks): ModuleList(
        (0): SwinTransformerBlock(
          (norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
          (attn): WindowAttention(
            (cpb_mlp): Sequential(
              (0): Linear1D(in_features=2, out_features=512, bias=True, parallel=data)
              (1): ReLU(inplace=True)
              (2): Linear1D(in_features=512, out_features=24, bias=False, parallel=data)
            )
            (qkv): Linear1D(in_features=768, out_features=2304, bias=False, parallel=data)
            (attn_drop): Dropout(p=0.0, inplace=False)
            (proj): Linear1D(in_features=768, out_features=768, bias=True, parallel=data)
            (proj_drop): Dropout(p=0.0, inplace=False)
            (softmax): Softmax(dim=-1)
          )
          (drop_path): DropPath()
          (norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
          (mlp): MLP(
            bias_gelu_fusion=True, bias_dropout_fusion=True, dropout=0.0
            (dense_h_to_4h): Linear1D(in_features=768, out_features=3072, bias=True, parallel=col)
            (dense_4h_to_h): Linear1D(in_features=3072, out_features=768, bias=True, parallel=row)
          )
        )
        (1): SwinTransformerBlock(
          (norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
          (attn): WindowAttention(
            (cpb_mlp): Sequential(
              (0): Linear1D(in_features=2, out_features=512, bias=True, parallel=data)
              (1): ReLU(inplace=True)
              (2): Linear1D(in_features=512, out_features=24, bias=False, parallel=data)
            )
            (qkv): Linear1D(in_features=768, out_features=2304, bias=False, parallel=data)
            (attn_drop): Dropout(p=0.0, inplace=False)
            (proj): Linear1D(in_features=768, out_features=768, bias=True, parallel=data)
            (proj_drop): Dropout(p=0.0, inplace=False)
            (softmax): Softmax(dim=-1)
          )
          (drop_path): DropPath()
          (norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
          (mlp): MLP(
            bias_gelu_fusion=True, bias_dropout_fusion=True, dropout=0.0
            (dense_h_to_4h): Linear1D(in_features=768, out_features=3072, bias=True, parallel=col)
            (dense_4h_to_h): Linear1D(in_features=3072, out_features=768, bias=True, parallel=row)
          )
        )
      )
    )
  )
  (norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
  (avgpool): AdaptiveAvgPool1d()
  (head): Linear1D(in_features=768, out_features=1000, bias=True, parallel=data)
  (loss_func): SoftTargetCrossEntropy()
)
[08/29 22:23:47] lb.engine.trainer INFO: Starting training from iteration 0
[08/29 22:24:41] lb.utils.events INFO:  eta: 2 days, 3:58:22  iteration: 99/375342  consumed_samples: 102400  total_loss: 0.8713  time: 0.5051 s/iter  data_time: 0.0366 s/iter total_throughput: 2027.23 samples/s lr: 4.91e-06  
[08/29 22:25:31] lb.utils.events INFO:  eta: 2 days, 3:53:10  iteration: 199/375342  consumed_samples: 204800  total_loss: 0.8684  time: 0.5037 s/iter  data_time: 0.0395 s/iter total_throughput: 2032.95 samples/s lr: 8.86e-06  
[08/29 22:26:21] lb.utils.events INFO:  eta: 2 days, 3:48:17  iteration: 299/375342  consumed_samples: 307200  total_loss: 0.8641  time: 0.5023 s/iter  data_time: 0.0358 s/iter total_throughput: 2038.74 samples/s lr: 1.28e-05  
[08/29 22:27:11] lb.utils.events INFO:  eta: 2 days, 3:41:40  iteration: 399/375342  consumed_samples: 409600  total_loss: 0.861  time: 0.5011 s/iter  data_time: 0.0366 s/iter total_throughput: 2043.65 samples/s lr: 1.68e-05  
[08/29 22:28:01] lb.utils.events INFO:  eta: 2 days, 3:39:39  iteration: 499/375342  consumed_samples: 512000  total_loss: 0.859  time: 0.5004 s/iter  data_time: 0.0409 s/iter total_throughput: 2046.32 samples/s lr: 2.07e-05  
[08/29 22:28:50] lb.utils.events INFO:  eta: 2 days, 3:38:41  iteration: 599/375342  consumed_samples: 614400  total_loss: 0.857  time: 0.5001 s/iter  data_time: 0.0341 s/iter total_throughput: 2047.59 samples/s lr: 2.47e-05  
[08/29 22:29:40] lb.utils.events INFO:  eta: 2 days, 3:38:13  iteration: 699/375342  consumed_samples: 716800  total_loss: 0.8559  time: 0.4997 s/iter  data_time: 0.0364 s/iter total_throughput: 2049.24 samples/s lr: 2.86e-05  
[08/29 22:30:30] lb.utils.events INFO:  eta: 2 days, 3:36:33  iteration: 799/375342  consumed_samples: 819200  total_loss: 0.8534  time: 0.4993 s/iter  data_time: 0.0357 s/iter total_throughput: 2051.05 samples/s lr: 3.26e-05  
[08/29 22:31:19] lb.utils.events INFO:  eta: 2 days, 3:34:48  iteration: 899/375342  consumed_samples: 921600  total_loss: 0.8505  time: 0.4989 s/iter  data_time: 0.0367 s/iter total_throughput: 2052.65 samples/s lr: 3.65e-05  
[08/29 22:32:09] lb.utils.events INFO:  eta: 2 days, 3:33:08  iteration: 999/375342  consumed_samples: 1024000  total_loss: 0.8476  time: 0.4987 s/iter  data_time: 0.0371 s/iter total_throughput: 2053.51 samples/s lr: 4.05e-05  
[08/29 22:32:59] lb.utils.events INFO:  eta: 2 days, 3:28:54  iteration: 1099/375342  consumed_samples: 1126400  total_loss: 0.842  time: 0.4986 s/iter  data_time: 0.0373 s/iter total_throughput: 2053.77 samples/s lr: 4.44e-05  
[08/29 22:33:49] lb.utils.events INFO:  eta: 2 days, 3:26:19  iteration: 1199/375342  consumed_samples: 1228800  total_loss: 0.8351  time: 0.4985 s/iter  data_time: 0.0362 s/iter total_throughput: 2054.21 samples/s lr: 4.83e-05  
[08/29 22:34:38] lb.utils.events INFO:  eta: 2 days, 3:25:30  iteration: 1299/375342  consumed_samples: 1331200  total_loss: 0.8287  time: 0.4984 s/iter  data_time: 0.0368 s/iter total_throughput: 2054.43 samples/s lr: 5.23e-05  
[08/29 22:35:28] lb.utils.events INFO:  eta: 2 days, 3:25:18  iteration: 1399/375342  consumed_samples: 1433600  total_loss: 0.824  time: 0.4984 s/iter  data_time: 0.0385 s/iter total_throughput: 2054.70 samples/s lr: 5.62e-05  
[08/29 22:36:18] lb.utils.events INFO:  eta: 2 days, 3:28:08  iteration: 1499/375342  consumed_samples: 1536000  total_loss: 0.8208  time: 0.4984 s/iter  data_time: 0.0376 s/iter total_throughput: 2054.47 samples/s lr: 6.02e-05  
[08/29 22:37:08] lb.utils.events INFO:  eta: 2 days, 3:28:28  iteration: 1599/375342  consumed_samples: 1638400  total_loss: 0.818  time: 0.4985 s/iter  data_time: 0.0384 s/iter total_throughput: 2054.21 samples/s lr: 6.41e-05  
[08/29 22:37:58] lb.utils.events INFO:  eta: 2 days, 3:27:39  iteration: 1699/375342  consumed_samples: 1740800  total_loss: 0.8153  time: 0.4985 s/iter  data_time: 0.0371 s/iter total_throughput: 2054.09 samples/s lr: 6.81e-05  
[08/29 22:38:48] lb.utils.events INFO:  eta: 2 days, 3:27:36  iteration: 1799/375342  consumed_samples: 1843200  total_loss: 0.8099  time: 0.4985 s/iter  data_time: 0.0385 s/iter total_throughput: 2054.16 samples/s lr: 7.20e-05  
[08/29 22:39:38] lb.utils.events INFO:  eta: 2 days, 3:29:05  iteration: 1899/375342  consumed_samples: 1945600  total_loss: 0.8002  time: 0.4986 s/iter  data_time: 0.0368 s/iter total_throughput: 2053.87 samples/s lr: 7.60e-05  
[08/29 22:40:28] lb.utils.events INFO:  eta: 2 days, 3:29:14  iteration: 1999/375342  consumed_samples: 2048000  total_loss: 0.7934  time: 0.4986 s/iter  data_time: 0.0358 s/iter total_throughput: 2053.58 samples/s lr: 7.99e-05  
[08/29 22:41:18] lb.utils.events INFO:  eta: 2 days, 3:30:47  iteration: 2099/375342  consumed_samples: 2150400  total_loss: 0.7876  time: 0.4987 s/iter  data_time: 0.0383 s/iter total_throughput: 2053.29 samples/s lr: 8.39e-05  
[08/29 22:42:08] lb.utils.events INFO:  eta: 2 days, 3:32:02  iteration: 2199/375342  consumed_samples: 2252800  total_loss: 0.7841  time: 0.4988 s/iter  data_time: 0.0380 s/iter total_throughput: 2053.02 samples/s lr: 8.78e-05  
[08/29 22:42:58] lb.utils.events INFO:  eta: 2 days, 3:32:04  iteration: 2299/375342  consumed_samples: 2355200  total_loss: 0.7813  time: 0.4988 s/iter  data_time: 0.0387 s/iter total_throughput: 2053.00 samples/s lr: 9.18e-05  
[08/29 22:43:48] lb.utils.events INFO:  eta: 2 days, 3:31:52  iteration: 2399/375342  consumed_samples: 2457600  total_loss: 0.7776  time: 0.4988 s/iter  data_time: 0.0398 s/iter total_throughput: 2052.97 samples/s lr: 9.57e-05  
[08/29 22:44:38] lb.utils.events INFO:  eta: 2 days, 3:30:34  iteration: 2499/375342  consumed_samples: 2560000  total_loss: 0.7748  time: 0.4989 s/iter  data_time: 0.0387 s/iter total_throughput: 2052.59 samples/s lr: 9.97e-05  
[08/29 22:45:28] lb.utils.events INFO:  eta: 2 days, 3:30:22  iteration: 2599/375342  consumed_samples: 2662400  total_loss: 0.7703  time: 0.4989 s/iter  data_time: 0.0393 s/iter total_throughput: 2052.36 samples/s lr: 1.04e-04  
[08/29 22:46:18] lb.utils.events INFO:  eta: 2 days, 3:30:22  iteration: 2699/375342  consumed_samples: 2764800  total_loss: 0.7671  time: 0.4989 s/iter  data_time: 0.0385 s/iter total_throughput: 2052.36 samples/s lr: 1.08e-04  
[08/29 22:47:08] lb.utils.events INFO:  eta: 2 days, 3:31:13  iteration: 2799/375342  consumed_samples: 2867200  total_loss: 0.7649  time: 0.4990 s/iter  data_time: 0.0385 s/iter total_throughput: 2052.14 samples/s lr: 1.12e-04  
[08/29 22:47:58] lb.utils.events INFO:  eta: 2 days, 3:30:30  iteration: 2899/375342  consumed_samples: 2969600  total_loss: 0.7626  time: 0.4990 s/iter  data_time: 0.0371 s/iter total_throughput: 2052.02 samples/s lr: 1.15e-04  
[08/29 22:48:48] lb.utils.events INFO:  eta: 2 days, 3:30:17  iteration: 2999/375342  consumed_samples: 3072000  total_loss: 0.7612  time: 0.4991 s/iter  data_time: 0.0383 s/iter total_throughput: 2051.87 samples/s lr: 1.19e-04  
[08/29 22:49:38] lb.utils.events INFO:  eta: 2 days, 3:27:39  iteration: 3099/375342  consumed_samples: 3174400  total_loss: 0.7561  time: 0.4991 s/iter  data_time: 0.0360 s/iter total_throughput: 2051.82 samples/s lr: 1.23e-04  
[08/29 22:50:28] lb.utils.events INFO:  eta: 2 days, 3:28:05  iteration: 3199/375342  consumed_samples: 3276800  total_loss: 0.7505  time: 0.4991 s/iter  data_time: 0.0405 s/iter total_throughput: 2051.68 samples/s lr: 1.27e-04  
[08/29 22:51:18] lb.utils.events INFO:  eta: 2 days, 3:27:41  iteration: 3299/375342  consumed_samples: 3379200  total_loss: 0.7481  time: 0.4991 s/iter  data_time: 0.0382 s/iter total_throughput: 2051.56 samples/s lr: 1.31e-04  
[08/29 22:52:08] lb.utils.events INFO:  eta: 2 days, 3:27:32  iteration: 3399/375342  consumed_samples: 3481600  total_loss: 0.7453  time: 0.4992 s/iter  data_time: 0.0375 s/iter total_throughput: 2051.49 samples/s lr: 1.35e-04  
[08/29 22:52:58] lb.utils.events INFO:  eta: 2 days, 3:26:56  iteration: 3499/375342  consumed_samples: 3584000  total_loss: 0.7434  time: 0.4992 s/iter  data_time: 0.0387 s/iter total_throughput: 2051.16 samples/s lr: 1.39e-04  
[08/29 22:53:48] lb.utils.events INFO:  eta: 2 days, 3:26:06  iteration: 3599/375342  consumed_samples: 3686400  total_loss: 0.7414  time: 0.4993 s/iter  data_time: 0.0387 s/iter total_throughput: 2051.05 samples/s lr: 1.43e-04  
[08/29 22:54:38] lb.utils.events INFO:  eta: 2 days, 3:25:16  iteration: 3699/375342  consumed_samples: 3788800  total_loss: 0.7399  time: 0.4993 s/iter  data_time: 0.0391 s/iter total_throughput: 2050.92 samples/s lr: 1.47e-04  
[08/29 22:55:28] lb.utils.events INFO:  eta: 2 days, 3:23:59  iteration: 3799/375342  consumed_samples: 3891200  total_loss: 0.7346  time: 0.4993 s/iter  data_time: 0.0385 s/iter total_throughput: 2050.83 samples/s lr: 1.51e-04  
[08/29 22:56:18] lb.utils.events INFO:  eta: 2 days, 3:23:23  iteration: 3899/375342  consumed_samples: 3993600  total_loss: 0.7309  time: 0.4993 s/iter  data_time: 0.0394 s/iter total_throughput: 2050.69 samples/s lr: 1.55e-04  
[08/29 22:57:09] lb.utils.events INFO:  eta: 2 days, 3:23:19  iteration: 3999/375342  consumed_samples: 4096000  total_loss: 0.7305  time: 0.4994 s/iter  data_time: 0.0404 s/iter total_throughput: 2050.44 samples/s lr: 1.59e-04  
[08/29 22:57:59] lb.utils.events INFO:  eta: 2 days, 3:22:59  iteration: 4099/375342  consumed_samples: 4198400  total_loss: 0.7296  time: 0.4994 s/iter  data_time: 0.0396 s/iter total_throughput: 2050.35 samples/s lr: 1.63e-04  
[08/29 22:58:49] lb.utils.events INFO:  eta: 2 days, 3:21:59  iteration: 4199/375342  consumed_samples: 4300800  total_loss: 0.7263  time: 0.4994 s/iter  data_time: 0.0379 s/iter total_throughput: 2050.39 samples/s lr: 1.67e-04  
[08/29 22:59:38] lb.utils.events INFO:  eta: 2 days, 3:21:04  iteration: 4299/375342  consumed_samples: 4403200  total_loss: 0.7188  time: 0.4994 s/iter  data_time: 0.0405 s/iter total_throughput: 2050.38 samples/s lr: 1.71e-04  
[08/29 23:00:29] lb.utils.events INFO:  eta: 2 days, 3:21:19  iteration: 4399/375342  consumed_samples: 4505600  total_loss: 0.7159  time: 0.4995 s/iter  data_time: 0.0398 s/iter total_throughput: 2050.23 samples/s lr: 1.75e-04  
[08/29 23:01:19] lb.utils.events INFO:  eta: 2 days, 3:19:45  iteration: 4499/375342  consumed_samples: 4608000  total_loss: 0.7136  time: 0.4995 s/iter  data_time: 0.0406 s/iter total_throughput: 2050.13 samples/s lr: 1.79e-04  
[08/29 23:02:09] lb.utils.events INFO:  eta: 2 days, 3:18:38  iteration: 4599/375342  consumed_samples: 4710400  total_loss: 0.7115  time: 0.4995 s/iter  data_time: 0.0376 s/iter total_throughput: 2050.12 samples/s lr: 1.83e-04  
[08/29 23:02:59] lb.utils.events INFO:  eta: 2 days, 3:17:48  iteration: 4699/375342  consumed_samples: 4812800  total_loss: 0.7087  time: 0.4995 s/iter  data_time: 0.0378 s/iter total_throughput: 2050.17 samples/s lr: 1.87e-04  
[08/29 23:03:48] lb.utils.events INFO:  eta: 2 days, 3:17:10  iteration: 4799/375342  consumed_samples: 4915200  total_loss: 0.7042  time: 0.4995 s/iter  data_time: 0.0391 s/iter total_throughput: 2050.19 samples/s lr: 1.91e-04  
[08/29 23:04:39] lb.utils.events INFO:  eta: 2 days, 3:16:10  iteration: 4899/375342  consumed_samples: 5017600  total_loss: 0.7062  time: 0.4995 s/iter  data_time: 0.0383 s/iter total_throughput: 2049.91 samples/s lr: 1.94e-04  
[08/29 23:05:29] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0004999
[08/29 23:05:30] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/29 23:05:30] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/29 23:05:34] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0562 s/iter. Inference: 0.2521 s/iter. Eval: 0.0024 s/iter. Total: 0.3107 s/iter. ETA=0:00:11
[08/29 23:05:40] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0839 s/iter. Inference: 0.2455 s/iter. Eval: 0.0023 s/iter. Total: 0.3319 s/iter. ETA=0:00:07
[08/29 23:05:45] lb.evaluation.evaluator INFO: Inference done 41984/50000. Dataloading: 0.0836 s/iter. Inference: 0.2466 s/iter. Eval: 0.0023 s/iter. Total: 0.3327 s/iter. ETA=0:00:02
[08/29 23:05:47] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/29 23:05:47] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.225659 (0.000285 s / iter per device, on 8 devices)
[08/29 23:05:47] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000217 s / iter per device, on 8 devices)
[08/29 23:05:47] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/29 23:05:47] lb.evaluation.utils INFO: copypaste: Acc@1=10.592
[08/29 23:05:47] lb.evaluation.utils INFO: copypaste: Acc@5=24.478
[08/29 23:05:47] lb.engine.hooks INFO: Saved first model at 10.59200 @ 4999 steps
[08/29 23:05:47] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/29 23:05:47] lb.utils.events INFO:  eta: 2 days, 3:15:46  iteration: 4999/375342  consumed_samples: 5120000  total_loss: 0.7032  time: 0.4996 s/iter  data_time: 0.0406 s/iter total_throughput: 2049.72 samples/s lr: 1.98e-04  
[08/29 23:06:38] lb.utils.events INFO:  eta: 2 days, 3:15:07  iteration: 5099/375342  consumed_samples: 5222400  total_loss: 0.6981  time: 0.4996 s/iter  data_time: 0.0381 s/iter total_throughput: 2049.44 samples/s lr: 2.02e-04  
[08/29 23:07:28] lb.utils.events INFO:  eta: 2 days, 3:16:05  iteration: 5199/375342  consumed_samples: 5324800  total_loss: 0.6941  time: 0.4997 s/iter  data_time: 0.0408 s/iter total_throughput: 2049.24 samples/s lr: 2.06e-04  
[08/29 23:08:18] lb.utils.events INFO:  eta: 2 days, 3:16:38  iteration: 5299/375342  consumed_samples: 5427200  total_loss: 0.6909  time: 0.4997 s/iter  data_time: 0.0398 s/iter total_throughput: 2049.07 samples/s lr: 2.10e-04  
[08/29 23:09:08] lb.utils.events INFO:  eta: 2 days, 3:16:03  iteration: 5399/375342  consumed_samples: 5529600  total_loss: 0.6911  time: 0.4998 s/iter  data_time: 0.0399 s/iter total_throughput: 2048.90 samples/s lr: 2.14e-04  
[08/29 23:09:59] lb.utils.events INFO:  eta: 2 days, 3:15:45  iteration: 5499/375342  consumed_samples: 5632000  total_loss: 0.6839  time: 0.4998 s/iter  data_time: 0.0400 s/iter total_throughput: 2048.66 samples/s lr: 2.18e-04  
[08/29 23:10:49] lb.utils.events INFO:  eta: 2 days, 3:15:44  iteration: 5599/375342  consumed_samples: 5734400  total_loss: 0.6808  time: 0.4999 s/iter  data_time: 0.0401 s/iter total_throughput: 2048.46 samples/s lr: 2.22e-04  
[08/29 23:11:39] lb.utils.events INFO:  eta: 2 days, 3:17:28  iteration: 5699/375342  consumed_samples: 5836800  total_loss: 0.6789  time: 0.4999 s/iter  data_time: 0.0394 s/iter total_throughput: 2048.27 samples/s lr: 2.26e-04  
[08/29 23:12:29] lb.utils.events INFO:  eta: 2 days, 3:20:04  iteration: 5799/375342  consumed_samples: 5939200  total_loss: 0.6794  time: 0.5000 s/iter  data_time: 0.0407 s/iter total_throughput: 2048.19 samples/s lr: 2.30e-04  
[08/29 23:13:20] lb.utils.events INFO:  eta: 2 days, 3:20:28  iteration: 5899/375342  consumed_samples: 6041600  total_loss: 0.6731  time: 0.5000 s/iter  data_time: 0.0400 s/iter total_throughput: 2047.96 samples/s lr: 2.34e-04  
[08/29 23:14:10] lb.utils.events INFO:  eta: 2 days, 3:19:22  iteration: 5999/375342  consumed_samples: 6144000  total_loss: 0.672  time: 0.5001 s/iter  data_time: 0.0397 s/iter total_throughput: 2047.72 samples/s lr: 2.38e-04  
[08/29 23:15:00] lb.utils.events INFO:  eta: 2 days, 3:18:39  iteration: 6099/375342  consumed_samples: 6246400  total_loss: 0.6723  time: 0.5001 s/iter  data_time: 0.0397 s/iter total_throughput: 2047.58 samples/s lr: 2.42e-04  
[08/29 23:15:50] lb.utils.events INFO:  eta: 2 days, 3:16:44  iteration: 6199/375342  consumed_samples: 6348800  total_loss: 0.6715  time: 0.5001 s/iter  data_time: 0.0407 s/iter total_throughput: 2047.45 samples/s lr: 2.46e-04  
[08/29 23:16:41] lb.utils.events INFO:  eta: 2 days, 3:15:04  iteration: 6299/375342  consumed_samples: 6451200  total_loss: 0.6717  time: 0.5002 s/iter  data_time: 0.0395 s/iter total_throughput: 2047.32 samples/s lr: 2.50e-04  
[08/29 23:17:31] lb.utils.events INFO:  eta: 2 days, 3:12:32  iteration: 6399/375342  consumed_samples: 6553600  total_loss: 0.6704  time: 0.5002 s/iter  data_time: 0.0399 s/iter total_throughput: 2047.19 samples/s lr: 2.54e-04  
[08/29 23:18:21] lb.utils.events INFO:  eta: 2 days, 3:12:22  iteration: 6499/375342  consumed_samples: 6656000  total_loss: 0.6563  time: 0.5002 s/iter  data_time: 0.0406 s/iter total_throughput: 2046.99 samples/s lr: 2.58e-04  
[08/29 23:19:11] lb.utils.events INFO:  eta: 2 days, 3:11:27  iteration: 6599/375342  consumed_samples: 6758400  total_loss: 0.6531  time: 0.5003 s/iter  data_time: 0.0397 s/iter total_throughput: 2046.94 samples/s lr: 2.62e-04  
[08/29 23:20:02] lb.utils.events INFO:  eta: 2 days, 3:09:51  iteration: 6699/375342  consumed_samples: 6860800  total_loss: 0.6505  time: 0.5003 s/iter  data_time: 0.0388 s/iter total_throughput: 2046.84 samples/s lr: 2.66e-04  
[08/29 23:20:52] lb.utils.events INFO:  eta: 2 days, 3:09:35  iteration: 6799/375342  consumed_samples: 6963200  total_loss: 0.6519  time: 0.5003 s/iter  data_time: 0.0382 s/iter total_throughput: 2046.76 samples/s lr: 2.69e-04  
[08/29 23:21:42] lb.utils.events INFO:  eta: 2 days, 3:08:11  iteration: 6899/375342  consumed_samples: 7065600  total_loss: 0.6541  time: 0.5003 s/iter  data_time: 0.0379 s/iter total_throughput: 2046.64 samples/s lr: 2.73e-04  
[08/29 23:22:32] lb.utils.events INFO:  eta: 2 days, 3:06:46  iteration: 6999/375342  consumed_samples: 7168000  total_loss: 0.6504  time: 0.5004 s/iter  data_time: 0.0399 s/iter total_throughput: 2046.54 samples/s lr: 2.77e-04  
[08/29 23:23:22] lb.utils.events INFO:  eta: 2 days, 3:06:32  iteration: 7099/375342  consumed_samples: 7270400  total_loss: 0.6466  time: 0.5004 s/iter  data_time: 0.0408 s/iter total_throughput: 2046.44 samples/s lr: 2.81e-04  
[08/29 23:24:13] lb.utils.events INFO:  eta: 2 days, 3:06:04  iteration: 7199/375342  consumed_samples: 7372800  total_loss: 0.6453  time: 0.5004 s/iter  data_time: 0.0393 s/iter total_throughput: 2046.31 samples/s lr: 2.85e-04  
[08/29 23:25:03] lb.utils.events INFO:  eta: 2 days, 3:06:01  iteration: 7299/375342  consumed_samples: 7475200  total_loss: 0.6437  time: 0.5005 s/iter  data_time: 0.0400 s/iter total_throughput: 2046.12 samples/s lr: 2.89e-04  
[08/29 23:25:53] lb.utils.events INFO:  eta: 2 days, 3:06:09  iteration: 7399/375342  consumed_samples: 7577600  total_loss: 0.6447  time: 0.5005 s/iter  data_time: 0.0405 s/iter total_throughput: 2046.07 samples/s lr: 2.93e-04  
[08/29 23:26:44] lb.utils.events INFO:  eta: 2 days, 3:05:31  iteration: 7499/375342  consumed_samples: 7680000  total_loss: 0.6368  time: 0.5005 s/iter  data_time: 0.0407 s/iter total_throughput: 2045.95 samples/s lr: 2.97e-04  
[08/29 23:27:34] lb.utils.events INFO:  eta: 2 days, 3:06:43  iteration: 7599/375342  consumed_samples: 7782400  total_loss: 0.6384  time: 0.5005 s/iter  data_time: 0.0400 s/iter total_throughput: 2045.82 samples/s lr: 3.01e-04  
[08/29 23:28:24] lb.utils.events INFO:  eta: 2 days, 3:05:50  iteration: 7699/375342  consumed_samples: 7884800  total_loss: 0.6353  time: 0.5005 s/iter  data_time: 0.0388 s/iter total_throughput: 2045.79 samples/s lr: 3.05e-04  
[08/29 23:29:14] lb.utils.events INFO:  eta: 2 days, 3:05:06  iteration: 7799/375342  consumed_samples: 7987200  total_loss: 0.6318  time: 0.5006 s/iter  data_time: 0.0407 s/iter total_throughput: 2045.70 samples/s lr: 3.09e-04  
[08/29 23:30:04] lb.utils.events INFO:  eta: 2 days, 3:04:16  iteration: 7899/375342  consumed_samples: 8089600  total_loss: 0.6372  time: 0.5006 s/iter  data_time: 0.0410 s/iter total_throughput: 2045.67 samples/s lr: 3.13e-04  
[08/29 23:30:55] lb.utils.events INFO:  eta: 2 days, 3:03:50  iteration: 7999/375342  consumed_samples: 8192000  total_loss: 0.6373  time: 0.5006 s/iter  data_time: 0.0400 s/iter total_throughput: 2045.54 samples/s lr: 3.17e-04  
[08/29 23:31:45] lb.utils.events INFO:  eta: 2 days, 3:02:52  iteration: 8099/375342  consumed_samples: 8294400  total_loss: 0.6275  time: 0.5006 s/iter  data_time: 0.0393 s/iter total_throughput: 2045.41 samples/s lr: 3.21e-04  
[08/29 23:32:35] lb.utils.events INFO:  eta: 2 days, 3:02:19  iteration: 8199/375342  consumed_samples: 8396800  total_loss: 0.6225  time: 0.5007 s/iter  data_time: 0.0404 s/iter total_throughput: 2045.31 samples/s lr: 3.25e-04  
[08/29 23:33:25] lb.utils.events INFO:  eta: 2 days, 3:01:04  iteration: 8299/375342  consumed_samples: 8499200  total_loss: 0.6304  time: 0.5007 s/iter  data_time: 0.0399 s/iter total_throughput: 2045.30 samples/s lr: 3.29e-04  
[08/29 23:34:16] lb.utils.events INFO:  eta: 2 days, 3:00:22  iteration: 8399/375342  consumed_samples: 8601600  total_loss: 0.6279  time: 0.5007 s/iter  data_time: 0.0388 s/iter total_throughput: 2045.24 samples/s lr: 3.33e-04  
[08/29 23:35:06] lb.utils.events INFO:  eta: 2 days, 2:59:45  iteration: 8499/375342  consumed_samples: 8704000  total_loss: 0.6209  time: 0.5007 s/iter  data_time: 0.0394 s/iter total_throughput: 2045.13 samples/s lr: 3.37e-04  
[08/29 23:35:56] lb.utils.events INFO:  eta: 2 days, 2:58:38  iteration: 8599/375342  consumed_samples: 8806400  total_loss: 0.6201  time: 0.5007 s/iter  data_time: 0.0400 s/iter total_throughput: 2045.08 samples/s lr: 3.41e-04  
[08/29 23:36:46] lb.utils.events INFO:  eta: 2 days, 2:57:46  iteration: 8699/375342  consumed_samples: 8908800  total_loss: 0.614  time: 0.5007 s/iter  data_time: 0.0410 s/iter total_throughput: 2045.04 samples/s lr: 3.45e-04  
[08/29 23:37:37] lb.utils.events INFO:  eta: 2 days, 2:56:34  iteration: 8799/375342  consumed_samples: 9011200  total_loss: 0.6134  time: 0.5008 s/iter  data_time: 0.0383 s/iter total_throughput: 2044.92 samples/s lr: 3.48e-04  
[08/29 23:38:27] lb.utils.events INFO:  eta: 2 days, 2:55:58  iteration: 8899/375342  consumed_samples: 9113600  total_loss: 0.6162  time: 0.5008 s/iter  data_time: 0.0390 s/iter total_throughput: 2044.80 samples/s lr: 3.52e-04  
[08/29 23:39:17] lb.utils.events INFO:  eta: 2 days, 2:53:54  iteration: 8999/375342  consumed_samples: 9216000  total_loss: 0.6147  time: 0.5008 s/iter  data_time: 0.0416 s/iter total_throughput: 2044.78 samples/s lr: 3.56e-04  
[08/29 23:40:07] lb.utils.events INFO:  eta: 2 days, 2:54:00  iteration: 9099/375342  consumed_samples: 9318400  total_loss: 0.6171  time: 0.5008 s/iter  data_time: 0.0402 s/iter total_throughput: 2044.67 samples/s lr: 3.60e-04  
[08/29 23:40:58] lb.utils.events INFO:  eta: 2 days, 2:53:14  iteration: 9199/375342  consumed_samples: 9420800  total_loss: 0.6062  time: 0.5008 s/iter  data_time: 0.0399 s/iter total_throughput: 2044.62 samples/s lr: 3.64e-04  
[08/29 23:41:48] lb.utils.events INFO:  eta: 2 days, 2:51:11  iteration: 9299/375342  consumed_samples: 9523200  total_loss: 0.583  time: 0.5008 s/iter  data_time: 0.0394 s/iter total_throughput: 2044.60 samples/s lr: 3.68e-04  
[08/29 23:42:38] lb.utils.events INFO:  eta: 2 days, 2:50:11  iteration: 9399/375342  consumed_samples: 9625600  total_loss: 0.5712  time: 0.5009 s/iter  data_time: 0.0387 s/iter total_throughput: 2044.50 samples/s lr: 3.72e-04  
[08/29 23:43:28] lb.utils.events INFO:  eta: 2 days, 2:48:45  iteration: 9499/375342  consumed_samples: 9728000  total_loss: 0.5622  time: 0.5009 s/iter  data_time: 0.0394 s/iter total_throughput: 2044.47 samples/s lr: 3.76e-04  
[08/29 23:44:19] lb.utils.events INFO:  eta: 2 days, 2:48:02  iteration: 9599/375342  consumed_samples: 9830400  total_loss: 0.5682  time: 0.5009 s/iter  data_time: 0.0396 s/iter total_throughput: 2044.35 samples/s lr: 3.80e-04  
[08/29 23:45:09] lb.utils.events INFO:  eta: 2 days, 2:47:35  iteration: 9699/375342  consumed_samples: 9932800  total_loss: 0.5637  time: 0.5009 s/iter  data_time: 0.0381 s/iter total_throughput: 2044.29 samples/s lr: 3.84e-04  
[08/29 23:45:59] lb.utils.events INFO:  eta: 2 days, 2:46:50  iteration: 9799/375342  consumed_samples: 10035200  total_loss: 0.5561  time: 0.5009 s/iter  data_time: 0.0398 s/iter total_throughput: 2044.26 samples/s lr: 3.88e-04  
[08/29 23:46:49] lb.utils.events INFO:  eta: 2 days, 2:46:38  iteration: 9899/375342  consumed_samples: 10137600  total_loss: 0.5544  time: 0.5009 s/iter  data_time: 0.0377 s/iter total_throughput: 2044.23 samples/s lr: 3.92e-04  
[08/29 23:47:40] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0009999
[08/29 23:47:40] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/29 23:47:40] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/29 23:47:45] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0615 s/iter. Inference: 0.2460 s/iter. Eval: 0.0021 s/iter. Total: 0.3096 s/iter. ETA=0:00:11
[08/29 23:47:50] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0900 s/iter. Inference: 0.2412 s/iter. Eval: 0.0022 s/iter. Total: 0.3336 s/iter. ETA=0:00:07
[08/29 23:47:55] lb.evaluation.evaluator INFO: Inference done 43008/50000. Dataloading: 0.0863 s/iter. Inference: 0.2397 s/iter. Eval: 0.0024 s/iter. Total: 0.3285 s/iter. ETA=0:00:01
[08/29 23:47:57] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/29 23:47:57] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.161677 (0.000283 s / iter per device, on 8 devices)
[08/29 23:47:57] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000212 s / iter per device, on 8 devices)
[08/29 23:47:57] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/29 23:47:57] lb.evaluation.utils INFO: copypaste: Acc@1=34.412
[08/29 23:47:57] lb.evaluation.utils INFO: copypaste: Acc@5=58.278
[08/29 23:47:57] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 34.41200, better than last best score 10.59200 @ iteration 4999.
[08/29 23:47:57] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/29 23:47:58] lb.utils.events INFO:  eta: 2 days, 2:47:35  iteration: 9999/375342  consumed_samples: 10240000  total_loss: 0.5556  time: 0.5010 s/iter  data_time: 0.0393 s/iter total_throughput: 2044.09 samples/s lr: 3.96e-04  
[08/29 23:48:48] lb.utils.events INFO:  eta: 2 days, 2:46:36  iteration: 10099/375342  consumed_samples: 10342400  total_loss: 0.5531  time: 0.5010 s/iter  data_time: 0.0384 s/iter total_throughput: 2044.02 samples/s lr: 4.00e-04  
[08/29 23:49:39] lb.utils.events INFO:  eta: 2 days, 2:45:44  iteration: 10199/375342  consumed_samples: 10444800  total_loss: 0.5491  time: 0.5010 s/iter  data_time: 0.0366 s/iter total_throughput: 2043.91 samples/s lr: 4.04e-04  
[08/29 23:50:29] lb.utils.events INFO:  eta: 2 days, 2:46:23  iteration: 10299/375342  consumed_samples: 10547200  total_loss: 0.5495  time: 0.5010 s/iter  data_time: 0.0389 s/iter total_throughput: 2043.85 samples/s lr: 4.08e-04  
[08/29 23:51:19] lb.utils.events INFO:  eta: 2 days, 2:44:21  iteration: 10399/375342  consumed_samples: 10649600  total_loss: 0.5446  time: 0.5010 s/iter  data_time: 0.0372 s/iter total_throughput: 2043.78 samples/s lr: 4.12e-04  
[08/29 23:52:10] lb.utils.events INFO:  eta: 2 days, 2:44:32  iteration: 10499/375342  consumed_samples: 10752000  total_loss: 0.5511  time: 0.5010 s/iter  data_time: 0.0408 s/iter total_throughput: 2043.73 samples/s lr: 4.16e-04  
[08/29 23:53:00] lb.utils.events INFO:  eta: 2 days, 2:42:25  iteration: 10599/375342  consumed_samples: 10854400  total_loss: 0.5568  time: 0.5010 s/iter  data_time: 0.0387 s/iter total_throughput: 2043.73 samples/s lr: 4.20e-04  
[08/29 23:53:50] lb.utils.events INFO:  eta: 2 days, 2:40:56  iteration: 10699/375342  consumed_samples: 10956800  total_loss: 0.5471  time: 0.5010 s/iter  data_time: 0.0401 s/iter total_throughput: 2043.71 samples/s lr: 4.24e-04  
[08/29 23:54:40] lb.utils.events INFO:  eta: 2 days, 2:39:34  iteration: 10799/375342  consumed_samples: 11059200  total_loss: 0.5471  time: 0.5011 s/iter  data_time: 0.0381 s/iter total_throughput: 2043.68 samples/s lr: 4.27e-04  
[08/29 23:55:30] lb.utils.events INFO:  eta: 2 days, 2:36:42  iteration: 10899/375342  consumed_samples: 11161600  total_loss: 0.5467  time: 0.5011 s/iter  data_time: 0.0379 s/iter total_throughput: 2043.63 samples/s lr: 4.31e-04  
[08/29 23:56:21] lb.utils.events INFO:  eta: 2 days, 2:34:09  iteration: 10999/375342  consumed_samples: 11264000  total_loss: 0.5445  time: 0.5011 s/iter  data_time: 0.0404 s/iter total_throughput: 2043.63 samples/s lr: 4.35e-04  
[08/29 23:57:11] lb.utils.events INFO:  eta: 2 days, 2:33:13  iteration: 11099/375342  consumed_samples: 11366400  total_loss: 0.5481  time: 0.5011 s/iter  data_time: 0.0401 s/iter total_throughput: 2043.60 samples/s lr: 4.39e-04  
[08/29 23:58:01] lb.utils.events INFO:  eta: 2 days, 2:31:44  iteration: 11199/375342  consumed_samples: 11468800  total_loss: 0.5464  time: 0.5011 s/iter  data_time: 0.0395 s/iter total_throughput: 2043.61 samples/s lr: 4.43e-04  
[08/29 23:58:51] lb.utils.events INFO:  eta: 2 days, 2:30:21  iteration: 11299/375342  consumed_samples: 11571200  total_loss: 0.5422  time: 0.5011 s/iter  data_time: 0.0398 s/iter total_throughput: 2043.59 samples/s lr: 4.47e-04  
[08/29 23:59:41] lb.utils.events INFO:  eta: 2 days, 2:30:00  iteration: 11399/375342  consumed_samples: 11673600  total_loss: 0.5424  time: 0.5011 s/iter  data_time: 0.0395 s/iter total_throughput: 2043.54 samples/s lr: 4.51e-04  
[08/30 00:00:32] lb.utils.events INFO:  eta: 2 days, 2:29:43  iteration: 11499/375342  consumed_samples: 11776000  total_loss: 0.5436  time: 0.5011 s/iter  data_time: 0.0385 s/iter total_throughput: 2043.48 samples/s lr: 4.55e-04  
[08/30 00:01:22] lb.utils.events INFO:  eta: 2 days, 2:28:09  iteration: 11599/375342  consumed_samples: 11878400  total_loss: 0.5354  time: 0.5011 s/iter  data_time: 0.0384 s/iter total_throughput: 2043.48 samples/s lr: 4.59e-04  
[08/30 00:02:12] lb.utils.events INFO:  eta: 2 days, 2:27:19  iteration: 11699/375342  consumed_samples: 11980800  total_loss: 0.5362  time: 0.5011 s/iter  data_time: 0.0367 s/iter total_throughput: 2043.42 samples/s lr: 4.63e-04  
[08/30 00:03:02] lb.utils.events INFO:  eta: 2 days, 2:26:57  iteration: 11799/375342  consumed_samples: 12083200  total_loss: 0.5412  time: 0.5011 s/iter  data_time: 0.0385 s/iter total_throughput: 2043.38 samples/s lr: 4.67e-04  
[08/30 00:03:52] lb.utils.events INFO:  eta: 2 days, 2:27:15  iteration: 11899/375342  consumed_samples: 12185600  total_loss: 0.527  time: 0.5011 s/iter  data_time: 0.0387 s/iter total_throughput: 2043.39 samples/s lr: 4.71e-04  
[08/30 00:04:42] lb.utils.events INFO:  eta: 2 days, 2:26:38  iteration: 11999/375342  consumed_samples: 12288000  total_loss: 0.5269  time: 0.5011 s/iter  data_time: 0.0402 s/iter total_throughput: 2043.39 samples/s lr: 4.75e-04  
[08/30 00:05:33] lb.utils.events INFO:  eta: 2 days, 2:26:15  iteration: 12099/375342  consumed_samples: 12390400  total_loss: 0.5339  time: 0.5011 s/iter  data_time: 0.0402 s/iter total_throughput: 2043.36 samples/s lr: 4.79e-04  
[08/30 00:06:23] lb.utils.events INFO:  eta: 2 days, 2:25:18  iteration: 12199/375342  consumed_samples: 12492800  total_loss: 0.5375  time: 0.5011 s/iter  data_time: 0.0389 s/iter total_throughput: 2043.38 samples/s lr: 4.83e-04  
[08/30 00:07:13] lb.utils.events INFO:  eta: 2 days, 2:26:03  iteration: 12299/375342  consumed_samples: 12595200  total_loss: 0.5353  time: 0.5012 s/iter  data_time: 0.0380 s/iter total_throughput: 2043.29 samples/s lr: 4.87e-04  
[08/30 00:08:04] lb.utils.events INFO:  eta: 2 days, 2:24:22  iteration: 12399/375342  consumed_samples: 12697600  total_loss: 0.5219  time: 0.5012 s/iter  data_time: 0.0389 s/iter total_throughput: 2043.22 samples/s lr: 4.91e-04  
[08/30 00:08:54] lb.utils.events INFO:  eta: 2 days, 2:22:21  iteration: 12499/375342  consumed_samples: 12800000  total_loss: 0.5257  time: 0.5012 s/iter  data_time: 0.0390 s/iter total_throughput: 2043.17 samples/s lr: 4.95e-04  
[08/30 00:09:44] lb.utils.events INFO:  eta: 2 days, 2:21:39  iteration: 12599/375342  consumed_samples: 12902400  total_loss: 0.5292  time: 0.5012 s/iter  data_time: 0.0385 s/iter total_throughput: 2043.18 samples/s lr: 4.99e-04  
[08/30 00:10:34] lb.utils.events INFO:  eta: 2 days, 2:20:20  iteration: 12699/375342  consumed_samples: 13004800  total_loss: 0.5357  time: 0.5012 s/iter  data_time: 0.0386 s/iter total_throughput: 2043.14 samples/s lr: 5.02e-04  
[08/30 00:11:24] lb.utils.events INFO:  eta: 2 days, 2:19:03  iteration: 12799/375342  consumed_samples: 13107200  total_loss: 0.5284  time: 0.5012 s/iter  data_time: 0.0391 s/iter total_throughput: 2043.09 samples/s lr: 5.06e-04  
[08/30 00:12:15] lb.utils.events INFO:  eta: 2 days, 2:19:09  iteration: 12899/375342  consumed_samples: 13209600  total_loss: 0.5222  time: 0.5012 s/iter  data_time: 0.0387 s/iter total_throughput: 2043.02 samples/s lr: 5.10e-04  
[08/30 00:13:05] lb.utils.events INFO:  eta: 2 days, 2:18:22  iteration: 12999/375342  consumed_samples: 13312000  total_loss: 0.5271  time: 0.5012 s/iter  data_time: 0.0389 s/iter total_throughput: 2043.03 samples/s lr: 5.14e-04  
[08/30 00:13:55] lb.utils.events INFO:  eta: 2 days, 2:16:02  iteration: 13099/375342  consumed_samples: 13414400  total_loss: 0.5167  time: 0.5012 s/iter  data_time: 0.0391 s/iter total_throughput: 2043.04 samples/s lr: 5.18e-04  
[08/30 00:14:45] lb.utils.events INFO:  eta: 2 days, 2:15:54  iteration: 13199/375342  consumed_samples: 13516800  total_loss: 0.5196  time: 0.5012 s/iter  data_time: 0.0394 s/iter total_throughput: 2043.04 samples/s lr: 5.22e-04  
[08/30 00:15:36] lb.utils.events INFO:  eta: 2 days, 2:15:09  iteration: 13299/375342  consumed_samples: 13619200  total_loss: 0.5205  time: 0.5012 s/iter  data_time: 0.0379 s/iter total_throughput: 2042.94 samples/s lr: 5.26e-04  
[08/30 00:16:26] lb.utils.events INFO:  eta: 2 days, 2:14:14  iteration: 13399/375342  consumed_samples: 13721600  total_loss: 0.506  time: 0.5012 s/iter  data_time: 0.0376 s/iter total_throughput: 2042.89 samples/s lr: 5.30e-04  
[08/30 00:17:16] lb.utils.events INFO:  eta: 2 days, 2:14:04  iteration: 13499/375342  consumed_samples: 13824000  total_loss: 0.5066  time: 0.5013 s/iter  data_time: 0.0399 s/iter total_throughput: 2042.88 samples/s lr: 5.34e-04  
[08/30 00:18:06] lb.utils.events INFO:  eta: 2 days, 2:13:27  iteration: 13599/375342  consumed_samples: 13926400  total_loss: 0.5179  time: 0.5013 s/iter  data_time: 0.0384 s/iter total_throughput: 2042.86 samples/s lr: 5.38e-04  
[08/30 00:18:56] lb.utils.events INFO:  eta: 2 days, 2:12:46  iteration: 13699/375342  consumed_samples: 14028800  total_loss: 0.5215  time: 0.5013 s/iter  data_time: 0.0385 s/iter total_throughput: 2042.88 samples/s lr: 5.42e-04  
[08/30 00:19:47] lb.utils.events INFO:  eta: 2 days, 2:13:09  iteration: 13799/375342  consumed_samples: 14131200  total_loss: 0.5198  time: 0.5013 s/iter  data_time: 0.0366 s/iter total_throughput: 2042.80 samples/s lr: 5.46e-04  
[08/30 00:20:37] lb.utils.events INFO:  eta: 2 days, 2:12:20  iteration: 13899/375342  consumed_samples: 14233600  total_loss: 0.5158  time: 0.5013 s/iter  data_time: 0.0375 s/iter total_throughput: 2042.73 samples/s lr: 5.50e-04  
[08/30 00:21:27] lb.utils.events INFO:  eta: 2 days, 2:11:41  iteration: 13999/375342  consumed_samples: 14336000  total_loss: 0.5129  time: 0.5013 s/iter  data_time: 0.0397 s/iter total_throughput: 2042.71 samples/s lr: 5.54e-04  
[08/30 00:22:18] lb.utils.events INFO:  eta: 2 days, 2:11:48  iteration: 14099/375342  consumed_samples: 14438400  total_loss: 0.5118  time: 0.5013 s/iter  data_time: 0.0387 s/iter total_throughput: 2042.67 samples/s lr: 5.58e-04  
[08/30 00:23:08] lb.utils.events INFO:  eta: 2 days, 2:10:09  iteration: 14199/375342  consumed_samples: 14540800  total_loss: 0.5136  time: 0.5013 s/iter  data_time: 0.0384 s/iter total_throughput: 2042.69 samples/s lr: 5.62e-04  
[08/30 00:23:58] lb.utils.events INFO:  eta: 2 days, 2:09:08  iteration: 14299/375342  consumed_samples: 14643200  total_loss: 0.5074  time: 0.5013 s/iter  data_time: 0.0401 s/iter total_throughput: 2042.65 samples/s lr: 5.66e-04  
[08/30 00:24:48] lb.utils.events INFO:  eta: 2 days, 2:07:46  iteration: 14399/375342  consumed_samples: 14745600  total_loss: 0.5109  time: 0.5013 s/iter  data_time: 0.0401 s/iter total_throughput: 2042.61 samples/s lr: 5.70e-04  
[08/30 00:25:38] lb.utils.events INFO:  eta: 2 days, 2:07:06  iteration: 14499/375342  consumed_samples: 14848000  total_loss: 0.5156  time: 0.5013 s/iter  data_time: 0.0397 s/iter total_throughput: 2042.61 samples/s lr: 5.74e-04  
[08/30 00:26:29] lb.utils.events INFO:  eta: 2 days, 2:06:16  iteration: 14599/375342  consumed_samples: 14950400  total_loss: 0.5182  time: 0.5013 s/iter  data_time: 0.0415 s/iter total_throughput: 2042.56 samples/s lr: 5.78e-04  
[08/30 00:27:19] lb.utils.events INFO:  eta: 2 days, 2:05:39  iteration: 14699/375342  consumed_samples: 15052800  total_loss: 0.5198  time: 0.5013 s/iter  data_time: 0.0397 s/iter total_throughput: 2042.55 samples/s lr: 5.81e-04  
[08/30 00:28:09] lb.utils.events INFO:  eta: 2 days, 2:04:15  iteration: 14799/375342  consumed_samples: 15155200  total_loss: 0.5098  time: 0.5013 s/iter  data_time: 0.0389 s/iter total_throughput: 2042.53 samples/s lr: 5.85e-04  
[08/30 00:28:59] lb.utils.events INFO:  eta: 2 days, 2:01:23  iteration: 14899/375342  consumed_samples: 15257600  total_loss: 0.5083  time: 0.5013 s/iter  data_time: 0.0389 s/iter total_throughput: 2042.52 samples/s lr: 5.89e-04  
[08/30 00:29:50] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0014999
[08/30 00:29:50] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 00:29:50] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 00:29:55] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0521 s/iter. Inference: 0.2453 s/iter. Eval: 0.0024 s/iter. Total: 0.2999 s/iter. ETA=0:00:11
[08/30 00:30:00] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0812 s/iter. Inference: 0.2437 s/iter. Eval: 0.0022 s/iter. Total: 0.3275 s/iter. ETA=0:00:07
[08/30 00:30:05] lb.evaluation.evaluator INFO: Inference done 41984/50000. Dataloading: 0.0859 s/iter. Inference: 0.2440 s/iter. Eval: 0.0022 s/iter. Total: 0.3325 s/iter. ETA=0:00:02
[08/30 00:30:08] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 00:30:08] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.231225 (0.000285 s / iter per device, on 8 devices)
[08/30 00:30:08] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000215 s / iter per device, on 8 devices)
[08/30 00:30:08] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 00:30:08] lb.evaluation.utils INFO: copypaste: Acc@1=52.93
[08/30 00:30:08] lb.evaluation.utils INFO: copypaste: Acc@5=78.794
[08/30 00:30:08] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 52.93000, better than last best score 34.41200 @ iteration 9999.
[08/30 00:30:08] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 00:30:08] lb.utils.events INFO:  eta: 2 days, 2:02:03  iteration: 14999/375342  consumed_samples: 15360000  total_loss: 0.5148  time: 0.5014 s/iter  data_time: 0.0393 s/iter total_throughput: 2042.46 samples/s lr: 5.93e-04  
[08/30 00:30:58] lb.utils.events INFO:  eta: 2 days, 1:57:57  iteration: 15099/375342  consumed_samples: 15462400  total_loss: 0.5072  time: 0.5013 s/iter  data_time: 0.0369 s/iter total_throughput: 2042.50 samples/s lr: 5.97e-04  
[08/30 00:31:49] lb.utils.events INFO:  eta: 2 days, 1:57:19  iteration: 15199/375342  consumed_samples: 15564800  total_loss: 0.5032  time: 0.5014 s/iter  data_time: 0.0393 s/iter total_throughput: 2042.46 samples/s lr: 6.01e-04  
[08/30 00:32:39] lb.utils.events INFO:  eta: 2 days, 1:56:26  iteration: 15299/375342  consumed_samples: 15667200  total_loss: 0.4971  time: 0.5014 s/iter  data_time: 0.0388 s/iter total_throughput: 2042.48 samples/s lr: 6.05e-04  
[08/30 00:33:29] lb.utils.events INFO:  eta: 2 days, 1:56:19  iteration: 15399/375342  consumed_samples: 15769600  total_loss: 0.4953  time: 0.5014 s/iter  data_time: 0.0376 s/iter total_throughput: 2042.45 samples/s lr: 6.09e-04  
[08/30 00:34:19] lb.utils.events INFO:  eta: 2 days, 1:56:28  iteration: 15499/375342  consumed_samples: 15872000  total_loss: 0.4989  time: 0.5014 s/iter  data_time: 0.0407 s/iter total_throughput: 2042.43 samples/s lr: 6.13e-04  
[08/30 00:35:09] lb.utils.events INFO:  eta: 2 days, 1:56:32  iteration: 15599/375342  consumed_samples: 15974400  total_loss: 0.4992  time: 0.5014 s/iter  data_time: 0.0401 s/iter total_throughput: 2042.42 samples/s lr: 6.17e-04  
[08/30 00:35:59] lb.utils.events INFO:  eta: 2 days, 1:54:55  iteration: 15699/375342  consumed_samples: 16076800  total_loss: 0.4927  time: 0.5014 s/iter  data_time: 0.0390 s/iter total_throughput: 2042.43 samples/s lr: 6.21e-04  
[08/30 00:36:50] lb.utils.events INFO:  eta: 2 days, 1:54:05  iteration: 15799/375342  consumed_samples: 16179200  total_loss: 0.4902  time: 0.5014 s/iter  data_time: 0.0393 s/iter total_throughput: 2042.43 samples/s lr: 6.25e-04  
[08/30 00:37:40] lb.utils.events INFO:  eta: 2 days, 1:54:02  iteration: 15899/375342  consumed_samples: 16281600  total_loss: 0.4995  time: 0.5014 s/iter  data_time: 0.0374 s/iter total_throughput: 2042.42 samples/s lr: 6.29e-04  
[08/30 00:38:30] lb.utils.events INFO:  eta: 2 days, 1:52:21  iteration: 15999/375342  consumed_samples: 16384000  total_loss: 0.5019  time: 0.5014 s/iter  data_time: 0.0376 s/iter total_throughput: 2042.45 samples/s lr: 6.33e-04  
[08/30 00:39:20] lb.utils.events INFO:  eta: 2 days, 1:53:11  iteration: 16099/375342  consumed_samples: 16486400  total_loss: 0.5033  time: 0.5014 s/iter  data_time: 0.0360 s/iter total_throughput: 2042.43 samples/s lr: 6.37e-04  
[08/30 00:40:10] lb.utils.events INFO:  eta: 2 days, 1:52:37  iteration: 16199/375342  consumed_samples: 16588800  total_loss: 0.5028  time: 0.5014 s/iter  data_time: 0.0399 s/iter total_throughput: 2042.43 samples/s lr: 6.41e-04  
[08/30 00:41:00] lb.utils.events INFO:  eta: 2 days, 1:52:07  iteration: 16299/375342  consumed_samples: 16691200  total_loss: 0.4978  time: 0.5014 s/iter  data_time: 0.0391 s/iter total_throughput: 2042.41 samples/s lr: 6.45e-04  
[08/30 00:41:51] lb.utils.events INFO:  eta: 2 days, 1:52:18  iteration: 16399/375342  consumed_samples: 16793600  total_loss: 0.4975  time: 0.5014 s/iter  data_time: 0.0391 s/iter total_throughput: 2042.36 samples/s lr: 6.49e-04  
[08/30 00:42:41] lb.utils.events INFO:  eta: 2 days, 1:50:19  iteration: 16499/375342  consumed_samples: 16896000  total_loss: 0.4901  time: 0.5014 s/iter  data_time: 0.0388 s/iter total_throughput: 2042.39 samples/s lr: 6.53e-04  
[08/30 00:43:31] lb.utils.events INFO:  eta: 2 days, 1:49:32  iteration: 16599/375342  consumed_samples: 16998400  total_loss: 0.4889  time: 0.5014 s/iter  data_time: 0.0399 s/iter total_throughput: 2042.39 samples/s lr: 6.57e-04  
[08/30 00:44:21] lb.utils.events INFO:  eta: 2 days, 1:49:37  iteration: 16699/375342  consumed_samples: 17100800  total_loss: 0.4914  time: 0.5014 s/iter  data_time: 0.0399 s/iter total_throughput: 2042.38 samples/s lr: 6.60e-04  
[08/30 00:45:11] lb.utils.events INFO:  eta: 2 days, 1:48:53  iteration: 16799/375342  consumed_samples: 17203200  total_loss: 0.4922  time: 0.5014 s/iter  data_time: 0.0392 s/iter total_throughput: 2042.36 samples/s lr: 6.64e-04  
[08/30 00:46:02] lb.utils.events INFO:  eta: 2 days, 1:48:32  iteration: 16899/375342  consumed_samples: 17305600  total_loss: 0.5022  time: 0.5014 s/iter  data_time: 0.0392 s/iter total_throughput: 2042.32 samples/s lr: 6.68e-04  
[08/30 00:46:52] lb.utils.events INFO:  eta: 2 days, 1:48:17  iteration: 16999/375342  consumed_samples: 17408000  total_loss: 0.4974  time: 0.5014 s/iter  data_time: 0.0397 s/iter total_throughput: 2042.31 samples/s lr: 6.72e-04  
[08/30 00:47:42] lb.utils.events INFO:  eta: 2 days, 1:47:08  iteration: 17099/375342  consumed_samples: 17510400  total_loss: 0.4715  time: 0.5014 s/iter  data_time: 0.0397 s/iter total_throughput: 2042.31 samples/s lr: 6.76e-04  
[08/30 00:48:32] lb.utils.events INFO:  eta: 2 days, 1:45:56  iteration: 17199/375342  consumed_samples: 17612800  total_loss: 0.4834  time: 0.5014 s/iter  data_time: 0.0384 s/iter total_throughput: 2042.33 samples/s lr: 6.80e-04  
[08/30 00:49:22] lb.utils.events INFO:  eta: 2 days, 1:45:12  iteration: 17299/375342  consumed_samples: 17715200  total_loss: 0.4917  time: 0.5014 s/iter  data_time: 0.0395 s/iter total_throughput: 2042.30 samples/s lr: 6.84e-04  
[08/30 00:50:13] lb.utils.events INFO:  eta: 2 days, 1:43:33  iteration: 17399/375342  consumed_samples: 17817600  total_loss: 0.4866  time: 0.5014 s/iter  data_time: 0.0385 s/iter total_throughput: 2042.27 samples/s lr: 6.88e-04  
[08/30 00:51:03] lb.utils.events INFO:  eta: 2 days, 1:43:48  iteration: 17499/375342  consumed_samples: 17920000  total_loss: 0.483  time: 0.5014 s/iter  data_time: 0.0387 s/iter total_throughput: 2042.27 samples/s lr: 6.92e-04  
[08/30 00:51:53] lb.utils.events INFO:  eta: 2 days, 1:42:26  iteration: 17599/375342  consumed_samples: 18022400  total_loss: 0.4761  time: 0.5014 s/iter  data_time: 0.0407 s/iter total_throughput: 2042.26 samples/s lr: 6.96e-04  
[08/30 00:52:43] lb.utils.events INFO:  eta: 2 days, 1:41:31  iteration: 17699/375342  consumed_samples: 18124800  total_loss: 0.4802  time: 0.5014 s/iter  data_time: 0.0380 s/iter total_throughput: 2042.24 samples/s lr: 7.00e-04  
[08/30 00:53:33] lb.utils.events INFO:  eta: 2 days, 1:40:10  iteration: 17799/375342  consumed_samples: 18227200  total_loss: 0.4818  time: 0.5014 s/iter  data_time: 0.0380 s/iter total_throughput: 2042.24 samples/s lr: 7.04e-04  
[08/30 00:54:24] lb.utils.events INFO:  eta: 2 days, 1:37:18  iteration: 17899/375342  consumed_samples: 18329600  total_loss: 0.4833  time: 0.5014 s/iter  data_time: 0.0402 s/iter total_throughput: 2042.25 samples/s lr: 7.08e-04  
[08/30 00:55:14] lb.utils.events INFO:  eta: 2 days, 1:36:16  iteration: 17999/375342  consumed_samples: 18432000  total_loss: 0.4925  time: 0.5014 s/iter  data_time: 0.0401 s/iter total_throughput: 2042.24 samples/s lr: 7.12e-04  
[08/30 00:56:04] lb.utils.events INFO:  eta: 2 days, 1:34:43  iteration: 18099/375342  consumed_samples: 18534400  total_loss: 0.4881  time: 0.5014 s/iter  data_time: 0.0390 s/iter total_throughput: 2042.26 samples/s lr: 7.16e-04  
[08/30 00:56:54] lb.utils.events INFO:  eta: 2 days, 1:34:03  iteration: 18199/375342  consumed_samples: 18636800  total_loss: 0.4888  time: 0.5014 s/iter  data_time: 0.0379 s/iter total_throughput: 2042.27 samples/s lr: 7.20e-04  
[08/30 00:57:44] lb.utils.events INFO:  eta: 2 days, 1:32:39  iteration: 18299/375342  consumed_samples: 18739200  total_loss: 0.4816  time: 0.5014 s/iter  data_time: 0.0396 s/iter total_throughput: 2042.26 samples/s lr: 7.24e-04  
[08/30 00:58:34] lb.utils.events INFO:  eta: 2 days, 1:30:52  iteration: 18399/375342  consumed_samples: 18841600  total_loss: 0.4832  time: 0.5014 s/iter  data_time: 0.0392 s/iter total_throughput: 2042.25 samples/s lr: 7.28e-04  
[08/30 00:59:24] lb.utils.events INFO:  eta: 2 days, 1:29:01  iteration: 18499/375342  consumed_samples: 18944000  total_loss: 0.4923  time: 0.5014 s/iter  data_time: 0.0391 s/iter total_throughput: 2042.25 samples/s lr: 7.32e-04  
[08/30 01:00:14] lb.utils.events INFO:  eta: 2 days, 1:27:47  iteration: 18599/375342  consumed_samples: 19046400  total_loss: 0.4876  time: 0.5014 s/iter  data_time: 0.0386 s/iter total_throughput: 2042.28 samples/s lr: 7.35e-04  
[08/30 01:01:05] lb.utils.events INFO:  eta: 2 days, 1:28:07  iteration: 18699/375342  consumed_samples: 19148800  total_loss: 0.4809  time: 0.5014 s/iter  data_time: 0.0393 s/iter total_throughput: 2042.26 samples/s lr: 7.39e-04  
[08/30 01:01:55] lb.utils.events INFO:  eta: 2 days, 1:28:06  iteration: 18799/375342  consumed_samples: 19251200  total_loss: 0.4764  time: 0.5014 s/iter  data_time: 0.0388 s/iter total_throughput: 2042.24 samples/s lr: 7.43e-04  
[08/30 01:02:45] lb.utils.events INFO:  eta: 2 days, 1:28:15  iteration: 18899/375342  consumed_samples: 19353600  total_loss: 0.4785  time: 0.5014 s/iter  data_time: 0.0380 s/iter total_throughput: 2042.23 samples/s lr: 7.47e-04  
[08/30 01:03:35] lb.utils.events INFO:  eta: 2 days, 1:27:25  iteration: 18999/375342  consumed_samples: 19456000  total_loss: 0.4827  time: 0.5014 s/iter  data_time: 0.0400 s/iter total_throughput: 2042.21 samples/s lr: 7.51e-04  
[08/30 01:04:25] lb.utils.events INFO:  eta: 2 days, 1:27:04  iteration: 19099/375342  consumed_samples: 19558400  total_loss: 0.468  time: 0.5014 s/iter  data_time: 0.0392 s/iter total_throughput: 2042.24 samples/s lr: 7.55e-04  
[08/30 01:05:16] lb.utils.events INFO:  eta: 2 days, 1:26:17  iteration: 19199/375342  consumed_samples: 19660800  total_loss: 0.4692  time: 0.5014 s/iter  data_time: 0.0390 s/iter total_throughput: 2042.22 samples/s lr: 7.59e-04  
[08/30 01:06:06] lb.utils.events INFO:  eta: 2 days, 1:25:58  iteration: 19299/375342  consumed_samples: 19763200  total_loss: 0.4805  time: 0.5014 s/iter  data_time: 0.0388 s/iter total_throughput: 2042.20 samples/s lr: 7.63e-04  
[08/30 01:06:56] lb.utils.events INFO:  eta: 2 days, 1:25:51  iteration: 19399/375342  consumed_samples: 19865600  total_loss: 0.4711  time: 0.5014 s/iter  data_time: 0.0393 s/iter total_throughput: 2042.19 samples/s lr: 7.67e-04  
[08/30 01:07:46] lb.utils.events INFO:  eta: 2 days, 1:25:34  iteration: 19499/375342  consumed_samples: 19968000  total_loss: 0.4729  time: 0.5014 s/iter  data_time: 0.0379 s/iter total_throughput: 2042.18 samples/s lr: 7.71e-04  
[08/30 01:08:37] lb.utils.events INFO:  eta: 2 days, 1:25:34  iteration: 19599/375342  consumed_samples: 20070400  total_loss: 0.4709  time: 0.5014 s/iter  data_time: 0.0385 s/iter total_throughput: 2042.14 samples/s lr: 7.75e-04  
[08/30 01:09:27] lb.utils.events INFO:  eta: 2 days, 1:23:15  iteration: 19699/375342  consumed_samples: 20172800  total_loss: 0.4684  time: 0.5014 s/iter  data_time: 0.0389 s/iter total_throughput: 2042.15 samples/s lr: 7.79e-04  
[08/30 01:10:17] lb.utils.events INFO:  eta: 2 days, 1:21:48  iteration: 19799/375342  consumed_samples: 20275200  total_loss: 0.4724  time: 0.5014 s/iter  data_time: 0.0393 s/iter total_throughput: 2042.15 samples/s lr: 7.83e-04  
[08/30 01:11:07] lb.utils.events INFO:  eta: 2 days, 1:20:44  iteration: 19899/375342  consumed_samples: 20377600  total_loss: 0.4799  time: 0.5014 s/iter  data_time: 0.0370 s/iter total_throughput: 2042.13 samples/s lr: 7.87e-04  
[08/30 01:11:58] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0019999
[08/30 01:11:58] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 01:11:58] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 01:12:03] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0548 s/iter. Inference: 0.2464 s/iter. Eval: 0.0023 s/iter. Total: 0.3036 s/iter. ETA=0:00:11
[08/30 01:12:08] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0836 s/iter. Inference: 0.2403 s/iter. Eval: 0.0022 s/iter. Total: 0.3264 s/iter. ETA=0:00:07
[08/30 01:12:13] lb.evaluation.evaluator INFO: Inference done 43008/50000. Dataloading: 0.0798 s/iter. Inference: 0.2420 s/iter. Eval: 0.0022 s/iter. Total: 0.3242 s/iter. ETA=0:00:01
[08/30 01:12:15] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 01:12:15] lb.evaluation.evaluator INFO: Total inference time: 0:00:13.983675 (0.000280 s / iter per device, on 8 devices)
[08/30 01:12:15] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000213 s / iter per device, on 8 devices)
[08/30 01:12:15] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 01:12:15] lb.evaluation.utils INFO: copypaste: Acc@1=60.746
[08/30 01:12:15] lb.evaluation.utils INFO: copypaste: Acc@5=84.816
[08/30 01:12:15] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 60.74600, better than last best score 52.93000 @ iteration 14999.
[08/30 01:12:15] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 01:12:16] lb.utils.events INFO:  eta: 2 days, 1:20:36  iteration: 19999/375342  consumed_samples: 20480000  total_loss: 0.4774  time: 0.5014 s/iter  data_time: 0.0384 s/iter total_throughput: 2042.08 samples/s lr: 7.91e-04  
[08/30 01:13:06] lb.utils.events INFO:  eta: 2 days, 1:20:04  iteration: 20099/375342  consumed_samples: 20582400  total_loss: 0.474  time: 0.5015 s/iter  data_time: 0.0368 s/iter total_throughput: 2042.07 samples/s lr: 7.95e-04  
[08/30 01:13:56] lb.utils.events INFO:  eta: 2 days, 1:18:46  iteration: 20199/375342  consumed_samples: 20684800  total_loss: 0.4768  time: 0.5014 s/iter  data_time: 0.0385 s/iter total_throughput: 2042.10 samples/s lr: 7.99e-04  
[08/30 01:14:46] lb.utils.events INFO:  eta: 2 days, 1:16:17  iteration: 20299/375342  consumed_samples: 20787200  total_loss: 0.4738  time: 0.5014 s/iter  data_time: 0.0402 s/iter total_throughput: 2042.11 samples/s lr: 8.03e-04  
[08/30 01:15:36] lb.utils.events INFO:  eta: 2 days, 1:14:26  iteration: 20399/375342  consumed_samples: 20889600  total_loss: 0.4724  time: 0.5014 s/iter  data_time: 0.0396 s/iter total_throughput: 2042.11 samples/s lr: 8.07e-04  
[08/30 01:16:26] lb.utils.events INFO:  eta: 2 days, 1:13:47  iteration: 20499/375342  consumed_samples: 20992000  total_loss: 0.4709  time: 0.5015 s/iter  data_time: 0.0384 s/iter total_throughput: 2042.08 samples/s lr: 8.11e-04  
[08/30 01:17:17] lb.utils.events INFO:  eta: 2 days, 1:12:57  iteration: 20599/375342  consumed_samples: 21094400  total_loss: 0.4723  time: 0.5015 s/iter  data_time: 0.0391 s/iter total_throughput: 2042.07 samples/s lr: 8.14e-04  
[08/30 01:18:07] lb.utils.events INFO:  eta: 2 days, 1:13:33  iteration: 20699/375342  consumed_samples: 21196800  total_loss: 0.4734  time: 0.5015 s/iter  data_time: 0.0380 s/iter total_throughput: 2042.04 samples/s lr: 8.18e-04  
[08/30 01:18:57] lb.utils.events INFO:  eta: 2 days, 1:12:58  iteration: 20799/375342  consumed_samples: 21299200  total_loss: 0.4654  time: 0.5015 s/iter  data_time: 0.0376 s/iter total_throughput: 2042.02 samples/s lr: 8.22e-04  
[08/30 01:19:47] lb.utils.events INFO:  eta: 2 days, 1:12:08  iteration: 20899/375342  consumed_samples: 21401600  total_loss: 0.467  time: 0.5015 s/iter  data_time: 0.0390 s/iter total_throughput: 2042.01 samples/s lr: 8.26e-04  
[08/30 01:20:38] lb.utils.events INFO:  eta: 2 days, 1:10:58  iteration: 20999/375342  consumed_samples: 21504000  total_loss: 0.4713  time: 0.5015 s/iter  data_time: 0.0387 s/iter total_throughput: 2041.99 samples/s lr: 8.30e-04  
[08/30 01:21:28] lb.utils.events INFO:  eta: 2 days, 1:09:59  iteration: 21099/375342  consumed_samples: 21606400  total_loss: 0.4685  time: 0.5015 s/iter  data_time: 0.0399 s/iter total_throughput: 2041.97 samples/s lr: 8.34e-04  
[08/30 01:22:18] lb.utils.events INFO:  eta: 2 days, 1:09:27  iteration: 21199/375342  consumed_samples: 21708800  total_loss: 0.477  time: 0.5015 s/iter  data_time: 0.0394 s/iter total_throughput: 2041.98 samples/s lr: 8.38e-04  
[08/30 01:23:08] lb.utils.events INFO:  eta: 2 days, 1:09:36  iteration: 21299/375342  consumed_samples: 21811200  total_loss: 0.4606  time: 0.5015 s/iter  data_time: 0.0366 s/iter total_throughput: 2041.95 samples/s lr: 8.42e-04  
[08/30 01:23:59] lb.utils.events INFO:  eta: 2 days, 1:10:00  iteration: 21399/375342  consumed_samples: 21913600  total_loss: 0.4486  time: 0.5015 s/iter  data_time: 0.0396 s/iter total_throughput: 2041.92 samples/s lr: 8.46e-04  
[08/30 01:24:49] lb.utils.events INFO:  eta: 2 days, 1:10:30  iteration: 21499/375342  consumed_samples: 22016000  total_loss: 0.465  time: 0.5015 s/iter  data_time: 0.0411 s/iter total_throughput: 2041.90 samples/s lr: 8.50e-04  
[08/30 01:25:39] lb.utils.events INFO:  eta: 2 days, 1:10:46  iteration: 21599/375342  consumed_samples: 22118400  total_loss: 0.465  time: 0.5015 s/iter  data_time: 0.0395 s/iter total_throughput: 2041.87 samples/s lr: 8.54e-04  
[08/30 01:26:29] lb.utils.events INFO:  eta: 2 days, 1:08:03  iteration: 21699/375342  consumed_samples: 22220800  total_loss: 0.4697  time: 0.5015 s/iter  data_time: 0.0389 s/iter total_throughput: 2041.88 samples/s lr: 8.58e-04  
[08/30 01:27:20] lb.utils.events INFO:  eta: 2 days, 1:05:28  iteration: 21799/375342  consumed_samples: 22323200  total_loss: 0.4676  time: 0.5015 s/iter  data_time: 0.0381 s/iter total_throughput: 2041.87 samples/s lr: 8.62e-04  
[08/30 01:28:10] lb.utils.events INFO:  eta: 2 days, 1:06:06  iteration: 21899/375342  consumed_samples: 22425600  total_loss: 0.4575  time: 0.5015 s/iter  data_time: 0.0386 s/iter total_throughput: 2041.86 samples/s lr: 8.66e-04  
[08/30 01:29:00] lb.utils.events INFO:  eta: 2 days, 1:04:55  iteration: 21999/375342  consumed_samples: 22528000  total_loss: 0.4649  time: 0.5015 s/iter  data_time: 0.0384 s/iter total_throughput: 2041.84 samples/s lr: 8.70e-04  
[08/30 01:29:50] lb.utils.events INFO:  eta: 2 days, 1:04:25  iteration: 22099/375342  consumed_samples: 22630400  total_loss: 0.4613  time: 0.5015 s/iter  data_time: 0.0407 s/iter total_throughput: 2041.83 samples/s lr: 8.74e-04  
[08/30 01:30:40] lb.utils.events INFO:  eta: 2 days, 1:03:21  iteration: 22199/375342  consumed_samples: 22732800  total_loss: 0.4539  time: 0.5015 s/iter  data_time: 0.0389 s/iter total_throughput: 2041.85 samples/s lr: 8.78e-04  
[08/30 01:31:31] lb.utils.events INFO:  eta: 2 days, 1:04:10  iteration: 22299/375342  consumed_samples: 22835200  total_loss: 0.4548  time: 0.5015 s/iter  data_time: 0.0381 s/iter total_throughput: 2041.83 samples/s lr: 8.82e-04  
[08/30 01:32:21] lb.utils.events INFO:  eta: 2 days, 1:02:26  iteration: 22399/375342  consumed_samples: 22937600  total_loss: 0.4546  time: 0.5015 s/iter  data_time: 0.0388 s/iter total_throughput: 2041.82 samples/s lr: 8.86e-04  
[08/30 01:33:11] lb.utils.events INFO:  eta: 2 days, 1:01:15  iteration: 22499/375342  consumed_samples: 23040000  total_loss: 0.4526  time: 0.5015 s/iter  data_time: 0.0398 s/iter total_throughput: 2041.80 samples/s lr: 8.90e-04  
[08/30 01:34:02] lb.utils.events INFO:  eta: 2 days, 0:59:22  iteration: 22599/375342  consumed_samples: 23142400  total_loss: 0.4647  time: 0.5015 s/iter  data_time: 0.0389 s/iter total_throughput: 2041.78 samples/s lr: 8.93e-04  
[08/30 01:34:52] lb.utils.events INFO:  eta: 2 days, 0:58:50  iteration: 22699/375342  consumed_samples: 23244800  total_loss: 0.462  time: 0.5015 s/iter  data_time: 0.0385 s/iter total_throughput: 2041.77 samples/s lr: 8.97e-04  
[08/30 01:35:42] lb.utils.events INFO:  eta: 2 days, 0:58:17  iteration: 22799/375342  consumed_samples: 23347200  total_loss: 0.4634  time: 0.5015 s/iter  data_time: 0.0388 s/iter total_throughput: 2041.77 samples/s lr: 9.01e-04  
[08/30 01:36:32] lb.utils.events INFO:  eta: 2 days, 0:57:35  iteration: 22899/375342  consumed_samples: 23449600  total_loss: 0.4665  time: 0.5015 s/iter  data_time: 0.0411 s/iter total_throughput: 2041.76 samples/s lr: 9.05e-04  
[08/30 01:37:22] lb.utils.events INFO:  eta: 2 days, 0:56:54  iteration: 22999/375342  consumed_samples: 23552000  total_loss: 0.461  time: 0.5015 s/iter  data_time: 0.0406 s/iter total_throughput: 2041.74 samples/s lr: 9.09e-04  
[08/30 01:38:13] lb.utils.events INFO:  eta: 2 days, 0:55:51  iteration: 23099/375342  consumed_samples: 23654400  total_loss: 0.4571  time: 0.5015 s/iter  data_time: 0.0404 s/iter total_throughput: 2041.72 samples/s lr: 9.13e-04  
[08/30 01:39:03] lb.utils.events INFO:  eta: 2 days, 0:56:47  iteration: 23199/375342  consumed_samples: 23756800  total_loss: 0.4598  time: 0.5015 s/iter  data_time: 0.0405 s/iter total_throughput: 2041.71 samples/s lr: 9.17e-04  
[08/30 01:39:53] lb.utils.events INFO:  eta: 2 days, 0:54:31  iteration: 23299/375342  consumed_samples: 23859200  total_loss: 0.4633  time: 0.5015 s/iter  data_time: 0.0368 s/iter total_throughput: 2041.70 samples/s lr: 9.21e-04  
[08/30 01:40:43] lb.utils.events INFO:  eta: 2 days, 0:56:53  iteration: 23399/375342  consumed_samples: 23961600  total_loss: 0.4697  time: 0.5015 s/iter  data_time: 0.0394 s/iter total_throughput: 2041.69 samples/s lr: 9.25e-04  
[08/30 01:41:33] lb.utils.events INFO:  eta: 2 days, 0:54:56  iteration: 23499/375342  consumed_samples: 24064000  total_loss: 0.4677  time: 0.5015 s/iter  data_time: 0.0401 s/iter total_throughput: 2041.70 samples/s lr: 9.29e-04  
[08/30 01:42:24] lb.utils.events INFO:  eta: 2 days, 0:54:53  iteration: 23599/375342  consumed_samples: 24166400  total_loss: 0.4679  time: 0.5015 s/iter  data_time: 0.0406 s/iter total_throughput: 2041.69 samples/s lr: 9.33e-04  
[08/30 01:43:14] lb.utils.events INFO:  eta: 2 days, 0:51:48  iteration: 23699/375342  consumed_samples: 24268800  total_loss: 0.4606  time: 0.5015 s/iter  data_time: 0.0391 s/iter total_throughput: 2041.69 samples/s lr: 9.37e-04  
[08/30 01:44:04] lb.utils.events INFO:  eta: 2 days, 0:52:48  iteration: 23799/375342  consumed_samples: 24371200  total_loss: 0.4581  time: 0.5016 s/iter  data_time: 0.0398 s/iter total_throughput: 2041.63 samples/s lr: 9.41e-04  
[08/30 01:44:55] lb.utils.events INFO:  eta: 2 days, 0:49:36  iteration: 23899/375342  consumed_samples: 24473600  total_loss: 0.4641  time: 0.5016 s/iter  data_time: 0.0386 s/iter total_throughput: 2041.62 samples/s lr: 9.45e-04  
[08/30 01:45:45] lb.utils.events INFO:  eta: 2 days, 0:48:46  iteration: 23999/375342  consumed_samples: 24576000  total_loss: 0.4538  time: 0.5016 s/iter  data_time: 0.0378 s/iter total_throughput: 2041.61 samples/s lr: 9.49e-04  
[08/30 01:46:35] lb.utils.events INFO:  eta: 2 days, 0:47:41  iteration: 24099/375342  consumed_samples: 24678400  total_loss: 0.4562  time: 0.5016 s/iter  data_time: 0.0394 s/iter total_throughput: 2041.61 samples/s lr: 9.53e-04  
[08/30 01:47:25] lb.utils.events INFO:  eta: 2 days, 0:47:20  iteration: 24199/375342  consumed_samples: 24780800  total_loss: 0.4579  time: 0.5016 s/iter  data_time: 0.0393 s/iter total_throughput: 2041.59 samples/s lr: 9.57e-04  
[08/30 01:48:16] lb.utils.events INFO:  eta: 2 days, 0:47:49  iteration: 24299/375342  consumed_samples: 24883200  total_loss: 0.4647  time: 0.5016 s/iter  data_time: 0.0377 s/iter total_throughput: 2041.57 samples/s lr: 9.61e-04  
[08/30 01:49:06] lb.utils.events INFO:  eta: 2 days, 0:44:52  iteration: 24399/375342  consumed_samples: 24985600  total_loss: 0.4661  time: 0.5016 s/iter  data_time: 0.0408 s/iter total_throughput: 2041.59 samples/s lr: 9.65e-04  
[08/30 01:49:56] lb.utils.events INFO:  eta: 2 days, 0:44:58  iteration: 24499/375342  consumed_samples: 25088000  total_loss: 0.4606  time: 0.5016 s/iter  data_time: 0.0399 s/iter total_throughput: 2041.59 samples/s lr: 9.68e-04  
[08/30 01:50:46] lb.utils.events INFO:  eta: 2 days, 0:43:05  iteration: 24599/375342  consumed_samples: 25190400  total_loss: 0.4544  time: 0.5016 s/iter  data_time: 0.0404 s/iter total_throughput: 2041.59 samples/s lr: 9.72e-04  
[08/30 01:51:36] lb.utils.events INFO:  eta: 2 days, 0:42:45  iteration: 24699/375342  consumed_samples: 25292800  total_loss: 0.4519  time: 0.5016 s/iter  data_time: 0.0400 s/iter total_throughput: 2041.60 samples/s lr: 9.76e-04  
[08/30 01:52:26] lb.utils.events INFO:  eta: 2 days, 0:41:54  iteration: 24799/375342  consumed_samples: 25395200  total_loss: 0.4511  time: 0.5016 s/iter  data_time: 0.0376 s/iter total_throughput: 2041.58 samples/s lr: 9.80e-04  
[08/30 01:53:17] lb.utils.events INFO:  eta: 2 days, 0:41:00  iteration: 24899/375342  consumed_samples: 25497600  total_loss: 0.4431  time: 0.5016 s/iter  data_time: 0.0384 s/iter total_throughput: 2041.57 samples/s lr: 9.84e-04  
[08/30 01:54:07] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0024999
[08/30 01:54:07] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 01:54:07] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 01:54:12] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0709 s/iter. Inference: 0.2363 s/iter. Eval: 0.0023 s/iter. Total: 0.3094 s/iter. ETA=0:00:11
[08/30 01:54:17] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0887 s/iter. Inference: 0.2396 s/iter. Eval: 0.0022 s/iter. Total: 0.3308 s/iter. ETA=0:00:07
[08/30 01:54:22] lb.evaluation.evaluator INFO: Inference done 43008/50000. Dataloading: 0.0863 s/iter. Inference: 0.2374 s/iter. Eval: 0.0022 s/iter. Total: 0.3261 s/iter. ETA=0:00:01
[08/30 01:54:24] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 01:54:24] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.192002 (0.000284 s / iter per device, on 8 devices)
[08/30 01:54:24] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000209 s / iter per device, on 8 devices)
[08/30 01:54:24] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 01:54:24] lb.evaluation.utils INFO: copypaste: Acc@1=65.298
[08/30 01:54:24] lb.evaluation.utils INFO: copypaste: Acc@5=87.67
[08/30 01:54:24] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 65.29800, better than last best score 60.74600 @ iteration 19999.
[08/30 01:54:24] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 01:54:25] lb.utils.events INFO:  eta: 2 days, 0:40:10  iteration: 24999/375342  consumed_samples: 25600000  total_loss: 0.4503  time: 0.5016 s/iter  data_time: 0.0390 s/iter total_throughput: 2041.58 samples/s lr: 9.88e-04  
[08/30 01:55:15] lb.utils.events INFO:  eta: 2 days, 0:38:38  iteration: 25099/375342  consumed_samples: 25702400  total_loss: 0.452  time: 0.5016 s/iter  data_time: 0.0386 s/iter total_throughput: 2041.56 samples/s lr: 9.89e-04  
[08/30 01:56:06] lb.utils.events INFO:  eta: 2 days, 0:37:51  iteration: 25199/375342  consumed_samples: 25804800  total_loss: 0.4607  time: 0.5016 s/iter  data_time: 0.0398 s/iter total_throughput: 2041.54 samples/s lr: 9.89e-04  
[08/30 01:56:56] lb.utils.events INFO:  eta: 2 days, 0:36:51  iteration: 25299/375342  consumed_samples: 25907200  total_loss: 0.4602  time: 0.5016 s/iter  data_time: 0.0386 s/iter total_throughput: 2041.54 samples/s lr: 9.89e-04  
[08/30 01:57:46] lb.utils.events INFO:  eta: 2 days, 0:34:45  iteration: 25399/375342  consumed_samples: 26009600  total_loss: 0.4589  time: 0.5016 s/iter  data_time: 0.0404 s/iter total_throughput: 2041.55 samples/s lr: 9.89e-04  
[08/30 01:58:36] lb.utils.events INFO:  eta: 2 days, 0:34:22  iteration: 25499/375342  consumed_samples: 26112000  total_loss: 0.4566  time: 0.5016 s/iter  data_time: 0.0399 s/iter total_throughput: 2041.54 samples/s lr: 9.89e-04  
[08/30 01:59:26] lb.utils.events INFO:  eta: 2 days, 0:34:15  iteration: 25599/375342  consumed_samples: 26214400  total_loss: 0.4477  time: 0.5016 s/iter  data_time: 0.0392 s/iter total_throughput: 2041.53 samples/s lr: 9.89e-04  
[08/30 02:00:17] lb.utils.events INFO:  eta: 2 days, 0:33:18  iteration: 25699/375342  consumed_samples: 26316800  total_loss: 0.4446  time: 0.5016 s/iter  data_time: 0.0403 s/iter total_throughput: 2041.54 samples/s lr: 9.89e-04  
[08/30 02:01:07] lb.utils.events INFO:  eta: 2 days, 0:30:56  iteration: 25799/375342  consumed_samples: 26419200  total_loss: 0.4508  time: 0.5016 s/iter  data_time: 0.0386 s/iter total_throughput: 2041.55 samples/s lr: 9.89e-04  
[08/30 02:01:57] lb.utils.events INFO:  eta: 2 days, 0:31:38  iteration: 25899/375342  consumed_samples: 26521600  total_loss: 0.4537  time: 0.5016 s/iter  data_time: 0.0395 s/iter total_throughput: 2041.54 samples/s lr: 9.88e-04  
[08/30 02:02:47] lb.utils.events INFO:  eta: 2 days, 0:31:14  iteration: 25999/375342  consumed_samples: 26624000  total_loss: 0.4566  time: 0.5016 s/iter  data_time: 0.0397 s/iter total_throughput: 2041.53 samples/s lr: 9.88e-04  
[08/30 02:03:37] lb.utils.events INFO:  eta: 2 days, 0:30:53  iteration: 26099/375342  consumed_samples: 26726400  total_loss: 0.4555  time: 0.5016 s/iter  data_time: 0.0403 s/iter total_throughput: 2041.51 samples/s lr: 9.88e-04  
[08/30 02:04:28] lb.utils.events INFO:  eta: 2 days, 0:30:18  iteration: 26199/375342  consumed_samples: 26828800  total_loss: 0.4508  time: 0.5016 s/iter  data_time: 0.0394 s/iter total_throughput: 2041.48 samples/s lr: 9.88e-04  
[08/30 02:05:18] lb.utils.events INFO:  eta: 2 days, 0:29:31  iteration: 26299/375342  consumed_samples: 26931200  total_loss: 0.4569  time: 0.5016 s/iter  data_time: 0.0385 s/iter total_throughput: 2041.46 samples/s lr: 9.88e-04  
[08/30 02:06:08] lb.utils.events INFO:  eta: 2 days, 0:30:06  iteration: 26399/375342  consumed_samples: 27033600  total_loss: 0.4504  time: 0.5016 s/iter  data_time: 0.0383 s/iter total_throughput: 2041.43 samples/s lr: 9.88e-04  
[08/30 02:06:59] lb.utils.events INFO:  eta: 2 days, 0:29:35  iteration: 26499/375342  consumed_samples: 27136000  total_loss: 0.4411  time: 0.5016 s/iter  data_time: 0.0380 s/iter total_throughput: 2041.40 samples/s lr: 9.88e-04  
[08/30 02:07:49] lb.utils.events INFO:  eta: 2 days, 0:28:28  iteration: 26599/375342  consumed_samples: 27238400  total_loss: 0.4417  time: 0.5016 s/iter  data_time: 0.0416 s/iter total_throughput: 2041.40 samples/s lr: 9.88e-04  
[08/30 02:08:39] lb.utils.events INFO:  eta: 2 days, 0:28:26  iteration: 26699/375342  consumed_samples: 27340800  total_loss: 0.4525  time: 0.5016 s/iter  data_time: 0.0386 s/iter total_throughput: 2041.39 samples/s lr: 9.88e-04  
[08/30 02:09:30] lb.utils.events INFO:  eta: 2 days, 0:28:52  iteration: 26799/375342  consumed_samples: 27443200  total_loss: 0.4508  time: 0.5016 s/iter  data_time: 0.0406 s/iter total_throughput: 2041.37 samples/s lr: 9.88e-04  
[08/30 02:10:20] lb.utils.events INFO:  eta: 2 days, 0:27:12  iteration: 26899/375342  consumed_samples: 27545600  total_loss: 0.4502  time: 0.5016 s/iter  data_time: 0.0374 s/iter total_throughput: 2041.38 samples/s lr: 9.88e-04  
[08/30 02:11:10] lb.utils.events INFO:  eta: 2 days, 0:26:33  iteration: 26999/375342  consumed_samples: 27648000  total_loss: 0.4501  time: 0.5016 s/iter  data_time: 0.0402 s/iter total_throughput: 2041.37 samples/s lr: 9.87e-04  
[08/30 02:12:00] lb.utils.events INFO:  eta: 2 days, 0:25:30  iteration: 27099/375342  consumed_samples: 27750400  total_loss: 0.4455  time: 0.5016 s/iter  data_time: 0.0398 s/iter total_throughput: 2041.38 samples/s lr: 9.87e-04  
[08/30 02:12:50] lb.utils.events INFO:  eta: 2 days, 0:23:39  iteration: 27199/375342  consumed_samples: 27852800  total_loss: 0.4533  time: 0.5016 s/iter  data_time: 0.0384 s/iter total_throughput: 2041.39 samples/s lr: 9.87e-04  
[08/30 02:13:40] lb.utils.events INFO:  eta: 2 days, 0:22:01  iteration: 27299/375342  consumed_samples: 27955200  total_loss: 0.4542  time: 0.5016 s/iter  data_time: 0.0398 s/iter total_throughput: 2041.39 samples/s lr: 9.87e-04  
[08/30 02:14:31] lb.utils.events INFO:  eta: 2 days, 0:21:12  iteration: 27399/375342  consumed_samples: 28057600  total_loss: 0.4449  time: 0.5016 s/iter  data_time: 0.0391 s/iter total_throughput: 2041.38 samples/s lr: 9.87e-04  
[08/30 02:15:21] lb.utils.events INFO:  eta: 2 days, 0:20:10  iteration: 27499/375342  consumed_samples: 28160000  total_loss: 0.4427  time: 0.5016 s/iter  data_time: 0.0387 s/iter total_throughput: 2041.37 samples/s lr: 9.87e-04  
[08/30 02:16:11] lb.utils.events INFO:  eta: 2 days, 0:19:36  iteration: 27599/375342  consumed_samples: 28262400  total_loss: 0.4426  time: 0.5016 s/iter  data_time: 0.0395 s/iter total_throughput: 2041.36 samples/s lr: 9.87e-04  
[08/30 02:17:01] lb.utils.events INFO:  eta: 2 days, 0:16:59  iteration: 27699/375342  consumed_samples: 28364800  total_loss: 0.4446  time: 0.5016 s/iter  data_time: 0.0390 s/iter total_throughput: 2041.37 samples/s lr: 9.87e-04  
[08/30 02:17:51] lb.utils.events INFO:  eta: 2 days, 0:15:59  iteration: 27799/375342  consumed_samples: 28467200  total_loss: 0.4446  time: 0.5016 s/iter  data_time: 0.0406 s/iter total_throughput: 2041.38 samples/s lr: 9.87e-04  
[08/30 02:18:41] lb.utils.events INFO:  eta: 2 days, 0:16:39  iteration: 27899/375342  consumed_samples: 28569600  total_loss: 0.4513  time: 0.5016 s/iter  data_time: 0.0394 s/iter total_throughput: 2041.37 samples/s lr: 9.87e-04  
[08/30 02:19:32] lb.utils.events INFO:  eta: 2 days, 0:14:56  iteration: 27999/375342  consumed_samples: 28672000  total_loss: 0.4518  time: 0.5016 s/iter  data_time: 0.0374 s/iter total_throughput: 2041.38 samples/s lr: 9.86e-04  
[08/30 02:20:22] lb.utils.events INFO:  eta: 2 days, 0:14:18  iteration: 28099/375342  consumed_samples: 28774400  total_loss: 0.4511  time: 0.5016 s/iter  data_time: 0.0399 s/iter total_throughput: 2041.39 samples/s lr: 9.86e-04  
[08/30 02:21:12] lb.utils.events INFO:  eta: 2 days, 0:13:00  iteration: 28199/375342  consumed_samples: 28876800  total_loss: 0.4484  time: 0.5016 s/iter  data_time: 0.0379 s/iter total_throughput: 2041.39 samples/s lr: 9.86e-04  
[08/30 02:22:02] lb.utils.events INFO:  eta: 2 days, 0:13:26  iteration: 28299/375342  consumed_samples: 28979200  total_loss: 0.4369  time: 0.5016 s/iter  data_time: 0.0380 s/iter total_throughput: 2041.36 samples/s lr: 9.86e-04  
[08/30 02:22:52] lb.utils.events INFO:  eta: 2 days, 0:11:28  iteration: 28399/375342  consumed_samples: 29081600  total_loss: 0.4375  time: 0.5016 s/iter  data_time: 0.0405 s/iter total_throughput: 2041.35 samples/s lr: 9.86e-04  
[08/30 02:23:43] lb.utils.events INFO:  eta: 2 days, 0:10:07  iteration: 28499/375342  consumed_samples: 29184000  total_loss: 0.4332  time: 0.5016 s/iter  data_time: 0.0394 s/iter total_throughput: 2041.36 samples/s lr: 9.86e-04  
[08/30 02:24:33] lb.utils.events INFO:  eta: 2 days, 0:09:33  iteration: 28599/375342  consumed_samples: 29286400  total_loss: 0.4339  time: 0.5016 s/iter  data_time: 0.0393 s/iter total_throughput: 2041.33 samples/s lr: 9.86e-04  
[08/30 02:25:23] lb.utils.events INFO:  eta: 2 days, 0:09:37  iteration: 28699/375342  consumed_samples: 29388800  total_loss: 0.4451  time: 0.5016 s/iter  data_time: 0.0404 s/iter total_throughput: 2041.32 samples/s lr: 9.86e-04  
[08/30 02:26:13] lb.utils.events INFO:  eta: 2 days, 0:08:04  iteration: 28799/375342  consumed_samples: 29491200  total_loss: 0.4344  time: 0.5016 s/iter  data_time: 0.0387 s/iter total_throughput: 2041.31 samples/s lr: 9.86e-04  
[08/30 02:27:04] lb.utils.events INFO:  eta: 2 days, 0:06:56  iteration: 28899/375342  consumed_samples: 29593600  total_loss: 0.4315  time: 0.5016 s/iter  data_time: 0.0386 s/iter total_throughput: 2041.33 samples/s lr: 9.86e-04  
[08/30 02:27:54] lb.utils.events INFO:  eta: 2 days, 0:06:33  iteration: 28999/375342  consumed_samples: 29696000  total_loss: 0.437  time: 0.5016 s/iter  data_time: 0.0396 s/iter total_throughput: 2041.30 samples/s lr: 9.85e-04  
[08/30 02:28:44] lb.utils.events INFO:  eta: 2 days, 0:06:11  iteration: 29099/375342  consumed_samples: 29798400  total_loss: 0.4295  time: 0.5016 s/iter  data_time: 0.0388 s/iter total_throughput: 2041.31 samples/s lr: 9.85e-04  
[08/30 02:29:34] lb.utils.events INFO:  eta: 2 days, 0:06:03  iteration: 29199/375342  consumed_samples: 29900800  total_loss: 0.4298  time: 0.5016 s/iter  data_time: 0.0383 s/iter total_throughput: 2041.28 samples/s lr: 9.85e-04  
[08/30 02:30:25] lb.utils.events INFO:  eta: 2 days, 0:03:29  iteration: 29299/375342  consumed_samples: 30003200  total_loss: 0.4362  time: 0.5016 s/iter  data_time: 0.0382 s/iter total_throughput: 2041.27 samples/s lr: 9.85e-04  
[08/30 02:31:15] lb.utils.events INFO:  eta: 2 days, 0:03:51  iteration: 29399/375342  consumed_samples: 30105600  total_loss: 0.4445  time: 0.5017 s/iter  data_time: 0.0403 s/iter total_throughput: 2041.25 samples/s lr: 9.85e-04  
[08/30 02:32:05] lb.utils.events INFO:  eta: 2 days, 0:02:57  iteration: 29499/375342  consumed_samples: 30208000  total_loss: 0.4445  time: 0.5017 s/iter  data_time: 0.0393 s/iter total_throughput: 2041.25 samples/s lr: 9.85e-04  
[08/30 02:32:55] lb.utils.events INFO:  eta: 2 days, 0:02:05  iteration: 29599/375342  consumed_samples: 30310400  total_loss: 0.4359  time: 0.5017 s/iter  data_time: 0.0385 s/iter total_throughput: 2041.25 samples/s lr: 9.85e-04  
[08/30 02:33:46] lb.utils.events INFO:  eta: 2 days, 0:01:35  iteration: 29699/375342  consumed_samples: 30412800  total_loss: 0.4354  time: 0.5017 s/iter  data_time: 0.0392 s/iter total_throughput: 2041.23 samples/s lr: 9.85e-04  
[08/30 02:34:36] lb.utils.events INFO:  eta: 2 days, 0:00:40  iteration: 29799/375342  consumed_samples: 30515200  total_loss: 0.4272  time: 0.5017 s/iter  data_time: 0.0395 s/iter total_throughput: 2041.26 samples/s lr: 9.85e-04  
[08/30 02:35:26] lb.utils.events INFO:  eta: 2 days, 0:00:55  iteration: 29899/375342  consumed_samples: 30617600  total_loss: 0.4257  time: 0.5017 s/iter  data_time: 0.0380 s/iter total_throughput: 2041.23 samples/s lr: 9.85e-04  
[08/30 02:36:16] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0029999
[08/30 02:36:17] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 02:36:17] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 02:36:22] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0719 s/iter. Inference: 0.2435 s/iter. Eval: 0.0023 s/iter. Total: 0.3177 s/iter. ETA=0:00:11
[08/30 02:36:27] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0915 s/iter. Inference: 0.2427 s/iter. Eval: 0.0022 s/iter. Total: 0.3366 s/iter. ETA=0:00:07
[08/30 02:36:32] lb.evaluation.evaluator INFO: Inference done 43008/50000. Dataloading: 0.0874 s/iter. Inference: 0.2414 s/iter. Eval: 0.0022 s/iter. Total: 0.3312 s/iter. ETA=0:00:01
[08/30 02:36:34] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 02:36:34] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.263925 (0.000285 s / iter per device, on 8 devices)
[08/30 02:36:34] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000213 s / iter per device, on 8 devices)
[08/30 02:36:34] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 02:36:34] lb.evaluation.utils INFO: copypaste: Acc@1=68.066
[08/30 02:36:34] lb.evaluation.utils INFO: copypaste: Acc@5=89.64800000000001
[08/30 02:36:34] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 68.06600, better than last best score 65.29800 @ iteration 24999.
[08/30 02:36:34] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 02:36:35] lb.utils.events INFO:  eta: 2 days, 0:00:03  iteration: 29999/375342  consumed_samples: 30720000  total_loss: 0.4372  time: 0.5017 s/iter  data_time: 0.0392 s/iter total_throughput: 2041.22 samples/s lr: 9.84e-04  
[08/30 02:37:25] lb.utils.events INFO:  eta: 1 day, 23:59:04  iteration: 30099/375342  consumed_samples: 30822400  total_loss: 0.4401  time: 0.5017 s/iter  data_time: 0.0379 s/iter total_throughput: 2041.21 samples/s lr: 9.84e-04  
[08/30 02:38:15] lb.utils.events INFO:  eta: 1 day, 23:58:52  iteration: 30199/375342  consumed_samples: 30924800  total_loss: 0.4303  time: 0.5017 s/iter  data_time: 0.0388 s/iter total_throughput: 2041.17 samples/s lr: 9.84e-04  
[08/30 02:39:06] lb.utils.events INFO:  eta: 2 days, 0:00:30  iteration: 30299/375342  consumed_samples: 31027200  total_loss: 0.433  time: 0.5017 s/iter  data_time: 0.0412 s/iter total_throughput: 2041.13 samples/s lr: 9.84e-04  
[08/30 02:39:56] lb.utils.events INFO:  eta: 2 days, 0:00:42  iteration: 30399/375342  consumed_samples: 31129600  total_loss: 0.4356  time: 0.5017 s/iter  data_time: 0.0372 s/iter total_throughput: 2041.08 samples/s lr: 9.84e-04  
[08/30 02:40:47] lb.utils.events INFO:  eta: 2 days, 0:01:16  iteration: 30499/375342  consumed_samples: 31232000  total_loss: 0.4397  time: 0.5017 s/iter  data_time: 0.0390 s/iter total_throughput: 2041.05 samples/s lr: 9.84e-04  
[08/30 02:41:37] lb.utils.events INFO:  eta: 2 days, 0:02:05  iteration: 30599/375342  consumed_samples: 31334400  total_loss: 0.4407  time: 0.5017 s/iter  data_time: 0.0406 s/iter total_throughput: 2041.04 samples/s lr: 9.84e-04  
[08/30 02:42:28] lb.utils.events INFO:  eta: 2 days, 0:01:19  iteration: 30699/375342  consumed_samples: 31436800  total_loss: 0.4341  time: 0.5017 s/iter  data_time: 0.0380 s/iter total_throughput: 2041.01 samples/s lr: 9.84e-04  
[08/30 02:43:18] lb.utils.events INFO:  eta: 2 days, 0:03:26  iteration: 30799/375342  consumed_samples: 31539200  total_loss: 0.4283  time: 0.5017 s/iter  data_time: 0.0403 s/iter total_throughput: 2040.99 samples/s lr: 9.84e-04  
[08/30 02:44:08] lb.utils.events INFO:  eta: 2 days, 0:02:19  iteration: 30899/375342  consumed_samples: 31641600  total_loss: 0.434  time: 0.5017 s/iter  data_time: 0.0398 s/iter total_throughput: 2040.97 samples/s lr: 9.84e-04  
[08/30 02:44:58] lb.utils.events INFO:  eta: 2 days, 0:02:22  iteration: 30999/375342  consumed_samples: 31744000  total_loss: 0.4309  time: 0.5017 s/iter  data_time: 0.0398 s/iter total_throughput: 2040.96 samples/s lr: 9.83e-04  
[08/30 02:45:49] lb.utils.events INFO:  eta: 2 days, 0:02:34  iteration: 31099/375342  consumed_samples: 31846400  total_loss: 0.4334  time: 0.5017 s/iter  data_time: 0.0381 s/iter total_throughput: 2040.94 samples/s lr: 9.83e-04  
[08/30 02:46:39] lb.utils.events INFO:  eta: 2 days, 0:01:00  iteration: 31199/375342  consumed_samples: 31948800  total_loss: 0.4334  time: 0.5017 s/iter  data_time: 0.0377 s/iter total_throughput: 2040.93 samples/s lr: 9.83e-04  
[08/30 02:47:29] lb.utils.events INFO:  eta: 1 day, 23:57:25  iteration: 31299/375342  consumed_samples: 32051200  total_loss: 0.4322  time: 0.5017 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.92 samples/s lr: 9.83e-04  
[08/30 02:48:20] lb.utils.events INFO:  eta: 1 day, 23:55:22  iteration: 31399/375342  consumed_samples: 32153600  total_loss: 0.4347  time: 0.5017 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.90 samples/s lr: 9.83e-04  
[08/30 02:49:10] lb.utils.events INFO:  eta: 1 day, 23:55:24  iteration: 31499/375342  consumed_samples: 32256000  total_loss: 0.4264  time: 0.5017 s/iter  data_time: 0.0401 s/iter total_throughput: 2040.88 samples/s lr: 9.83e-04  
[08/30 02:50:00] lb.utils.events INFO:  eta: 1 day, 23:52:50  iteration: 31599/375342  consumed_samples: 32358400  total_loss: 0.4285  time: 0.5017 s/iter  data_time: 0.0397 s/iter total_throughput: 2040.87 samples/s lr: 9.83e-04  
[08/30 02:50:51] lb.utils.events INFO:  eta: 1 day, 23:51:31  iteration: 31699/375342  consumed_samples: 32460800  total_loss: 0.4311  time: 0.5017 s/iter  data_time: 0.0382 s/iter total_throughput: 2040.87 samples/s lr: 9.83e-04  
[08/30 02:51:41] lb.utils.events INFO:  eta: 1 day, 23:50:35  iteration: 31799/375342  consumed_samples: 32563200  total_loss: 0.425  time: 0.5017 s/iter  data_time: 0.0382 s/iter total_throughput: 2040.86 samples/s lr: 9.83e-04  
[08/30 02:52:31] lb.utils.events INFO:  eta: 1 day, 23:50:04  iteration: 31899/375342  consumed_samples: 32665600  total_loss: 0.4293  time: 0.5018 s/iter  data_time: 0.0395 s/iter total_throughput: 2040.85 samples/s lr: 9.82e-04  
[08/30 02:53:21] lb.utils.events INFO:  eta: 1 day, 23:47:41  iteration: 31999/375342  consumed_samples: 32768000  total_loss: 0.4367  time: 0.5018 s/iter  data_time: 0.0398 s/iter total_throughput: 2040.85 samples/s lr: 9.82e-04  
[08/30 02:54:11] lb.utils.events INFO:  eta: 1 day, 23:45:14  iteration: 32099/375342  consumed_samples: 32870400  total_loss: 0.4323  time: 0.5018 s/iter  data_time: 0.0411 s/iter total_throughput: 2040.85 samples/s lr: 9.82e-04  
[08/30 02:55:02] lb.utils.events INFO:  eta: 1 day, 23:43:59  iteration: 32199/375342  consumed_samples: 32972800  total_loss: 0.419  time: 0.5018 s/iter  data_time: 0.0384 s/iter total_throughput: 2040.84 samples/s lr: 9.82e-04  
[08/30 02:55:52] lb.utils.events INFO:  eta: 1 day, 23:43:24  iteration: 32299/375342  consumed_samples: 33075200  total_loss: 0.4261  time: 0.5018 s/iter  data_time: 0.0382 s/iter total_throughput: 2040.86 samples/s lr: 9.82e-04  
[08/30 02:56:42] lb.utils.events INFO:  eta: 1 day, 23:43:52  iteration: 32399/375342  consumed_samples: 33177600  total_loss: 0.432  time: 0.5018 s/iter  data_time: 0.0392 s/iter total_throughput: 2040.82 samples/s lr: 9.82e-04  
[08/30 02:57:32] lb.utils.events INFO:  eta: 1 day, 23:41:54  iteration: 32499/375342  consumed_samples: 33280000  total_loss: 0.4306  time: 0.5018 s/iter  data_time: 0.0404 s/iter total_throughput: 2040.81 samples/s lr: 9.82e-04  
[08/30 02:58:23] lb.utils.events INFO:  eta: 1 day, 23:40:37  iteration: 32599/375342  consumed_samples: 33382400  total_loss: 0.4245  time: 0.5018 s/iter  data_time: 0.0386 s/iter total_throughput: 2040.80 samples/s lr: 9.82e-04  
[08/30 02:59:13] lb.utils.events INFO:  eta: 1 day, 23:38:31  iteration: 32699/375342  consumed_samples: 33484800  total_loss: 0.4274  time: 0.5018 s/iter  data_time: 0.0395 s/iter total_throughput: 2040.79 samples/s lr: 9.82e-04  
[08/30 03:00:03] lb.utils.events INFO:  eta: 1 day, 23:38:32  iteration: 32799/375342  consumed_samples: 33587200  total_loss: 0.4317  time: 0.5018 s/iter  data_time: 0.0393 s/iter total_throughput: 2040.79 samples/s lr: 9.81e-04  
[08/30 03:00:54] lb.utils.events INFO:  eta: 1 day, 23:36:55  iteration: 32899/375342  consumed_samples: 33689600  total_loss: 0.4336  time: 0.5018 s/iter  data_time: 0.0395 s/iter total_throughput: 2040.77 samples/s lr: 9.81e-04  
[08/30 03:01:44] lb.utils.events INFO:  eta: 1 day, 23:35:48  iteration: 32999/375342  consumed_samples: 33792000  total_loss: 0.4367  time: 0.5018 s/iter  data_time: 0.0390 s/iter total_throughput: 2040.77 samples/s lr: 9.81e-04  
[08/30 03:02:34] lb.utils.events INFO:  eta: 1 day, 23:34:42  iteration: 33099/375342  consumed_samples: 33894400  total_loss: 0.4378  time: 0.5018 s/iter  data_time: 0.0395 s/iter total_throughput: 2040.78 samples/s lr: 9.81e-04  
[08/30 03:03:24] lb.utils.events INFO:  eta: 1 day, 23:34:23  iteration: 33199/375342  consumed_samples: 33996800  total_loss: 0.4363  time: 0.5018 s/iter  data_time: 0.0381 s/iter total_throughput: 2040.76 samples/s lr: 9.81e-04  
[08/30 03:04:14] lb.utils.events INFO:  eta: 1 day, 23:33:39  iteration: 33299/375342  consumed_samples: 34099200  total_loss: 0.4315  time: 0.5018 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.76 samples/s lr: 9.81e-04  
[08/30 03:05:05] lb.utils.events INFO:  eta: 1 day, 23:30:48  iteration: 33399/375342  consumed_samples: 34201600  total_loss: 0.4322  time: 0.5018 s/iter  data_time: 0.0392 s/iter total_throughput: 2040.76 samples/s lr: 9.81e-04  
[08/30 03:05:55] lb.utils.events INFO:  eta: 1 day, 23:29:58  iteration: 33499/375342  consumed_samples: 34304000  total_loss: 0.4322  time: 0.5018 s/iter  data_time: 0.0393 s/iter total_throughput: 2040.74 samples/s lr: 9.81e-04  
[08/30 03:06:45] lb.utils.events INFO:  eta: 1 day, 23:30:37  iteration: 33599/375342  consumed_samples: 34406400  total_loss: 0.4273  time: 0.5018 s/iter  data_time: 0.0404 s/iter total_throughput: 2040.73 samples/s lr: 9.81e-04  
[08/30 03:07:35] lb.utils.events INFO:  eta: 1 day, 23:30:16  iteration: 33699/375342  consumed_samples: 34508800  total_loss: 0.4289  time: 0.5018 s/iter  data_time: 0.0380 s/iter total_throughput: 2040.72 samples/s lr: 9.80e-04  
[08/30 03:08:26] lb.utils.events INFO:  eta: 1 day, 23:27:48  iteration: 33799/375342  consumed_samples: 34611200  total_loss: 0.4305  time: 0.5018 s/iter  data_time: 0.0405 s/iter total_throughput: 2040.72 samples/s lr: 9.80e-04  
[08/30 03:09:16] lb.utils.events INFO:  eta: 1 day, 23:26:16  iteration: 33899/375342  consumed_samples: 34713600  total_loss: 0.4278  time: 0.5018 s/iter  data_time: 0.0381 s/iter total_throughput: 2040.74 samples/s lr: 9.80e-04  
[08/30 03:10:06] lb.utils.events INFO:  eta: 1 day, 23:24:50  iteration: 33999/375342  consumed_samples: 34816000  total_loss: 0.4282  time: 0.5018 s/iter  data_time: 0.0399 s/iter total_throughput: 2040.74 samples/s lr: 9.80e-04  
[08/30 03:10:56] lb.utils.events INFO:  eta: 1 day, 23:24:20  iteration: 34099/375342  consumed_samples: 34918400  total_loss: 0.4167  time: 0.5018 s/iter  data_time: 0.0399 s/iter total_throughput: 2040.74 samples/s lr: 9.80e-04  
[08/30 03:11:47] lb.utils.events INFO:  eta: 1 day, 23:23:49  iteration: 34199/375342  consumed_samples: 35020800  total_loss: 0.4213  time: 0.5018 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.71 samples/s lr: 9.80e-04  
[08/30 03:12:37] lb.utils.events INFO:  eta: 1 day, 23:22:59  iteration: 34299/375342  consumed_samples: 35123200  total_loss: 0.4226  time: 0.5018 s/iter  data_time: 0.0390 s/iter total_throughput: 2040.72 samples/s lr: 9.80e-04  
[08/30 03:13:27] lb.utils.events INFO:  eta: 1 day, 23:23:09  iteration: 34399/375342  consumed_samples: 35225600  total_loss: 0.4136  time: 0.5018 s/iter  data_time: 0.0394 s/iter total_throughput: 2040.72 samples/s lr: 9.80e-04  
[08/30 03:14:17] lb.utils.events INFO:  eta: 1 day, 23:22:28  iteration: 34499/375342  consumed_samples: 35328000  total_loss: 0.4309  time: 0.5018 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.71 samples/s lr: 9.80e-04  
[08/30 03:15:07] lb.utils.events INFO:  eta: 1 day, 23:19:50  iteration: 34599/375342  consumed_samples: 35430400  total_loss: 0.4279  time: 0.5018 s/iter  data_time: 0.0400 s/iter total_throughput: 2040.71 samples/s lr: 9.79e-04  
[08/30 03:15:58] lb.utils.events INFO:  eta: 1 day, 23:18:20  iteration: 34699/375342  consumed_samples: 35532800  total_loss: 0.4181  time: 0.5018 s/iter  data_time: 0.0369 s/iter total_throughput: 2040.70 samples/s lr: 9.79e-04  
[08/30 03:16:48] lb.utils.events INFO:  eta: 1 day, 23:16:42  iteration: 34799/375342  consumed_samples: 35635200  total_loss: 0.4204  time: 0.5018 s/iter  data_time: 0.0391 s/iter total_throughput: 2040.70 samples/s lr: 9.79e-04  
[08/30 03:17:38] lb.utils.events INFO:  eta: 1 day, 23:15:40  iteration: 34899/375342  consumed_samples: 35737600  total_loss: 0.4159  time: 0.5018 s/iter  data_time: 0.0377 s/iter total_throughput: 2040.71 samples/s lr: 9.79e-04  
[08/30 03:18:28] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0034999
[08/30 03:18:29] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 03:18:29] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 03:18:33] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0502 s/iter. Inference: 0.2449 s/iter. Eval: 0.0021 s/iter. Total: 0.2972 s/iter. ETA=0:00:10
[08/30 03:18:38] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0829 s/iter. Inference: 0.2394 s/iter. Eval: 0.0022 s/iter. Total: 0.3248 s/iter. ETA=0:00:07
[08/30 03:18:43] lb.evaluation.evaluator INFO: Inference done 41984/50000. Dataloading: 0.0863 s/iter. Inference: 0.2410 s/iter. Eval: 0.0022 s/iter. Total: 0.3297 s/iter. ETA=0:00:02
[08/30 03:18:46] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 03:18:46] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.108919 (0.000282 s / iter per device, on 8 devices)
[08/30 03:18:46] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000214 s / iter per device, on 8 devices)
[08/30 03:18:46] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 03:18:46] lb.evaluation.utils INFO: copypaste: Acc@1=70.27799999999999
[08/30 03:18:46] lb.evaluation.utils INFO: copypaste: Acc@5=90.786
[08/30 03:18:46] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 70.27800, better than last best score 68.06600 @ iteration 29999.
[08/30 03:18:46] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 03:18:46] lb.utils.events INFO:  eta: 1 day, 23:14:39  iteration: 34999/375342  consumed_samples: 35840000  total_loss: 0.4192  time: 0.5018 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.71 samples/s lr: 9.79e-04  
[08/30 03:19:37] lb.utils.events INFO:  eta: 1 day, 23:15:03  iteration: 35099/375342  consumed_samples: 35942400  total_loss: 0.4255  time: 0.5018 s/iter  data_time: 0.0390 s/iter total_throughput: 2040.66 samples/s lr: 9.79e-04  
[08/30 03:20:27] lb.utils.events INFO:  eta: 1 day, 23:13:02  iteration: 35199/375342  consumed_samples: 36044800  total_loss: 0.4236  time: 0.5018 s/iter  data_time: 0.0395 s/iter total_throughput: 2040.65 samples/s lr: 9.79e-04  
[08/30 03:21:18] lb.utils.events INFO:  eta: 1 day, 23:12:31  iteration: 35299/375342  consumed_samples: 36147200  total_loss: 0.4145  time: 0.5018 s/iter  data_time: 0.0383 s/iter total_throughput: 2040.63 samples/s lr: 9.79e-04  
[08/30 03:22:08] lb.utils.events INFO:  eta: 1 day, 23:12:22  iteration: 35399/375342  consumed_samples: 36249600  total_loss: 0.4117  time: 0.5018 s/iter  data_time: 0.0377 s/iter total_throughput: 2040.61 samples/s lr: 9.78e-04  
[08/30 03:22:58] lb.utils.events INFO:  eta: 1 day, 23:10:43  iteration: 35499/375342  consumed_samples: 36352000  total_loss: 0.4166  time: 0.5018 s/iter  data_time: 0.0406 s/iter total_throughput: 2040.61 samples/s lr: 9.78e-04  
[08/30 03:23:48] lb.utils.events INFO:  eta: 1 day, 23:10:42  iteration: 35599/375342  consumed_samples: 36454400  total_loss: 0.4268  time: 0.5018 s/iter  data_time: 0.0399 s/iter total_throughput: 2040.61 samples/s lr: 9.78e-04  
[08/30 03:24:39] lb.utils.events INFO:  eta: 1 day, 23:10:02  iteration: 35699/375342  consumed_samples: 36556800  total_loss: 0.431  time: 0.5018 s/iter  data_time: 0.0386 s/iter total_throughput: 2040.59 samples/s lr: 9.78e-04  
[08/30 03:25:29] lb.utils.events INFO:  eta: 1 day, 23:11:02  iteration: 35799/375342  consumed_samples: 36659200  total_loss: 0.4167  time: 0.5018 s/iter  data_time: 0.0393 s/iter total_throughput: 2040.57 samples/s lr: 9.78e-04  
[08/30 03:26:19] lb.utils.events INFO:  eta: 1 day, 23:11:22  iteration: 35899/375342  consumed_samples: 36761600  total_loss: 0.4134  time: 0.5018 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.57 samples/s lr: 9.78e-04  
[08/30 03:27:10] lb.utils.events INFO:  eta: 1 day, 23:11:46  iteration: 35999/375342  consumed_samples: 36864000  total_loss: 0.4239  time: 0.5018 s/iter  data_time: 0.0401 s/iter total_throughput: 2040.57 samples/s lr: 9.78e-04  
[08/30 03:28:00] lb.utils.events INFO:  eta: 1 day, 23:09:32  iteration: 36099/375342  consumed_samples: 36966400  total_loss: 0.4243  time: 0.5018 s/iter  data_time: 0.0393 s/iter total_throughput: 2040.56 samples/s lr: 9.78e-04  
[08/30 03:28:50] lb.utils.events INFO:  eta: 1 day, 23:09:52  iteration: 36199/375342  consumed_samples: 37068800  total_loss: 0.4203  time: 0.5018 s/iter  data_time: 0.0382 s/iter total_throughput: 2040.55 samples/s lr: 9.77e-04  
[08/30 03:29:40] lb.utils.events INFO:  eta: 1 day, 23:07:56  iteration: 36299/375342  consumed_samples: 37171200  total_loss: 0.4231  time: 0.5018 s/iter  data_time: 0.0379 s/iter total_throughput: 2040.56 samples/s lr: 9.77e-04  
[08/30 03:30:30] lb.utils.events INFO:  eta: 1 day, 23:06:01  iteration: 36399/375342  consumed_samples: 37273600  total_loss: 0.4243  time: 0.5018 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.57 samples/s lr: 9.77e-04  
[08/30 03:31:21] lb.utils.events INFO:  eta: 1 day, 23:05:23  iteration: 36499/375342  consumed_samples: 37376000  total_loss: 0.4239  time: 0.5018 s/iter  data_time: 0.0396 s/iter total_throughput: 2040.56 samples/s lr: 9.77e-04  
[08/30 03:32:11] lb.utils.events INFO:  eta: 1 day, 23:04:02  iteration: 36599/375342  consumed_samples: 37478400  total_loss: 0.427  time: 0.5018 s/iter  data_time: 0.0376 s/iter total_throughput: 2040.55 samples/s lr: 9.77e-04  
[08/30 03:33:01] lb.utils.events INFO:  eta: 1 day, 23:02:40  iteration: 36699/375342  consumed_samples: 37580800  total_loss: 0.4201  time: 0.5018 s/iter  data_time: 0.0397 s/iter total_throughput: 2040.55 samples/s lr: 9.77e-04  
[08/30 03:33:51] lb.utils.events INFO:  eta: 1 day, 22:59:46  iteration: 36799/375342  consumed_samples: 37683200  total_loss: 0.411  time: 0.5018 s/iter  data_time: 0.0372 s/iter total_throughput: 2040.54 samples/s lr: 9.77e-04  
[08/30 03:34:42] lb.utils.events INFO:  eta: 1 day, 22:57:28  iteration: 36899/375342  consumed_samples: 37785600  total_loss: 0.42  time: 0.5018 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.54 samples/s lr: 9.77e-04  
[08/30 03:35:32] lb.utils.events INFO:  eta: 1 day, 22:56:20  iteration: 36999/375342  consumed_samples: 37888000  total_loss: 0.4227  time: 0.5018 s/iter  data_time: 0.0373 s/iter total_throughput: 2040.55 samples/s lr: 9.76e-04  
[08/30 03:36:22] lb.utils.events INFO:  eta: 1 day, 22:56:50  iteration: 37099/375342  consumed_samples: 37990400  total_loss: 0.4167  time: 0.5018 s/iter  data_time: 0.0399 s/iter total_throughput: 2040.53 samples/s lr: 9.76e-04  
[08/30 03:37:12] lb.utils.events INFO:  eta: 1 day, 22:56:21  iteration: 37199/375342  consumed_samples: 38092800  total_loss: 0.42  time: 0.5018 s/iter  data_time: 0.0402 s/iter total_throughput: 2040.52 samples/s lr: 9.76e-04  
[08/30 03:38:03] lb.utils.events INFO:  eta: 1 day, 22:56:40  iteration: 37299/375342  consumed_samples: 38195200  total_loss: 0.4275  time: 0.5018 s/iter  data_time: 0.0397 s/iter total_throughput: 2040.51 samples/s lr: 9.76e-04  
[08/30 03:38:53] lb.utils.events INFO:  eta: 1 day, 22:54:52  iteration: 37399/375342  consumed_samples: 38297600  total_loss: 0.4245  time: 0.5018 s/iter  data_time: 0.0390 s/iter total_throughput: 2040.51 samples/s lr: 9.76e-04  
[08/30 03:39:43] lb.utils.events INFO:  eta: 1 day, 22:55:36  iteration: 37499/375342  consumed_samples: 38400000  total_loss: 0.4201  time: 0.5018 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.50 samples/s lr: 9.76e-04  
[08/30 03:40:33] lb.utils.events INFO:  eta: 1 day, 22:55:17  iteration: 37599/375342  consumed_samples: 38502400  total_loss: 0.4126  time: 0.5018 s/iter  data_time: 0.0397 s/iter total_throughput: 2040.50 samples/s lr: 9.76e-04  
[08/30 03:41:24] lb.utils.events INFO:  eta: 1 day, 22:56:10  iteration: 37699/375342  consumed_samples: 38604800  total_loss: 0.4172  time: 0.5018 s/iter  data_time: 0.0397 s/iter total_throughput: 2040.47 samples/s lr: 9.76e-04  
[08/30 03:42:14] lb.utils.events INFO:  eta: 1 day, 22:55:37  iteration: 37799/375342  consumed_samples: 38707200  total_loss: 0.4212  time: 0.5018 s/iter  data_time: 0.0378 s/iter total_throughput: 2040.48 samples/s lr: 9.75e-04  
[08/30 03:43:04] lb.utils.events INFO:  eta: 1 day, 22:53:38  iteration: 37899/375342  consumed_samples: 38809600  total_loss: 0.4171  time: 0.5018 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.49 samples/s lr: 9.75e-04  
[08/30 03:43:54] lb.utils.events INFO:  eta: 1 day, 22:52:10  iteration: 37999/375342  consumed_samples: 38912000  total_loss: 0.4122  time: 0.5018 s/iter  data_time: 0.0378 s/iter total_throughput: 2040.48 samples/s lr: 9.75e-04  
[08/30 03:44:45] lb.utils.events INFO:  eta: 1 day, 22:51:31  iteration: 38099/375342  consumed_samples: 39014400  total_loss: 0.4035  time: 0.5018 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.47 samples/s lr: 9.75e-04  
[08/30 03:45:35] lb.utils.events INFO:  eta: 1 day, 22:51:01  iteration: 38199/375342  consumed_samples: 39116800  total_loss: 0.4094  time: 0.5018 s/iter  data_time: 0.0406 s/iter total_throughput: 2040.46 samples/s lr: 9.75e-04  
[08/30 03:46:25] lb.utils.events INFO:  eta: 1 day, 22:50:12  iteration: 38299/375342  consumed_samples: 39219200  total_loss: 0.4244  time: 0.5018 s/iter  data_time: 0.0404 s/iter total_throughput: 2040.45 samples/s lr: 9.75e-04  
[08/30 03:47:16] lb.utils.events INFO:  eta: 1 day, 22:51:24  iteration: 38399/375342  consumed_samples: 39321600  total_loss: 0.4231  time: 0.5019 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.44 samples/s lr: 9.75e-04  
[08/30 03:48:06] lb.utils.events INFO:  eta: 1 day, 22:48:18  iteration: 38499/375342  consumed_samples: 39424000  total_loss: 0.4194  time: 0.5018 s/iter  data_time: 0.0380 s/iter total_throughput: 2040.45 samples/s lr: 9.75e-04  
[08/30 03:48:56] lb.utils.events INFO:  eta: 1 day, 22:47:03  iteration: 38599/375342  consumed_samples: 39526400  total_loss: 0.4232  time: 0.5018 s/iter  data_time: 0.0390 s/iter total_throughput: 2040.46 samples/s lr: 9.74e-04  
[08/30 03:49:46] lb.utils.events INFO:  eta: 1 day, 22:45:07  iteration: 38699/375342  consumed_samples: 39628800  total_loss: 0.4122  time: 0.5019 s/iter  data_time: 0.0377 s/iter total_throughput: 2040.44 samples/s lr: 9.74e-04  
[08/30 03:50:36] lb.utils.events INFO:  eta: 1 day, 22:44:53  iteration: 38799/375342  consumed_samples: 39731200  total_loss: 0.412  time: 0.5019 s/iter  data_time: 0.0385 s/iter total_throughput: 2040.43 samples/s lr: 9.74e-04  
[08/30 03:51:27] lb.utils.events INFO:  eta: 1 day, 22:44:46  iteration: 38899/375342  consumed_samples: 39833600  total_loss: 0.4193  time: 0.5019 s/iter  data_time: 0.0386 s/iter total_throughput: 2040.44 samples/s lr: 9.74e-04  
[08/30 03:52:17] lb.utils.events INFO:  eta: 1 day, 22:44:29  iteration: 38999/375342  consumed_samples: 39936000  total_loss: 0.4249  time: 0.5019 s/iter  data_time: 0.0397 s/iter total_throughput: 2040.44 samples/s lr: 9.74e-04  
[08/30 03:53:07] lb.utils.events INFO:  eta: 1 day, 22:42:44  iteration: 39099/375342  consumed_samples: 40038400  total_loss: 0.4215  time: 0.5019 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.44 samples/s lr: 9.74e-04  
[08/30 03:53:57] lb.utils.events INFO:  eta: 1 day, 22:39:54  iteration: 39199/375342  consumed_samples: 40140800  total_loss: 0.4132  time: 0.5019 s/iter  data_time: 0.0397 s/iter total_throughput: 2040.43 samples/s lr: 9.74e-04  
[08/30 03:54:47] lb.utils.events INFO:  eta: 1 day, 22:38:25  iteration: 39299/375342  consumed_samples: 40243200  total_loss: 0.404  time: 0.5019 s/iter  data_time: 0.0376 s/iter total_throughput: 2040.43 samples/s lr: 9.73e-04  
[08/30 03:55:38] lb.utils.events INFO:  eta: 1 day, 22:36:51  iteration: 39399/375342  consumed_samples: 40345600  total_loss: 0.4108  time: 0.5019 s/iter  data_time: 0.0400 s/iter total_throughput: 2040.42 samples/s lr: 9.73e-04  
[08/30 03:56:28] lb.utils.events INFO:  eta: 1 day, 22:36:05  iteration: 39499/375342  consumed_samples: 40448000  total_loss: 0.4201  time: 0.5019 s/iter  data_time: 0.0382 s/iter total_throughput: 2040.42 samples/s lr: 9.73e-04  
[08/30 03:57:18] lb.utils.events INFO:  eta: 1 day, 22:35:15  iteration: 39599/375342  consumed_samples: 40550400  total_loss: 0.4248  time: 0.5019 s/iter  data_time: 0.0384 s/iter total_throughput: 2040.41 samples/s lr: 9.73e-04  
[08/30 03:58:09] lb.utils.events INFO:  eta: 1 day, 22:34:35  iteration: 39699/375342  consumed_samples: 40652800  total_loss: 0.4195  time: 0.5019 s/iter  data_time: 0.0390 s/iter total_throughput: 2040.40 samples/s lr: 9.73e-04  
[08/30 03:58:59] lb.utils.events INFO:  eta: 1 day, 22:33:38  iteration: 39799/375342  consumed_samples: 40755200  total_loss: 0.4137  time: 0.5019 s/iter  data_time: 0.0384 s/iter total_throughput: 2040.39 samples/s lr: 9.73e-04  
[08/30 03:59:49] lb.utils.events INFO:  eta: 1 day, 22:32:48  iteration: 39899/375342  consumed_samples: 40857600  total_loss: 0.418  time: 0.5019 s/iter  data_time: 0.0391 s/iter total_throughput: 2040.40 samples/s lr: 9.73e-04  
[08/30 04:00:39] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0039999
[08/30 04:00:40] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 04:00:40] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 04:00:44] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0430 s/iter. Inference: 0.2504 s/iter. Eval: 0.0023 s/iter. Total: 0.2958 s/iter. ETA=0:00:10
[08/30 04:00:49] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0792 s/iter. Inference: 0.2423 s/iter. Eval: 0.0021 s/iter. Total: 0.3239 s/iter. ETA=0:00:07
[08/30 04:00:55] lb.evaluation.evaluator INFO: Inference done 43008/50000. Dataloading: 0.0783 s/iter. Inference: 0.2445 s/iter. Eval: 0.0021 s/iter. Total: 0.3252 s/iter. ETA=0:00:01
[08/30 04:00:57] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 04:00:57] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.006190 (0.000280 s / iter per device, on 8 devices)
[08/30 04:00:57] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000215 s / iter per device, on 8 devices)
[08/30 04:00:57] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 04:00:57] lb.evaluation.utils INFO: copypaste: Acc@1=71.64399999999999
[08/30 04:00:57] lb.evaluation.utils INFO: copypaste: Acc@5=91.00399999999999
[08/30 04:00:57] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 71.64400, better than last best score 70.27800 @ iteration 34999.
[08/30 04:00:57] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 04:00:57] lb.utils.events INFO:  eta: 1 day, 22:31:54  iteration: 39999/375342  consumed_samples: 40960000  total_loss: 0.4192  time: 0.5019 s/iter  data_time: 0.0393 s/iter total_throughput: 2040.40 samples/s lr: 9.73e-04  
[08/30 04:01:47] lb.utils.events INFO:  eta: 1 day, 22:31:05  iteration: 40099/375342  consumed_samples: 41062400  total_loss: 0.42  time: 0.5019 s/iter  data_time: 0.0354 s/iter total_throughput: 2040.40 samples/s lr: 9.72e-04  
[08/30 04:02:38] lb.utils.events INFO:  eta: 1 day, 22:31:35  iteration: 40199/375342  consumed_samples: 41164800  total_loss: 0.4133  time: 0.5019 s/iter  data_time: 0.0400 s/iter total_throughput: 2040.39 samples/s lr: 9.72e-04  
[08/30 04:03:28] lb.utils.events INFO:  eta: 1 day, 22:30:45  iteration: 40299/375342  consumed_samples: 41267200  total_loss: 0.4148  time: 0.5019 s/iter  data_time: 0.0403 s/iter total_throughput: 2040.39 samples/s lr: 9.72e-04  
[08/30 04:04:18] lb.utils.events INFO:  eta: 1 day, 22:28:46  iteration: 40399/375342  consumed_samples: 41369600  total_loss: 0.4256  time: 0.5019 s/iter  data_time: 0.0381 s/iter total_throughput: 2040.39 samples/s lr: 9.72e-04  
[08/30 04:05:08] lb.utils.events INFO:  eta: 1 day, 22:27:28  iteration: 40499/375342  consumed_samples: 41472000  total_loss: 0.4257  time: 0.5019 s/iter  data_time: 0.0393 s/iter total_throughput: 2040.39 samples/s lr: 9.72e-04  
[08/30 04:05:58] lb.utils.events INFO:  eta: 1 day, 22:26:02  iteration: 40599/375342  consumed_samples: 41574400  total_loss: 0.4244  time: 0.5019 s/iter  data_time: 0.0396 s/iter total_throughput: 2040.40 samples/s lr: 9.72e-04  
[08/30 04:06:49] lb.utils.events INFO:  eta: 1 day, 22:23:31  iteration: 40699/375342  consumed_samples: 41676800  total_loss: 0.4166  time: 0.5019 s/iter  data_time: 0.0391 s/iter total_throughput: 2040.41 samples/s lr: 9.72e-04  
[08/30 04:07:39] lb.utils.events INFO:  eta: 1 day, 22:23:51  iteration: 40799/375342  consumed_samples: 41779200  total_loss: 0.4115  time: 0.5019 s/iter  data_time: 0.0402 s/iter total_throughput: 2040.42 samples/s lr: 9.71e-04  
[08/30 04:08:29] lb.utils.events INFO:  eta: 1 day, 22:24:01  iteration: 40899/375342  consumed_samples: 41881600  total_loss: 0.4131  time: 0.5019 s/iter  data_time: 0.0409 s/iter total_throughput: 2040.41 samples/s lr: 9.71e-04  
[08/30 04:09:19] lb.utils.events INFO:  eta: 1 day, 22:24:28  iteration: 40999/375342  consumed_samples: 41984000  total_loss: 0.4154  time: 0.5019 s/iter  data_time: 0.0393 s/iter total_throughput: 2040.40 samples/s lr: 9.71e-04  
[08/30 04:10:09] lb.utils.events INFO:  eta: 1 day, 22:23:21  iteration: 41099/375342  consumed_samples: 42086400  total_loss: 0.4155  time: 0.5019 s/iter  data_time: 0.0386 s/iter total_throughput: 2040.40 samples/s lr: 9.71e-04  
[08/30 04:11:00] lb.utils.events INFO:  eta: 1 day, 22:21:26  iteration: 41199/375342  consumed_samples: 42188800  total_loss: 0.4194  time: 0.5019 s/iter  data_time: 0.0398 s/iter total_throughput: 2040.41 samples/s lr: 9.71e-04  
[08/30 04:11:50] lb.utils.events INFO:  eta: 1 day, 22:21:29  iteration: 41299/375342  consumed_samples: 42291200  total_loss: 0.4192  time: 0.5019 s/iter  data_time: 0.0395 s/iter total_throughput: 2040.40 samples/s lr: 9.71e-04  
[08/30 04:12:40] lb.utils.events INFO:  eta: 1 day, 22:21:51  iteration: 41399/375342  consumed_samples: 42393600  total_loss: 0.4113  time: 0.5019 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.40 samples/s lr: 9.71e-04  
[08/30 04:13:30] lb.utils.events INFO:  eta: 1 day, 22:23:36  iteration: 41499/375342  consumed_samples: 42496000  total_loss: 0.411  time: 0.5019 s/iter  data_time: 0.0384 s/iter total_throughput: 2040.39 samples/s lr: 9.70e-04  
[08/30 04:14:20] lb.utils.events INFO:  eta: 1 day, 22:22:07  iteration: 41599/375342  consumed_samples: 42598400  total_loss: 0.4194  time: 0.5019 s/iter  data_time: 0.0385 s/iter total_throughput: 2040.41 samples/s lr: 9.70e-04  
[08/30 04:15:11] lb.utils.events INFO:  eta: 1 day, 22:22:31  iteration: 41699/375342  consumed_samples: 42700800  total_loss: 0.4183  time: 0.5019 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.40 samples/s lr: 9.70e-04  
[08/30 04:16:01] lb.utils.events INFO:  eta: 1 day, 22:21:56  iteration: 41799/375342  consumed_samples: 42803200  total_loss: 0.4075  time: 0.5019 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.39 samples/s lr: 9.70e-04  
[08/30 04:16:51] lb.utils.events INFO:  eta: 1 day, 22:19:30  iteration: 41899/375342  consumed_samples: 42905600  total_loss: 0.4086  time: 0.5019 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.38 samples/s lr: 9.70e-04  
[08/30 04:17:42] lb.utils.events INFO:  eta: 1 day, 22:18:53  iteration: 41999/375342  consumed_samples: 43008000  total_loss: 0.41  time: 0.5019 s/iter  data_time: 0.0396 s/iter total_throughput: 2040.37 samples/s lr: 9.70e-04  
[08/30 04:18:32] lb.utils.events INFO:  eta: 1 day, 22:17:50  iteration: 42099/375342  consumed_samples: 43110400  total_loss: 0.4134  time: 0.5019 s/iter  data_time: 0.0384 s/iter total_throughput: 2040.36 samples/s lr: 9.70e-04  
[08/30 04:19:22] lb.utils.events INFO:  eta: 1 day, 22:17:14  iteration: 42199/375342  consumed_samples: 43212800  total_loss: 0.4108  time: 0.5019 s/iter  data_time: 0.0390 s/iter total_throughput: 2040.36 samples/s lr: 9.69e-04  
[08/30 04:20:12] lb.utils.events INFO:  eta: 1 day, 22:16:25  iteration: 42299/375342  consumed_samples: 43315200  total_loss: 0.4093  time: 0.5019 s/iter  data_time: 0.0397 s/iter total_throughput: 2040.35 samples/s lr: 9.69e-04  
[08/30 04:21:03] lb.utils.events INFO:  eta: 1 day, 22:15:37  iteration: 42399/375342  consumed_samples: 43417600  total_loss: 0.4161  time: 0.5019 s/iter  data_time: 0.0393 s/iter total_throughput: 2040.33 samples/s lr: 9.69e-04  
[08/30 04:21:53] lb.utils.events INFO:  eta: 1 day, 22:14:17  iteration: 42499/375342  consumed_samples: 43520000  total_loss: 0.4133  time: 0.5019 s/iter  data_time: 0.0380 s/iter total_throughput: 2040.32 samples/s lr: 9.69e-04  
[08/30 04:22:43] lb.utils.events INFO:  eta: 1 day, 22:15:34  iteration: 42599/375342  consumed_samples: 43622400  total_loss: 0.4103  time: 0.5019 s/iter  data_time: 0.0403 s/iter total_throughput: 2040.31 samples/s lr: 9.69e-04  
[08/30 04:23:34] lb.utils.events INFO:  eta: 1 day, 22:14:54  iteration: 42699/375342  consumed_samples: 43724800  total_loss: 0.4098  time: 0.5019 s/iter  data_time: 0.0393 s/iter total_throughput: 2040.30 samples/s lr: 9.69e-04  
[08/30 04:24:24] lb.utils.events INFO:  eta: 1 day, 22:14:52  iteration: 42799/375342  consumed_samples: 43827200  total_loss: 0.4132  time: 0.5019 s/iter  data_time: 0.0380 s/iter total_throughput: 2040.29 samples/s lr: 9.69e-04  
[08/30 04:25:14] lb.utils.events INFO:  eta: 1 day, 22:15:31  iteration: 42899/375342  consumed_samples: 43929600  total_loss: 0.4081  time: 0.5019 s/iter  data_time: 0.0375 s/iter total_throughput: 2040.27 samples/s lr: 9.68e-04  
[08/30 04:26:05] lb.utils.events INFO:  eta: 1 day, 22:14:54  iteration: 42999/375342  consumed_samples: 44032000  total_loss: 0.4116  time: 0.5019 s/iter  data_time: 0.0393 s/iter total_throughput: 2040.27 samples/s lr: 9.68e-04  
[08/30 04:26:55] lb.utils.events INFO:  eta: 1 day, 22:14:18  iteration: 43099/375342  consumed_samples: 44134400  total_loss: 0.4126  time: 0.5019 s/iter  data_time: 0.0370 s/iter total_throughput: 2040.27 samples/s lr: 9.68e-04  
[08/30 04:27:45] lb.utils.events INFO:  eta: 1 day, 22:13:14  iteration: 43199/375342  consumed_samples: 44236800  total_loss: 0.4119  time: 0.5019 s/iter  data_time: 0.0383 s/iter total_throughput: 2040.25 samples/s lr: 9.68e-04  
[08/30 04:28:36] lb.utils.events INFO:  eta: 1 day, 22:14:32  iteration: 43299/375342  consumed_samples: 44339200  total_loss: 0.4202  time: 0.5019 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.23 samples/s lr: 9.68e-04  
[08/30 04:29:26] lb.utils.events INFO:  eta: 1 day, 22:13:08  iteration: 43399/375342  consumed_samples: 44441600  total_loss: 0.4192  time: 0.5019 s/iter  data_time: 0.0394 s/iter total_throughput: 2040.22 samples/s lr: 9.68e-04  
[08/30 04:30:16] lb.utils.events INFO:  eta: 1 day, 22:12:03  iteration: 43499/375342  consumed_samples: 44544000  total_loss: 0.4179  time: 0.5019 s/iter  data_time: 0.0396 s/iter total_throughput: 2040.21 samples/s lr: 9.68e-04  
[08/30 04:31:07] lb.utils.events INFO:  eta: 1 day, 22:10:07  iteration: 43599/375342  consumed_samples: 44646400  total_loss: 0.4165  time: 0.5019 s/iter  data_time: 0.0373 s/iter total_throughput: 2040.20 samples/s lr: 9.67e-04  
[08/30 04:31:57] lb.utils.events INFO:  eta: 1 day, 22:08:24  iteration: 43699/375342  consumed_samples: 44748800  total_loss: 0.4101  time: 0.5019 s/iter  data_time: 0.0377 s/iter total_throughput: 2040.18 samples/s lr: 9.67e-04  
[08/30 04:32:47] lb.utils.events INFO:  eta: 1 day, 22:07:03  iteration: 43799/375342  consumed_samples: 44851200  total_loss: 0.4071  time: 0.5019 s/iter  data_time: 0.0373 s/iter total_throughput: 2040.17 samples/s lr: 9.67e-04  
[08/30 04:33:38] lb.utils.events INFO:  eta: 1 day, 22:06:30  iteration: 43899/375342  consumed_samples: 44953600  total_loss: 0.4096  time: 0.5019 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.16 samples/s lr: 9.67e-04  
[08/30 04:34:28] lb.utils.events INFO:  eta: 1 day, 22:06:12  iteration: 43999/375342  consumed_samples: 45056000  total_loss: 0.4155  time: 0.5019 s/iter  data_time: 0.0390 s/iter total_throughput: 2040.14 samples/s lr: 9.67e-04  
[08/30 04:35:19] lb.utils.events INFO:  eta: 1 day, 22:09:02  iteration: 44099/375342  consumed_samples: 45158400  total_loss: 0.4177  time: 0.5019 s/iter  data_time: 0.0375 s/iter total_throughput: 2040.12 samples/s lr: 9.67e-04  
[08/30 04:36:09] lb.utils.events INFO:  eta: 1 day, 22:07:53  iteration: 44199/375342  consumed_samples: 45260800  total_loss: 0.4145  time: 0.5019 s/iter  data_time: 0.0394 s/iter total_throughput: 2040.11 samples/s lr: 9.67e-04  
[08/30 04:36:59] lb.utils.events INFO:  eta: 1 day, 22:02:55  iteration: 44299/375342  consumed_samples: 45363200  total_loss: 0.4173  time: 0.5019 s/iter  data_time: 0.0399 s/iter total_throughput: 2040.13 samples/s lr: 9.66e-04  
[08/30 04:37:49] lb.utils.events INFO:  eta: 1 day, 22:02:00  iteration: 44399/375342  consumed_samples: 45465600  total_loss: 0.4115  time: 0.5019 s/iter  data_time: 0.0399 s/iter total_throughput: 2040.13 samples/s lr: 9.66e-04  
[08/30 04:38:39] lb.utils.events INFO:  eta: 1 day, 22:01:25  iteration: 44499/375342  consumed_samples: 45568000  total_loss: 0.4082  time: 0.5019 s/iter  data_time: 0.0392 s/iter total_throughput: 2040.11 samples/s lr: 9.66e-04  
[08/30 04:39:30] lb.utils.events INFO:  eta: 1 day, 22:01:38  iteration: 44599/375342  consumed_samples: 45670400  total_loss: 0.4094  time: 0.5019 s/iter  data_time: 0.0401 s/iter total_throughput: 2040.09 samples/s lr: 9.66e-04  
[08/30 04:40:20] lb.utils.events INFO:  eta: 1 day, 21:59:37  iteration: 44699/375342  consumed_samples: 45772800  total_loss: 0.4078  time: 0.5019 s/iter  data_time: 0.0385 s/iter total_throughput: 2040.09 samples/s lr: 9.66e-04  
[08/30 04:41:11] lb.utils.events INFO:  eta: 1 day, 22:00:17  iteration: 44799/375342  consumed_samples: 45875200  total_loss: 0.4185  time: 0.5019 s/iter  data_time: 0.0384 s/iter total_throughput: 2040.05 samples/s lr: 9.66e-04  
[08/30 04:42:01] lb.utils.events INFO:  eta: 1 day, 21:58:13  iteration: 44899/375342  consumed_samples: 45977600  total_loss: 0.4186  time: 0.5019 s/iter  data_time: 0.0412 s/iter total_throughput: 2040.05 samples/s lr: 9.65e-04  
[08/30 04:42:51] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0044999
[08/30 04:42:52] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 04:42:52] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 04:42:57] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0713 s/iter. Inference: 0.2422 s/iter. Eval: 0.0022 s/iter. Total: 0.3158 s/iter. ETA=0:00:11
[08/30 04:43:02] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0875 s/iter. Inference: 0.2428 s/iter. Eval: 0.0021 s/iter. Total: 0.3327 s/iter. ETA=0:00:07
[08/30 04:43:07] lb.evaluation.evaluator INFO: Inference done 43008/50000. Dataloading: 0.0832 s/iter. Inference: 0.2415 s/iter. Eval: 0.0023 s/iter. Total: 0.3271 s/iter. ETA=0:00:01
[08/30 04:43:09] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 04:43:09] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.097672 (0.000282 s / iter per device, on 8 devices)
[08/30 04:43:09] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000213 s / iter per device, on 8 devices)
[08/30 04:43:09] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 04:43:09] lb.evaluation.utils INFO: copypaste: Acc@1=72.964
[08/30 04:43:09] lb.evaluation.utils INFO: copypaste: Acc@5=91.848
[08/30 04:43:09] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 72.96400, better than last best score 71.64400 @ iteration 39999.
[08/30 04:43:09] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 04:43:09] lb.utils.events INFO:  eta: 1 day, 21:57:21  iteration: 44999/375342  consumed_samples: 46080000  total_loss: 0.404  time: 0.5019 s/iter  data_time: 0.0395 s/iter total_throughput: 2040.05 samples/s lr: 9.65e-04  
[08/30 04:44:00] lb.utils.events INFO:  eta: 1 day, 21:54:47  iteration: 45099/375342  consumed_samples: 46182400  total_loss: 0.404  time: 0.5020 s/iter  data_time: 0.0382 s/iter total_throughput: 2040.01 samples/s lr: 9.65e-04  
[08/30 04:44:50] lb.utils.events INFO:  eta: 1 day, 21:54:50  iteration: 45199/375342  consumed_samples: 46284800  total_loss: 0.4093  time: 0.5020 s/iter  data_time: 0.0394 s/iter total_throughput: 2040.01 samples/s lr: 9.65e-04  
[08/30 04:45:41] lb.utils.events INFO:  eta: 1 day, 21:55:19  iteration: 45299/375342  consumed_samples: 46387200  total_loss: 0.406  time: 0.5020 s/iter  data_time: 0.0385 s/iter total_throughput: 2040.00 samples/s lr: 9.65e-04  
[08/30 04:46:31] lb.utils.events INFO:  eta: 1 day, 21:55:11  iteration: 45399/375342  consumed_samples: 46489600  total_loss: 0.4044  time: 0.5020 s/iter  data_time: 0.0399 s/iter total_throughput: 2039.99 samples/s lr: 9.65e-04  
[08/30 04:47:21] lb.utils.events INFO:  eta: 1 day, 21:53:41  iteration: 45499/375342  consumed_samples: 46592000  total_loss: 0.4134  time: 0.5020 s/iter  data_time: 0.0388 s/iter total_throughput: 2039.99 samples/s lr: 9.65e-04  
[08/30 04:48:11] lb.utils.events INFO:  eta: 1 day, 21:52:49  iteration: 45599/375342  consumed_samples: 46694400  total_loss: 0.4118  time: 0.5020 s/iter  data_time: 0.0405 s/iter total_throughput: 2039.98 samples/s lr: 9.64e-04  
[08/30 04:49:02] lb.utils.events INFO:  eta: 1 day, 21:53:00  iteration: 45699/375342  consumed_samples: 46796800  total_loss: 0.4078  time: 0.5020 s/iter  data_time: 0.0399 s/iter total_throughput: 2039.98 samples/s lr: 9.64e-04  
[08/30 04:49:52] lb.utils.events INFO:  eta: 1 day, 21:51:33  iteration: 45799/375342  consumed_samples: 46899200  total_loss: 0.4091  time: 0.5020 s/iter  data_time: 0.0396 s/iter total_throughput: 2039.96 samples/s lr: 9.64e-04  
[08/30 04:50:43] lb.utils.events INFO:  eta: 1 day, 21:52:37  iteration: 45899/375342  consumed_samples: 47001600  total_loss: 0.4043  time: 0.5020 s/iter  data_time: 0.0400 s/iter total_throughput: 2039.94 samples/s lr: 9.64e-04  
[08/30 04:51:33] lb.utils.events INFO:  eta: 1 day, 21:51:47  iteration: 45999/375342  consumed_samples: 47104000  total_loss: 0.4119  time: 0.5020 s/iter  data_time: 0.0400 s/iter total_throughput: 2039.94 samples/s lr: 9.64e-04  
[08/30 04:52:23] lb.utils.events INFO:  eta: 1 day, 21:51:29  iteration: 46099/375342  consumed_samples: 47206400  total_loss: 0.3967  time: 0.5020 s/iter  data_time: 0.0405 s/iter total_throughput: 2039.93 samples/s lr: 9.64e-04  
[08/30 04:53:14] lb.utils.events INFO:  eta: 1 day, 21:50:39  iteration: 46199/375342  consumed_samples: 47308800  total_loss: 0.3985  time: 0.5020 s/iter  data_time: 0.0389 s/iter total_throughput: 2039.91 samples/s lr: 9.63e-04  
[08/30 04:54:04] lb.utils.events INFO:  eta: 1 day, 21:50:42  iteration: 46299/375342  consumed_samples: 47411200  total_loss: 0.4082  time: 0.5020 s/iter  data_time: 0.0419 s/iter total_throughput: 2039.89 samples/s lr: 9.63e-04  
[08/30 04:54:54] lb.utils.events INFO:  eta: 1 day, 21:48:31  iteration: 46399/375342  consumed_samples: 47513600  total_loss: 0.4057  time: 0.5020 s/iter  data_time: 0.0380 s/iter total_throughput: 2039.88 samples/s lr: 9.63e-04  
[08/30 04:55:45] lb.utils.events INFO:  eta: 1 day, 21:49:51  iteration: 46499/375342  consumed_samples: 47616000  total_loss: 0.406  time: 0.5020 s/iter  data_time: 0.0398 s/iter total_throughput: 2039.87 samples/s lr: 9.63e-04  
[08/30 04:56:35] lb.utils.events INFO:  eta: 1 day, 21:46:45  iteration: 46599/375342  consumed_samples: 47718400  total_loss: 0.4132  time: 0.5020 s/iter  data_time: 0.0395 s/iter total_throughput: 2039.87 samples/s lr: 9.63e-04  
[08/30 04:57:25] lb.utils.events INFO:  eta: 1 day, 21:46:00  iteration: 46699/375342  consumed_samples: 47820800  total_loss: 0.405  time: 0.5020 s/iter  data_time: 0.0396 s/iter total_throughput: 2039.86 samples/s lr: 9.63e-04  
[08/30 04:58:15] lb.utils.events INFO:  eta: 1 day, 21:43:45  iteration: 46799/375342  consumed_samples: 47923200  total_loss: 0.4015  time: 0.5020 s/iter  data_time: 0.0395 s/iter total_throughput: 2039.87 samples/s lr: 9.63e-04  
[08/30 04:59:06] lb.utils.events INFO:  eta: 1 day, 21:42:14  iteration: 46899/375342  consumed_samples: 48025600  total_loss: 0.3995  time: 0.5020 s/iter  data_time: 0.0381 s/iter total_throughput: 2039.85 samples/s lr: 9.62e-04  
[08/30 04:59:56] lb.utils.events INFO:  eta: 1 day, 21:41:31  iteration: 46999/375342  consumed_samples: 48128000  total_loss: 0.4086  time: 0.5020 s/iter  data_time: 0.0391 s/iter total_throughput: 2039.84 samples/s lr: 9.62e-04  
[08/30 05:00:46] lb.utils.events INFO:  eta: 1 day, 21:40:27  iteration: 47099/375342  consumed_samples: 48230400  total_loss: 0.4156  time: 0.5020 s/iter  data_time: 0.0396 s/iter total_throughput: 2039.83 samples/s lr: 9.62e-04  
[08/30 05:01:37] lb.utils.events INFO:  eta: 1 day, 21:38:31  iteration: 47199/375342  consumed_samples: 48332800  total_loss: 0.4135  time: 0.5020 s/iter  data_time: 0.0387 s/iter total_throughput: 2039.82 samples/s lr: 9.62e-04  
[08/30 05:02:27] lb.utils.events INFO:  eta: 1 day, 21:38:05  iteration: 47299/375342  consumed_samples: 48435200  total_loss: 0.4084  time: 0.5020 s/iter  data_time: 0.0388 s/iter total_throughput: 2039.80 samples/s lr: 9.62e-04  
[08/30 05:03:17] lb.utils.events INFO:  eta: 1 day, 21:37:32  iteration: 47399/375342  consumed_samples: 48537600  total_loss: 0.4083  time: 0.5020 s/iter  data_time: 0.0389 s/iter total_throughput: 2039.80 samples/s lr: 9.62e-04  
[08/30 05:04:08] lb.utils.events INFO:  eta: 1 day, 21:35:55  iteration: 47499/375342  consumed_samples: 48640000  total_loss: 0.4088  time: 0.5020 s/iter  data_time: 0.0397 s/iter total_throughput: 2039.79 samples/s lr: 9.61e-04  
[08/30 05:04:58] lb.utils.events INFO:  eta: 1 day, 21:37:04  iteration: 47599/375342  consumed_samples: 48742400  total_loss: 0.4012  time: 0.5020 s/iter  data_time: 0.0413 s/iter total_throughput: 2039.78 samples/s lr: 9.61e-04  
[08/30 05:05:49] lb.utils.events INFO:  eta: 1 day, 21:36:26  iteration: 47699/375342  consumed_samples: 48844800  total_loss: 0.4004  time: 0.5020 s/iter  data_time: 0.0400 s/iter total_throughput: 2039.75 samples/s lr: 9.61e-04  
[08/30 05:06:39] lb.utils.events INFO:  eta: 1 day, 21:36:20  iteration: 47799/375342  consumed_samples: 48947200  total_loss: 0.4032  time: 0.5020 s/iter  data_time: 0.0405 s/iter total_throughput: 2039.75 samples/s lr: 9.61e-04  
[08/30 05:07:29] lb.utils.events INFO:  eta: 1 day, 21:35:30  iteration: 47899/375342  consumed_samples: 49049600  total_loss: 0.4043  time: 0.5020 s/iter  data_time: 0.0398 s/iter total_throughput: 2039.75 samples/s lr: 9.61e-04  
[08/30 05:08:19] lb.utils.events INFO:  eta: 1 day, 21:34:14  iteration: 47999/375342  consumed_samples: 49152000  total_loss: 0.4098  time: 0.5020 s/iter  data_time: 0.0404 s/iter total_throughput: 2039.74 samples/s lr: 9.61e-04  
[08/30 05:09:10] lb.utils.events INFO:  eta: 1 day, 21:33:01  iteration: 48099/375342  consumed_samples: 49254400  total_loss: 0.408  time: 0.5020 s/iter  data_time: 0.0390 s/iter total_throughput: 2039.74 samples/s lr: 9.60e-04  
[08/30 05:10:00] lb.utils.events INFO:  eta: 1 day, 21:33:58  iteration: 48199/375342  consumed_samples: 49356800  total_loss: 0.4025  time: 0.5020 s/iter  data_time: 0.0397 s/iter total_throughput: 2039.73 samples/s lr: 9.60e-04  
[08/30 05:10:51] lb.utils.events INFO:  eta: 1 day, 21:31:03  iteration: 48299/375342  consumed_samples: 49459200  total_loss: 0.4021  time: 0.5020 s/iter  data_time: 0.0370 s/iter total_throughput: 2039.67 samples/s lr: 9.60e-04  
[08/30 05:11:41] lb.utils.events INFO:  eta: 1 day, 21:28:10  iteration: 48399/375342  consumed_samples: 49561600  total_loss: 0.4095  time: 0.5020 s/iter  data_time: 0.0385 s/iter total_throughput: 2039.68 samples/s lr: 9.60e-04  
[08/30 05:12:31] lb.utils.events INFO:  eta: 1 day, 21:27:20  iteration: 48499/375342  consumed_samples: 49664000  total_loss: 0.4109  time: 0.5020 s/iter  data_time: 0.0396 s/iter total_throughput: 2039.68 samples/s lr: 9.60e-04  
[08/30 05:13:21] lb.utils.events INFO:  eta: 1 day, 21:26:28  iteration: 48599/375342  consumed_samples: 49766400  total_loss: 0.4067  time: 0.5020 s/iter  data_time: 0.0397 s/iter total_throughput: 2039.68 samples/s lr: 9.60e-04  
[08/30 05:14:12] lb.utils.events INFO:  eta: 1 day, 21:23:48  iteration: 48699/375342  consumed_samples: 49868800  total_loss: 0.4034  time: 0.5020 s/iter  data_time: 0.0396 s/iter total_throughput: 2039.68 samples/s lr: 9.59e-04  
[08/30 05:15:02] lb.utils.events INFO:  eta: 1 day, 21:23:43  iteration: 48799/375342  consumed_samples: 49971200  total_loss: 0.4042  time: 0.5020 s/iter  data_time: 0.0387 s/iter total_throughput: 2039.68 samples/s lr: 9.59e-04  
[08/30 05:15:52] lb.utils.events INFO:  eta: 1 day, 21:22:08  iteration: 48899/375342  consumed_samples: 50073600  total_loss: 0.3975  time: 0.5020 s/iter  data_time: 0.0385 s/iter total_throughput: 2039.67 samples/s lr: 9.59e-04  
[08/30 05:16:42] lb.utils.events INFO:  eta: 1 day, 21:21:13  iteration: 48999/375342  consumed_samples: 50176000  total_loss: 0.4011  time: 0.5020 s/iter  data_time: 0.0405 s/iter total_throughput: 2039.68 samples/s lr: 9.59e-04  
[08/30 05:17:33] lb.utils.events INFO:  eta: 1 day, 21:19:23  iteration: 49099/375342  consumed_samples: 50278400  total_loss: 0.4115  time: 0.5020 s/iter  data_time: 0.0401 s/iter total_throughput: 2039.68 samples/s lr: 9.59e-04  
[08/30 05:18:23] lb.utils.events INFO:  eta: 1 day, 21:15:33  iteration: 49199/375342  consumed_samples: 50380800  total_loss: 0.4066  time: 0.5020 s/iter  data_time: 0.0386 s/iter total_throughput: 2039.69 samples/s lr: 9.59e-04  
[08/30 05:19:13] lb.utils.events INFO:  eta: 1 day, 21:16:03  iteration: 49299/375342  consumed_samples: 50483200  total_loss: 0.4004  time: 0.5020 s/iter  data_time: 0.0415 s/iter total_throughput: 2039.69 samples/s lr: 9.58e-04  
[08/30 05:20:03] lb.utils.events INFO:  eta: 1 day, 21:15:43  iteration: 49399/375342  consumed_samples: 50585600  total_loss: 0.4001  time: 0.5020 s/iter  data_time: 0.0400 s/iter total_throughput: 2039.69 samples/s lr: 9.58e-04  
[08/30 05:20:53] lb.utils.events INFO:  eta: 1 day, 21:14:22  iteration: 49499/375342  consumed_samples: 50688000  total_loss: 0.4073  time: 0.5020 s/iter  data_time: 0.0393 s/iter total_throughput: 2039.70 samples/s lr: 9.58e-04  
[08/30 05:21:43] lb.utils.events INFO:  eta: 1 day, 21:14:08  iteration: 49599/375342  consumed_samples: 50790400  total_loss: 0.413  time: 0.5020 s/iter  data_time: 0.0403 s/iter total_throughput: 2039.70 samples/s lr: 9.58e-04  
[08/30 05:22:34] lb.utils.events INFO:  eta: 1 day, 21:14:26  iteration: 49699/375342  consumed_samples: 50892800  total_loss: 0.3977  time: 0.5020 s/iter  data_time: 0.0399 s/iter total_throughput: 2039.69 samples/s lr: 9.58e-04  
[08/30 05:23:24] lb.utils.events INFO:  eta: 1 day, 21:14:20  iteration: 49799/375342  consumed_samples: 50995200  total_loss: 0.4045  time: 0.5020 s/iter  data_time: 0.0397 s/iter total_throughput: 2039.67 samples/s lr: 9.58e-04  
[08/30 05:24:15] lb.utils.events INFO:  eta: 1 day, 21:13:38  iteration: 49899/375342  consumed_samples: 51097600  total_loss: 0.4053  time: 0.5020 s/iter  data_time: 0.0390 s/iter total_throughput: 2039.65 samples/s lr: 9.57e-04  
[08/30 05:25:05] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0049999
[08/30 05:25:05] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 05:25:05] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 05:25:10] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0640 s/iter. Inference: 0.2425 s/iter. Eval: 0.0023 s/iter. Total: 0.3089 s/iter. ETA=0:00:11
[08/30 05:25:15] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0936 s/iter. Inference: 0.2372 s/iter. Eval: 0.0023 s/iter. Total: 0.3333 s/iter. ETA=0:00:07
[08/30 05:25:21] lb.evaluation.evaluator INFO: Inference done 43008/50000. Dataloading: 0.0899 s/iter. Inference: 0.2386 s/iter. Eval: 0.0023 s/iter. Total: 0.3310 s/iter. ETA=0:00:01
[08/30 05:25:23] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 05:25:23] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.198500 (0.000284 s / iter per device, on 8 devices)
[08/30 05:25:23] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000213 s / iter per device, on 8 devices)
[08/30 05:25:23] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 05:25:23] lb.evaluation.utils INFO: copypaste: Acc@1=73.36
[08/30 05:25:23] lb.evaluation.utils INFO: copypaste: Acc@5=92.16
[08/30 05:25:23] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 73.36000, better than last best score 72.96400 @ iteration 44999.
[08/30 05:25:23] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 05:25:23] lb.utils.events INFO:  eta: 1 day, 21:12:48  iteration: 49999/375342  consumed_samples: 51200000  total_loss: 0.4086  time: 0.5020 s/iter  data_time: 0.0389 s/iter total_throughput: 2039.64 samples/s lr: 9.57e-04  
[08/30 05:26:14] lb.utils.events INFO:  eta: 1 day, 21:14:17  iteration: 50099/375342  consumed_samples: 51302400  total_loss: 0.4105  time: 0.5021 s/iter  data_time: 0.0391 s/iter total_throughput: 2039.62 samples/s lr: 9.57e-04  
[08/30 05:27:04] lb.utils.events INFO:  eta: 1 day, 21:14:25  iteration: 50199/375342  consumed_samples: 51404800  total_loss: 0.4123  time: 0.5021 s/iter  data_time: 0.0393 s/iter total_throughput: 2039.62 samples/s lr: 9.57e-04  
[08/30 05:27:54] lb.utils.events INFO:  eta: 1 day, 21:14:01  iteration: 50299/375342  consumed_samples: 51507200  total_loss: 0.4136  time: 0.5021 s/iter  data_time: 0.0409 s/iter total_throughput: 2039.61 samples/s lr: 9.57e-04  
[08/30 05:28:45] lb.utils.events INFO:  eta: 1 day, 21:13:52  iteration: 50399/375342  consumed_samples: 51609600  total_loss: 0.407  time: 0.5021 s/iter  data_time: 0.0382 s/iter total_throughput: 2039.59 samples/s lr: 9.57e-04  
[08/30 05:29:35] lb.utils.events INFO:  eta: 1 day, 21:14:46  iteration: 50499/375342  consumed_samples: 51712000  total_loss: 0.3993  time: 0.5021 s/iter  data_time: 0.0399 s/iter total_throughput: 2039.58 samples/s lr: 9.56e-04  
[08/30 05:30:25] lb.utils.events INFO:  eta: 1 day, 21:14:20  iteration: 50599/375342  consumed_samples: 51814400  total_loss: 0.4052  time: 0.5021 s/iter  data_time: 0.0391 s/iter total_throughput: 2039.57 samples/s lr: 9.56e-04  
[08/30 05:31:16] lb.utils.events INFO:  eta: 1 day, 21:13:10  iteration: 50699/375342  consumed_samples: 51916800  total_loss: 0.415  time: 0.5021 s/iter  data_time: 0.0390 s/iter total_throughput: 2039.57 samples/s lr: 9.56e-04  
[08/30 05:32:06] lb.utils.events INFO:  eta: 1 day, 21:10:48  iteration: 50799/375342  consumed_samples: 52019200  total_loss: 0.4111  time: 0.5021 s/iter  data_time: 0.0401 s/iter total_throughput: 2039.56 samples/s lr: 9.56e-04  
[08/30 05:32:56] lb.utils.events INFO:  eta: 1 day, 21:09:45  iteration: 50899/375342  consumed_samples: 52121600  total_loss: 0.403  time: 0.5021 s/iter  data_time: 0.0398 s/iter total_throughput: 2039.55 samples/s lr: 9.56e-04  
[08/30 05:33:47] lb.utils.events INFO:  eta: 1 day, 21:08:18  iteration: 50999/375342  consumed_samples: 52224000  total_loss: 0.4111  time: 0.5021 s/iter  data_time: 0.0393 s/iter total_throughput: 2039.55 samples/s lr: 9.56e-04  
[08/30 05:34:37] lb.utils.events INFO:  eta: 1 day, 21:06:15  iteration: 51099/375342  consumed_samples: 52326400  total_loss: 0.4041  time: 0.5021 s/iter  data_time: 0.0407 s/iter total_throughput: 2039.54 samples/s lr: 9.55e-04  
[08/30 05:35:27] lb.utils.events INFO:  eta: 1 day, 21:04:39  iteration: 51199/375342  consumed_samples: 52428800  total_loss: 0.4012  time: 0.5021 s/iter  data_time: 0.0399 s/iter total_throughput: 2039.53 samples/s lr: 9.55e-04  
[08/30 05:36:17] lb.utils.events INFO:  eta: 1 day, 21:03:22  iteration: 51299/375342  consumed_samples: 52531200  total_loss: 0.4079  time: 0.5021 s/iter  data_time: 0.0370 s/iter total_throughput: 2039.54 samples/s lr: 9.55e-04  
[08/30 05:37:08] lb.utils.events INFO:  eta: 1 day, 21:02:46  iteration: 51399/375342  consumed_samples: 52633600  total_loss: 0.409  time: 0.5021 s/iter  data_time: 0.0395 s/iter total_throughput: 2039.53 samples/s lr: 9.55e-04  
[08/30 05:37:58] lb.utils.events INFO:  eta: 1 day, 21:01:34  iteration: 51499/375342  consumed_samples: 52736000  total_loss: 0.4056  time: 0.5021 s/iter  data_time: 0.0399 s/iter total_throughput: 2039.52 samples/s lr: 9.55e-04  
[08/30 05:38:48] lb.utils.events INFO:  eta: 1 day, 20:58:59  iteration: 51599/375342  consumed_samples: 52838400  total_loss: 0.3959  time: 0.5021 s/iter  data_time: 0.0381 s/iter total_throughput: 2039.52 samples/s lr: 9.55e-04  
[08/30 05:39:39] lb.utils.events INFO:  eta: 1 day, 20:58:59  iteration: 51699/375342  consumed_samples: 52940800  total_loss: 0.3924  time: 0.5021 s/iter  data_time: 0.0408 s/iter total_throughput: 2039.51 samples/s lr: 9.54e-04  
[08/30 05:40:29] lb.utils.events INFO:  eta: 1 day, 20:59:16  iteration: 51799/375342  consumed_samples: 53043200  total_loss: 0.3974  time: 0.5021 s/iter  data_time: 0.0398 s/iter total_throughput: 2039.49 samples/s lr: 9.54e-04  
[08/30 05:41:19] lb.utils.events INFO:  eta: 1 day, 20:59:12  iteration: 51899/375342  consumed_samples: 53145600  total_loss: 0.4083  time: 0.5021 s/iter  data_time: 0.0390 s/iter total_throughput: 2039.48 samples/s lr: 9.54e-04  
[08/30 05:42:10] lb.utils.events INFO:  eta: 1 day, 21:00:39  iteration: 51999/375342  consumed_samples: 53248000  total_loss: 0.4158  time: 0.5021 s/iter  data_time: 0.0386 s/iter total_throughput: 2039.46 samples/s lr: 9.54e-04  
[08/30 05:43:00] lb.utils.events INFO:  eta: 1 day, 20:59:56  iteration: 52099/375342  consumed_samples: 53350400  total_loss: 0.4053  time: 0.5021 s/iter  data_time: 0.0395 s/iter total_throughput: 2039.44 samples/s lr: 9.54e-04  
[08/30 05:43:51] lb.utils.events INFO:  eta: 1 day, 20:59:24  iteration: 52199/375342  consumed_samples: 53452800  total_loss: 0.403  time: 0.5021 s/iter  data_time: 0.0387 s/iter total_throughput: 2039.44 samples/s lr: 9.54e-04  
[08/30 05:44:41] lb.utils.events INFO:  eta: 1 day, 20:59:43  iteration: 52299/375342  consumed_samples: 53555200  total_loss: 0.4069  time: 0.5021 s/iter  data_time: 0.0374 s/iter total_throughput: 2039.43 samples/s lr: 9.53e-04  
[08/30 05:45:31] lb.utils.events INFO:  eta: 1 day, 20:57:24  iteration: 52399/375342  consumed_samples: 53657600  total_loss: 0.4046  time: 0.5021 s/iter  data_time: 0.0409 s/iter total_throughput: 2039.43 samples/s lr: 9.53e-04  
[08/30 05:46:22] lb.utils.events INFO:  eta: 1 day, 20:56:11  iteration: 52499/375342  consumed_samples: 53760000  total_loss: 0.4008  time: 0.5021 s/iter  data_time: 0.0390 s/iter total_throughput: 2039.41 samples/s lr: 9.53e-04  
[08/30 05:47:12] lb.utils.events INFO:  eta: 1 day, 20:55:49  iteration: 52599/375342  consumed_samples: 53862400  total_loss: 0.4028  time: 0.5021 s/iter  data_time: 0.0388 s/iter total_throughput: 2039.41 samples/s lr: 9.53e-04  
[08/30 05:48:02] lb.utils.events INFO:  eta: 1 day, 20:54:22  iteration: 52699/375342  consumed_samples: 53964800  total_loss: 0.402  time: 0.5021 s/iter  data_time: 0.0399 s/iter total_throughput: 2039.42 samples/s lr: 9.53e-04  
[08/30 05:48:52] lb.utils.events INFO:  eta: 1 day, 20:52:33  iteration: 52799/375342  consumed_samples: 54067200  total_loss: 0.4017  time: 0.5021 s/iter  data_time: 0.0397 s/iter total_throughput: 2039.42 samples/s lr: 9.52e-04  
[08/30 05:49:42] lb.utils.events INFO:  eta: 1 day, 20:50:44  iteration: 52899/375342  consumed_samples: 54169600  total_loss: 0.4088  time: 0.5021 s/iter  data_time: 0.0409 s/iter total_throughput: 2039.43 samples/s lr: 9.52e-04  
[08/30 05:50:33] lb.utils.events INFO:  eta: 1 day, 20:49:12  iteration: 52999/375342  consumed_samples: 54272000  total_loss: 0.4059  time: 0.5021 s/iter  data_time: 0.0404 s/iter total_throughput: 2039.44 samples/s lr: 9.52e-04  
[08/30 05:51:23] lb.utils.events INFO:  eta: 1 day, 20:46:15  iteration: 53099/375342  consumed_samples: 54374400  total_loss: 0.4066  time: 0.5021 s/iter  data_time: 0.0388 s/iter total_throughput: 2039.45 samples/s lr: 9.52e-04  
[08/30 05:52:13] lb.utils.events INFO:  eta: 1 day, 20:45:11  iteration: 53199/375342  consumed_samples: 54476800  total_loss: 0.4089  time: 0.5021 s/iter  data_time: 0.0379 s/iter total_throughput: 2039.45 samples/s lr: 9.52e-04  
[08/30 05:53:03] lb.utils.events INFO:  eta: 1 day, 20:43:43  iteration: 53299/375342  consumed_samples: 54579200  total_loss: 0.3971  time: 0.5021 s/iter  data_time: 0.0384 s/iter total_throughput: 2039.45 samples/s lr: 9.52e-04  
[08/30 05:53:54] lb.utils.events INFO:  eta: 1 day, 20:43:39  iteration: 53399/375342  consumed_samples: 54681600  total_loss: 0.393  time: 0.5021 s/iter  data_time: 0.0403 s/iter total_throughput: 2039.43 samples/s lr: 9.51e-04  
[08/30 05:54:44] lb.utils.events INFO:  eta: 1 day, 20:42:58  iteration: 53499/375342  consumed_samples: 54784000  total_loss: 0.3959  time: 0.5021 s/iter  data_time: 0.0390 s/iter total_throughput: 2039.42 samples/s lr: 9.51e-04  
[08/30 05:55:34] lb.utils.events INFO:  eta: 1 day, 20:42:35  iteration: 53599/375342  consumed_samples: 54886400  total_loss: 0.3996  time: 0.5021 s/iter  data_time: 0.0394 s/iter total_throughput: 2039.42 samples/s lr: 9.51e-04  
[08/30 05:56:25] lb.utils.events INFO:  eta: 1 day, 20:42:33  iteration: 53699/375342  consumed_samples: 54988800  total_loss: 0.3964  time: 0.5021 s/iter  data_time: 0.0393 s/iter total_throughput: 2039.40 samples/s lr: 9.51e-04  
[08/30 05:57:15] lb.utils.events INFO:  eta: 1 day, 20:41:09  iteration: 53799/375342  consumed_samples: 55091200  total_loss: 0.4055  time: 0.5021 s/iter  data_time: 0.0374 s/iter total_throughput: 2039.41 samples/s lr: 9.51e-04  
[08/30 05:58:05] lb.utils.events INFO:  eta: 1 day, 20:41:37  iteration: 53899/375342  consumed_samples: 55193600  total_loss: 0.4123  time: 0.5021 s/iter  data_time: 0.0392 s/iter total_throughput: 2039.39 samples/s lr: 9.50e-04  
[08/30 05:58:55] lb.utils.events INFO:  eta: 1 day, 20:41:03  iteration: 53999/375342  consumed_samples: 55296000  total_loss: 0.3991  time: 0.5021 s/iter  data_time: 0.0402 s/iter total_throughput: 2039.38 samples/s lr: 9.50e-04  
[08/30 05:59:46] lb.utils.events INFO:  eta: 1 day, 20:43:36  iteration: 54099/375342  consumed_samples: 55398400  total_loss: 0.3899  time: 0.5021 s/iter  data_time: 0.0406 s/iter total_throughput: 2039.36 samples/s lr: 9.50e-04  
[08/30 06:00:36] lb.utils.events INFO:  eta: 1 day, 20:44:11  iteration: 54199/375342  consumed_samples: 55500800  total_loss: 0.3983  time: 0.5021 s/iter  data_time: 0.0394 s/iter total_throughput: 2039.36 samples/s lr: 9.50e-04  
[08/30 06:01:27] lb.utils.events INFO:  eta: 1 day, 20:43:58  iteration: 54299/375342  consumed_samples: 55603200  total_loss: 0.3966  time: 0.5021 s/iter  data_time: 0.0398 s/iter total_throughput: 2039.36 samples/s lr: 9.50e-04  
[08/30 06:02:17] lb.utils.events INFO:  eta: 1 day, 20:40:24  iteration: 54399/375342  consumed_samples: 55705600  total_loss: 0.3895  time: 0.5021 s/iter  data_time: 0.0396 s/iter total_throughput: 2039.36 samples/s lr: 9.50e-04  
[08/30 06:03:07] lb.utils.events INFO:  eta: 1 day, 20:39:22  iteration: 54499/375342  consumed_samples: 55808000  total_loss: 0.3913  time: 0.5021 s/iter  data_time: 0.0385 s/iter total_throughput: 2039.35 samples/s lr: 9.49e-04  
[08/30 06:03:57] lb.utils.events INFO:  eta: 1 day, 20:37:28  iteration: 54599/375342  consumed_samples: 55910400  total_loss: 0.4003  time: 0.5021 s/iter  data_time: 0.0397 s/iter total_throughput: 2039.35 samples/s lr: 9.49e-04  
[08/30 06:04:47] lb.utils.events INFO:  eta: 1 day, 20:34:50  iteration: 54699/375342  consumed_samples: 56012800  total_loss: 0.4123  time: 0.5021 s/iter  data_time: 0.0398 s/iter total_throughput: 2039.36 samples/s lr: 9.49e-04  
[08/30 06:05:38] lb.utils.events INFO:  eta: 1 day, 20:34:46  iteration: 54799/375342  consumed_samples: 56115200  total_loss: 0.4043  time: 0.5021 s/iter  data_time: 0.0383 s/iter total_throughput: 2039.35 samples/s lr: 9.49e-04  
[08/30 06:06:28] lb.utils.events INFO:  eta: 1 day, 20:32:01  iteration: 54899/375342  consumed_samples: 56217600  total_loss: 0.3949  time: 0.5021 s/iter  data_time: 0.0391 s/iter total_throughput: 2039.35 samples/s lr: 9.49e-04  
[08/30 06:07:18] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0054999
[08/30 06:07:19] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 06:07:19] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 06:07:24] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0494 s/iter. Inference: 0.2463 s/iter. Eval: 0.0023 s/iter. Total: 0.2979 s/iter. ETA=0:00:11
[08/30 06:07:29] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0847 s/iter. Inference: 0.2409 s/iter. Eval: 0.0022 s/iter. Total: 0.3281 s/iter. ETA=0:00:07
[08/30 06:07:34] lb.evaluation.evaluator INFO: Inference done 41984/50000. Dataloading: 0.0876 s/iter. Inference: 0.2431 s/iter. Eval: 0.0022 s/iter. Total: 0.3331 s/iter. ETA=0:00:02
[08/30 06:07:36] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 06:07:36] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.229284 (0.000285 s / iter per device, on 8 devices)
[08/30 06:07:36] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000214 s / iter per device, on 8 devices)
[08/30 06:07:36] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 06:07:36] lb.evaluation.utils INFO: copypaste: Acc@1=74.08200000000001
[08/30 06:07:36] lb.evaluation.utils INFO: copypaste: Acc@5=92.366
[08/30 06:07:36] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 74.08200, better than last best score 73.36000 @ iteration 49999.
[08/30 06:07:36] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 06:07:37] lb.utils.events INFO:  eta: 1 day, 20:31:50  iteration: 54999/375342  consumed_samples: 56320000  total_loss: 0.3965  time: 0.5021 s/iter  data_time: 0.0398 s/iter total_throughput: 2039.35 samples/s lr: 9.48e-04  
[08/30 06:08:27] lb.utils.events INFO:  eta: 1 day, 20:29:39  iteration: 55099/375342  consumed_samples: 56422400  total_loss: 0.3968  time: 0.5021 s/iter  data_time: 0.0366 s/iter total_throughput: 2039.33 samples/s lr: 9.48e-04  
[08/30 06:09:18] lb.utils.events INFO:  eta: 1 day, 20:31:17  iteration: 55199/375342  consumed_samples: 56524800  total_loss: 0.3944  time: 0.5021 s/iter  data_time: 0.0405 s/iter total_throughput: 2039.32 samples/s lr: 9.48e-04  
[08/30 06:10:08] lb.utils.events INFO:  eta: 1 day, 20:30:15  iteration: 55299/375342  consumed_samples: 56627200  total_loss: 0.3931  time: 0.5021 s/iter  data_time: 0.0388 s/iter total_throughput: 2039.31 samples/s lr: 9.48e-04  
[08/30 06:10:58] lb.utils.events INFO:  eta: 1 day, 20:31:42  iteration: 55399/375342  consumed_samples: 56729600  total_loss: 0.3976  time: 0.5021 s/iter  data_time: 0.0394 s/iter total_throughput: 2039.30 samples/s lr: 9.48e-04  
[08/30 06:11:49] lb.utils.events INFO:  eta: 1 day, 20:29:09  iteration: 55499/375342  consumed_samples: 56832000  total_loss: 0.4054  time: 0.5021 s/iter  data_time: 0.0395 s/iter total_throughput: 2039.30 samples/s lr: 9.48e-04  
[08/30 06:12:39] lb.utils.events INFO:  eta: 1 day, 20:28:20  iteration: 55599/375342  consumed_samples: 56934400  total_loss: 0.4024  time: 0.5021 s/iter  data_time: 0.0394 s/iter total_throughput: 2039.30 samples/s lr: 9.47e-04  
[08/30 06:13:29] lb.utils.events INFO:  eta: 1 day, 20:27:56  iteration: 55699/375342  consumed_samples: 57036800  total_loss: 0.4059  time: 0.5021 s/iter  data_time: 0.0369 s/iter total_throughput: 2039.30 samples/s lr: 9.47e-04  
[08/30 06:14:19] lb.utils.events INFO:  eta: 1 day, 20:28:13  iteration: 55799/375342  consumed_samples: 57139200  total_loss: 0.411  time: 0.5021 s/iter  data_time: 0.0402 s/iter total_throughput: 2039.30 samples/s lr: 9.47e-04  
[08/30 06:15:09] lb.utils.events INFO:  eta: 1 day, 20:27:07  iteration: 55899/375342  consumed_samples: 57241600  total_loss: 0.3976  time: 0.5021 s/iter  data_time: 0.0397 s/iter total_throughput: 2039.30 samples/s lr: 9.47e-04  
[08/30 06:16:00] lb.utils.events INFO:  eta: 1 day, 20:25:40  iteration: 55999/375342  consumed_samples: 57344000  total_loss: 0.3884  time: 0.5021 s/iter  data_time: 0.0380 s/iter total_throughput: 2039.29 samples/s lr: 9.47e-04  
[08/30 06:16:50] lb.utils.events INFO:  eta: 1 day, 20:25:59  iteration: 56099/375342  consumed_samples: 57446400  total_loss: 0.3999  time: 0.5021 s/iter  data_time: 0.0375 s/iter total_throughput: 2039.28 samples/s lr: 9.46e-04  
[08/30 06:17:40] lb.utils.events INFO:  eta: 1 day, 20:24:06  iteration: 56199/375342  consumed_samples: 57548800  total_loss: 0.4012  time: 0.5021 s/iter  data_time: 0.0380 s/iter total_throughput: 2039.28 samples/s lr: 9.46e-04  
[08/30 06:18:31] lb.utils.events INFO:  eta: 1 day, 20:23:19  iteration: 56299/375342  consumed_samples: 57651200  total_loss: 0.4019  time: 0.5021 s/iter  data_time: 0.0371 s/iter total_throughput: 2039.27 samples/s lr: 9.46e-04  
[08/30 06:19:21] lb.utils.events INFO:  eta: 1 day, 20:21:38  iteration: 56399/375342  consumed_samples: 57753600  total_loss: 0.3925  time: 0.5021 s/iter  data_time: 0.0405 s/iter total_throughput: 2039.28 samples/s lr: 9.46e-04  
[08/30 06:20:11] lb.utils.events INFO:  eta: 1 day, 20:21:41  iteration: 56499/375342  consumed_samples: 57856000  total_loss: 0.3907  time: 0.5021 s/iter  data_time: 0.0399 s/iter total_throughput: 2039.28 samples/s lr: 9.46e-04  
[08/30 06:21:02] lb.utils.events INFO:  eta: 1 day, 20:21:50  iteration: 56599/375342  consumed_samples: 57958400  total_loss: 0.3945  time: 0.5021 s/iter  data_time: 0.0388 s/iter total_throughput: 2039.27 samples/s lr: 9.45e-04  
[08/30 06:21:52] lb.utils.events INFO:  eta: 1 day, 20:20:23  iteration: 56699/375342  consumed_samples: 58060800  total_loss: 0.393  time: 0.5021 s/iter  data_time: 0.0391 s/iter total_throughput: 2039.27 samples/s lr: 9.45e-04  
[08/30 06:22:42] lb.utils.events INFO:  eta: 1 day, 20:18:59  iteration: 56799/375342  consumed_samples: 58163200  total_loss: 0.4007  time: 0.5021 s/iter  data_time: 0.0376 s/iter total_throughput: 2039.27 samples/s lr: 9.45e-04  
[08/30 06:23:32] lb.utils.events INFO:  eta: 1 day, 20:18:23  iteration: 56899/375342  consumed_samples: 58265600  total_loss: 0.3978  time: 0.5021 s/iter  data_time: 0.0398 s/iter total_throughput: 2039.27 samples/s lr: 9.45e-04  
[08/30 06:24:22] lb.utils.events INFO:  eta: 1 day, 20:17:30  iteration: 56999/375342  consumed_samples: 58368000  total_loss: 0.4017  time: 0.5021 s/iter  data_time: 0.0409 s/iter total_throughput: 2039.27 samples/s lr: 9.45e-04  
[08/30 06:25:13] lb.utils.events INFO:  eta: 1 day, 20:14:48  iteration: 57099/375342  consumed_samples: 58470400  total_loss: 0.4044  time: 0.5021 s/iter  data_time: 0.0398 s/iter total_throughput: 2039.26 samples/s lr: 9.45e-04  
[08/30 06:26:03] lb.utils.events INFO:  eta: 1 day, 20:14:40  iteration: 57199/375342  consumed_samples: 58572800  total_loss: 0.4043  time: 0.5021 s/iter  data_time: 0.0397 s/iter total_throughput: 2039.25 samples/s lr: 9.44e-04  
[08/30 06:26:54] lb.utils.events INFO:  eta: 1 day, 20:14:42  iteration: 57299/375342  consumed_samples: 58675200  total_loss: 0.4043  time: 0.5021 s/iter  data_time: 0.0393 s/iter total_throughput: 2039.23 samples/s lr: 9.44e-04  
[08/30 06:27:44] lb.utils.events INFO:  eta: 1 day, 20:13:12  iteration: 57399/375342  consumed_samples: 58777600  total_loss: 0.4041  time: 0.5021 s/iter  data_time: 0.0382 s/iter total_throughput: 2039.24 samples/s lr: 9.44e-04  
[08/30 06:28:34] lb.utils.events INFO:  eta: 1 day, 20:11:33  iteration: 57499/375342  consumed_samples: 58880000  total_loss: 0.4054  time: 0.5021 s/iter  data_time: 0.0387 s/iter total_throughput: 2039.23 samples/s lr: 9.44e-04  
[08/30 06:29:24] lb.utils.events INFO:  eta: 1 day, 20:10:05  iteration: 57599/375342  consumed_samples: 58982400  total_loss: 0.4001  time: 0.5021 s/iter  data_time: 0.0388 s/iter total_throughput: 2039.24 samples/s lr: 9.44e-04  
[08/30 06:30:15] lb.utils.events INFO:  eta: 1 day, 20:09:54  iteration: 57699/375342  consumed_samples: 59084800  total_loss: 0.3982  time: 0.5021 s/iter  data_time: 0.0392 s/iter total_throughput: 2039.24 samples/s lr: 9.43e-04  
[08/30 06:31:05] lb.utils.events INFO:  eta: 1 day, 20:09:46  iteration: 57799/375342  consumed_samples: 59187200  total_loss: 0.4004  time: 0.5022 s/iter  data_time: 0.0403 s/iter total_throughput: 2039.23 samples/s lr: 9.43e-04  
[08/30 06:31:55] lb.utils.events INFO:  eta: 1 day, 20:08:43  iteration: 57899/375342  consumed_samples: 59289600  total_loss: 0.4025  time: 0.5022 s/iter  data_time: 0.0387 s/iter total_throughput: 2039.23 samples/s lr: 9.43e-04  
[08/30 06:32:45] lb.utils.events INFO:  eta: 1 day, 20:07:45  iteration: 57999/375342  consumed_samples: 59392000  total_loss: 0.3967  time: 0.5022 s/iter  data_time: 0.0393 s/iter total_throughput: 2039.23 samples/s lr: 9.43e-04  
[08/30 06:33:36] lb.utils.events INFO:  eta: 1 day, 20:06:23  iteration: 58099/375342  consumed_samples: 59494400  total_loss: 0.4011  time: 0.5022 s/iter  data_time: 0.0387 s/iter total_throughput: 2039.23 samples/s lr: 9.43e-04  
[08/30 06:34:26] lb.utils.events INFO:  eta: 1 day, 20:05:05  iteration: 58199/375342  consumed_samples: 59596800  total_loss: 0.4148  time: 0.5022 s/iter  data_time: 0.0399 s/iter total_throughput: 2039.23 samples/s lr: 9.42e-04  
[08/30 06:35:16] lb.utils.events INFO:  eta: 1 day, 20:03:21  iteration: 58299/375342  consumed_samples: 59699200  total_loss: 0.4043  time: 0.5022 s/iter  data_time: 0.0403 s/iter total_throughput: 2039.23 samples/s lr: 9.42e-04  
[08/30 06:36:06] lb.utils.events INFO:  eta: 1 day, 20:03:22  iteration: 58399/375342  consumed_samples: 59801600  total_loss: 0.3974  time: 0.5021 s/iter  data_time: 0.0400 s/iter total_throughput: 2039.23 samples/s lr: 9.42e-04  
[08/30 06:36:57] lb.utils.events INFO:  eta: 1 day, 20:03:09  iteration: 58499/375342  consumed_samples: 59904000  total_loss: 0.4028  time: 0.5022 s/iter  data_time: 0.0405 s/iter total_throughput: 2039.23 samples/s lr: 9.42e-04  
[08/30 06:37:47] lb.utils.events INFO:  eta: 1 day, 20:02:21  iteration: 58599/375342  consumed_samples: 60006400  total_loss: 0.4028  time: 0.5022 s/iter  data_time: 0.0397 s/iter total_throughput: 2039.23 samples/s lr: 9.42e-04  
[08/30 06:38:37] lb.utils.events INFO:  eta: 1 day, 20:00:35  iteration: 58699/375342  consumed_samples: 60108800  total_loss: 0.4005  time: 0.5021 s/iter  data_time: 0.0395 s/iter total_throughput: 2039.24 samples/s lr: 9.41e-04  
[08/30 06:39:27] lb.utils.events INFO:  eta: 1 day, 19:57:33  iteration: 58799/375342  consumed_samples: 60211200  total_loss: 0.4003  time: 0.5021 s/iter  data_time: 0.0425 s/iter total_throughput: 2039.25 samples/s lr: 9.41e-04  
[08/30 06:40:17] lb.utils.events INFO:  eta: 1 day, 19:57:07  iteration: 58899/375342  consumed_samples: 60313600  total_loss: 0.3977  time: 0.5021 s/iter  data_time: 0.0399 s/iter total_throughput: 2039.26 samples/s lr: 9.41e-04  
[08/30 06:41:07] lb.utils.events INFO:  eta: 1 day, 19:55:07  iteration: 58999/375342  consumed_samples: 60416000  total_loss: 0.3951  time: 0.5021 s/iter  data_time: 0.0395 s/iter total_throughput: 2039.26 samples/s lr: 9.41e-04  
[08/30 06:41:57] lb.utils.events INFO:  eta: 1 day, 19:54:17  iteration: 59099/375342  consumed_samples: 60518400  total_loss: 0.397  time: 0.5021 s/iter  data_time: 0.0375 s/iter total_throughput: 2039.26 samples/s lr: 9.41e-04  
[08/30 06:42:47] lb.utils.events INFO:  eta: 1 day, 19:52:31  iteration: 59199/375342  consumed_samples: 60620800  total_loss: 0.3969  time: 0.5021 s/iter  data_time: 0.0382 s/iter total_throughput: 2039.27 samples/s lr: 9.40e-04  
[08/30 06:43:38] lb.utils.events INFO:  eta: 1 day, 19:51:36  iteration: 59299/375342  consumed_samples: 60723200  total_loss: 0.3967  time: 0.5021 s/iter  data_time: 0.0388 s/iter total_throughput: 2039.28 samples/s lr: 9.40e-04  
[08/30 06:44:28] lb.utils.events INFO:  eta: 1 day, 19:50:18  iteration: 59399/375342  consumed_samples: 60825600  total_loss: 0.3946  time: 0.5021 s/iter  data_time: 0.0396 s/iter total_throughput: 2039.29 samples/s lr: 9.40e-04  
[08/30 06:45:18] lb.utils.events INFO:  eta: 1 day, 19:47:49  iteration: 59499/375342  consumed_samples: 60928000  total_loss: 0.3949  time: 0.5021 s/iter  data_time: 0.0395 s/iter total_throughput: 2039.31 samples/s lr: 9.40e-04  
[08/30 06:46:08] lb.utils.events INFO:  eta: 1 day, 19:46:17  iteration: 59599/375342  consumed_samples: 61030400  total_loss: 0.4046  time: 0.5021 s/iter  data_time: 0.0402 s/iter total_throughput: 2039.32 samples/s lr: 9.40e-04  
[08/30 06:46:58] lb.utils.events INFO:  eta: 1 day, 19:44:52  iteration: 59699/375342  consumed_samples: 61132800  total_loss: 0.4055  time: 0.5021 s/iter  data_time: 0.0400 s/iter total_throughput: 2039.33 samples/s lr: 9.39e-04  
[08/30 06:47:48] lb.utils.events INFO:  eta: 1 day, 19:44:06  iteration: 59799/375342  consumed_samples: 61235200  total_loss: 0.3987  time: 0.5021 s/iter  data_time: 0.0404 s/iter total_throughput: 2039.34 samples/s lr: 9.39e-04  
[08/30 06:48:38] lb.utils.events INFO:  eta: 1 day, 19:43:16  iteration: 59899/375342  consumed_samples: 61337600  total_loss: 0.394  time: 0.5021 s/iter  data_time: 0.0408 s/iter total_throughput: 2039.35 samples/s lr: 9.39e-04  
[08/30 06:49:28] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0059999
[08/30 06:49:28] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 06:49:28] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 06:49:33] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0512 s/iter. Inference: 0.2479 s/iter. Eval: 0.0031 s/iter. Total: 0.3022 s/iter. ETA=0:00:11
[08/30 06:49:38] lb.evaluation.evaluator INFO: Inference done 27648/50000. Dataloading: 0.0744 s/iter. Inference: 0.2434 s/iter. Eval: 0.0024 s/iter. Total: 0.3205 s/iter. ETA=0:00:06
[08/30 06:49:43] lb.evaluation.evaluator INFO: Inference done 43008/50000. Dataloading: 0.0805 s/iter. Inference: 0.2441 s/iter. Eval: 0.0023 s/iter. Total: 0.3271 s/iter. ETA=0:00:01
[08/30 06:49:45] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 06:49:45] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.039037 (0.000281 s / iter per device, on 8 devices)
[08/30 06:49:45] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000215 s / iter per device, on 8 devices)
[08/30 06:49:45] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 06:49:45] lb.evaluation.utils INFO: copypaste: Acc@1=74.722
[08/30 06:49:45] lb.evaluation.utils INFO: copypaste: Acc@5=92.654
[08/30 06:49:45] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 74.72200, better than last best score 74.08200 @ iteration 54999.
[08/30 06:49:45] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 06:49:46] lb.utils.events INFO:  eta: 1 day, 19:42:27  iteration: 59999/375342  consumed_samples: 61440000  total_loss: 0.3992  time: 0.5021 s/iter  data_time: 0.0385 s/iter total_throughput: 2039.37 samples/s lr: 9.39e-04  
[08/30 06:50:36] lb.utils.events INFO:  eta: 1 day, 19:41:24  iteration: 60099/375342  consumed_samples: 61542400  total_loss: 0.4022  time: 0.5021 s/iter  data_time: 0.0375 s/iter total_throughput: 2039.39 samples/s lr: 9.39e-04  
[08/30 06:51:26] lb.utils.events INFO:  eta: 1 day, 19:39:28  iteration: 60199/375342  consumed_samples: 61644800  total_loss: 0.3917  time: 0.5021 s/iter  data_time: 0.0400 s/iter total_throughput: 2039.41 samples/s lr: 9.38e-04  
[08/30 06:52:16] lb.utils.events INFO:  eta: 1 day, 19:39:16  iteration: 60299/375342  consumed_samples: 61747200  total_loss: 0.3933  time: 0.5021 s/iter  data_time: 0.0386 s/iter total_throughput: 2039.42 samples/s lr: 9.38e-04  
[08/30 06:53:06] lb.utils.events INFO:  eta: 1 day, 19:37:49  iteration: 60399/375342  consumed_samples: 61849600  total_loss: 0.4028  time: 0.5021 s/iter  data_time: 0.0396 s/iter total_throughput: 2039.43 samples/s lr: 9.38e-04  
[08/30 06:53:56] lb.utils.events INFO:  eta: 1 day, 19:36:52  iteration: 60499/375342  consumed_samples: 61952000  total_loss: 0.4055  time: 0.5021 s/iter  data_time: 0.0392 s/iter total_throughput: 2039.44 samples/s lr: 9.38e-04  
[08/30 06:54:46] lb.utils.events INFO:  eta: 1 day, 19:36:13  iteration: 60599/375342  consumed_samples: 62054400  total_loss: 0.4025  time: 0.5021 s/iter  data_time: 0.0390 s/iter total_throughput: 2039.45 samples/s lr: 9.38e-04  
[08/30 06:55:36] lb.utils.events INFO:  eta: 1 day, 19:35:18  iteration: 60699/375342  consumed_samples: 62156800  total_loss: 0.3944  time: 0.5021 s/iter  data_time: 0.0393 s/iter total_throughput: 2039.46 samples/s lr: 9.37e-04  
[08/30 06:56:26] lb.utils.events INFO:  eta: 1 day, 19:34:14  iteration: 60799/375342  consumed_samples: 62259200  total_loss: 0.3855  time: 0.5021 s/iter  data_time: 0.0404 s/iter total_throughput: 2039.47 samples/s lr: 9.37e-04  
[08/30 06:57:16] lb.utils.events INFO:  eta: 1 day, 19:32:35  iteration: 60899/375342  consumed_samples: 62361600  total_loss: 0.3949  time: 0.5021 s/iter  data_time: 0.0404 s/iter total_throughput: 2039.48 samples/s lr: 9.37e-04  
[08/30 06:58:06] lb.utils.events INFO:  eta: 1 day, 19:31:29  iteration: 60999/375342  consumed_samples: 62464000  total_loss: 0.4027  time: 0.5021 s/iter  data_time: 0.0377 s/iter total_throughput: 2039.49 samples/s lr: 9.37e-04  
[08/30 06:58:56] lb.utils.events INFO:  eta: 1 day, 19:30:33  iteration: 61099/375342  consumed_samples: 62566400  total_loss: 0.3989  time: 0.5021 s/iter  data_time: 0.0398 s/iter total_throughput: 2039.51 samples/s lr: 9.37e-04  
[08/30 06:59:47] lb.utils.events INFO:  eta: 1 day, 19:29:50  iteration: 61199/375342  consumed_samples: 62668800  total_loss: 0.3959  time: 0.5021 s/iter  data_time: 0.0393 s/iter total_throughput: 2039.51 samples/s lr: 9.36e-04  
[08/30 07:00:37] lb.utils.events INFO:  eta: 1 day, 19:28:51  iteration: 61299/375342  consumed_samples: 62771200  total_loss: 0.3948  time: 0.5021 s/iter  data_time: 0.0374 s/iter total_throughput: 2039.52 samples/s lr: 9.36e-04  
[08/30 07:01:27] lb.utils.events INFO:  eta: 1 day, 19:28:07  iteration: 61399/375342  consumed_samples: 62873600  total_loss: 0.3924  time: 0.5021 s/iter  data_time: 0.0390 s/iter total_throughput: 2039.52 samples/s lr: 9.36e-04  
[08/30 07:02:17] lb.utils.events INFO:  eta: 1 day, 19:28:07  iteration: 61499/375342  consumed_samples: 62976000  total_loss: 0.3929  time: 0.5021 s/iter  data_time: 0.0396 s/iter total_throughput: 2039.52 samples/s lr: 9.36e-04  
[08/30 07:03:07] lb.utils.events INFO:  eta: 1 day, 19:27:17  iteration: 61599/375342  consumed_samples: 63078400  total_loss: 0.3923  time: 0.5021 s/iter  data_time: 0.0368 s/iter total_throughput: 2039.52 samples/s lr: 9.36e-04  
[08/30 07:03:57] lb.utils.events INFO:  eta: 1 day, 19:26:41  iteration: 61699/375342  consumed_samples: 63180800  total_loss: 0.3989  time: 0.5021 s/iter  data_time: 0.0383 s/iter total_throughput: 2039.53 samples/s lr: 9.35e-04  
[08/30 07:04:47] lb.utils.events INFO:  eta: 1 day, 19:25:56  iteration: 61799/375342  consumed_samples: 63283200  total_loss: 0.4022  time: 0.5021 s/iter  data_time: 0.0391 s/iter total_throughput: 2039.54 samples/s lr: 9.35e-04  
[08/30 07:05:38] lb.utils.events INFO:  eta: 1 day, 19:25:56  iteration: 61899/375342  consumed_samples: 63385600  total_loss: 0.3958  time: 0.5021 s/iter  data_time: 0.0384 s/iter total_throughput: 2039.55 samples/s lr: 9.35e-04  
[08/30 07:06:28] lb.utils.events INFO:  eta: 1 day, 19:25:52  iteration: 61999/375342  consumed_samples: 63488000  total_loss: 0.3956  time: 0.5021 s/iter  data_time: 0.0386 s/iter total_throughput: 2039.56 samples/s lr: 9.35e-04  
[08/30 07:07:18] lb.utils.events INFO:  eta: 1 day, 19:25:28  iteration: 62099/375342  consumed_samples: 63590400  total_loss: 0.3989  time: 0.5021 s/iter  data_time: 0.0401 s/iter total_throughput: 2039.57 samples/s lr: 9.35e-04  
[08/30 07:08:08] lb.utils.events INFO:  eta: 1 day, 19:24:48  iteration: 62199/375342  consumed_samples: 63692800  total_loss: 0.3905  time: 0.5021 s/iter  data_time: 0.0389 s/iter total_throughput: 2039.59 samples/s lr: 9.34e-04  
[08/30 07:08:58] lb.utils.events INFO:  eta: 1 day, 19:24:16  iteration: 62299/375342  consumed_samples: 63795200  total_loss: 0.3913  time: 0.5021 s/iter  data_time: 0.0395 s/iter total_throughput: 2039.59 samples/s lr: 9.34e-04  
[08/30 07:09:48] lb.utils.events INFO:  eta: 1 day, 19:22:48  iteration: 62399/375342  consumed_samples: 63897600  total_loss: 0.3945  time: 0.5021 s/iter  data_time: 0.0402 s/iter total_throughput: 2039.60 samples/s lr: 9.34e-04  
[08/30 07:10:38] lb.utils.events INFO:  eta: 1 day, 19:21:30  iteration: 62499/375342  consumed_samples: 64000000  total_loss: 0.3921  time: 0.5021 s/iter  data_time: 0.0394 s/iter total_throughput: 2039.61 samples/s lr: 9.34e-04  
[08/30 07:11:28] lb.utils.events INFO:  eta: 1 day, 19:20:05  iteration: 62599/375342  consumed_samples: 64102400  total_loss: 0.3946  time: 0.5021 s/iter  data_time: 0.0392 s/iter total_throughput: 2039.62 samples/s lr: 9.34e-04  
[08/30 07:12:18] lb.utils.events INFO:  eta: 1 day, 19:20:24  iteration: 62699/375342  consumed_samples: 64204800  total_loss: 0.3969  time: 0.5021 s/iter  data_time: 0.0399 s/iter total_throughput: 2039.62 samples/s lr: 9.33e-04  
[08/30 07:13:08] lb.utils.events INFO:  eta: 1 day, 19:18:32  iteration: 62799/375342  consumed_samples: 64307200  total_loss: 0.3959  time: 0.5020 s/iter  data_time: 0.0401 s/iter total_throughput: 2039.64 samples/s lr: 9.33e-04  
[08/30 07:13:58] lb.utils.events INFO:  eta: 1 day, 19:18:01  iteration: 62899/375342  consumed_samples: 64409600  total_loss: 0.389  time: 0.5021 s/iter  data_time: 0.0383 s/iter total_throughput: 2039.64 samples/s lr: 9.33e-04  
[08/30 07:14:49] lb.utils.events INFO:  eta: 1 day, 19:16:56  iteration: 62999/375342  consumed_samples: 64512000  total_loss: 0.391  time: 0.5020 s/iter  data_time: 0.0386 s/iter total_throughput: 2039.64 samples/s lr: 9.33e-04  
[08/30 07:15:39] lb.utils.events INFO:  eta: 1 day, 19:16:06  iteration: 63099/375342  consumed_samples: 64614400  total_loss: 0.3992  time: 0.5020 s/iter  data_time: 0.0386 s/iter total_throughput: 2039.64 samples/s lr: 9.33e-04  
[08/30 07:16:29] lb.utils.events INFO:  eta: 1 day, 19:17:23  iteration: 63199/375342  consumed_samples: 64716800  total_loss: 0.3928  time: 0.5020 s/iter  data_time: 0.0395 s/iter total_throughput: 2039.65 samples/s lr: 9.32e-04  
[08/30 07:17:19] lb.utils.events INFO:  eta: 1 day, 19:15:04  iteration: 63299/375342  consumed_samples: 64819200  total_loss: 0.3845  time: 0.5020 s/iter  data_time: 0.0388 s/iter total_throughput: 2039.66 samples/s lr: 9.32e-04  
[08/30 07:18:09] lb.utils.events INFO:  eta: 1 day, 19:13:36  iteration: 63399/375342  consumed_samples: 64921600  total_loss: 0.3876  time: 0.5020 s/iter  data_time: 0.0395 s/iter total_throughput: 2039.67 samples/s lr: 9.32e-04  
[08/30 07:18:59] lb.utils.events INFO:  eta: 1 day, 19:12:27  iteration: 63499/375342  consumed_samples: 65024000  total_loss: 0.3941  time: 0.5020 s/iter  data_time: 0.0396 s/iter total_throughput: 2039.68 samples/s lr: 9.32e-04  
[08/30 07:19:49] lb.utils.events INFO:  eta: 1 day, 19:12:13  iteration: 63599/375342  consumed_samples: 65126400  total_loss: 0.3919  time: 0.5020 s/iter  data_time: 0.0388 s/iter total_throughput: 2039.67 samples/s lr: 9.32e-04  
[08/30 07:20:40] lb.utils.events INFO:  eta: 1 day, 19:11:04  iteration: 63699/375342  consumed_samples: 65228800  total_loss: 0.3947  time: 0.5020 s/iter  data_time: 0.0376 s/iter total_throughput: 2039.68 samples/s lr: 9.31e-04  
[08/30 07:21:30] lb.utils.events INFO:  eta: 1 day, 19:13:32  iteration: 63799/375342  consumed_samples: 65331200  total_loss: 0.4076  time: 0.5020 s/iter  data_time: 0.0393 s/iter total_throughput: 2039.67 samples/s lr: 9.31e-04  
[08/30 07:22:20] lb.utils.events INFO:  eta: 1 day, 19:12:08  iteration: 63899/375342  consumed_samples: 65433600  total_loss: 0.4077  time: 0.5020 s/iter  data_time: 0.0403 s/iter total_throughput: 2039.68 samples/s lr: 9.31e-04  
[08/30 07:23:10] lb.utils.events INFO:  eta: 1 day, 19:10:22  iteration: 63999/375342  consumed_samples: 65536000  total_loss: 0.3998  time: 0.5020 s/iter  data_time: 0.0392 s/iter total_throughput: 2039.69 samples/s lr: 9.31e-04  
[08/30 07:24:00] lb.utils.events INFO:  eta: 1 day, 19:08:35  iteration: 64099/375342  consumed_samples: 65638400  total_loss: 0.3931  time: 0.5020 s/iter  data_time: 0.0388 s/iter total_throughput: 2039.70 samples/s lr: 9.30e-04  
[08/30 07:24:50] lb.utils.events INFO:  eta: 1 day, 19:05:28  iteration: 64199/375342  consumed_samples: 65740800  total_loss: 0.3958  time: 0.5020 s/iter  data_time: 0.0381 s/iter total_throughput: 2039.72 samples/s lr: 9.30e-04  
[08/30 07:25:40] lb.utils.events INFO:  eta: 1 day, 19:04:28  iteration: 64299/375342  consumed_samples: 65843200  total_loss: 0.394  time: 0.5020 s/iter  data_time: 0.0380 s/iter total_throughput: 2039.73 samples/s lr: 9.30e-04  
[08/30 07:26:30] lb.utils.events INFO:  eta: 1 day, 19:04:41  iteration: 64399/375342  consumed_samples: 65945600  total_loss: 0.3909  time: 0.5020 s/iter  data_time: 0.0388 s/iter total_throughput: 2039.73 samples/s lr: 9.30e-04  
[08/30 07:27:20] lb.utils.events INFO:  eta: 1 day, 19:02:51  iteration: 64499/375342  consumed_samples: 66048000  total_loss: 0.404  time: 0.5020 s/iter  data_time: 0.0391 s/iter total_throughput: 2039.75 samples/s lr: 9.30e-04  
[08/30 07:28:10] lb.utils.events INFO:  eta: 1 day, 19:02:01  iteration: 64599/375342  consumed_samples: 66150400  total_loss: 0.4004  time: 0.5020 s/iter  data_time: 0.0389 s/iter total_throughput: 2039.76 samples/s lr: 9.29e-04  
[08/30 07:29:00] lb.utils.events INFO:  eta: 1 day, 19:01:14  iteration: 64699/375342  consumed_samples: 66252800  total_loss: 0.3904  time: 0.5020 s/iter  data_time: 0.0376 s/iter total_throughput: 2039.76 samples/s lr: 9.29e-04  
[08/30 07:29:51] lb.utils.events INFO:  eta: 1 day, 19:00:15  iteration: 64799/375342  consumed_samples: 66355200  total_loss: 0.3918  time: 0.5020 s/iter  data_time: 0.0393 s/iter total_throughput: 2039.75 samples/s lr: 9.29e-04  
[08/30 07:30:41] lb.utils.events INFO:  eta: 1 day, 18:59:29  iteration: 64899/375342  consumed_samples: 66457600  total_loss: 0.3989  time: 0.5020 s/iter  data_time: 0.0398 s/iter total_throughput: 2039.76 samples/s lr: 9.29e-04  
[08/30 07:31:31] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0064999
[08/30 07:31:31] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 07:31:31] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 07:31:36] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0721 s/iter. Inference: 0.2364 s/iter. Eval: 0.0021 s/iter. Total: 0.3106 s/iter. ETA=0:00:11
[08/30 07:31:41] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0853 s/iter. Inference: 0.2424 s/iter. Eval: 0.0023 s/iter. Total: 0.3302 s/iter. ETA=0:00:07
[08/30 07:31:46] lb.evaluation.evaluator INFO: Inference done 43008/50000. Dataloading: 0.0824 s/iter. Inference: 0.2407 s/iter. Eval: 0.0022 s/iter. Total: 0.3255 s/iter. ETA=0:00:01
[08/30 07:31:48] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 07:31:48] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.103355 (0.000282 s / iter per device, on 8 devices)
[08/30 07:31:48] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000212 s / iter per device, on 8 devices)
[08/30 07:31:48] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 07:31:48] lb.evaluation.utils INFO: copypaste: Acc@1=74.776
[08/30 07:31:48] lb.evaluation.utils INFO: copypaste: Acc@5=92.792
[08/30 07:31:48] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 74.77600, better than last best score 74.72200 @ iteration 59999.
[08/30 07:31:48] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 07:31:49] lb.utils.events INFO:  eta: 1 day, 18:58:14  iteration: 64999/375342  consumed_samples: 66560000  total_loss: 0.3989  time: 0.5020 s/iter  data_time: 0.0377 s/iter total_throughput: 2039.77 samples/s lr: 9.29e-04  
[08/30 07:32:39] lb.utils.events INFO:  eta: 1 day, 18:57:49  iteration: 65099/375342  consumed_samples: 66662400  total_loss: 0.3931  time: 0.5020 s/iter  data_time: 0.0372 s/iter total_throughput: 2039.77 samples/s lr: 9.28e-04  
[08/30 07:33:29] lb.utils.events INFO:  eta: 1 day, 18:57:07  iteration: 65199/375342  consumed_samples: 66764800  total_loss: 0.3995  time: 0.5020 s/iter  data_time: 0.0396 s/iter total_throughput: 2039.78 samples/s lr: 9.28e-04  
[08/30 07:34:19] lb.utils.events INFO:  eta: 1 day, 18:58:16  iteration: 65299/375342  consumed_samples: 66867200  total_loss: 0.3949  time: 0.5020 s/iter  data_time: 0.0399 s/iter total_throughput: 2039.79 samples/s lr: 9.28e-04  
[08/30 07:35:09] lb.utils.events INFO:  eta: 1 day, 18:57:10  iteration: 65399/375342  consumed_samples: 66969600  total_loss: 0.3924  time: 0.5020 s/iter  data_time: 0.0395 s/iter total_throughput: 2039.80 samples/s lr: 9.28e-04  
[08/30 07:36:00] lb.utils.events INFO:  eta: 1 day, 18:56:36  iteration: 65499/375342  consumed_samples: 67072000  total_loss: 0.3961  time: 0.5020 s/iter  data_time: 0.0390 s/iter total_throughput: 2039.80 samples/s lr: 9.27e-04  
[08/30 07:36:50] lb.utils.events INFO:  eta: 1 day, 18:55:55  iteration: 65599/375342  consumed_samples: 67174400  total_loss: 0.3948  time: 0.5020 s/iter  data_time: 0.0394 s/iter total_throughput: 2039.81 samples/s lr: 9.27e-04  
[08/30 07:37:40] lb.utils.events INFO:  eta: 1 day, 18:53:32  iteration: 65699/375342  consumed_samples: 67276800  total_loss: 0.3956  time: 0.5020 s/iter  data_time: 0.0393 s/iter total_throughput: 2039.82 samples/s lr: 9.27e-04  
[08/30 07:38:30] lb.utils.events INFO:  eta: 1 day, 18:52:36  iteration: 65799/375342  consumed_samples: 67379200  total_loss: 0.4048  time: 0.5020 s/iter  data_time: 0.0400 s/iter total_throughput: 2039.83 samples/s lr: 9.27e-04  
[08/30 07:39:20] lb.utils.events INFO:  eta: 1 day, 18:51:47  iteration: 65899/375342  consumed_samples: 67481600  total_loss: 0.3984  time: 0.5020 s/iter  data_time: 0.0397 s/iter total_throughput: 2039.84 samples/s lr: 9.27e-04  
[08/30 07:40:10] lb.utils.events INFO:  eta: 1 day, 18:51:37  iteration: 65999/375342  consumed_samples: 67584000  total_loss: 0.3988  time: 0.5020 s/iter  data_time: 0.0400 s/iter total_throughput: 2039.85 samples/s lr: 9.26e-04  
[08/30 07:41:00] lb.utils.events INFO:  eta: 1 day, 18:51:28  iteration: 66099/375342  consumed_samples: 67686400  total_loss: 0.3981  time: 0.5020 s/iter  data_time: 0.0395 s/iter total_throughput: 2039.85 samples/s lr: 9.26e-04  
[08/30 07:41:50] lb.utils.events INFO:  eta: 1 day, 18:51:04  iteration: 66199/375342  consumed_samples: 67788800  total_loss: 0.389  time: 0.5020 s/iter  data_time: 0.0392 s/iter total_throughput: 2039.86 samples/s lr: 9.26e-04  
[08/30 07:42:40] lb.utils.events INFO:  eta: 1 day, 18:49:30  iteration: 66299/375342  consumed_samples: 67891200  total_loss: 0.3886  time: 0.5020 s/iter  data_time: 0.0410 s/iter total_throughput: 2039.87 samples/s lr: 9.26e-04  
[08/30 07:43:30] lb.utils.events INFO:  eta: 1 day, 18:49:09  iteration: 66399/375342  consumed_samples: 67993600  total_loss: 0.3874  time: 0.5020 s/iter  data_time: 0.0400 s/iter total_throughput: 2039.87 samples/s lr: 9.26e-04  
[08/30 07:44:21] lb.utils.events INFO:  eta: 1 day, 18:48:35  iteration: 66499/375342  consumed_samples: 68096000  total_loss: 0.3885  time: 0.5020 s/iter  data_time: 0.0381 s/iter total_throughput: 2039.88 samples/s lr: 9.25e-04  
[08/30 07:45:11] lb.utils.events INFO:  eta: 1 day, 18:47:19  iteration: 66599/375342  consumed_samples: 68198400  total_loss: 0.3859  time: 0.5020 s/iter  data_time: 0.0394 s/iter total_throughput: 2039.88 samples/s lr: 9.25e-04  
[08/30 07:46:01] lb.utils.events INFO:  eta: 1 day, 18:46:19  iteration: 66699/375342  consumed_samples: 68300800  total_loss: 0.3942  time: 0.5020 s/iter  data_time: 0.0395 s/iter total_throughput: 2039.89 samples/s lr: 9.25e-04  
[08/30 07:46:51] lb.utils.events INFO:  eta: 1 day, 18:45:50  iteration: 66799/375342  consumed_samples: 68403200  total_loss: 0.3987  time: 0.5020 s/iter  data_time: 0.0389 s/iter total_throughput: 2039.89 samples/s lr: 9.25e-04  
[08/30 07:47:41] lb.utils.events INFO:  eta: 1 day, 18:45:28  iteration: 66899/375342  consumed_samples: 68505600  total_loss: 0.3956  time: 0.5020 s/iter  data_time: 0.0380 s/iter total_throughput: 2039.89 samples/s lr: 9.24e-04  
[08/30 07:48:31] lb.utils.events INFO:  eta: 1 day, 18:44:21  iteration: 66999/375342  consumed_samples: 68608000  total_loss: 0.3928  time: 0.5020 s/iter  data_time: 0.0384 s/iter total_throughput: 2039.90 samples/s lr: 9.24e-04  
[08/30 07:49:21] lb.utils.events INFO:  eta: 1 day, 18:42:13  iteration: 67099/375342  consumed_samples: 68710400  total_loss: 0.3893  time: 0.5020 s/iter  data_time: 0.0387 s/iter total_throughput: 2039.92 samples/s lr: 9.24e-04  
[08/30 07:50:11] lb.utils.events INFO:  eta: 1 day, 18:40:49  iteration: 67199/375342  consumed_samples: 68812800  total_loss: 0.3907  time: 0.5020 s/iter  data_time: 0.0392 s/iter total_throughput: 2039.93 samples/s lr: 9.24e-04  
[08/30 07:51:01] lb.utils.events INFO:  eta: 1 day, 18:40:17  iteration: 67299/375342  consumed_samples: 68915200  total_loss: 0.4017  time: 0.5020 s/iter  data_time: 0.0375 s/iter total_throughput: 2039.93 samples/s lr: 9.24e-04  
[08/30 07:51:52] lb.utils.events INFO:  eta: 1 day, 18:38:58  iteration: 67399/375342  consumed_samples: 69017600  total_loss: 0.4063  time: 0.5020 s/iter  data_time: 0.0377 s/iter total_throughput: 2039.94 samples/s lr: 9.23e-04  
[08/30 07:52:42] lb.utils.events INFO:  eta: 1 day, 18:38:00  iteration: 67499/375342  consumed_samples: 69120000  total_loss: 0.4037  time: 0.5020 s/iter  data_time: 0.0402 s/iter total_throughput: 2039.94 samples/s lr: 9.23e-04  
[08/30 07:53:32] lb.utils.events INFO:  eta: 1 day, 18:37:56  iteration: 67599/375342  consumed_samples: 69222400  total_loss: 0.4063  time: 0.5020 s/iter  data_time: 0.0403 s/iter total_throughput: 2039.95 samples/s lr: 9.23e-04  
[08/30 07:54:22] lb.utils.events INFO:  eta: 1 day, 18:37:44  iteration: 67699/375342  consumed_samples: 69324800  total_loss: 0.3882  time: 0.5020 s/iter  data_time: 0.0396 s/iter total_throughput: 2039.95 samples/s lr: 9.23e-04  
[08/30 07:55:12] lb.utils.events INFO:  eta: 1 day, 18:35:13  iteration: 67799/375342  consumed_samples: 69427200  total_loss: 0.3889  time: 0.5020 s/iter  data_time: 0.0383 s/iter total_throughput: 2039.95 samples/s lr: 9.22e-04  
[08/30 07:56:02] lb.utils.events INFO:  eta: 1 day, 18:34:01  iteration: 67899/375342  consumed_samples: 69529600  total_loss: 0.3902  time: 0.5020 s/iter  data_time: 0.0386 s/iter total_throughput: 2039.96 samples/s lr: 9.22e-04  
[08/30 07:56:52] lb.utils.events INFO:  eta: 1 day, 18:33:03  iteration: 67999/375342  consumed_samples: 69632000  total_loss: 0.3882  time: 0.5020 s/iter  data_time: 0.0382 s/iter total_throughput: 2039.97 samples/s lr: 9.22e-04  
[08/30 07:57:42] lb.utils.events INFO:  eta: 1 day, 18:32:27  iteration: 68099/375342  consumed_samples: 69734400  total_loss: 0.3903  time: 0.5020 s/iter  data_time: 0.0392 s/iter total_throughput: 2039.98 samples/s lr: 9.22e-04  
[08/30 07:58:32] lb.utils.events INFO:  eta: 1 day, 18:32:11  iteration: 68199/375342  consumed_samples: 69836800  total_loss: 0.399  time: 0.5020 s/iter  data_time: 0.0375 s/iter total_throughput: 2039.99 samples/s lr: 9.22e-04  
[08/30 07:59:22] lb.utils.events INFO:  eta: 1 day, 18:32:41  iteration: 68299/375342  consumed_samples: 69939200  total_loss: 0.3966  time: 0.5020 s/iter  data_time: 0.0380 s/iter total_throughput: 2039.99 samples/s lr: 9.21e-04  
[08/30 08:00:12] lb.utils.events INFO:  eta: 1 day, 18:31:12  iteration: 68399/375342  consumed_samples: 70041600  total_loss: 0.3927  time: 0.5020 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.01 samples/s lr: 9.21e-04  
[08/30 08:01:03] lb.utils.events INFO:  eta: 1 day, 18:30:02  iteration: 68499/375342  consumed_samples: 70144000  total_loss: 0.3912  time: 0.5020 s/iter  data_time: 0.0391 s/iter total_throughput: 2040.01 samples/s lr: 9.21e-04  
[08/30 08:01:53] lb.utils.events INFO:  eta: 1 day, 18:28:53  iteration: 68599/375342  consumed_samples: 70246400  total_loss: 0.3973  time: 0.5020 s/iter  data_time: 0.0384 s/iter total_throughput: 2040.02 samples/s lr: 9.21e-04  
[08/30 08:02:43] lb.utils.events INFO:  eta: 1 day, 18:28:22  iteration: 68699/375342  consumed_samples: 70348800  total_loss: 0.3976  time: 0.5020 s/iter  data_time: 0.0395 s/iter total_throughput: 2040.03 samples/s lr: 9.20e-04  
[08/30 08:03:33] lb.utils.events INFO:  eta: 1 day, 18:28:11  iteration: 68799/375342  consumed_samples: 70451200  total_loss: 0.3966  time: 0.5020 s/iter  data_time: 0.0379 s/iter total_throughput: 2040.03 samples/s lr: 9.20e-04  
[08/30 08:04:23] lb.utils.events INFO:  eta: 1 day, 18:27:52  iteration: 68899/375342  consumed_samples: 70553600  total_loss: 0.3966  time: 0.5020 s/iter  data_time: 0.0383 s/iter total_throughput: 2040.03 samples/s lr: 9.20e-04  
[08/30 08:05:13] lb.utils.events INFO:  eta: 1 day, 18:27:02  iteration: 68999/375342  consumed_samples: 70656000  total_loss: 0.3884  time: 0.5019 s/iter  data_time: 0.0395 s/iter total_throughput: 2040.05 samples/s lr: 9.20e-04  
[08/30 08:06:03] lb.utils.events INFO:  eta: 1 day, 18:27:41  iteration: 69099/375342  consumed_samples: 70758400  total_loss: 0.3835  time: 0.5019 s/iter  data_time: 0.0379 s/iter total_throughput: 2040.05 samples/s lr: 9.19e-04  
[08/30 08:06:53] lb.utils.events INFO:  eta: 1 day, 18:27:20  iteration: 69199/375342  consumed_samples: 70860800  total_loss: 0.389  time: 0.5019 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.05 samples/s lr: 9.19e-04  
[08/30 08:07:43] lb.utils.events INFO:  eta: 1 day, 18:25:54  iteration: 69299/375342  consumed_samples: 70963200  total_loss: 0.3926  time: 0.5019 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.06 samples/s lr: 9.19e-04  
[08/30 08:08:33] lb.utils.events INFO:  eta: 1 day, 18:26:27  iteration: 69399/375342  consumed_samples: 71065600  total_loss: 0.39  time: 0.5019 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.07 samples/s lr: 9.19e-04  
[08/30 08:09:24] lb.utils.events INFO:  eta: 1 day, 18:25:42  iteration: 69499/375342  consumed_samples: 71168000  total_loss: 0.397  time: 0.5019 s/iter  data_time: 0.0366 s/iter total_throughput: 2040.07 samples/s lr: 9.19e-04  
[08/30 08:10:14] lb.utils.events INFO:  eta: 1 day, 18:23:52  iteration: 69599/375342  consumed_samples: 71270400  total_loss: 0.3954  time: 0.5019 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.08 samples/s lr: 9.18e-04  
[08/30 08:11:04] lb.utils.events INFO:  eta: 1 day, 18:23:02  iteration: 69699/375342  consumed_samples: 71372800  total_loss: 0.3962  time: 0.5019 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.08 samples/s lr: 9.18e-04  
[08/30 08:11:54] lb.utils.events INFO:  eta: 1 day, 18:22:13  iteration: 69799/375342  consumed_samples: 71475200  total_loss: 0.3968  time: 0.5019 s/iter  data_time: 0.0392 s/iter total_throughput: 2040.08 samples/s lr: 9.18e-04  
[08/30 08:12:44] lb.utils.events INFO:  eta: 1 day, 18:20:08  iteration: 69899/375342  consumed_samples: 71577600  total_loss: 0.3887  time: 0.5019 s/iter  data_time: 0.0369 s/iter total_throughput: 2040.09 samples/s lr: 9.18e-04  
[08/30 08:13:34] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0069999
[08/30 08:13:35] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 08:13:35] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 08:13:40] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0563 s/iter. Inference: 0.2470 s/iter. Eval: 0.0023 s/iter. Total: 0.3056 s/iter. ETA=0:00:11
[08/30 08:13:45] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0857 s/iter. Inference: 0.2422 s/iter. Eval: 0.0024 s/iter. Total: 0.3305 s/iter. ETA=0:00:07
[08/30 08:13:50] lb.evaluation.evaluator INFO: Inference done 43008/50000. Dataloading: 0.0820 s/iter. Inference: 0.2411 s/iter. Eval: 0.0024 s/iter. Total: 0.3258 s/iter. ETA=0:00:01
[08/30 08:13:52] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 08:13:52] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.074448 (0.000282 s / iter per device, on 8 devices)
[08/30 08:13:52] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000213 s / iter per device, on 8 devices)
[08/30 08:13:52] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 08:13:52] lb.evaluation.utils INFO: copypaste: Acc@1=75.31
[08/30 08:13:52] lb.evaluation.utils INFO: copypaste: Acc@5=93.048
[08/30 08:13:52] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 75.31000, better than last best score 74.77600 @ iteration 64999.
[08/30 08:13:52] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 08:13:53] lb.utils.events INFO:  eta: 1 day, 18:19:35  iteration: 69999/375342  consumed_samples: 71680000  total_loss: 0.3868  time: 0.5019 s/iter  data_time: 0.0402 s/iter total_throughput: 2040.10 samples/s lr: 9.17e-04  
[08/30 08:14:43] lb.utils.events INFO:  eta: 1 day, 18:19:11  iteration: 70099/375342  consumed_samples: 71782400  total_loss: 0.3868  time: 0.5019 s/iter  data_time: 0.0372 s/iter total_throughput: 2040.09 samples/s lr: 9.17e-04  
[08/30 08:15:33] lb.utils.events INFO:  eta: 1 day, 18:17:18  iteration: 70199/375342  consumed_samples: 71884800  total_loss: 0.3873  time: 0.5019 s/iter  data_time: 0.0383 s/iter total_throughput: 2040.10 samples/s lr: 9.17e-04  
[08/30 08:16:23] lb.utils.events INFO:  eta: 1 day, 18:16:29  iteration: 70299/375342  consumed_samples: 71987200  total_loss: 0.4011  time: 0.5019 s/iter  data_time: 0.0386 s/iter total_throughput: 2040.11 samples/s lr: 9.17e-04  
[08/30 08:17:13] lb.utils.events INFO:  eta: 1 day, 18:16:19  iteration: 70399/375342  consumed_samples: 72089600  total_loss: 0.4032  time: 0.5019 s/iter  data_time: 0.0393 s/iter total_throughput: 2040.11 samples/s lr: 9.17e-04  
[08/30 08:18:03] lb.utils.events INFO:  eta: 1 day, 18:15:29  iteration: 70499/375342  consumed_samples: 72192000  total_loss: 0.3963  time: 0.5019 s/iter  data_time: 0.0380 s/iter total_throughput: 2040.12 samples/s lr: 9.16e-04  
[08/30 08:18:53] lb.utils.events INFO:  eta: 1 day, 18:14:34  iteration: 70599/375342  consumed_samples: 72294400  total_loss: 0.3919  time: 0.5019 s/iter  data_time: 0.0372 s/iter total_throughput: 2040.13 samples/s lr: 9.16e-04  
[08/30 08:19:43] lb.utils.events INFO:  eta: 1 day, 18:12:45  iteration: 70699/375342  consumed_samples: 72396800  total_loss: 0.3957  time: 0.5019 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.13 samples/s lr: 9.16e-04  
[08/30 08:20:33] lb.utils.events INFO:  eta: 1 day, 18:11:03  iteration: 70799/375342  consumed_samples: 72499200  total_loss: 0.3963  time: 0.5019 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.14 samples/s lr: 9.16e-04  
[08/30 08:21:23] lb.utils.events INFO:  eta: 1 day, 18:10:42  iteration: 70899/375342  consumed_samples: 72601600  total_loss: 0.3935  time: 0.5019 s/iter  data_time: 0.0395 s/iter total_throughput: 2040.15 samples/s lr: 9.15e-04  
[08/30 08:22:13] lb.utils.events INFO:  eta: 1 day, 18:09:08  iteration: 70999/375342  consumed_samples: 72704000  total_loss: 0.3873  time: 0.5019 s/iter  data_time: 0.0401 s/iter total_throughput: 2040.17 samples/s lr: 9.15e-04  
[08/30 08:23:03] lb.utils.events INFO:  eta: 1 day, 18:07:14  iteration: 71099/375342  consumed_samples: 72806400  total_loss: 0.3939  time: 0.5019 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.17 samples/s lr: 9.15e-04  
[08/30 08:23:53] lb.utils.events INFO:  eta: 1 day, 18:05:52  iteration: 71199/375342  consumed_samples: 72908800  total_loss: 0.398  time: 0.5019 s/iter  data_time: 0.0380 s/iter total_throughput: 2040.19 samples/s lr: 9.15e-04  
[08/30 08:24:44] lb.utils.events INFO:  eta: 1 day, 18:05:02  iteration: 71299/375342  consumed_samples: 73011200  total_loss: 0.3903  time: 0.5019 s/iter  data_time: 0.0391 s/iter total_throughput: 2040.19 samples/s lr: 9.14e-04  
[08/30 08:25:34] lb.utils.events INFO:  eta: 1 day, 18:03:53  iteration: 71399/375342  consumed_samples: 73113600  total_loss: 0.3939  time: 0.5019 s/iter  data_time: 0.0390 s/iter total_throughput: 2040.19 samples/s lr: 9.14e-04  
[08/30 08:26:24] lb.utils.events INFO:  eta: 1 day, 18:03:26  iteration: 71499/375342  consumed_samples: 73216000  total_loss: 0.3913  time: 0.5019 s/iter  data_time: 0.0386 s/iter total_throughput: 2040.20 samples/s lr: 9.14e-04  
[08/30 08:27:14] lb.utils.events INFO:  eta: 1 day, 18:02:49  iteration: 71599/375342  consumed_samples: 73318400  total_loss: 0.3906  time: 0.5019 s/iter  data_time: 0.0377 s/iter total_throughput: 2040.20 samples/s lr: 9.14e-04  
[08/30 08:28:04] lb.utils.events INFO:  eta: 1 day, 18:01:32  iteration: 71699/375342  consumed_samples: 73420800  total_loss: 0.4029  time: 0.5019 s/iter  data_time: 0.0390 s/iter total_throughput: 2040.22 samples/s lr: 9.14e-04  
[08/30 08:28:54] lb.utils.events INFO:  eta: 1 day, 18:02:29  iteration: 71799/375342  consumed_samples: 73523200  total_loss: 0.3966  time: 0.5019 s/iter  data_time: 0.0416 s/iter total_throughput: 2040.22 samples/s lr: 9.13e-04  
[08/30 08:29:44] lb.utils.events INFO:  eta: 1 day, 18:01:10  iteration: 71899/375342  consumed_samples: 73625600  total_loss: 0.3895  time: 0.5019 s/iter  data_time: 0.0384 s/iter total_throughput: 2040.22 samples/s lr: 9.13e-04  
[08/30 08:30:34] lb.utils.events INFO:  eta: 1 day, 18:01:22  iteration: 71999/375342  consumed_samples: 73728000  total_loss: 0.3905  time: 0.5019 s/iter  data_time: 0.0379 s/iter total_throughput: 2040.23 samples/s lr: 9.13e-04  
[08/30 08:31:24] lb.utils.events INFO:  eta: 1 day, 18:00:03  iteration: 72099/375342  consumed_samples: 73830400  total_loss: 0.3905  time: 0.5019 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.24 samples/s lr: 9.13e-04  
[08/30 08:32:14] lb.utils.events INFO:  eta: 1 day, 17:59:43  iteration: 72199/375342  consumed_samples: 73932800  total_loss: 0.3979  time: 0.5019 s/iter  data_time: 0.0396 s/iter total_throughput: 2040.25 samples/s lr: 9.12e-04  
[08/30 08:33:05] lb.utils.events INFO:  eta: 1 day, 17:59:47  iteration: 72299/375342  consumed_samples: 74035200  total_loss: 0.3996  time: 0.5019 s/iter  data_time: 0.0372 s/iter total_throughput: 2040.24 samples/s lr: 9.12e-04  
[08/30 08:33:55] lb.utils.events INFO:  eta: 1 day, 17:59:04  iteration: 72399/375342  consumed_samples: 74137600  total_loss: 0.3914  time: 0.5019 s/iter  data_time: 0.0406 s/iter total_throughput: 2040.26 samples/s lr: 9.12e-04  
[08/30 08:34:45] lb.utils.events INFO:  eta: 1 day, 17:58:10  iteration: 72499/375342  consumed_samples: 74240000  total_loss: 0.3873  time: 0.5019 s/iter  data_time: 0.0386 s/iter total_throughput: 2040.27 samples/s lr: 9.12e-04  
[08/30 08:35:35] lb.utils.events INFO:  eta: 1 day, 17:56:47  iteration: 72599/375342  consumed_samples: 74342400  total_loss: 0.3941  time: 0.5019 s/iter  data_time: 0.0384 s/iter total_throughput: 2040.28 samples/s lr: 9.11e-04  
[08/30 08:36:25] lb.utils.events INFO:  eta: 1 day, 17:56:23  iteration: 72699/375342  consumed_samples: 74444800  total_loss: 0.3939  time: 0.5019 s/iter  data_time: 0.0376 s/iter total_throughput: 2040.29 samples/s lr: 9.11e-04  
[08/30 08:37:15] lb.utils.events INFO:  eta: 1 day, 17:54:46  iteration: 72799/375342  consumed_samples: 74547200  total_loss: 0.389  time: 0.5019 s/iter  data_time: 0.0382 s/iter total_throughput: 2040.29 samples/s lr: 9.11e-04  
[08/30 08:38:05] lb.utils.events INFO:  eta: 1 day, 17:52:40  iteration: 72899/375342  consumed_samples: 74649600  total_loss: 0.3846  time: 0.5019 s/iter  data_time: 0.0366 s/iter total_throughput: 2040.30 samples/s lr: 9.11e-04  
[08/30 08:38:55] lb.utils.events INFO:  eta: 1 day, 17:51:45  iteration: 72999/375342  consumed_samples: 74752000  total_loss: 0.3974  time: 0.5019 s/iter  data_time: 0.0390 s/iter total_throughput: 2040.31 samples/s lr: 9.10e-04  
[08/30 08:39:45] lb.utils.events INFO:  eta: 1 day, 17:50:58  iteration: 73099/375342  consumed_samples: 74854400  total_loss: 0.3982  time: 0.5019 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.32 samples/s lr: 9.10e-04  
[08/30 08:40:35] lb.utils.events INFO:  eta: 1 day, 17:49:30  iteration: 73199/375342  consumed_samples: 74956800  total_loss: 0.395  time: 0.5019 s/iter  data_time: 0.0398 s/iter total_throughput: 2040.33 samples/s lr: 9.10e-04  
[08/30 08:41:25] lb.utils.events INFO:  eta: 1 day, 17:48:39  iteration: 73299/375342  consumed_samples: 75059200  total_loss: 0.3953  time: 0.5019 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.34 samples/s lr: 9.10e-04  
[08/30 08:42:15] lb.utils.events INFO:  eta: 1 day, 17:46:50  iteration: 73399/375342  consumed_samples: 75161600  total_loss: 0.3859  time: 0.5019 s/iter  data_time: 0.0401 s/iter total_throughput: 2040.35 samples/s lr: 9.09e-04  
[08/30 08:43:05] lb.utils.events INFO:  eta: 1 day, 17:45:55  iteration: 73499/375342  consumed_samples: 75264000  total_loss: 0.3879  time: 0.5019 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.36 samples/s lr: 9.09e-04  
[08/30 08:43:55] lb.utils.events INFO:  eta: 1 day, 17:44:36  iteration: 73599/375342  consumed_samples: 75366400  total_loss: 0.3932  time: 0.5019 s/iter  data_time: 0.0371 s/iter total_throughput: 2040.36 samples/s lr: 9.09e-04  
[08/30 08:44:45] lb.utils.events INFO:  eta: 1 day, 17:43:29  iteration: 73699/375342  consumed_samples: 75468800  total_loss: 0.3919  time: 0.5019 s/iter  data_time: 0.0385 s/iter total_throughput: 2040.37 samples/s lr: 9.09e-04  
[08/30 08:45:35] lb.utils.events INFO:  eta: 1 day, 17:42:00  iteration: 73799/375342  consumed_samples: 75571200  total_loss: 0.3908  time: 0.5019 s/iter  data_time: 0.0395 s/iter total_throughput: 2040.37 samples/s lr: 9.09e-04  
[08/30 08:46:25] lb.utils.events INFO:  eta: 1 day, 17:41:53  iteration: 73899/375342  consumed_samples: 75673600  total_loss: 0.3888  time: 0.5019 s/iter  data_time: 0.0384 s/iter total_throughput: 2040.38 samples/s lr: 9.08e-04  
[08/30 08:47:15] lb.utils.events INFO:  eta: 1 day, 17:40:43  iteration: 73999/375342  consumed_samples: 75776000  total_loss: 0.3899  time: 0.5019 s/iter  data_time: 0.0380 s/iter total_throughput: 2040.40 samples/s lr: 9.08e-04  
[08/30 08:48:05] lb.utils.events INFO:  eta: 1 day, 17:39:28  iteration: 74099/375342  consumed_samples: 75878400  total_loss: 0.3961  time: 0.5019 s/iter  data_time: 0.0382 s/iter total_throughput: 2040.41 samples/s lr: 9.08e-04  
[08/30 08:48:55] lb.utils.events INFO:  eta: 1 day, 17:39:44  iteration: 74199/375342  consumed_samples: 75980800  total_loss: 0.389  time: 0.5019 s/iter  data_time: 0.0380 s/iter total_throughput: 2040.42 samples/s lr: 9.08e-04  
[08/30 08:49:45] lb.utils.events INFO:  eta: 1 day, 17:38:21  iteration: 74299/375342  consumed_samples: 76083200  total_loss: 0.3915  time: 0.5019 s/iter  data_time: 0.0386 s/iter total_throughput: 2040.42 samples/s lr: 9.07e-04  
[08/30 08:50:35] lb.utils.events INFO:  eta: 1 day, 17:37:45  iteration: 74399/375342  consumed_samples: 76185600  total_loss: 0.3832  time: 0.5019 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.44 samples/s lr: 9.07e-04  
[08/30 08:51:25] lb.utils.events INFO:  eta: 1 day, 17:36:48  iteration: 74499/375342  consumed_samples: 76288000  total_loss: 0.3892  time: 0.5019 s/iter  data_time: 0.0374 s/iter total_throughput: 2040.45 samples/s lr: 9.07e-04  
[08/30 08:52:16] lb.utils.events INFO:  eta: 1 day, 17:35:32  iteration: 74599/375342  consumed_samples: 76390400  total_loss: 0.3833  time: 0.5018 s/iter  data_time: 0.0380 s/iter total_throughput: 2040.46 samples/s lr: 9.07e-04  
[08/30 08:53:06] lb.utils.events INFO:  eta: 1 day, 17:35:48  iteration: 74699/375342  consumed_samples: 76492800  total_loss: 0.3849  time: 0.5018 s/iter  data_time: 0.0381 s/iter total_throughput: 2040.46 samples/s lr: 9.06e-04  
[08/30 08:53:56] lb.utils.events INFO:  eta: 1 day, 17:35:50  iteration: 74799/375342  consumed_samples: 76595200  total_loss: 0.4031  time: 0.5018 s/iter  data_time: 0.0379 s/iter total_throughput: 2040.47 samples/s lr: 9.06e-04  
[08/30 08:54:46] lb.utils.events INFO:  eta: 1 day, 17:34:18  iteration: 74899/375342  consumed_samples: 76697600  total_loss: 0.3988  time: 0.5018 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.48 samples/s lr: 9.06e-04  
[08/30 08:55:36] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0074999
[08/30 08:55:36] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 08:55:36] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 08:55:41] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0537 s/iter. Inference: 0.2465 s/iter. Eval: 0.0023 s/iter. Total: 0.3026 s/iter. ETA=0:00:11
[08/30 08:55:46] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0826 s/iter. Inference: 0.2415 s/iter. Eval: 0.0025 s/iter. Total: 0.3269 s/iter. ETA=0:00:07
[08/30 08:55:51] lb.evaluation.evaluator INFO: Inference done 41984/50000. Dataloading: 0.0866 s/iter. Inference: 0.2436 s/iter. Eval: 0.0024 s/iter. Total: 0.3330 s/iter. ETA=0:00:02
[08/30 08:55:53] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 08:55:53] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.243953 (0.000285 s / iter per device, on 8 devices)
[08/30 08:55:53] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000215 s / iter per device, on 8 devices)
[08/30 08:55:53] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 08:55:53] lb.evaluation.utils INFO: copypaste: Acc@1=75.32
[08/30 08:55:53] lb.evaluation.utils INFO: copypaste: Acc@5=92.81
[08/30 08:55:53] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 75.32000, better than last best score 75.31000 @ iteration 69999.
[08/30 08:55:53] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 08:55:54] lb.utils.events INFO:  eta: 1 day, 17:34:36  iteration: 74999/375342  consumed_samples: 76800000  total_loss: 0.3845  time: 0.5018 s/iter  data_time: 0.0378 s/iter total_throughput: 2040.49 samples/s lr: 9.06e-04  
[08/30 08:56:44] lb.utils.events INFO:  eta: 1 day, 17:33:57  iteration: 75099/375342  consumed_samples: 76902400  total_loss: 0.3891  time: 0.5018 s/iter  data_time: 0.0373 s/iter total_throughput: 2040.50 samples/s lr: 9.05e-04  
[08/30 08:57:34] lb.utils.events INFO:  eta: 1 day, 17:33:07  iteration: 75199/375342  consumed_samples: 77004800  total_loss: 0.3955  time: 0.5018 s/iter  data_time: 0.0404 s/iter total_throughput: 2040.51 samples/s lr: 9.05e-04  
[08/30 08:58:24] lb.utils.events INFO:  eta: 1 day, 17:31:46  iteration: 75299/375342  consumed_samples: 77107200  total_loss: 0.3932  time: 0.5018 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.51 samples/s lr: 9.05e-04  
[08/30 08:59:14] lb.utils.events INFO:  eta: 1 day, 17:30:52  iteration: 75399/375342  consumed_samples: 77209600  total_loss: 0.3854  time: 0.5018 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.53 samples/s lr: 9.05e-04  
[08/30 09:00:04] lb.utils.events INFO:  eta: 1 day, 17:30:29  iteration: 75499/375342  consumed_samples: 77312000  total_loss: 0.3837  time: 0.5018 s/iter  data_time: 0.0395 s/iter total_throughput: 2040.53 samples/s lr: 9.04e-04  
[08/30 09:00:54] lb.utils.events INFO:  eta: 1 day, 17:30:29  iteration: 75599/375342  consumed_samples: 77414400  total_loss: 0.3853  time: 0.5018 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.54 samples/s lr: 9.04e-04  
[08/30 09:01:44] lb.utils.events INFO:  eta: 1 day, 17:28:55  iteration: 75699/375342  consumed_samples: 77516800  total_loss: 0.3851  time: 0.5018 s/iter  data_time: 0.0375 s/iter total_throughput: 2040.55 samples/s lr: 9.04e-04  
[08/30 09:02:35] lb.utils.events INFO:  eta: 1 day, 17:27:49  iteration: 75799/375342  consumed_samples: 77619200  total_loss: 0.3874  time: 0.5018 s/iter  data_time: 0.0369 s/iter total_throughput: 2040.54 samples/s lr: 9.04e-04  
[08/30 09:03:25] lb.utils.events INFO:  eta: 1 day, 17:27:09  iteration: 75899/375342  consumed_samples: 77721600  total_loss: 0.3918  time: 0.5018 s/iter  data_time: 0.0390 s/iter total_throughput: 2040.55 samples/s lr: 9.03e-04  
[08/30 09:04:15] lb.utils.events INFO:  eta: 1 day, 17:26:10  iteration: 75999/375342  consumed_samples: 77824000  total_loss: 0.3892  time: 0.5018 s/iter  data_time: 0.0404 s/iter total_throughput: 2040.56 samples/s lr: 9.03e-04  
[08/30 09:05:05] lb.utils.events INFO:  eta: 1 day, 17:25:21  iteration: 76099/375342  consumed_samples: 77926400  total_loss: 0.384  time: 0.5018 s/iter  data_time: 0.0393 s/iter total_throughput: 2040.57 samples/s lr: 9.03e-04  
[08/30 09:05:55] lb.utils.events INFO:  eta: 1 day, 17:24:44  iteration: 76199/375342  consumed_samples: 78028800  total_loss: 0.3894  time: 0.5018 s/iter  data_time: 0.0399 s/iter total_throughput: 2040.57 samples/s lr: 9.03e-04  
[08/30 09:06:45] lb.utils.events INFO:  eta: 1 day, 17:25:23  iteration: 76299/375342  consumed_samples: 78131200  total_loss: 0.3892  time: 0.5018 s/iter  data_time: 0.0403 s/iter total_throughput: 2040.57 samples/s lr: 9.02e-04  
[08/30 09:07:35] lb.utils.events INFO:  eta: 1 day, 17:25:21  iteration: 76399/375342  consumed_samples: 78233600  total_loss: 0.3914  time: 0.5018 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.58 samples/s lr: 9.02e-04  
[08/30 09:08:25] lb.utils.events INFO:  eta: 1 day, 17:24:11  iteration: 76499/375342  consumed_samples: 78336000  total_loss: 0.3952  time: 0.5018 s/iter  data_time: 0.0377 s/iter total_throughput: 2040.59 samples/s lr: 9.02e-04  
[08/30 09:09:15] lb.utils.events INFO:  eta: 1 day, 17:23:11  iteration: 76599/375342  consumed_samples: 78438400  total_loss: 0.3932  time: 0.5018 s/iter  data_time: 0.0398 s/iter total_throughput: 2040.60 samples/s lr: 9.02e-04  
[08/30 09:10:05] lb.utils.events INFO:  eta: 1 day, 17:23:03  iteration: 76699/375342  consumed_samples: 78540800  total_loss: 0.3913  time: 0.5018 s/iter  data_time: 0.0378 s/iter total_throughput: 2040.60 samples/s lr: 9.01e-04  
[08/30 09:10:56] lb.utils.events INFO:  eta: 1 day, 17:22:59  iteration: 76799/375342  consumed_samples: 78643200  total_loss: 0.3929  time: 0.5018 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.60 samples/s lr: 9.01e-04  
[08/30 09:11:46] lb.utils.events INFO:  eta: 1 day, 17:22:36  iteration: 76899/375342  consumed_samples: 78745600  total_loss: 0.3924  time: 0.5018 s/iter  data_time: 0.0385 s/iter total_throughput: 2040.61 samples/s lr: 9.01e-04  
[08/30 09:12:36] lb.utils.events INFO:  eta: 1 day, 17:20:52  iteration: 76999/375342  consumed_samples: 78848000  total_loss: 0.3888  time: 0.5018 s/iter  data_time: 0.0378 s/iter total_throughput: 2040.62 samples/s lr: 9.01e-04  
[08/30 09:13:26] lb.utils.events INFO:  eta: 1 day, 17:20:16  iteration: 77099/375342  consumed_samples: 78950400  total_loss: 0.3887  time: 0.5018 s/iter  data_time: 0.0386 s/iter total_throughput: 2040.63 samples/s lr: 9.00e-04  
[08/30 09:14:16] lb.utils.events INFO:  eta: 1 day, 17:18:16  iteration: 77199/375342  consumed_samples: 79052800  total_loss: 0.3873  time: 0.5018 s/iter  data_time: 0.0383 s/iter total_throughput: 2040.64 samples/s lr: 9.00e-04  
[08/30 09:15:06] lb.utils.events INFO:  eta: 1 day, 17:17:17  iteration: 77299/375342  consumed_samples: 79155200  total_loss: 0.3861  time: 0.5018 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.64 samples/s lr: 9.00e-04  
[08/30 09:15:56] lb.utils.events INFO:  eta: 1 day, 17:15:48  iteration: 77399/375342  consumed_samples: 79257600  total_loss: 0.3858  time: 0.5018 s/iter  data_time: 0.0395 s/iter total_throughput: 2040.64 samples/s lr: 9.00e-04  
[08/30 09:16:46] lb.utils.events INFO:  eta: 1 day, 17:15:05  iteration: 77499/375342  consumed_samples: 79360000  total_loss: 0.3864  time: 0.5018 s/iter  data_time: 0.0400 s/iter total_throughput: 2040.65 samples/s lr: 8.99e-04  
[08/30 09:17:36] lb.utils.events INFO:  eta: 1 day, 17:15:07  iteration: 77599/375342  consumed_samples: 79462400  total_loss: 0.3825  time: 0.5018 s/iter  data_time: 0.0374 s/iter total_throughput: 2040.65 samples/s lr: 8.99e-04  
[08/30 09:18:26] lb.utils.events INFO:  eta: 1 day, 17:12:56  iteration: 77699/375342  consumed_samples: 79564800  total_loss: 0.381  time: 0.5018 s/iter  data_time: 0.0375 s/iter total_throughput: 2040.66 samples/s lr: 8.99e-04  
[08/30 09:19:16] lb.utils.events INFO:  eta: 1 day, 17:11:46  iteration: 77799/375342  consumed_samples: 79667200  total_loss: 0.3872  time: 0.5018 s/iter  data_time: 0.0391 s/iter total_throughput: 2040.66 samples/s lr: 8.99e-04  
[08/30 09:20:06] lb.utils.events INFO:  eta: 1 day, 17:10:47  iteration: 77899/375342  consumed_samples: 79769600  total_loss: 0.3807  time: 0.5018 s/iter  data_time: 0.0384 s/iter total_throughput: 2040.67 samples/s lr: 8.98e-04  
[08/30 09:20:56] lb.utils.events INFO:  eta: 1 day, 17:10:20  iteration: 77999/375342  consumed_samples: 79872000  total_loss: 0.3836  time: 0.5018 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.68 samples/s lr: 8.98e-04  
[08/30 09:21:46] lb.utils.events INFO:  eta: 1 day, 17:09:12  iteration: 78099/375342  consumed_samples: 79974400  total_loss: 0.3804  time: 0.5018 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.70 samples/s lr: 8.98e-04  
[08/30 09:22:36] lb.utils.events INFO:  eta: 1 day, 17:08:40  iteration: 78199/375342  consumed_samples: 80076800  total_loss: 0.3789  time: 0.5018 s/iter  data_time: 0.0400 s/iter total_throughput: 2040.71 samples/s lr: 8.98e-04  
[08/30 09:23:26] lb.utils.events INFO:  eta: 1 day, 17:07:11  iteration: 78299/375342  consumed_samples: 80179200  total_loss: 0.3809  time: 0.5018 s/iter  data_time: 0.0392 s/iter total_throughput: 2040.71 samples/s lr: 8.97e-04  
[08/30 09:24:17] lb.utils.events INFO:  eta: 1 day, 17:06:29  iteration: 78399/375342  consumed_samples: 80281600  total_loss: 0.3832  time: 0.5018 s/iter  data_time: 0.0394 s/iter total_throughput: 2040.71 samples/s lr: 8.97e-04  
[08/30 09:25:07] lb.utils.events INFO:  eta: 1 day, 17:05:11  iteration: 78499/375342  consumed_samples: 80384000  total_loss: 0.3875  time: 0.5018 s/iter  data_time: 0.0393 s/iter total_throughput: 2040.72 samples/s lr: 8.97e-04  
[08/30 09:25:57] lb.utils.events INFO:  eta: 1 day, 17:03:19  iteration: 78599/375342  consumed_samples: 80486400  total_loss: 0.3901  time: 0.5018 s/iter  data_time: 0.0398 s/iter total_throughput: 2040.73 samples/s lr: 8.97e-04  
[08/30 09:26:47] lb.utils.events INFO:  eta: 1 day, 17:02:31  iteration: 78699/375342  consumed_samples: 80588800  total_loss: 0.3923  time: 0.5018 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.74 samples/s lr: 8.96e-04  
[08/30 09:27:37] lb.utils.events INFO:  eta: 1 day, 17:01:20  iteration: 78799/375342  consumed_samples: 80691200  total_loss: 0.3952  time: 0.5018 s/iter  data_time: 0.0389 s/iter total_throughput: 2040.74 samples/s lr: 8.96e-04  
[08/30 09:28:27] lb.utils.events INFO:  eta: 1 day, 17:01:18  iteration: 78899/375342  consumed_samples: 80793600  total_loss: 0.3906  time: 0.5018 s/iter  data_time: 0.0382 s/iter total_throughput: 2040.75 samples/s lr: 8.96e-04  
[08/30 09:29:17] lb.utils.events INFO:  eta: 1 day, 17:00:28  iteration: 78999/375342  consumed_samples: 80896000  total_loss: 0.3918  time: 0.5018 s/iter  data_time: 0.0396 s/iter total_throughput: 2040.75 samples/s lr: 8.96e-04  
[08/30 09:30:07] lb.utils.events INFO:  eta: 1 day, 17:00:27  iteration: 79099/375342  consumed_samples: 80998400  total_loss: 0.3905  time: 0.5018 s/iter  data_time: 0.0386 s/iter total_throughput: 2040.75 samples/s lr: 8.95e-04  
[08/30 09:30:57] lb.utils.events INFO:  eta: 1 day, 16:59:07  iteration: 79199/375342  consumed_samples: 81100800  total_loss: 0.3888  time: 0.5018 s/iter  data_time: 0.0397 s/iter total_throughput: 2040.76 samples/s lr: 8.95e-04  
[08/30 09:31:47] lb.utils.events INFO:  eta: 1 day, 16:57:31  iteration: 79299/375342  consumed_samples: 81203200  total_loss: 0.3894  time: 0.5018 s/iter  data_time: 0.0382 s/iter total_throughput: 2040.77 samples/s lr: 8.95e-04  
[08/30 09:32:37] lb.utils.events INFO:  eta: 1 day, 16:56:55  iteration: 79399/375342  consumed_samples: 81305600  total_loss: 0.3892  time: 0.5018 s/iter  data_time: 0.0391 s/iter total_throughput: 2040.78 samples/s lr: 8.95e-04  
[08/30 09:33:27] lb.utils.events INFO:  eta: 1 day, 16:55:47  iteration: 79499/375342  consumed_samples: 81408000  total_loss: 0.3898  time: 0.5018 s/iter  data_time: 0.0365 s/iter total_throughput: 2040.79 samples/s lr: 8.94e-04  
[08/30 09:34:17] lb.utils.events INFO:  eta: 1 day, 16:55:16  iteration: 79599/375342  consumed_samples: 81510400  total_loss: 0.3916  time: 0.5018 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.80 samples/s lr: 8.94e-04  
[08/30 09:35:07] lb.utils.events INFO:  eta: 1 day, 16:54:44  iteration: 79699/375342  consumed_samples: 81612800  total_loss: 0.3889  time: 0.5018 s/iter  data_time: 0.0384 s/iter total_throughput: 2040.81 samples/s lr: 8.94e-04  
[08/30 09:35:57] lb.utils.events INFO:  eta: 1 day, 16:55:24  iteration: 79799/375342  consumed_samples: 81715200  total_loss: 0.386  time: 0.5018 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.81 samples/s lr: 8.94e-04  
[08/30 09:36:47] lb.utils.events INFO:  eta: 1 day, 16:53:10  iteration: 79899/375342  consumed_samples: 81817600  total_loss: 0.3908  time: 0.5018 s/iter  data_time: 0.0388 s/iter total_throughput: 2040.82 samples/s lr: 8.93e-04  
[08/30 09:37:37] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0079999
[08/30 09:37:38] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 09:37:38] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 09:37:43] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0779 s/iter. Inference: 0.2386 s/iter. Eval: 0.0021 s/iter. Total: 0.3186 s/iter. ETA=0:00:11
[08/30 09:37:48] lb.evaluation.evaluator INFO: Inference done 27648/50000. Dataloading: 0.0923 s/iter. Inference: 0.2330 s/iter. Eval: 0.0021 s/iter. Total: 0.3275 s/iter. ETA=0:00:06
[08/30 09:37:53] lb.evaluation.evaluator INFO: Inference done 44032/50000. Dataloading: 0.0890 s/iter. Inference: 0.2358 s/iter. Eval: 0.0021 s/iter. Total: 0.3271 s/iter. ETA=0:00:01
[08/30 09:37:55] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 09:37:55] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.228858 (0.000285 s / iter per device, on 8 devices)
[08/30 09:37:55] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000209 s / iter per device, on 8 devices)
[08/30 09:37:55] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 09:37:55] lb.evaluation.utils INFO: copypaste: Acc@1=75.486
[08/30 09:37:55] lb.evaluation.utils INFO: copypaste: Acc@5=92.89
[08/30 09:37:55] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 75.48600, better than last best score 75.32000 @ iteration 74999.
[08/30 09:37:55] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 09:37:56] lb.utils.events INFO:  eta: 1 day, 16:51:40  iteration: 79999/375342  consumed_samples: 81920000  total_loss: 0.3904  time: 0.5018 s/iter  data_time: 0.0383 s/iter total_throughput: 2040.83 samples/s lr: 8.93e-04  
[08/30 09:38:46] lb.utils.events INFO:  eta: 1 day, 16:49:55  iteration: 80099/375342  consumed_samples: 82022400  total_loss: 0.3854  time: 0.5018 s/iter  data_time: 0.0396 s/iter total_throughput: 2040.84 samples/s lr: 8.93e-04  
[08/30 09:39:36] lb.utils.events INFO:  eta: 1 day, 16:50:07  iteration: 80199/375342  consumed_samples: 82124800  total_loss: 0.388  time: 0.5018 s/iter  data_time: 0.0386 s/iter total_throughput: 2040.84 samples/s lr: 8.93e-04  
[08/30 09:40:26] lb.utils.events INFO:  eta: 1 day, 16:49:35  iteration: 80299/375342  consumed_samples: 82227200  total_loss: 0.3873  time: 0.5018 s/iter  data_time: 0.0380 s/iter total_throughput: 2040.85 samples/s lr: 8.92e-04  
[08/30 09:41:16] lb.utils.events INFO:  eta: 1 day, 16:47:43  iteration: 80399/375342  consumed_samples: 82329600  total_loss: 0.3729  time: 0.5017 s/iter  data_time: 0.0391 s/iter total_throughput: 2040.86 samples/s lr: 8.92e-04  
[08/30 09:42:06] lb.utils.events INFO:  eta: 1 day, 16:47:02  iteration: 80499/375342  consumed_samples: 82432000  total_loss: 0.3891  time: 0.5017 s/iter  data_time: 0.0393 s/iter total_throughput: 2040.87 samples/s lr: 8.92e-04  
[08/30 09:42:56] lb.utils.events INFO:  eta: 1 day, 16:46:12  iteration: 80599/375342  consumed_samples: 82534400  total_loss: 0.3984  time: 0.5017 s/iter  data_time: 0.0383 s/iter total_throughput: 2040.87 samples/s lr: 8.92e-04  
[08/30 09:43:46] lb.utils.events INFO:  eta: 1 day, 16:45:00  iteration: 80699/375342  consumed_samples: 82636800  total_loss: 0.3887  time: 0.5017 s/iter  data_time: 0.0396 s/iter total_throughput: 2040.88 samples/s lr: 8.91e-04  
[08/30 09:44:36] lb.utils.events INFO:  eta: 1 day, 16:44:25  iteration: 80799/375342  consumed_samples: 82739200  total_loss: 0.3874  time: 0.5017 s/iter  data_time: 0.0404 s/iter total_throughput: 2040.89 samples/s lr: 8.91e-04  
[08/30 09:45:26] lb.utils.events INFO:  eta: 1 day, 16:43:25  iteration: 80899/375342  consumed_samples: 82841600  total_loss: 0.3915  time: 0.5017 s/iter  data_time: 0.0391 s/iter total_throughput: 2040.91 samples/s lr: 8.91e-04  
[08/30 09:46:16] lb.utils.events INFO:  eta: 1 day, 16:43:08  iteration: 80999/375342  consumed_samples: 82944000  total_loss: 0.3867  time: 0.5017 s/iter  data_time: 0.0386 s/iter total_throughput: 2040.91 samples/s lr: 8.91e-04  
[08/30 09:47:06] lb.utils.events INFO:  eta: 1 day, 16:42:34  iteration: 81099/375342  consumed_samples: 83046400  total_loss: 0.3864  time: 0.5017 s/iter  data_time: 0.0377 s/iter total_throughput: 2040.91 samples/s lr: 8.90e-04  
[08/30 09:47:56] lb.utils.events INFO:  eta: 1 day, 16:41:36  iteration: 81199/375342  consumed_samples: 83148800  total_loss: 0.3921  time: 0.5017 s/iter  data_time: 0.0387 s/iter total_throughput: 2040.91 samples/s lr: 8.90e-04  
[08/30 09:48:47] lb.utils.events INFO:  eta: 1 day, 16:42:38  iteration: 81299/375342  consumed_samples: 83251200  total_loss: 0.3857  time: 0.5017 s/iter  data_time: 0.0375 s/iter total_throughput: 2040.91 samples/s lr: 8.90e-04  
[08/30 09:49:37] lb.utils.events INFO:  eta: 1 day, 16:42:43  iteration: 81399/375342  consumed_samples: 83353600  total_loss: 0.386  time: 0.5017 s/iter  data_time: 0.0380 s/iter total_throughput: 2040.91 samples/s lr: 8.89e-04  
[08/30 09:50:27] lb.utils.events INFO:  eta: 1 day, 16:42:02  iteration: 81499/375342  consumed_samples: 83456000  total_loss: 0.3912  time: 0.5017 s/iter  data_time: 0.0392 s/iter total_throughput: 2040.92 samples/s lr: 8.89e-04  
[08/30 09:51:17] lb.utils.events INFO:  eta: 1 day, 16:40:37  iteration: 81599/375342  consumed_samples: 83558400  total_loss: 0.39  time: 0.5017 s/iter  data_time: 0.0398 s/iter total_throughput: 2040.93 samples/s lr: 8.89e-04  
[08/30 09:52:07] lb.utils.events INFO:  eta: 1 day, 16:39:25  iteration: 81699/375342  consumed_samples: 83660800  total_loss: 0.3908  time: 0.5017 s/iter  data_time: 0.0381 s/iter total_throughput: 2040.95 samples/s lr: 8.89e-04  
[08/30 09:52:56] lb.utils.events INFO:  eta: 1 day, 16:37:16  iteration: 81799/375342  consumed_samples: 83763200  total_loss: 0.3956  time: 0.5017 s/iter  data_time: 0.0378 s/iter total_throughput: 2040.96 samples/s lr: 8.88e-04  
[08/30 09:53:47] lb.utils.events INFO:  eta: 1 day, 16:37:22  iteration: 81899/375342  consumed_samples: 83865600  total_loss: 0.386  time: 0.5017 s/iter  data_time: 0.0381 s/iter total_throughput: 2040.97 samples/s lr: 8.88e-04  
[08/30 09:54:37] lb.utils.events INFO:  eta: 1 day, 16:36:05  iteration: 81999/375342  consumed_samples: 83968000  total_loss: 0.389  time: 0.5017 s/iter  data_time: 0.0378 s/iter total_throughput: 2040.98 samples/s lr: 8.88e-04  
[08/30 09:55:27] lb.utils.events INFO:  eta: 1 day, 16:34:47  iteration: 82099/375342  consumed_samples: 84070400  total_loss: 0.3873  time: 0.5017 s/iter  data_time: 0.0390 s/iter total_throughput: 2040.99 samples/s lr: 8.88e-04  
[08/30 09:56:17] lb.utils.events INFO:  eta: 1 day, 16:32:54  iteration: 82199/375342  consumed_samples: 84172800  total_loss: 0.3832  time: 0.5017 s/iter  data_time: 0.0395 s/iter total_throughput: 2041.00 samples/s lr: 8.87e-04  
[08/30 09:57:07] lb.utils.events INFO:  eta: 1 day, 16:30:25  iteration: 82299/375342  consumed_samples: 84275200  total_loss: 0.3852  time: 0.5017 s/iter  data_time: 0.0388 s/iter total_throughput: 2041.00 samples/s lr: 8.87e-04  
[08/30 09:57:57] lb.utils.events INFO:  eta: 1 day, 16:29:28  iteration: 82399/375342  consumed_samples: 84377600  total_loss: 0.3802  time: 0.5017 s/iter  data_time: 0.0385 s/iter total_throughput: 2041.01 samples/s lr: 8.87e-04  
[08/30 09:58:47] lb.utils.events INFO:  eta: 1 day, 16:28:43  iteration: 82499/375342  consumed_samples: 84480000  total_loss: 0.3825  time: 0.5017 s/iter  data_time: 0.0386 s/iter total_throughput: 2041.01 samples/s lr: 8.87e-04  
[08/30 09:59:37] lb.utils.events INFO:  eta: 1 day, 16:27:49  iteration: 82599/375342  consumed_samples: 84582400  total_loss: 0.3869  time: 0.5017 s/iter  data_time: 0.0390 s/iter total_throughput: 2041.02 samples/s lr: 8.86e-04  
[08/30 10:00:27] lb.utils.events INFO:  eta: 1 day, 16:26:57  iteration: 82699/375342  consumed_samples: 84684800  total_loss: 0.3864  time: 0.5017 s/iter  data_time: 0.0389 s/iter total_throughput: 2041.03 samples/s lr: 8.86e-04  
[08/30 10:01:17] lb.utils.events INFO:  eta: 1 day, 16:27:09  iteration: 82799/375342  consumed_samples: 84787200  total_loss: 0.3872  time: 0.5017 s/iter  data_time: 0.0395 s/iter total_throughput: 2041.04 samples/s lr: 8.86e-04  
[08/30 10:02:07] lb.utils.events INFO:  eta: 1 day, 16:26:51  iteration: 82899/375342  consumed_samples: 84889600  total_loss: 0.3891  time: 0.5017 s/iter  data_time: 0.0379 s/iter total_throughput: 2041.04 samples/s lr: 8.86e-04  
[08/30 10:02:57] lb.utils.events INFO:  eta: 1 day, 16:26:22  iteration: 82999/375342  consumed_samples: 84992000  total_loss: 0.386  time: 0.5017 s/iter  data_time: 0.0388 s/iter total_throughput: 2041.05 samples/s lr: 8.85e-04  
[08/30 10:03:47] lb.utils.events INFO:  eta: 1 day, 16:25:57  iteration: 83099/375342  consumed_samples: 85094400  total_loss: 0.3814  time: 0.5017 s/iter  data_time: 0.0381 s/iter total_throughput: 2041.06 samples/s lr: 8.85e-04  
[08/30 10:04:37] lb.utils.events INFO:  eta: 1 day, 16:24:24  iteration: 83199/375342  consumed_samples: 85196800  total_loss: 0.3816  time: 0.5017 s/iter  data_time: 0.0381 s/iter total_throughput: 2041.07 samples/s lr: 8.85e-04  
[08/30 10:05:27] lb.utils.events INFO:  eta: 1 day, 16:23:23  iteration: 83299/375342  consumed_samples: 85299200  total_loss: 0.3901  time: 0.5017 s/iter  data_time: 0.0374 s/iter total_throughput: 2041.08 samples/s lr: 8.84e-04  
[08/30 10:06:17] lb.utils.events INFO:  eta: 1 day, 16:21:37  iteration: 83399/375342  consumed_samples: 85401600  total_loss: 0.3877  time: 0.5017 s/iter  data_time: 0.0383 s/iter total_throughput: 2041.09 samples/s lr: 8.84e-04  
[08/30 10:07:07] lb.utils.events INFO:  eta: 1 day, 16:19:47  iteration: 83499/375342  consumed_samples: 85504000  total_loss: 0.3858  time: 0.5017 s/iter  data_time: 0.0384 s/iter total_throughput: 2041.10 samples/s lr: 8.84e-04  
[08/30 10:07:57] lb.utils.events INFO:  eta: 1 day, 16:20:21  iteration: 83599/375342  consumed_samples: 85606400  total_loss: 0.3852  time: 0.5017 s/iter  data_time: 0.0390 s/iter total_throughput: 2041.11 samples/s lr: 8.84e-04  
[08/30 10:08:47] lb.utils.events INFO:  eta: 1 day, 16:19:56  iteration: 83699/375342  consumed_samples: 85708800  total_loss: 0.3886  time: 0.5017 s/iter  data_time: 0.0396 s/iter total_throughput: 2041.11 samples/s lr: 8.83e-04  
[08/30 10:09:37] lb.utils.events INFO:  eta: 1 day, 16:19:28  iteration: 83799/375342  consumed_samples: 85811200  total_loss: 0.3852  time: 0.5017 s/iter  data_time: 0.0390 s/iter total_throughput: 2041.11 samples/s lr: 8.83e-04  
[08/30 10:10:27] lb.utils.events INFO:  eta: 1 day, 16:17:43  iteration: 83899/375342  consumed_samples: 85913600  total_loss: 0.378  time: 0.5017 s/iter  data_time: 0.0378 s/iter total_throughput: 2041.12 samples/s lr: 8.83e-04  
[08/30 10:11:17] lb.utils.events INFO:  eta: 1 day, 16:16:47  iteration: 83999/375342  consumed_samples: 86016000  total_loss: 0.3772  time: 0.5017 s/iter  data_time: 0.0388 s/iter total_throughput: 2041.13 samples/s lr: 8.83e-04  
[08/30 10:12:07] lb.utils.events INFO:  eta: 1 day, 16:16:14  iteration: 84099/375342  consumed_samples: 86118400  total_loss: 0.3806  time: 0.5017 s/iter  data_time: 0.0390 s/iter total_throughput: 2041.14 samples/s lr: 8.82e-04  
[08/30 10:12:57] lb.utils.events INFO:  eta: 1 day, 16:15:59  iteration: 84199/375342  consumed_samples: 86220800  total_loss: 0.3842  time: 0.5017 s/iter  data_time: 0.0394 s/iter total_throughput: 2041.14 samples/s lr: 8.82e-04  
[08/30 10:13:47] lb.utils.events INFO:  eta: 1 day, 16:14:55  iteration: 84299/375342  consumed_samples: 86323200  total_loss: 0.3809  time: 0.5017 s/iter  data_time: 0.0387 s/iter total_throughput: 2041.15 samples/s lr: 8.82e-04  
[08/30 10:14:37] lb.utils.events INFO:  eta: 1 day, 16:15:41  iteration: 84399/375342  consumed_samples: 86425600  total_loss: 0.3806  time: 0.5017 s/iter  data_time: 0.0378 s/iter total_throughput: 2041.15 samples/s lr: 8.82e-04  
[08/30 10:15:27] lb.utils.events INFO:  eta: 1 day, 16:16:19  iteration: 84499/375342  consumed_samples: 86528000  total_loss: 0.3851  time: 0.5017 s/iter  data_time: 0.0380 s/iter total_throughput: 2041.16 samples/s lr: 8.81e-04  
[08/30 10:16:17] lb.utils.events INFO:  eta: 1 day, 16:14:04  iteration: 84599/375342  consumed_samples: 86630400  total_loss: 0.3875  time: 0.5017 s/iter  data_time: 0.0384 s/iter total_throughput: 2041.17 samples/s lr: 8.81e-04  
[08/30 10:17:07] lb.utils.events INFO:  eta: 1 day, 16:13:20  iteration: 84699/375342  consumed_samples: 86732800  total_loss: 0.3886  time: 0.5017 s/iter  data_time: 0.0388 s/iter total_throughput: 2041.18 samples/s lr: 8.81e-04  
[08/30 10:17:58] lb.utils.events INFO:  eta: 1 day, 16:11:50  iteration: 84799/375342  consumed_samples: 86835200  total_loss: 0.3888  time: 0.5017 s/iter  data_time: 0.0402 s/iter total_throughput: 2041.18 samples/s lr: 8.80e-04  
[08/30 10:18:48] lb.utils.events INFO:  eta: 1 day, 16:11:38  iteration: 84899/375342  consumed_samples: 86937600  total_loss: 0.3921  time: 0.5017 s/iter  data_time: 0.0376 s/iter total_throughput: 2041.19 samples/s lr: 8.80e-04  
[08/30 10:19:38] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0084999
[08/30 10:19:38] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 10:19:38] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 10:19:43] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0700 s/iter. Inference: 0.2451 s/iter. Eval: 0.0023 s/iter. Total: 0.3175 s/iter. ETA=0:00:11
[08/30 10:19:48] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0877 s/iter. Inference: 0.2414 s/iter. Eval: 0.0024 s/iter. Total: 0.3318 s/iter. ETA=0:00:07
[08/30 10:19:53] lb.evaluation.evaluator INFO: Inference done 43008/50000. Dataloading: 0.0833 s/iter. Inference: 0.2423 s/iter. Eval: 0.0023 s/iter. Total: 0.3282 s/iter. ETA=0:00:01
[08/30 10:19:55] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 10:19:55] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.112189 (0.000282 s / iter per device, on 8 devices)
[08/30 10:19:55] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000213 s / iter per device, on 8 devices)
[08/30 10:19:55] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 10:19:55] lb.evaluation.utils INFO: copypaste: Acc@1=75.978
[08/30 10:19:55] lb.evaluation.utils INFO: copypaste: Acc@5=93.308
[08/30 10:19:55] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 75.97800, better than last best score 75.48600 @ iteration 79999.
[08/30 10:19:55] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 10:19:56] lb.utils.events INFO:  eta: 1 day, 16:11:19  iteration: 84999/375342  consumed_samples: 87040000  total_loss: 0.3907  time: 0.5017 s/iter  data_time: 0.0385 s/iter total_throughput: 2041.20 samples/s lr: 8.80e-04  
[08/30 10:20:46] lb.utils.events INFO:  eta: 1 day, 16:10:04  iteration: 85099/375342  consumed_samples: 87142400  total_loss: 0.3871  time: 0.5017 s/iter  data_time: 0.0369 s/iter total_throughput: 2041.19 samples/s lr: 8.80e-04  
[08/30 10:21:36] lb.utils.events INFO:  eta: 1 day, 16:08:49  iteration: 85199/375342  consumed_samples: 87244800  total_loss: 0.3869  time: 0.5017 s/iter  data_time: 0.0381 s/iter total_throughput: 2041.20 samples/s lr: 8.79e-04  
[08/30 10:22:26] lb.utils.events INFO:  eta: 1 day, 16:09:37  iteration: 85299/375342  consumed_samples: 87347200  total_loss: 0.3881  time: 0.5017 s/iter  data_time: 0.0405 s/iter total_throughput: 2041.21 samples/s lr: 8.79e-04  
[08/30 10:23:16] lb.utils.events INFO:  eta: 1 day, 16:08:45  iteration: 85399/375342  consumed_samples: 87449600  total_loss: 0.3805  time: 0.5017 s/iter  data_time: 0.0379 s/iter total_throughput: 2041.21 samples/s lr: 8.79e-04  
[08/30 10:24:06] lb.utils.events INFO:  eta: 1 day, 16:08:10  iteration: 85499/375342  consumed_samples: 87552000  total_loss: 0.3855  time: 0.5017 s/iter  data_time: 0.0383 s/iter total_throughput: 2041.21 samples/s lr: 8.79e-04  
[08/30 10:24:57] lb.utils.events INFO:  eta: 1 day, 16:08:52  iteration: 85599/375342  consumed_samples: 87654400  total_loss: 0.3894  time: 0.5017 s/iter  data_time: 0.0390 s/iter total_throughput: 2041.22 samples/s lr: 8.78e-04  
[08/30 10:25:47] lb.utils.events INFO:  eta: 1 day, 16:07:29  iteration: 85699/375342  consumed_samples: 87756800  total_loss: 0.3844  time: 0.5017 s/iter  data_time: 0.0383 s/iter total_throughput: 2041.22 samples/s lr: 8.78e-04  
[08/30 10:26:37] lb.utils.events INFO:  eta: 1 day, 16:07:07  iteration: 85799/375342  consumed_samples: 87859200  total_loss: 0.3843  time: 0.5017 s/iter  data_time: 0.0392 s/iter total_throughput: 2041.22 samples/s lr: 8.78e-04  
[08/30 10:27:27] lb.utils.events INFO:  eta: 1 day, 16:05:23  iteration: 85899/375342  consumed_samples: 87961600  total_loss: 0.382  time: 0.5017 s/iter  data_time: 0.0383 s/iter total_throughput: 2041.23 samples/s lr: 8.77e-04  
[08/30 10:28:17] lb.utils.events INFO:  eta: 1 day, 16:05:14  iteration: 85999/375342  consumed_samples: 88064000  total_loss: 0.3814  time: 0.5017 s/iter  data_time: 0.0390 s/iter total_throughput: 2041.23 samples/s lr: 8.77e-04  
[08/30 10:29:07] lb.utils.events INFO:  eta: 1 day, 16:03:39  iteration: 86099/375342  consumed_samples: 88166400  total_loss: 0.3861  time: 0.5017 s/iter  data_time: 0.0379 s/iter total_throughput: 2041.24 samples/s lr: 8.77e-04  
[08/30 10:29:57] lb.utils.events INFO:  eta: 1 day, 16:03:20  iteration: 86199/375342  consumed_samples: 88268800  total_loss: 0.3795  time: 0.5017 s/iter  data_time: 0.0384 s/iter total_throughput: 2041.25 samples/s lr: 8.77e-04  
[08/30 10:30:47] lb.utils.events INFO:  eta: 1 day, 16:01:24  iteration: 86299/375342  consumed_samples: 88371200  total_loss: 0.3753  time: 0.5017 s/iter  data_time: 0.0397 s/iter total_throughput: 2041.25 samples/s lr: 8.76e-04  
[08/30 10:31:37] lb.utils.events INFO:  eta: 1 day, 16:00:58  iteration: 86399/375342  consumed_samples: 88473600  total_loss: 0.3901  time: 0.5017 s/iter  data_time: 0.0376 s/iter total_throughput: 2041.26 samples/s lr: 8.76e-04  
[08/30 10:32:27] lb.utils.events INFO:  eta: 1 day, 16:00:19  iteration: 86499/375342  consumed_samples: 88576000  total_loss: 0.3988  time: 0.5017 s/iter  data_time: 0.0373 s/iter total_throughput: 2041.26 samples/s lr: 8.76e-04  
[08/30 10:33:17] lb.utils.events INFO:  eta: 1 day, 15:58:37  iteration: 86599/375342  consumed_samples: 88678400  total_loss: 0.3941  time: 0.5017 s/iter  data_time: 0.0379 s/iter total_throughput: 2041.26 samples/s lr: 8.76e-04  
[08/30 10:34:08] lb.utils.events INFO:  eta: 1 day, 15:58:07  iteration: 86699/375342  consumed_samples: 88780800  total_loss: 0.389  time: 0.5017 s/iter  data_time: 0.0385 s/iter total_throughput: 2041.26 samples/s lr: 8.75e-04  
[08/30 10:34:58] lb.utils.events INFO:  eta: 1 day, 15:56:27  iteration: 86799/375342  consumed_samples: 88883200  total_loss: 0.3867  time: 0.5016 s/iter  data_time: 0.0380 s/iter total_throughput: 2041.27 samples/s lr: 8.75e-04  
[08/30 10:35:48] lb.utils.events INFO:  eta: 1 day, 15:56:53  iteration: 86899/375342  consumed_samples: 88985600  total_loss: 0.3888  time: 0.5016 s/iter  data_time: 0.0386 s/iter total_throughput: 2041.27 samples/s lr: 8.75e-04  
[08/30 10:36:38] lb.utils.events INFO:  eta: 1 day, 15:55:54  iteration: 86999/375342  consumed_samples: 89088000  total_loss: 0.3881  time: 0.5016 s/iter  data_time: 0.0388 s/iter total_throughput: 2041.28 samples/s lr: 8.74e-04  
[08/30 10:37:28] lb.utils.events INFO:  eta: 1 day, 15:55:34  iteration: 87099/375342  consumed_samples: 89190400  total_loss: 0.3894  time: 0.5016 s/iter  data_time: 0.0396 s/iter total_throughput: 2041.28 samples/s lr: 8.74e-04  
[08/30 10:38:18] lb.utils.events INFO:  eta: 1 day, 15:52:55  iteration: 87199/375342  consumed_samples: 89292800  total_loss: 0.374  time: 0.5016 s/iter  data_time: 0.0389 s/iter total_throughput: 2041.29 samples/s lr: 8.74e-04  
[08/30 10:39:08] lb.utils.events INFO:  eta: 1 day, 15:51:48  iteration: 87299/375342  consumed_samples: 89395200  total_loss: 0.3722  time: 0.5016 s/iter  data_time: 0.0399 s/iter total_throughput: 2041.31 samples/s lr: 8.74e-04  
[08/30 10:39:58] lb.utils.events INFO:  eta: 1 day, 15:50:39  iteration: 87399/375342  consumed_samples: 89497600  total_loss: 0.3837  time: 0.5016 s/iter  data_time: 0.0388 s/iter total_throughput: 2041.31 samples/s lr: 8.73e-04  
[08/30 10:40:48] lb.utils.events INFO:  eta: 1 day, 15:48:40  iteration: 87499/375342  consumed_samples: 89600000  total_loss: 0.3789  time: 0.5016 s/iter  data_time: 0.0375 s/iter total_throughput: 2041.32 samples/s lr: 8.73e-04  
[08/30 10:41:38] lb.utils.events INFO:  eta: 1 day, 15:48:29  iteration: 87599/375342  consumed_samples: 89702400  total_loss: 0.3842  time: 0.5016 s/iter  data_time: 0.0376 s/iter total_throughput: 2041.32 samples/s lr: 8.73e-04  
[08/30 10:42:28] lb.utils.events INFO:  eta: 1 day, 15:47:39  iteration: 87699/375342  consumed_samples: 89804800  total_loss: 0.3891  time: 0.5016 s/iter  data_time: 0.0391 s/iter total_throughput: 2041.32 samples/s lr: 8.73e-04  
[08/30 10:43:18] lb.utils.events INFO:  eta: 1 day, 15:47:27  iteration: 87799/375342  consumed_samples: 89907200  total_loss: 0.3916  time: 0.5016 s/iter  data_time: 0.0396 s/iter total_throughput: 2041.32 samples/s lr: 8.72e-04  
[08/30 10:44:08] lb.utils.events INFO:  eta: 1 day, 15:46:46  iteration: 87899/375342  consumed_samples: 90009600  total_loss: 0.3912  time: 0.5016 s/iter  data_time: 0.0399 s/iter total_throughput: 2041.33 samples/s lr: 8.72e-04  
[08/30 10:44:58] lb.utils.events INFO:  eta: 1 day, 15:45:21  iteration: 87999/375342  consumed_samples: 90112000  total_loss: 0.3802  time: 0.5016 s/iter  data_time: 0.0385 s/iter total_throughput: 2041.34 samples/s lr: 8.72e-04  
[08/30 10:45:48] lb.utils.events INFO:  eta: 1 day, 15:44:31  iteration: 88099/375342  consumed_samples: 90214400  total_loss: 0.3815  time: 0.5016 s/iter  data_time: 0.0391 s/iter total_throughput: 2041.34 samples/s lr: 8.71e-04  
[08/30 10:46:38] lb.utils.events INFO:  eta: 1 day, 15:45:02  iteration: 88199/375342  consumed_samples: 90316800  total_loss: 0.3824  time: 0.5016 s/iter  data_time: 0.0385 s/iter total_throughput: 2041.35 samples/s lr: 8.71e-04  
[08/30 10:47:28] lb.utils.events INFO:  eta: 1 day, 15:43:29  iteration: 88299/375342  consumed_samples: 90419200  total_loss: 0.3823  time: 0.5016 s/iter  data_time: 0.0403 s/iter total_throughput: 2041.36 samples/s lr: 8.71e-04  
[08/30 10:48:18] lb.utils.events INFO:  eta: 1 day, 15:42:01  iteration: 88399/375342  consumed_samples: 90521600  total_loss: 0.3851  time: 0.5016 s/iter  data_time: 0.0381 s/iter total_throughput: 2041.37 samples/s lr: 8.71e-04  
[08/30 10:49:08] lb.utils.events INFO:  eta: 1 day, 15:40:54  iteration: 88499/375342  consumed_samples: 90624000  total_loss: 0.3855  time: 0.5016 s/iter  data_time: 0.0387 s/iter total_throughput: 2041.38 samples/s lr: 8.70e-04  
[08/30 10:49:59] lb.utils.events INFO:  eta: 1 day, 15:40:39  iteration: 88599/375342  consumed_samples: 90726400  total_loss: 0.3846  time: 0.5016 s/iter  data_time: 0.0392 s/iter total_throughput: 2041.38 samples/s lr: 8.70e-04  
[08/30 10:50:49] lb.utils.events INFO:  eta: 1 day, 15:39:21  iteration: 88699/375342  consumed_samples: 90828800  total_loss: 0.3818  time: 0.5016 s/iter  data_time: 0.0392 s/iter total_throughput: 2041.37 samples/s lr: 8.70e-04  
[08/30 10:51:39] lb.utils.events INFO:  eta: 1 day, 15:37:08  iteration: 88799/375342  consumed_samples: 90931200  total_loss: 0.3801  time: 0.5016 s/iter  data_time: 0.0398 s/iter total_throughput: 2041.38 samples/s lr: 8.69e-04  
[08/30 10:52:29] lb.utils.events INFO:  eta: 1 day, 15:36:14  iteration: 88899/375342  consumed_samples: 91033600  total_loss: 0.384  time: 0.5016 s/iter  data_time: 0.0381 s/iter total_throughput: 2041.38 samples/s lr: 8.69e-04  
[08/30 10:53:19] lb.utils.events INFO:  eta: 1 day, 15:35:31  iteration: 88999/375342  consumed_samples: 91136000  total_loss: 0.3833  time: 0.5016 s/iter  data_time: 0.0393 s/iter total_throughput: 2041.38 samples/s lr: 8.69e-04  
[08/30 10:54:09] lb.utils.events INFO:  eta: 1 day, 15:34:42  iteration: 89099/375342  consumed_samples: 91238400  total_loss: 0.393  time: 0.5016 s/iter  data_time: 0.0389 s/iter total_throughput: 2041.39 samples/s lr: 8.69e-04  
[08/30 10:54:59] lb.utils.events INFO:  eta: 1 day, 15:34:23  iteration: 89199/375342  consumed_samples: 91340800  total_loss: 0.3955  time: 0.5016 s/iter  data_time: 0.0398 s/iter total_throughput: 2041.38 samples/s lr: 8.68e-04  
[08/30 10:55:49] lb.utils.events INFO:  eta: 1 day, 15:35:45  iteration: 89299/375342  consumed_samples: 91443200  total_loss: 0.3823  time: 0.5016 s/iter  data_time: 0.0387 s/iter total_throughput: 2041.39 samples/s lr: 8.68e-04  
[08/30 10:56:39] lb.utils.events INFO:  eta: 1 day, 15:33:50  iteration: 89399/375342  consumed_samples: 91545600  total_loss: 0.3839  time: 0.5016 s/iter  data_time: 0.0393 s/iter total_throughput: 2041.40 samples/s lr: 8.68e-04  
[08/30 10:57:29] lb.utils.events INFO:  eta: 1 day, 15:33:07  iteration: 89499/375342  consumed_samples: 91648000  total_loss: 0.3813  time: 0.5016 s/iter  data_time: 0.0384 s/iter total_throughput: 2041.41 samples/s lr: 8.67e-04  
[08/30 10:58:19] lb.utils.events INFO:  eta: 1 day, 15:31:04  iteration: 89599/375342  consumed_samples: 91750400  total_loss: 0.3815  time: 0.5016 s/iter  data_time: 0.0374 s/iter total_throughput: 2041.42 samples/s lr: 8.67e-04  
[08/30 10:59:09] lb.utils.events INFO:  eta: 1 day, 15:30:02  iteration: 89699/375342  consumed_samples: 91852800  total_loss: 0.3861  time: 0.5016 s/iter  data_time: 0.0376 s/iter total_throughput: 2041.43 samples/s lr: 8.67e-04  
[08/30 10:59:59] lb.utils.events INFO:  eta: 1 day, 15:29:23  iteration: 89799/375342  consumed_samples: 91955200  total_loss: 0.386  time: 0.5016 s/iter  data_time: 0.0408 s/iter total_throughput: 2041.45 samples/s lr: 8.67e-04  
[08/30 11:00:49] lb.utils.events INFO:  eta: 1 day, 15:29:41  iteration: 89899/375342  consumed_samples: 92057600  total_loss: 0.3845  time: 0.5016 s/iter  data_time: 0.0385 s/iter total_throughput: 2041.45 samples/s lr: 8.66e-04  
[08/30 11:01:39] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_0089999
[08/30 11:01:40] lb.evaluation.evaluator INFO: with eval_iter 100000.0, reset total samples 50000 to 50000
[08/30 11:01:40] lb.evaluation.evaluator INFO: Start inference on 50000 samples
[08/30 11:01:45] lb.evaluation.evaluator INFO: Inference done 11264/50000. Dataloading: 0.0761 s/iter. Inference: 0.2397 s/iter. Eval: 0.0020 s/iter. Total: 0.3179 s/iter. ETA=0:00:11
[08/30 11:01:50] lb.evaluation.evaluator INFO: Inference done 26624/50000. Dataloading: 0.0925 s/iter. Inference: 0.2377 s/iter. Eval: 0.0023 s/iter. Total: 0.3327 s/iter. ETA=0:00:07
[08/30 11:01:55] lb.evaluation.evaluator INFO: Inference done 43008/50000. Dataloading: 0.0887 s/iter. Inference: 0.2373 s/iter. Eval: 0.0023 s/iter. Total: 0.3284 s/iter. ETA=0:00:01
[08/30 11:01:57] lb.evaluation.evaluator INFO: Total valid samples: 50000
[08/30 11:01:57] lb.evaluation.evaluator INFO: Total inference time: 0:00:14.192459 (0.000284 s / iter per device, on 8 devices)
[08/30 11:01:57] lb.evaluation.evaluator INFO: Total inference pure compute time: 0:00:10 (0.000210 s / iter per device, on 8 devices)
[08/30 11:01:57] lb.engine.default INFO: Evaluation results for ImageNetDataset in csv format:
[08/30 11:01:57] lb.evaluation.utils INFO: copypaste: Acc@1=76.6
[08/30 11:01:57] lb.evaluation.utils INFO: copypaste: Acc@5=93.374
[08/30 11:01:57] lb.engine.hooks INFO: Saved best model as latest eval score for Acc@1 is 76.60000, better than last best score 75.97800 @ iteration 84999.
[08/30 11:01:57] lb.utils.checkpoint INFO: Saving checkpoint to ./output_20220829/model_best
[08/30 11:01:58] lb.utils.events INFO:  eta: 1 day, 15:28:14  iteration: 89999/375342  consumed_samples: 92160000  total_loss: 0.3893  time: 0.5016 s/iter  data_time: 0.0389 s/iter total_throughput: 2041.46 samples/s lr: 8.66e-04  
[08/30 11:02:48] lb.utils.events INFO:  eta: 1 day, 15:28:50  iteration: 90099/375342  consumed_samples: 92262400  total_loss: 0.3873  time: 0.5016 s/iter  data_time: 0.0374 s/iter total_throughput: 2041.45 samples/s lr: 8.66e-04  
[08/30 11:03:38] lb.utils.events INFO:  eta: 1 day, 15:27:04  iteration: 90199/375342  consumed_samples: 92364800  total_loss: 0.3739  time: 0.5016 s/iter  data_time: 0.0365 s/iter total_throughput: 2041.45 samples/s lr: 8.66e-04  
[08/30 11:04:28] lb.utils.events INFO:  eta: 1 day, 15:25:52  iteration: 90299/375342  consumed_samples: 92467200  total_loss: 0.3893  time: 0.5016 s/iter  data_time: 0.0393 s/iter total_throughput: 2041.46 samples/s lr: 8.65e-04  
[08/30 11:05:18] lb.utils.events INFO:  eta: 1 day, 15:25:54  iteration: 90399/375342  consumed_samples: 92569600  total_loss: 0.3927  time: 0.5016 s/iter  data_time: 0.0382 s/iter total_throughput: 2041.46 samples/s lr: 8.65e-04  
[08/30 11:06:08] lb.utils.events INFO:  eta: 1 day, 15:25:07  iteration: 90499/375342  consumed_samples: 92672000  total_loss: 0.3864  time: 0.5016 s/iter  data_time: 0.0386 s/iter total_throughput: 2041.46 samples/s lr: 8.65e-04  
[08/30 11:06:58] lb.utils.events INFO:  eta: 1 day, 15:24:18  iteration: 90599/375342  consumed_samples: 92774400  total_loss: 0.3832  time: 0.5016 s/iter  data_time: 0.0394 s/iter total_throughput: 2041.48 samples/s lr: 8.64e-04  
[08/30 11:07:48] lb.utils.events INFO:  eta: 1 day, 15:23:48  iteration: 90699/375342  consumed_samples: 92876800  total_loss: 0.3821  time: 0.5016 s/iter  data_time: 0.0395 s/iter total_throughput: 2041.48 samples/s lr: 8.64e-04  
[08/30 11:08:38] lb.utils.events INFO:  eta: 1 day, 15:23:12  iteration: 90799/375342  consumed_samples: 92979200  total_loss: 0.3805  time: 0.5016 s/iter  data_time: 0.0379 s/iter total_throughput: 2041.48 samples/s lr: 8.64e-04  
[08/30 11:09:28] lb.utils.events INFO:  eta: 1 day, 15:21:51  iteration: 90899/375342  consumed_samples: 93081600  total_loss: 0.3801  time: 0.5016 s/iter  data_time: 0.0376 s/iter total_throughput: 2041.49 samples/s lr: 8.64e-04  
[08/30 11:10:18] lb.utils.events INFO:  eta: 1 day, 15:21:04  iteration: 90999/375342  consumed_samples: 93184000  total_loss: 0.3862  time: 0.5016 s/iter  data_time: 0.0395 s/iter total_throughput: 2041.50 samples/s lr: 8.63e-04  
[08/30 11:11:08] lb.utils.events INFO:  eta: 1 day, 15:18:21  iteration: 91099/375342  consumed_samples: 93286400  total_loss: 0.3905  time: 0.5016 s/iter  data_time: 0.0374 s/iter total_throughput: 2041.51 samples/s lr: 8.63e-04  
[08/30 11:11:58] lb.utils.events INFO:  eta: 1 day, 15:19:02  iteration: 91199/375342  consumed_samples: 93388800  total_loss: 0.3875  time: 0.5016 s/iter  data_time: 0.0395 s/iter total_throughput: 2041.51 samples/s lr: 8.63e-04  
[08/30 11:12:48] lb.utils.events INFO:  eta: 1 day, 15:16:20  iteration: 91299/375342  consumed_samples: 93491200  total_loss: 0.3843  time: 0.5016 s/iter  data_time: 0.0399 s/iter total_throughput: 2041.52 samples/s lr: 8.62e-04  
[08/30 11:13:38] lb.utils.events INFO:  eta: 1 day, 15:15:52  iteration: 91399/375342  consumed_samples: 93593600  total_loss: 0.3868  time: 0.5016 s/iter  data_time: 0.0400 s/iter total_throughput: 2041.53 samples/s lr: 8.62e-04  
[08/30 11:14:29] lb.utils.events INFO:  eta: 1 day, 15:16:46  iteration: 91499/375342  consumed_samples: 93696000  total_loss: 0.3883  time: 0.5016 s/iter  data_time: 0.0390 s/iter total_throughput: 2041.53 samples/s lr: 8.62e-04  
[08/30 11:15:19] lb.utils.events INFO:  eta: 1 day, 15:17:10  iteration: 91599/375342  consumed_samples: 93798400  total_loss: 0.3873  time: 0.5016 s/iter  data_time: 0.0372 s/iter total_throughput: 2041.54 samples/s lr: 8.62e-04  
[08/30 11:16:09] lb.utils.events INFO:  eta: 1 day, 15:15:52  iteration: 91699/375342  consumed_samples: 93900800  total_loss: 0.3858  time: 0.5016 s/iter  data_time: 0.0378 s/iter total_throughput: 2041.54 samples/s lr: 8.61e-04  
[08/30 11:16:59] lb.utils.events INFO:  eta: 1 day, 15:15:55  iteration: 91799/375342  consumed_samples: 94003200  total_loss: 0.3825  time: 0.5016 s/iter  data_time: 0.0377 s/iter total_throughput: 2041.55 samples/s lr: 8.61e-04  
[08/30 11:17:49] lb.utils.events INFO:  eta: 1 day, 15:14:49  iteration: 91899/375342  consumed_samples: 94105600  total_loss: 0.3827  time: 0.5016 s/iter  data_time: 0.0374 s/iter total_throughput: 2041.55 samples/s lr: 8.61e-04