DL_Training_Base.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dhdbsrlw/Instruct-Tune-LLaMA-with-PEFT-Techniques/blob/main/DL_Training_Base.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uR-VwJwUAYGc",
        "outputId": "59d0bc83-3e0e-4467-9ba8-12d4054f00b3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive/\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sB0YxZbtA6Sh"
      },
      "source": [
        "# 초기 설정"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H_Ro_EwKAnlB",
        "outputId": "0bc7f59d-92d3-4b0b-c5c3-79a166ac2c4a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive\n"
          ]
        }
      ],
      "source": [
        "%cd /content/drive/MyDrive"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7j-IIXMMAgE8",
        "outputId": "438585bb-ea08-4583-ca10-948a406fd31a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloning into 'lit-llama'...\n",
            "remote: Enumerating objects: 1925, done.\u001b[K\n",
            "remote: Counting objects: 100% (664/664), done.\u001b[K\n",
            "remote: Compressing objects: 100% (165/165), done.\u001b[K\n",
            "remote: Total 1925 (delta 566), reused 510 (delta 495), pack-reused 1261\u001b[K\n",
            "Receiving objects: 100% (1925/1925), 1.64 MiB | 5.54 MiB/s, done.\n",
            "Resolving deltas: 100% (1201/1201), done.\n"
          ]
        }
      ],
      "source": [
        "# !git clone https://github.com/Lightning-AI/lit-llama.git"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/dhdbsrlw/Instruct-Tune-LLaMA-with-PEFT-Techniques.git"
      ],
      "metadata": {
        "id": "uIsQgrL5DsLp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OdUE5ikYBgRJ"
      },
      "source": [
        "#**파이프라인**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XAEPjdz98Y46",
        "outputId": "fba57d3f-0200-449e-8713-54d1c8ce46a6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive/lit-llama\n"
          ]
        }
      ],
      "source": [
        "# %cd /content/drive/MyDrive/lit-llama"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/drive/MyDrive/Instruct-Tune-LLaMA-with-PEFT-Techniques"
      ],
      "metadata": {
        "id": "2H0jXA4ED5p4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SFpMcCCTBniS",
        "outputId": "211fe605-ea9b-45da-cf38-44aa550ca0b2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m19.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m189.7/189.7 kB\u001b[0m \u001b[31m24.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.6/92.6 MB\u001b[0m \u001b[31m19.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m521.2/521.2 kB\u001b[0m \u001b[31m53.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m109.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m806.1/806.1 kB\u001b[0m \u001b[31m69.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m776.9/776.9 kB\u001b[0m \u001b[31m68.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m594.2/594.2 kB\u001b[0m \u001b[31m50.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m18.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for lightning (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ],
      "source": [
        "!pip install -r requirements.txt -q"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QR4RMQQRBpzr",
        "outputId": "b8051f73-d032-4d88-86a2-11b9e00f504f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Updated git hooks.\n",
            "Git LFS initialized.\n"
          ]
        }
      ],
      "source": [
        "!git-lfs install"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "quIRDK5_rjR6",
        "outputId": "36ff5cb5-5e7a-4d2b-f430-db258afd1b1e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (4.66.1)\n"
          ]
        }
      ],
      "source": [
        "!pip install tqdm"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XNZwFM7gBZm6"
      },
      "source": [
        "## STEP 1. 훈련 데이터셋 준비 (전처리)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8HFGcXeA9XQv"
      },
      "source": [
        "한 번 데이터 저장되었으므로, 다시 실행시킬 필요 없다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "peJIm6OIFMIP"
      },
      "source": [
        "### 1-1. OpenLLaMA 모델 체크포인트 받아오기"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "v1a2AJhGFASJ",
        "outputId": "eb2115aa-68db-4075-b4b5-72b226c2b3ca"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content/drive/MyDrive/lit-llama'"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "%pwd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0mSjpNYjDMeG",
        "outputId": "86b8a147-85e1-435c-d8b7-bd07b8db1ab0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloning into 'checkpoints/open-llama/7B'...\n",
            "remote: Enumerating objects: 21, done.\u001b[K\n",
            "remote: Total 21 (delta 0), reused 0 (delta 0), pack-reused 21\u001b[K\n",
            "Unpacking objects: 100% (21/21), 7.72 KiB | 28.00 KiB/s, done.\n",
            "Filtering content: 100% (3/3), 4.55 GiB | 5.42 MiB/s, done.\n",
            "fatal: cannot exec '/content/drive/MyDrive/lit-llama/checkpoints/open-llama/7B/.git/hooks/post-checkout': Permission denied\n",
            "Encountered 1 file(s) that may not have been copied correctly on Windows:\n",
            "\tpytorch_model-00001-of-00002.bin\n",
            "\n",
            "See: `git lfs help smudge` for more details.\n"
          ]
        }
      ],
      "source": [
        "# 모델 체크포인트 받아오기 (실제 LLaMA 혹은 OpenLLaMA)\n",
        "# 일단 OpenLLaMA 채택\n",
        "\n",
        "# Make sure you have git-lfs installed (https://git-lfs.com): git lfs install\n",
        "! git clone https://huggingface.co/openlm-research/open_llama_7b checkpoints/open-llama/7B\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python scripts/convert_hf_checkpoint.py --checkpoint_dir checkpoints/open-llama/7B --model_size 7B"
      ],
      "metadata": {
        "id": "hSoygZr2EHBN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "J8cUaUe7HWni"
      },
      "outputs": [],
      "source": [
        "# Memory 이슈로 더 작은 모델 받아오기\n",
        "# !git clone https://huggingface.co/openlm-research/open_llama_3b checkpoints/open-llama/3B"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-wSn0uR9FJ8u"
      },
      "outputs": [],
      "source": [
        "# !python scripts/convert_hf_checkpoint.py --checkpoint_dir checkpoints/open-llama/3B --model_size 3B"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KMiitPKkFRBd"
      },
      "source": [
        "### 1-2. 훈련 데이터셋 Prompt 전처리 및 Split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QVidzwdRBU4d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9b9ab38d-af0f-4566-8910-1b5a8694e5fc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive/lit-llama\n"
          ]
        }
      ],
      "source": [
        "%cd /content/drive/MyDrive/lit-llama"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "WLH_cz758zw-",
        "outputId": "65084349-1618-4cc0-dd18-8760ee99c914"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/content/drive/MyDrive/lit-llama'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 7
        }
      ],
      "source": [
        "%pwd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MCM-GR1VBduX",
        "outputId": "e474e841-a56b-4bc5-9aca-3d7ee58bf1e5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "train has 13,011 samples\n",
            "val has 2,000 samples\n",
            "Processing train split ...\n",
            "100% 13011/13011 [00:14<00:00, 870.90it/s]\n",
            "Processing test split ...\n",
            "100% 2000/2000 [00:02<00:00, 848.11it/s]\n"
          ]
        }
      ],
      "source": [
        "!python scripts/prepare_dolly.py\n",
        "\n",
        "# 전처리 완료된 데이터셋이 어느 경로에 저장되는지 살펴보기\n",
        "# (분할 전) 전체 데이터셋은 - lit-llama/data/dolly 폴더 내 저장\n",
        "# (분할 후) 데이터셋은 - lit-llama/data 폴더 내 .pt 형식으로 저장"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "20R2SwRxDyGL"
      },
      "source": [
        "## STEP 2. 파인튜닝"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PIWP9RtAzLNZ"
      },
      "source": [
        "### 2-1. wandb 연동"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oX6VFDLnzO19",
        "outputId": "81d31a7c-32af-4e6b-cc6c-e8c772a9fb6e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting wandb\n",
            "  Downloading wandb-0.16.1-py3-none-any.whl (2.1 MB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/2.1 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.1/2.1 MB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/2.1 MB\u001b[0m \u001b[31m14.7 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m22.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: Click!=8.0.0,>=7.1 in /usr/local/lib/python3.10/dist-packages (from wandb) (8.1.7)\n",
            "Collecting GitPython!=3.1.29,>=1.0.0 (from wandb)\n",
            "  Downloading GitPython-3.1.40-py3-none-any.whl (190 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.6/190.6 kB\u001b[0m \u001b[31m26.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (2.31.0)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (5.9.5)\n",
            "Collecting sentry-sdk>=1.0.0 (from wandb)\n",
            "  Downloading sentry_sdk-1.38.0-py2.py3-none-any.whl (252 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m252.8/252.8 kB\u001b[0m \u001b[31m31.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting docker-pycreds>=0.4.0 (from wandb)\n",
            "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from wandb) (6.0.1)\n",
            "Collecting setproctitle (from wandb)\n",
            "  Downloading setproctitle-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb) (67.7.2)\n",
            "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb) (1.4.4)\n",
            "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (3.20.3)\n",
            "Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)\n",
            "Collecting gitdb<5,>=4.0.1 (from GitPython!=3.1.29,>=1.0.0->wandb)\n",
            "  Downloading gitdb-4.0.11-py3-none-any.whl (62 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (2023.11.17)\n",
            "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb)\n",
            "  Downloading smmap-5.0.1-py3-none-any.whl (24 kB)\n",
            "Installing collected packages: smmap, setproctitle, sentry-sdk, docker-pycreds, gitdb, GitPython, wandb\n",
            "Successfully installed GitPython-3.1.40 docker-pycreds-0.4.0 gitdb-4.0.11 sentry-sdk-1.38.0 setproctitle-1.3.3 smmap-5.0.1 wandb-0.16.1\n"
          ]
        }
      ],
      "source": [
        "!pip install wandb"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        },
        "id": "o6oLsIxnzWRm",
        "outputId": "4403bc4e-4fe1-4c9f-e6d6-869ecb768a5a"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
            "wandb: Paste an API key from your profile and hit enter, or press ctrl+c to quit:"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " ··········\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ],
      "source": [
        "import wandb\n",
        "wandb.login()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0Uq3-p6Szp70"
      },
      "source": [
        "### 3-2. 파인튜닝 수행"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XYqYIt4s0Wc7"
      },
      "source": [
        "#### `common_1` config"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ip3AY5MDCRnX",
        "outputId": "e08102f1-b052-4023-dff7-89f5566fc05d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[1;30;43m스트리밍 출력 내용이 길어서 마지막 5000줄이 삭제되었습니다.\u001b[0m\n",
            " 70% 11310/16260 [1:04:30<26:45,  3.08it/s]iter 11310: loss 5.5753, time: 424.54ms\n",
            " 70% 11311/16260 [1:04:30<29:16,  2.82it/s]iter 11311: loss 5.2879, time: 220.29ms\n",
            " 70% 11312/16260 [1:04:30<25:58,  3.18it/s]iter 11312: loss 4.3855, time: 214.95ms\n",
            " 70% 11313/16260 [1:04:31<23:31,  3.50it/s]iter 11313: loss 5.5795, time: 376.93ms\n",
            " 70% 11314/16260 [1:04:31<25:49,  3.19it/s]iter 11314: loss 5.4654, time: 315.87ms\n",
            " 70% 11315/16260 [1:04:31<25:54,  3.18it/s]iter 11315: loss 4.4411, time: 151.27ms\n",
            " 70% 11316/16260 [1:04:31<21:54,  3.76it/s]iter 11316: loss 6.0873, time: 464.06ms\n",
            " 70% 11317/16260 [1:04:32<26:50,  3.07it/s]iter 11317: loss 4.8359, time: 236.22ms\n",
            " 70% 11318/16260 [1:04:32<24:39,  3.34it/s]iter 11318: loss 5.1508, time: 275.15ms\n",
            " 70% 11319/16260 [1:04:32<24:05,  3.42it/s]iter 11319: loss 5.5632, time: 284.32ms\n",
            " 70% 11320/16260 [1:04:33<23:55,  3.44it/s]iter 11320: loss 4.8736, time: 209.30ms\n",
            " 70% 11321/16260 [1:04:33<21:56,  3.75it/s]iter 11321: loss 5.6823, time: 464.15ms\n",
            " 70% 11322/16260 [1:04:33<26:51,  3.06it/s]iter 11322: loss 4.4702, time: 213.32ms\n",
            " 70% 11323/16260 [1:04:34<24:05,  3.42it/s]iter 11323: loss 5.2579, time: 464.11ms\n",
            " 70% 11324/16260 [1:04:34<28:20,  2.90it/s]iter 11324: loss 5.3179, time: 308.79ms\n",
            " 70% 11325/16260 [1:04:34<27:29,  2.99it/s]iter 11325: loss 5.4276, time: 465.30ms\n",
            " 70% 11326/16260 [1:04:35<30:45,  2.67it/s]iter 11326: loss 4.9725, time: 275.54ms\n",
            " 70% 11327/16260 [1:04:35<28:21,  2.90it/s]iter 11327: loss 5.5689, time: 455.08ms\n",
            " 70% 11328/16260 [1:04:35<31:06,  2.64it/s]iter 11328: loss 6.0464, time: 463.75ms\n",
            " 70% 11329/16260 [1:04:36<33:14,  2.47it/s]iter 11329: loss 5.7153, time: 419.90ms\n",
            " 70% 11330/16260 [1:04:36<33:38,  2.44it/s]iter 11330: loss 4.3575, time: 190.91ms\n",
            " 70% 11331/16260 [1:04:37<28:17,  2.90it/s]iter 11331: loss 4.7184, time: 167.05ms\n",
            " 70% 11332/16260 [1:04:37<23:56,  3.43it/s]iter 11332: loss 5.1506, time: 253.82ms\n",
            " 70% 11333/16260 [1:04:37<23:02,  3.56it/s]iter 11333: loss 5.6467, time: 439.32ms\n",
            " 70% 11334/16260 [1:04:37<26:59,  3.04it/s]iter 11334: loss 5.2514, time: 464.20ms\n",
            " 70% 11335/16260 [1:04:38<30:21,  2.70it/s]iter 11335: loss 4.9537, time: 258.63ms\n",
            " 70% 11336/16260 [1:04:38<27:39,  2.97it/s]iter 11336: loss 5.5102, time: 464.10ms\n",
            " 70% 11337/16260 [1:04:39<30:48,  2.66it/s]iter 11337: loss 4.5988, time: 230.87ms\n",
            " 70% 11338/16260 [1:04:39<27:16,  3.01it/s]iter 11338: loss 5.5551, time: 464.00ms\n",
            " 70% 11339/16260 [1:04:39<30:32,  2.69it/s]iter 11339: loss 5.8751, time: 464.28ms\n",
            " 70% 11340/16260 [1:04:40<32:50,  2.50it/s]iter 11340: loss 5.5166, time: 444.16ms\n",
            " 70% 11341/16260 [1:04:40<33:56,  2.42it/s]iter 11341: loss 5.8046, time: 432.28ms\n",
            " 70% 11342/16260 [1:04:41<34:25,  2.38it/s]iter 11342: loss 5.2813, time: 259.60ms\n",
            " 70% 11343/16260 [1:04:41<30:31,  2.69it/s]iter 11343: loss 3.1148, time: 135.79ms\n",
            " 70% 11344/16260 [1:04:41<24:43,  3.31it/s]iter 11344: loss 4.6495, time: 428.19ms\n",
            " 70% 11345/16260 [1:04:41<27:52,  2.94it/s]iter 11345: loss 5.2847, time: 424.48ms\n",
            " 70% 11346/16260 [1:04:42<29:58,  2.73it/s]iter 11346: loss 4.3016, time: 168.22ms\n",
            " 70% 11347/16260 [1:04:42<25:08,  3.26it/s]iter 11347: loss 4.7495, time: 214.23ms\n",
            " 70% 11348/16260 [1:04:42<22:53,  3.58it/s]iter 11348: loss 4.4731, time: 170.62ms\n",
            " 70% 11349/16260 [1:04:42<20:15,  4.04it/s]iter 11349: loss 4.7719, time: 273.07ms\n",
            " 70% 11350/16260 [1:04:43<20:54,  3.91it/s]iter 11350: loss 5.3540, time: 463.81ms\n",
            " 70% 11351/16260 [1:04:43<26:03,  3.14it/s]iter 11351: loss 5.3366, time: 468.64ms\n",
            " 70% 11352/16260 [1:04:44<29:47,  2.75it/s]iter 11352: loss 4.1667, time: 172.12ms\n",
            " 70% 11353/16260 [1:04:44<25:06,  3.26it/s]iter 11353: loss 5.2629, time: 439.84ms\n",
            " 70% 11354/16260 [1:04:44<28:23,  2.88it/s]iter 11354: loss 5.2950, time: 464.47ms\n",
            " 70% 11355/16260 [1:04:45<31:18,  2.61it/s]iter 11355: loss 4.8607, time: 256.43ms\n",
            " 70% 11356/16260 [1:04:45<28:14,  2.89it/s]iter 11356: loss 4.4890, time: 214.92ms\n",
            " 70% 11357/16260 [1:04:45<25:04,  3.26it/s]iter 11357: loss 6.0136, time: 464.85ms\n",
            " 70% 11358/16260 [1:04:46<28:58,  2.82it/s]iter 11358: loss 5.1829, time: 273.71ms\n",
            " 70% 11359/16260 [1:04:46<27:01,  3.02it/s]iter 11359: loss 5.8128, time: 467.15ms\n",
            " 70% 11360/16260 [1:04:46<30:22,  2.69it/s]iter 11360: loss 4.7140, time: 309.23ms\n",
            " 70% 11361/16260 [1:04:47<28:52,  2.83it/s]iter 11361: loss 5.5342, time: 316.50ms\n",
            " 70% 11362/16260 [1:04:47<27:59,  2.92it/s]iter 11362: loss 4.7284, time: 164.69ms\n",
            " 70% 11363/16260 [1:04:47<23:39,  3.45it/s]iter 11363: loss 5.0707, time: 308.77ms\n",
            " 70% 11364/16260 [1:04:48<24:09,  3.38it/s]iter 11364: loss 5.2132, time: 306.74ms\n",
            " 70% 11365/16260 [1:04:48<24:27,  3.34it/s]iter 11365: loss 4.9875, time: 214.83ms\n",
            " 70% 11366/16260 [1:04:48<22:24,  3.64it/s]iter 11366: loss 3.7945, time: 351.28ms\n",
            " 70% 11367/16260 [1:04:48<24:19,  3.35it/s]iter 11367: loss 4.4439, time: 319.64ms\n",
            " 70% 11368/16260 [1:04:49<24:52,  3.28it/s]iter 11368: loss 4.8684, time: 236.70ms\n",
            " 70% 11369/16260 [1:04:49<23:13,  3.51it/s]iter 11369: loss 4.5967, time: 422.77ms\n",
            " 70% 11370/16260 [1:04:49<26:38,  3.06it/s]iter 11370: loss 4.4916, time: 185.71ms\n",
            " 70% 11371/16260 [1:04:50<23:12,  3.51it/s]iter 11371: loss 5.1917, time: 378.29ms\n",
            " 70% 11372/16260 [1:04:50<25:31,  3.19it/s]iter 11372: loss 4.5315, time: 233.49ms\n",
            " 70% 11373/16260 [1:04:50<23:36,  3.45it/s]iter 11373: loss 5.9430, time: 464.12ms\n",
            " 70% 11374/16260 [1:04:51<27:53,  2.92it/s]iter 11374: loss 6.0689, time: 463.43ms\n",
            " 70% 11375/16260 [1:04:51<30:52,  2.64it/s]iter 11375: loss 5.5428, time: 341.55ms\n",
            " 70% 11376/16260 [1:04:51<29:59,  2.71it/s]iter 11376: loss 4.7390, time: 293.27ms\n",
            " 70% 11377/16260 [1:04:52<28:11,  2.89it/s]iter 11377: loss 5.7277, time: 425.00ms\n",
            " 70% 11378/16260 [1:04:52<30:08,  2.70it/s]iter 11378: loss 5.4239, time: 315.18ms\n",
            " 70% 11379/16260 [1:04:53<28:49,  2.82it/s]iter 11379: loss 4.9584, time: 464.55ms\n",
            " 70% 11380/16260 [1:04:53<31:32,  2.58it/s]iter 11380: loss 3.8062, time: 146.38ms\n",
            " 70% 11381/16260 [1:04:53<25:41,  3.17it/s]iter 11381: loss 6.1686, time: 465.95ms\n",
            " 70% 11382/16260 [1:04:54<29:22,  2.77it/s]iter 11382: loss 3.8029, time: 148.37ms\n",
            " 70% 11383/16260 [1:04:54<24:12,  3.36it/s]iter 11383: loss 4.1521, time: 167.49ms\n",
            " 70% 11384/16260 [1:04:54<21:03,  3.86it/s]iter 11384: loss 4.8944, time: 250.39ms\n",
            " 70% 11385/16260 [1:04:54<20:52,  3.89it/s]iter 11385: loss 5.6765, time: 412.72ms\n",
            " 70% 11386/16260 [1:04:55<24:42,  3.29it/s]iter 11386: loss 4.7476, time: 186.93ms\n",
            " 70% 11387/16260 [1:04:55<21:53,  3.71it/s]iter 11387: loss 5.8160, time: 464.75ms\n",
            " 70% 11388/16260 [1:04:55<26:40,  3.04it/s]iter 11388: loss 4.4210, time: 149.03ms\n",
            " 70% 11389/16260 [1:04:55<22:20,  3.63it/s]iter 11389: loss 4.1730, time: 167.20ms\n",
            " 70% 11390/16260 [1:04:56<19:44,  4.11it/s]iter 11390: loss 5.3550, time: 466.65ms\n",
            " 70% 11391/16260 [1:04:56<25:13,  3.22it/s]iter 11391: loss 5.0168, time: 261.91ms\n",
            " 70% 11392/16260 [1:04:56<24:03,  3.37it/s]iter 11392: loss 4.6340, time: 208.02ms\n",
            " 70% 11393/16260 [1:04:57<21:55,  3.70it/s]iter 11393: loss 5.2557, time: 272.80ms\n",
            " 70% 11394/16260 [1:04:57<22:01,  3.68it/s]iter 11394: loss 4.6561, time: 260.63ms\n",
            " 70% 11395/16260 [1:04:57<21:47,  3.72it/s]iter 11395: loss 5.0007, time: 334.52ms\n",
            " 70% 11396/16260 [1:04:57<23:25,  3.46it/s]iter 11396: loss 3.7085, time: 170.11ms\n",
            " 70% 11397/16260 [1:04:58<20:34,  3.94it/s]iter 11397: loss 5.8551, time: 443.55ms\n",
            " 70% 11398/16260 [1:04:58<25:13,  3.21it/s]iter 11398: loss 5.7151, time: 341.21ms\n",
            " 70% 11399/16260 [1:04:58<25:58,  3.12it/s]iter 11399: loss 5.9273, time: 445.30ms\n",
            " 70% 11400/16260 [1:04:59<29:02,  2.79it/s]iter 11400: loss 5.3164, time: 464.55ms\n",
            " 70% 11401/16260 [1:04:59<31:39,  2.56it/s]iter 11401: loss 5.6425, time: 474.62ms\n",
            " 70% 11402/16260 [1:05:00<33:43,  2.40it/s]iter 11402: loss 5.5890, time: 473.64ms\n",
            " 70% 11403/16260 [1:05:00<35:08,  2.30it/s]iter 11403: loss 5.7487, time: 464.05ms\n",
            " 70% 11404/16260 [1:05:01<35:54,  2.25it/s]iter 11404: loss 6.0136, time: 463.94ms\n",
            " 70% 11405/16260 [1:05:01<36:25,  2.22it/s]iter 11405: loss 5.5429, time: 336.51ms\n",
            " 70% 11406/16260 [1:05:01<33:41,  2.40it/s]iter 11406: loss 5.0500, time: 242.96ms\n",
            " 70% 11407/16260 [1:05:02<29:30,  2.74it/s]iter 11407: loss 4.9318, time: 209.99ms\n",
            " 70% 11408/16260 [1:05:02<25:47,  3.14it/s]iter 11408: loss 4.9266, time: 262.59ms\n",
            " 70% 11409/16260 [1:05:02<24:27,  3.31it/s]iter 11409: loss 5.7866, time: 464.21ms\n",
            " 70% 11410/16260 [1:05:03<28:24,  2.85it/s]iter 11410: loss 5.8708, time: 464.95ms\n",
            " 70% 11411/16260 [1:05:03<31:11,  2.59it/s]iter 11411: loss 4.8173, time: 318.07ms\n",
            " 70% 11412/16260 [1:05:03<29:35,  2.73it/s]iter 11412: loss 5.7520, time: 237.26ms\n",
            " 70% 11413/16260 [1:05:04<26:29,  3.05it/s]iter 11413: loss 6.1709, time: 465.23ms\n",
            " 70% 11414/16260 [1:05:04<29:51,  2.70it/s]iter 11414: loss 5.1935, time: 464.42ms\n",
            " 70% 11415/16260 [1:05:05<32:11,  2.51it/s]iter 11415: loss 4.9115, time: 260.09ms\n",
            " 70% 11416/16260 [1:05:05<28:51,  2.80it/s]iter 11416: loss 5.0367, time: 470.76ms\n",
            " 70% 11417/16260 [1:05:05<31:38,  2.55it/s]iter 11417: loss 5.5772, time: 337.29ms\n",
            " 70% 11418/16260 [1:05:06<30:21,  2.66it/s]iter 11418: loss 5.1145, time: 272.46ms\n",
            " 70% 11419/16260 [1:05:06<27:52,  2.89it/s]iter 11419: loss 4.2343, time: 248.84ms\n",
            " 70% 11420/16260 [1:05:06<25:34,  3.15it/s]iter 11420: loss 5.5115, time: 464.34ms\n",
            " 70% 11421/16260 [1:05:07<29:10,  2.76it/s]iter 11421: loss 5.7916, time: 463.64ms\n",
            " 70% 11422/16260 [1:05:07<31:40,  2.55it/s]iter 11422: loss 5.6060, time: 463.73ms\n",
            " 70% 11423/16260 [1:05:08<33:25,  2.41it/s]iter 11423: loss 5.3477, time: 318.74ms\n",
            " 70% 11424/16260 [1:05:08<31:07,  2.59it/s]iter 11424: loss 5.1423, time: 471.00ms\n",
            " 70% 11425/16260 [1:05:08<33:12,  2.43it/s]iter 11425: loss 4.6197, time: 194.48ms\n",
            " 70% 11426/16260 [1:05:09<27:58,  2.88it/s]iter 11426: loss 4.0814, time: 214.00ms\n",
            " 70% 11427/16260 [1:05:09<24:47,  3.25it/s]iter 11427: loss 5.2978, time: 421.72ms\n",
            " 70% 11428/16260 [1:05:09<27:34,  2.92it/s]iter 11428: loss 4.0263, time: 166.34ms\n",
            " 70% 11429/16260 [1:05:09<23:21,  3.45it/s]iter 11429: loss 5.6019, time: 372.94ms\n",
            " 70% 11430/16260 [1:05:10<25:23,  3.17it/s]iter 11430: loss 5.7267, time: 464.76ms\n",
            " 70% 11431/16260 [1:05:10<29:01,  2.77it/s]iter 11431: loss 5.7090, time: 464.31ms\n",
            " 70% 11432/16260 [1:05:11<31:34,  2.55it/s]iter 11432: loss 5.1147, time: 464.82ms\n",
            " 70% 11433/16260 [1:05:11<33:20,  2.41it/s]iter 11433: loss 4.0056, time: 170.85ms\n",
            " 70% 11434/16260 [1:05:11<27:30,  2.92it/s]iter 11434: loss 4.2475, time: 464.04ms\n",
            " 70% 11435/16260 [1:05:12<30:29,  2.64it/s]iter 11435: loss 5.0540, time: 209.93ms\n",
            " 70% 11436/16260 [1:05:12<26:26,  3.04it/s]iter 11436: loss 5.3305, time: 464.51ms\n",
            " 70% 11437/16260 [1:05:12<29:44,  2.70it/s]iter 11437: loss 5.7812, time: 464.09ms\n",
            " 70% 11438/16260 [1:05:13<32:02,  2.51it/s]iter 11438: loss 5.9162, time: 464.28ms\n",
            " 70% 11439/16260 [1:05:13<33:38,  2.39it/s]iter 11439: loss 6.0131, time: 463.72ms\n",
            " 70% 11440/16260 [1:05:14<34:45,  2.31it/s]iter 11440: loss 4.5947, time: 274.81ms\n",
            " 70% 11441/16260 [1:05:14<30:59,  2.59it/s]iter 11441: loss 4.4403, time: 252.93ms\n",
            " 70% 11442/16260 [1:05:14<27:48,  2.89it/s]iter 11442: loss 5.4772, time: 436.99ms\n",
            " 70% 11443/16260 [1:05:15<30:01,  2.67it/s]iter 11443: loss 6.0261, time: 441.08ms\n",
            " 70% 11444/16260 [1:05:15<31:39,  2.53it/s]iter 11444: loss 5.0690, time: 463.61ms\n",
            " 70% 11445/16260 [1:05:16<33:21,  2.41it/s]iter 11445: loss 5.6904, time: 463.43ms\n",
            " 70% 11446/16260 [1:05:16<34:55,  2.30it/s]iter 11446: loss 4.4864, time: 318.28ms\n",
            " 70% 11447/16260 [1:05:17<32:08,  2.50it/s]iter 11447: loss 5.5698, time: 463.78ms\n",
            " 70% 11448/16260 [1:05:17<33:41,  2.38it/s]iter 11448: loss 4.8362, time: 168.67ms\n",
            " 70% 11449/16260 [1:05:17<27:40,  2.90it/s]iter 11449: loss 5.5669, time: 464.03ms\n",
            " 70% 11450/16260 [1:05:18<30:34,  2.62it/s]iter 11450: loss 5.5465, time: 464.31ms\n",
            " 70% 11451/16260 [1:05:18<32:35,  2.46it/s]iter 11451: loss 4.3257, time: 167.26ms\n",
            " 70% 11452/16260 [1:05:18<26:51,  2.98it/s]iter 11452: loss 5.4730, time: 275.36ms\n",
            " 70% 11453/16260 [1:05:19<25:27,  3.15it/s]iter 11453: loss 4.6671, time: 381.98ms\n",
            " 70% 11454/16260 [1:05:19<27:01,  2.96it/s]iter 11454: loss 5.6585, time: 474.20ms\n",
            " 70% 11455/16260 [1:05:19<30:20,  2.64it/s]iter 11455: loss 5.4727, time: 314.25ms\n",
            " 70% 11456/16260 [1:05:20<28:48,  2.78it/s]iter 11456: loss 4.6336, time: 150.21ms\n",
            " 70% 11457/16260 [1:05:20<23:48,  3.36it/s]iter 11457: loss 5.0867, time: 255.24ms\n",
            " 70% 11458/16260 [1:05:20<22:49,  3.51it/s]iter 11458: loss 5.3095, time: 273.26ms\n",
            " 70% 11459/16260 [1:05:20<22:33,  3.55it/s]iter 11459: loss 5.3947, time: 464.62ms\n",
            " 70% 11460/16260 [1:05:21<26:58,  2.96it/s]iter 11460: loss 5.7599, time: 278.63ms\n",
            " 70% 11461/16260 [1:05:21<25:36,  3.12it/s]iter 11461: loss 4.2589, time: 137.06ms\n",
            " 70% 11462/16260 [1:05:21<21:15,  3.76it/s]iter 11462: loss 5.3621, time: 423.94ms\n",
            " 70% 11463/16260 [1:05:22<25:04,  3.19it/s]iter 11463: loss 5.7295, time: 464.53ms\n",
            " 71% 11464/16260 [1:05:22<28:43,  2.78it/s]iter 11464: loss 4.6162, time: 186.54ms\n",
            " 71% 11465/16260 [1:05:22<24:36,  3.25it/s]iter 11465: loss 5.8495, time: 380.26ms\n",
            " 71% 11466/16260 [1:05:23<26:22,  3.03it/s]iter 11466: loss 5.2932, time: 464.43ms\n",
            " 71% 11467/16260 [1:05:23<29:37,  2.70it/s]iter 11467: loss 5.2397, time: 478.55ms\n",
            " 71% 11468/16260 [1:05:24<32:14,  2.48it/s]iter 11468: loss 5.5889, time: 368.15ms\n",
            " 71% 11469/16260 [1:05:24<31:25,  2.54it/s]iter 11469: loss 5.3495, time: 212.60ms\n",
            " 71% 11470/16260 [1:05:24<27:06,  2.94it/s]iter 11470: loss 5.2369, time: 421.54ms\n",
            " 71% 11471/16260 [1:05:25<29:06,  2.74it/s]iter 11471: loss 4.2798, time: 232.35ms\n",
            " 71% 11472/16260 [1:05:25<25:58,  3.07it/s]iter 11472: loss 5.0029, time: 250.06ms\n",
            " 71% 11473/16260 [1:05:25<24:11,  3.30it/s]iter 11473: loss 5.3411, time: 309.04ms\n",
            " 71% 11474/16260 [1:05:26<24:22,  3.27it/s]iter 11474: loss 4.9206, time: 190.56ms\n",
            " 71% 11475/16260 [1:05:26<21:38,  3.68it/s]iter 11475: loss 5.7883, time: 463.69ms\n",
            " 71% 11476/16260 [1:05:26<26:16,  3.03it/s]iter 11476: loss 5.7083, time: 331.34ms\n",
            " 71% 11477/16260 [1:05:27<26:20,  3.03it/s]iter 11477: loss 4.9562, time: 237.80ms\n",
            " 71% 11478/16260 [1:05:27<24:10,  3.30it/s]iter 11478: loss 5.9275, time: 368.63ms\n",
            " 71% 11479/16260 [1:05:27<25:45,  3.09it/s]iter 11479: loss 4.7270, time: 232.84ms\n",
            " 71% 11480/16260 [1:05:27<23:38,  3.37it/s]iter 11480: loss 5.5361, time: 464.59ms\n",
            " 71% 11481/16260 [1:05:28<27:40,  2.88it/s]iter 11481: loss 4.4497, time: 169.67ms\n",
            " 71% 11482/16260 [1:05:28<23:28,  3.39it/s]iter 11482: loss 4.8159, time: 237.19ms\n",
            " 71% 11483/16260 [1:05:28<22:07,  3.60it/s]iter 11483: loss 5.1854, time: 481.93ms\n",
            " 71% 11484/16260 [1:05:29<27:02,  2.94it/s]iter 11484: loss 5.6027, time: 463.80ms\n",
            " 71% 11485/16260 [1:05:29<30:01,  2.65it/s]iter 11485: loss 4.9256, time: 255.18ms\n",
            " 71% 11486/16260 [1:05:29<27:08,  2.93it/s]iter 11486: loss 5.6566, time: 473.68ms\n",
            " 71% 11487/16260 [1:05:30<30:20,  2.62it/s]iter 11487: loss 4.9917, time: 219.40ms\n",
            " 71% 11488/16260 [1:05:30<26:29,  3.00it/s]iter 11488: loss 5.7037, time: 464.12ms\n",
            " 71% 11489/16260 [1:05:31<29:38,  2.68it/s]iter 11489: loss 6.0686, time: 464.40ms\n",
            " 71% 11490/16260 [1:05:31<31:51,  2.50it/s]iter 11490: loss 6.1735, time: 464.78ms\n",
            " 71% 11491/16260 [1:05:32<33:24,  2.38it/s]iter 11491: loss 4.3630, time: 175.09ms\n",
            " 71% 11492/16260 [1:05:32<27:35,  2.88it/s]iter 11492: loss 5.2459, time: 217.82ms\n",
            " 71% 11493/16260 [1:05:32<24:32,  3.24it/s]iter 11493: loss 5.0732, time: 463.95ms\n",
            " 71% 11494/16260 [1:05:32<28:15,  2.81it/s]iter 11494: loss 4.7883, time: 335.96ms\n",
            " 71% 11495/16260 [1:05:33<27:48,  2.86it/s]iter 11495: loss 4.6751, time: 274.05ms\n",
            " 71% 11496/16260 [1:05:33<26:01,  3.05it/s]iter 11496: loss 5.3654, time: 310.22ms\n",
            " 71% 11497/16260 [1:05:33<25:38,  3.10it/s]iter 11497: loss 5.7811, time: 338.74ms\n",
            " 71% 11498/16260 [1:05:34<26:02,  3.05it/s]iter 11498: loss 5.4818, time: 274.01ms\n",
            " 71% 11499/16260 [1:05:34<24:46,  3.20it/s]iter 11499: loss 4.2848, time: 238.43ms\n",
            " 71% 11500/16260 [1:05:34<23:02,  3.44it/s]iter 11500: loss 5.9108, time: 377.96ms\n",
            " 71% 11501/16260 [1:05:35<25:09,  3.15it/s]iter 11501: loss 5.6172, time: 345.71ms\n",
            " 71% 11502/16260 [1:05:35<25:52,  3.07it/s]iter 11502: loss 5.2451, time: 462.87ms\n",
            " 71% 11503/16260 [1:05:35<29:09,  2.72it/s]iter 11503: loss 5.6353, time: 437.56ms\n",
            " 71% 11504/16260 [1:05:36<30:50,  2.57it/s]iter 11504: loss 4.4652, time: 186.18ms\n",
            " 71% 11505/16260 [1:05:36<26:02,  3.04it/s]iter 11505: loss 4.5119, time: 169.48ms\n",
            " 71% 11506/16260 [1:05:36<22:16,  3.56it/s]iter 11506: loss 5.8955, time: 415.77ms\n",
            " 71% 11507/16260 [1:05:37<25:30,  3.11it/s]iter 11507: loss 4.9196, time: 228.17ms\n",
            " 71% 11508/16260 [1:05:37<23:18,  3.40it/s]iter 11508: loss 5.4282, time: 464.52ms\n",
            " 71% 11509/16260 [1:05:37<27:22,  2.89it/s]iter 11509: loss 5.3151, time: 315.21ms\n",
            " 71% 11510/16260 [1:05:38<26:40,  2.97it/s]iter 11510: loss 5.4153, time: 375.95ms\n",
            " 71% 11511/16260 [1:05:38<27:38,  2.86it/s]iter 11511: loss 5.4961, time: 464.15ms\n",
            " 71% 11512/16260 [1:05:38<30:23,  2.60it/s]iter 11512: loss 4.2168, time: 152.28ms\n",
            " 71% 11513/16260 [1:05:39<24:55,  3.17it/s]iter 11513: loss 4.6492, time: 236.13ms\n",
            " 71% 11514/16260 [1:05:39<23:04,  3.43it/s]iter 11514: loss 5.3911, time: 232.50ms\n",
            " 71% 11515/16260 [1:05:39<21:42,  3.64it/s]iter 11515: loss 5.6414, time: 306.38ms\n",
            " 71% 11516/16260 [1:05:39<22:29,  3.51it/s]iter 11516: loss 4.6547, time: 255.68ms\n",
            " 71% 11517/16260 [1:05:40<21:50,  3.62it/s]iter 11517: loss 5.6072, time: 346.02ms\n",
            " 71% 11518/16260 [1:05:40<23:31,  3.36it/s]iter 11518: loss 5.5057, time: 334.23ms\n",
            " 71% 11519/16260 [1:05:40<24:25,  3.23it/s]iter 11519: loss 5.6749, time: 423.13ms\n",
            " 71% 11520/16260 [1:05:41<27:09,  2.91it/s]iter 11520: loss 5.3917, time: 323.93ms\n",
            " 71% 11521/16260 [1:05:41<26:42,  2.96it/s]iter 11521: loss 4.7859, time: 464.01ms\n",
            " 71% 11522/16260 [1:05:42<29:43,  2.66it/s]iter 11522: loss 4.8507, time: 168.67ms\n",
            " 71% 11523/16260 [1:05:42<24:50,  3.18it/s]iter 11523: loss 5.1623, time: 308.58ms\n",
            " 71% 11524/16260 [1:05:42<24:43,  3.19it/s]iter 11524: loss 5.4409, time: 305.64ms\n",
            " 71% 11525/16260 [1:05:42<24:34,  3.21it/s]iter 11525: loss 6.0777, time: 464.13ms\n",
            " 71% 11526/16260 [1:05:43<28:12,  2.80it/s]iter 11526: loss 5.1843, time: 422.17ms\n",
            " 71% 11527/16260 [1:05:43<29:46,  2.65it/s]iter 11527: loss 2.1788, time: 142.12ms\n",
            " 71% 11528/16260 [1:05:43<24:14,  3.25it/s]iter 11528: loss 5.9128, time: 464.60ms\n",
            " 71% 11529/16260 [1:05:44<27:59,  2.82it/s]iter 11529: loss 5.6047, time: 463.59ms\n",
            " 71% 11530/16260 [1:05:44<30:35,  2.58it/s]iter 11530: loss 5.3968, time: 282.20ms\n",
            " 71% 11531/16260 [1:05:45<28:06,  2.80it/s]iter 11531: loss 5.8205, time: 463.49ms\n",
            " 71% 11532/16260 [1:05:45<30:39,  2.57it/s]iter 11532: loss 5.5232, time: 341.06ms\n",
            " 71% 11533/16260 [1:05:45<29:33,  2.67it/s]iter 11533: loss 4.2305, time: 149.61ms\n",
            " 71% 11534/16260 [1:05:46<24:15,  3.25it/s]iter 11534: loss 5.9686, time: 377.60ms\n",
            " 71% 11535/16260 [1:05:46<25:56,  3.04it/s]iter 11535: loss 5.6026, time: 238.31ms\n",
            " 71% 11536/16260 [1:05:46<23:48,  3.31it/s]iter 11536: loss 6.1020, time: 463.53ms\n",
            " 71% 11537/16260 [1:05:47<27:39,  2.85it/s]iter 11537: loss 5.1525, time: 332.07ms\n",
            " 71% 11538/16260 [1:05:47<27:13,  2.89it/s]iter 11538: loss 3.7850, time: 163.72ms\n",
            " 71% 11539/16260 [1:05:47<22:57,  3.43it/s]iter 11539: loss 5.2486, time: 463.61ms\n",
            " 71% 11540/16260 [1:05:48<27:02,  2.91it/s]iter 11540: loss 5.7255, time: 471.49ms\n",
            " 71% 11541/16260 [1:05:48<30:05,  2.61it/s]iter 11541: loss 4.9222, time: 237.07ms\n",
            " 71% 11542/16260 [1:05:48<26:41,  2.95it/s]iter 11542: loss 4.6559, time: 253.23ms\n",
            " 71% 11543/16260 [1:05:49<24:40,  3.19it/s]iter 11543: loss 5.2685, time: 465.10ms\n",
            " 71% 11544/16260 [1:05:49<28:16,  2.78it/s]iter 11544: loss 4.5261, time: 170.84ms\n",
            " 71% 11545/16260 [1:05:49<23:51,  3.29it/s]iter 11545: loss 5.3152, time: 253.58ms\n",
            " 71% 11546/16260 [1:05:49<22:42,  3.46it/s]iter 11546: loss 5.3448, time: 436.54ms\n",
            " 71% 11547/16260 [1:05:50<26:13,  3.00it/s]iter 11547: loss 5.9715, time: 463.95ms\n",
            " 71% 11548/16260 [1:05:50<29:18,  2.68it/s]iter 11548: loss 5.0387, time: 310.56ms\n",
            " 71% 11549/16260 [1:05:51<27:51,  2.82it/s]iter 11549: loss 6.0021, time: 418.72ms\n",
            " 71% 11550/16260 [1:05:51<29:23,  2.67it/s]iter 11550: loss 4.8480, time: 207.53ms\n",
            " 71% 11551/16260 [1:05:51<25:29,  3.08it/s]iter 11551: loss 6.1890, time: 468.15ms\n",
            " 71% 11552/16260 [1:05:52<28:53,  2.72it/s]iter 11552: loss 5.2424, time: 463.83ms\n",
            " 71% 11553/16260 [1:05:52<31:10,  2.52it/s]iter 11553: loss 5.8244, time: 463.79ms\n",
            " 71% 11554/16260 [1:05:53<32:46,  2.39it/s]iter 11554: loss 4.7978, time: 252.42ms\n",
            " 71% 11555/16260 [1:05:53<28:54,  2.71it/s]iter 11555: loss 5.6460, time: 472.67ms\n",
            " 71% 11556/16260 [1:05:53<31:23,  2.50it/s]iter 11556: loss 6.0100, time: 464.17ms\n",
            " 71% 11557/16260 [1:05:54<32:55,  2.38it/s]iter 11557: loss 4.3658, time: 289.80ms\n",
            " 71% 11558/16260 [1:05:54<29:53,  2.62it/s]iter 11558: loss 5.8556, time: 464.34ms\n",
            " 71% 11559/16260 [1:05:55<31:52,  2.46it/s]iter 11559: loss 4.5666, time: 238.55ms\n",
            " 71% 11560/16260 [1:05:55<27:56,  2.80it/s]iter 11560: loss 5.7406, time: 306.85ms\n",
            " 71% 11561/16260 [1:05:55<26:48,  2.92it/s]iter 11561: loss 5.1939, time: 282.20ms\n",
            " 71% 11562/16260 [1:05:55<25:25,  3.08it/s]iter 11562: loss 4.7841, time: 208.13ms\n",
            " 71% 11563/16260 [1:05:56<22:42,  3.45it/s]iter 11563: loss 6.0468, time: 463.94ms\n",
            " 71% 11564/16260 [1:05:56<26:49,  2.92it/s]iter 11564: loss 5.2022, time: 315.52ms\n",
            " 71% 11565/16260 [1:05:56<26:12,  2.99it/s]iter 11565: loss 5.2179, time: 371.06ms\n",
            " 71% 11566/16260 [1:05:57<27:05,  2.89it/s]iter 11566: loss 4.2037, time: 187.42ms\n",
            " 71% 11567/16260 [1:05:57<23:23,  3.34it/s]iter 11567: loss 4.7560, time: 434.43ms\n",
            " 71% 11568/16260 [1:05:57<26:35,  2.94it/s]iter 11568: loss 3.9463, time: 172.76ms\n",
            " 71% 11569/16260 [1:05:58<22:42,  3.44it/s]iter 11569: loss 4.6126, time: 193.48ms\n",
            " 71% 11570/16260 [1:05:58<20:27,  3.82it/s]iter 11570: loss 5.1408, time: 314.56ms\n",
            " 71% 11571/16260 [1:05:58<21:43,  3.60it/s]iter 11571: loss 5.7752, time: 462.33ms\n",
            " 71% 11572/16260 [1:05:59<26:04,  3.00it/s]iter 11572: loss 4.9722, time: 345.21ms\n",
            " 71% 11573/16260 [1:05:59<26:22,  2.96it/s]iter 11573: loss 5.6133, time: 432.59ms\n",
            " 71% 11574/16260 [1:05:59<28:38,  2.73it/s]iter 11574: loss 5.1212, time: 256.23ms\n",
            " 71% 11575/16260 [1:06:00<26:04,  2.99it/s]iter 11575: loss 5.9681, time: 442.17ms\n",
            " 71% 11576/16260 [1:06:00<28:38,  2.73it/s]iter 11576: loss 4.2272, time: 190.36ms\n",
            " 71% 11577/16260 [1:06:00<24:32,  3.18it/s]iter 11577: loss 6.2002, time: 464.05ms\n",
            " 71% 11578/16260 [1:06:01<28:04,  2.78it/s]iter 11578: loss 4.7291, time: 239.81ms\n",
            " 71% 11579/16260 [1:06:01<25:17,  3.08it/s]iter 11579: loss 5.4581, time: 377.61ms\n",
            " 71% 11580/16260 [1:06:01<26:34,  2.93it/s]iter 11580: loss 4.3374, time: 234.07ms\n",
            " 71% 11581/16260 [1:06:02<24:06,  3.23it/s]iter 11581: loss 5.9760, time: 464.79ms\n",
            " 71% 11582/16260 [1:06:02<27:46,  2.81it/s]iter 11582: loss 5.6503, time: 421.24ms\n",
            " 71% 11583/16260 [1:06:02<29:19,  2.66it/s]iter 11583: loss 5.3841, time: 446.36ms\n",
            " 71% 11584/16260 [1:06:03<30:59,  2.52it/s]iter 11584: loss 4.8138, time: 181.93ms\n",
            " 71% 11585/16260 [1:06:03<25:57,  3.00it/s]iter 11585: loss 4.8089, time: 306.37ms\n",
            " 71% 11586/16260 [1:06:03<25:21,  3.07it/s]iter 11586: loss 5.2790, time: 318.56ms\n",
            " 71% 11587/16260 [1:06:04<25:13,  3.09it/s]iter 11587: loss 5.6590, time: 463.36ms\n",
            " 71% 11588/16260 [1:06:04<28:31,  2.73it/s]iter 11588: loss 4.7710, time: 262.23ms\n",
            " 71% 11589/16260 [1:06:04<26:07,  2.98it/s]iter 11589: loss 4.7939, time: 191.29ms\n",
            " 71% 11590/16260 [1:06:05<22:46,  3.42it/s]iter 11590: loss 5.9578, time: 465.04ms\n",
            " 71% 11591/16260 [1:06:05<26:50,  2.90it/s]iter 11591: loss 6.1354, time: 463.39ms\n",
            " 71% 11592/16260 [1:06:06<29:37,  2.63it/s]iter 11592: loss 5.1452, time: 463.66ms\n",
            " 71% 11593/16260 [1:06:06<31:35,  2.46it/s]iter 11593: loss 4.7257, time: 188.59ms\n",
            " 71% 11594/16260 [1:06:06<26:32,  2.93it/s]iter 11594: loss 4.6666, time: 238.24ms\n",
            " 71% 11595/16260 [1:06:07<24:09,  3.22it/s]iter 11595: loss 5.0951, time: 464.71ms\n",
            " 71% 11596/16260 [1:06:07<27:47,  2.80it/s]iter 11596: loss 5.7608, time: 465.14ms\n",
            " 71% 11597/16260 [1:06:07<30:19,  2.56it/s]iter 11597: loss 4.4393, time: 209.70ms\n",
            " 71% 11598/16260 [1:06:08<26:09,  2.97it/s]iter 11598: loss 5.1785, time: 211.63ms\n",
            " 71% 11599/16260 [1:06:08<23:16,  3.34it/s]iter 11599: loss 4.9913, time: 369.43ms\n",
            " 71% 11600/16260 [1:06:08<24:55,  3.12it/s]iter 11600: loss 4.7137, time: 185.14ms\n",
            " 71% 11601/16260 [1:06:08<21:47,  3.56it/s]iter 11601: loss 5.1655, time: 339.20ms\n",
            " 71% 11602/16260 [1:06:09<23:11,  3.35it/s]iter 11602: loss 5.2991, time: 465.35ms\n",
            " 71% 11603/16260 [1:06:09<27:05,  2.86it/s]iter 11603: loss 5.0841, time: 470.83ms\n",
            " 71% 11604/16260 [1:06:10<29:57,  2.59it/s]iter 11604: loss 4.9766, time: 230.58ms\n",
            " 71% 11605/16260 [1:06:10<26:21,  2.94it/s]iter 11605: loss 6.1547, time: 463.81ms\n",
            " 71% 11606/16260 [1:06:10<29:16,  2.65it/s]iter 11606: loss 5.5339, time: 310.60ms\n",
            " 71% 11607/16260 [1:06:11<27:44,  2.79it/s]iter 11607: loss 4.3359, time: 189.02ms\n",
            " 71% 11608/16260 [1:06:11<23:51,  3.25it/s]iter 11608: loss 4.9350, time: 257.39ms\n",
            " 71% 11609/16260 [1:06:11<22:42,  3.41it/s]iter 11609: loss 5.0891, time: 256.46ms\n",
            " 71% 11610/16260 [1:06:11<21:53,  3.54it/s]iter 11610: loss 5.6501, time: 309.85ms\n",
            " 71% 11611/16260 [1:06:12<22:33,  3.43it/s]iter 11611: loss 5.1682, time: 278.07ms\n",
            " 71% 11612/16260 [1:06:12<22:16,  3.48it/s]iter 11612: loss 5.4125, time: 423.21ms\n",
            " 71% 11613/16260 [1:06:12<25:27,  3.04it/s]iter 11613: loss 5.6159, time: 248.16ms\n",
            " 71% 11614/16260 [1:06:13<23:37,  3.28it/s]iter 11614: loss 5.4113, time: 422.00ms\n",
            " 71% 11615/16260 [1:06:13<26:22,  2.93it/s]iter 11615: loss 5.0651, time: 239.16ms\n",
            " 71% 11616/16260 [1:06:13<24:02,  3.22it/s]iter 11616: loss 5.8041, time: 345.36ms\n",
            " 71% 11617/16260 [1:06:14<24:52,  3.11it/s]iter 11617: loss 5.4645, time: 370.60ms\n",
            " 71% 11618/16260 [1:06:14<26:02,  2.97it/s]iter 11618: loss 5.3206, time: 464.28ms\n",
            " 71% 11619/16260 [1:06:15<29:02,  2.66it/s]iter 11619: loss 4.9881, time: 213.47ms\n",
            " 71% 11620/16260 [1:06:15<25:18,  3.06it/s]iter 11620: loss 3.8696, time: 146.47ms\n",
            " 71% 11621/16260 [1:06:15<21:08,  3.66it/s]iter 11621: loss 4.7055, time: 191.41ms\n",
            " 71% 11622/16260 [1:06:15<19:15,  4.01it/s]iter 11622: loss 5.2393, time: 463.53ms\n",
            " 71% 11623/16260 [1:06:16<24:15,  3.19it/s]iter 11623: loss 5.7882, time: 446.07ms\n",
            " 71% 11624/16260 [1:06:16<27:21,  2.82it/s]iter 11624: loss 5.8412, time: 465.36ms\n",
            " 71% 11625/16260 [1:06:16<29:58,  2.58it/s]iter 11625: loss 4.9693, time: 190.25ms\n",
            " 72% 11626/16260 [1:06:17<25:27,  3.03it/s]iter 11626: loss 5.4204, time: 257.55ms\n",
            " 72% 11627/16260 [1:06:17<23:48,  3.24it/s]iter 11627: loss 2.9277, time: 132.85ms\n",
            " 72% 11628/16260 [1:06:17<19:46,  3.90it/s]iter 11628: loss 5.0898, time: 337.70ms\n",
            " 72% 11629/16260 [1:06:17<21:41,  3.56it/s]iter 11629: loss 5.2275, time: 468.50ms\n",
            " 72% 11630/16260 [1:06:18<26:04,  2.96it/s]iter 11630: loss 5.2198, time: 335.05ms\n",
            " 72% 11631/16260 [1:06:18<26:01,  2.96it/s]iter 11631: loss 4.4148, time: 187.55ms\n",
            " 72% 11632/16260 [1:06:18<22:35,  3.41it/s]iter 11632: loss 5.9733, time: 473.66ms\n",
            " 72% 11633/16260 [1:06:19<26:48,  2.88it/s]iter 11633: loss 5.7035, time: 334.33ms\n",
            " 72% 11634/16260 [1:06:19<26:32,  2.91it/s]iter 11634: loss 4.5849, time: 191.57ms\n",
            " 72% 11635/16260 [1:06:19<23:02,  3.35it/s]iter 11635: loss 4.3685, time: 148.62ms\n",
            " 72% 11636/16260 [1:06:20<19:35,  3.93it/s]iter 11636: loss 4.5879, time: 216.54ms\n",
            " 72% 11637/16260 [1:06:20<18:45,  4.11it/s]iter 11637: loss 5.1211, time: 262.40ms\n",
            " 72% 11638/16260 [1:06:20<19:13,  4.01it/s]iter 11638: loss 4.1139, time: 149.35ms\n",
            " 72% 11639/16260 [1:06:20<16:56,  4.55it/s]iter 11639: loss 5.8217, time: 465.25ms\n",
            " 72% 11640/16260 [1:06:21<22:38,  3.40it/s]iter 11640: loss 5.7123, time: 469.09ms\n",
            " 72% 11641/16260 [1:06:21<26:42,  2.88it/s]iter 11641: loss 5.5353, time: 464.48ms\n",
            " 72% 11642/16260 [1:06:22<29:27,  2.61it/s]iter 11642: loss 4.1151, time: 152.52ms\n",
            " 72% 11643/16260 [1:06:22<24:10,  3.18it/s]iter 11643: loss 4.4830, time: 238.93ms\n",
            " 72% 11644/16260 [1:06:22<22:28,  3.42it/s]iter 11644: loss 5.2982, time: 308.50ms\n",
            " 72% 11645/16260 [1:06:22<22:53,  3.36it/s]iter 11645: loss 4.3317, time: 234.27ms\n",
            " 72% 11646/16260 [1:06:23<21:27,  3.58it/s]iter 11646: loss 5.8196, time: 445.61ms\n",
            " 72% 11647/16260 [1:06:23<25:19,  3.04it/s]iter 11647: loss 4.4162, time: 171.76ms\n",
            " 72% 11648/16260 [1:06:23<21:42,  3.54it/s]iter 11648: loss 5.1114, time: 281.18ms\n",
            " 72% 11649/16260 [1:06:23<21:42,  3.54it/s]iter 11649: loss 5.6785, time: 370.84ms\n",
            " 72% 11650/16260 [1:06:24<23:46,  3.23it/s]iter 11650: loss 5.4431, time: 465.85ms\n",
            " 72% 11651/16260 [1:06:24<27:24,  2.80it/s]iter 11651: loss 5.8840, time: 439.59ms\n",
            " 72% 11652/16260 [1:06:25<29:21,  2.62it/s]iter 11652: loss 5.1409, time: 445.17ms\n",
            " 72% 11653/16260 [1:06:25<30:49,  2.49it/s]iter 11653: loss 5.5242, time: 464.49ms\n",
            " 72% 11654/16260 [1:06:26<32:18,  2.38it/s]iter 11654: loss 3.0264, time: 135.14ms\n",
            " 72% 11655/16260 [1:06:26<25:45,  2.98it/s]iter 11655: loss 4.8716, time: 212.93ms\n",
            " 72% 11656/16260 [1:06:26<22:57,  3.34it/s]iter 11656: loss 6.1019, time: 464.53ms\n",
            " 72% 11657/16260 [1:06:26<26:47,  2.86it/s]iter 11657: loss 4.7783, time: 463.39ms\n",
            " 72% 11658/16260 [1:06:27<29:27,  2.60it/s]iter 11658: loss 5.3161, time: 208.35ms\n",
            " 72% 11659/16260 [1:06:27<25:26,  3.01it/s]iter 11659: loss 4.5903, time: 237.83ms\n",
            " 72% 11660/16260 [1:06:27<23:18,  3.29it/s]iter 11660: loss 5.4051, time: 464.26ms\n",
            " 72% 11661/16260 [1:06:28<27:01,  2.84it/s]iter 11661: loss 4.0138, time: 146.73ms\n",
            " 72% 11662/16260 [1:06:28<22:18,  3.43it/s]iter 11662: loss 5.4779, time: 463.87ms\n",
            " 72% 11663/16260 [1:06:28<26:18,  2.91it/s]iter 11663: loss 5.1658, time: 309.75ms\n",
            " 72% 11664/16260 [1:06:29<25:34,  3.00it/s]iter 11664: loss 5.3983, time: 471.84ms\n",
            " 72% 11665/16260 [1:06:29<28:46,  2.66it/s]iter 11665: loss 5.6010, time: 380.88ms\n",
            " 72% 11666/16260 [1:06:30<28:55,  2.65it/s]iter 11666: loss 6.1407, time: 436.44ms\n",
            " 72% 11667/16260 [1:06:30<30:17,  2.53it/s]iter 11667: loss 4.9756, time: 314.28ms\n",
            " 72% 11668/16260 [1:06:30<28:27,  2.69it/s]iter 11668: loss 6.1683, time: 464.12ms\n",
            " 72% 11669/16260 [1:06:31<30:36,  2.50it/s]iter 11669: loss 5.2782, time: 346.26ms\n",
            " 72% 11670/16260 [1:06:31<29:23,  2.60it/s]iter 11670: loss 4.6414, time: 464.39ms\n",
            " 72% 11671/16260 [1:06:32<31:16,  2.45it/s]iter 11671: loss 4.8831, time: 257.35ms\n",
            " 72% 11672/16260 [1:06:32<27:49,  2.75it/s]iter 11672: loss 5.4103, time: 307.07ms\n",
            " 72% 11673/16260 [1:06:32<26:32,  2.88it/s]iter 11673: loss 5.3233, time: 237.59ms\n",
            " 72% 11674/16260 [1:06:32<24:03,  3.18it/s]iter 11674: loss 4.6390, time: 337.09ms\n",
            " 72% 11675/16260 [1:06:33<24:36,  3.11it/s]iter 11675: loss 4.9991, time: 235.35ms\n",
            " 72% 11676/16260 [1:06:33<22:38,  3.37it/s]iter 11676: loss 4.3082, time: 234.56ms\n",
            " 72% 11677/16260 [1:06:33<21:15,  3.59it/s]iter 11677: loss 5.3958, time: 464.47ms\n",
            " 72% 11678/16260 [1:06:34<25:33,  2.99it/s]iter 11678: loss 5.7757, time: 463.65ms\n",
            " 72% 11679/16260 [1:06:34<28:32,  2.68it/s]iter 11679: loss 5.7712, time: 467.28ms\n",
            " 72% 11680/16260 [1:06:35<30:41,  2.49it/s]iter 11680: loss 5.3322, time: 257.63ms\n",
            " 72% 11681/16260 [1:06:35<27:24,  2.78it/s]iter 11681: loss 6.0635, time: 468.29ms\n",
            " 72% 11682/16260 [1:06:35<29:56,  2.55it/s]iter 11682: loss 2.3995, time: 145.63ms\n",
            " 72% 11683/16260 [1:06:36<24:19,  3.14it/s]iter 11683: loss 5.4324, time: 342.27ms\n",
            " 72% 11684/16260 [1:06:36<24:53,  3.06it/s]iter 11684: loss 4.7062, time: 188.57ms\n",
            " 72% 11685/16260 [1:06:36<21:45,  3.50it/s]iter 11685: loss 4.6429, time: 462.95ms\n",
            " 72% 11686/16260 [1:06:37<25:51,  2.95it/s]iter 11686: loss 4.8645, time: 279.94ms\n",
            " 72% 11687/16260 [1:06:37<24:31,  3.11it/s]iter 11687: loss 4.4297, time: 209.34ms\n",
            " 72% 11688/16260 [1:06:37<21:59,  3.46it/s]iter 11688: loss 4.8305, time: 228.72ms\n",
            " 72% 11689/16260 [1:06:37<20:38,  3.69it/s]iter 11689: loss 5.3593, time: 463.51ms\n",
            " 72% 11690/16260 [1:06:38<25:04,  3.04it/s]iter 11690: loss 5.1359, time: 379.11ms\n",
            " 72% 11691/16260 [1:06:38<26:14,  2.90it/s]iter 11691: loss 5.7830, time: 381.23ms\n",
            " 72% 11692/16260 [1:06:38<27:06,  2.81it/s]iter 11692: loss 4.7014, time: 212.86ms\n",
            " 72% 11693/16260 [1:06:39<23:52,  3.19it/s]iter 11693: loss 4.1585, time: 144.87ms\n",
            " 72% 11694/16260 [1:06:39<20:02,  3.80it/s]iter 11694: loss 5.6809, time: 463.17ms\n",
            " 72% 11695/16260 [1:06:39<24:37,  3.09it/s]iter 11695: loss 4.5668, time: 217.17ms\n",
            " 72% 11696/16260 [1:06:40<22:12,  3.42it/s]iter 11696: loss 6.2111, time: 463.93ms\n",
            " 72% 11697/16260 [1:06:40<26:10,  2.91it/s]iter 11697: loss 5.4341, time: 277.93ms\n",
            " 72% 11698/16260 [1:06:40<24:41,  3.08it/s]iter 11698: loss 4.5543, time: 238.24ms\n",
            " 72% 11699/16260 [1:06:40<22:44,  3.34it/s]iter 11699: loss 5.4267, time: 420.82ms\n",
            " 72% 11700/16260 [1:06:41<25:32,  2.97it/s]iter 11700: loss 5.4051, time: 334.53ms\n",
            " 72% 11701/16260 [1:06:41<25:32,  2.97it/s]iter 11701: loss 5.5521, time: 464.25ms\n",
            " 72% 11702/16260 [1:06:42<28:29,  2.67it/s]iter 11702: loss 5.1063, time: 276.52ms\n",
            " 72% 11703/16260 [1:06:42<26:16,  2.89it/s]iter 11703: loss 5.2786, time: 234.15ms\n",
            " 72% 11704/16260 [1:06:42<23:45,  3.20it/s]iter 11704: loss 5.0284, time: 250.16ms\n",
            " 72% 11705/16260 [1:06:42<22:21,  3.39it/s]iter 11705: loss 4.2110, time: 213.36ms\n",
            " 72% 11706/16260 [1:06:43<20:32,  3.69it/s]iter 11706: loss 4.9418, time: 343.05ms\n",
            " 72% 11707/16260 [1:06:43<22:13,  3.41it/s]iter 11707: loss 5.3932, time: 475.29ms\n",
            " 72% 11708/16260 [1:06:44<26:24,  2.87it/s]iter 11708: loss 5.1708, time: 272.89ms\n",
            " 72% 11709/16260 [1:06:44<24:43,  3.07it/s]iter 11709: loss 4.6535, time: 231.23ms\n",
            " 72% 11710/16260 [1:06:44<22:36,  3.35it/s]iter 11710: loss 5.5238, time: 207.38ms\n",
            " 72% 11711/16260 [1:06:44<20:34,  3.69it/s]iter 11711: loss 5.6858, time: 468.21ms\n",
            " 72% 11712/16260 [1:06:45<25:04,  3.02it/s]iter 11712: loss 5.3644, time: 380.32ms\n",
            " 72% 11713/16260 [1:06:45<26:13,  2.89it/s]iter 11713: loss 4.0336, time: 166.28ms\n",
            " 72% 11714/16260 [1:06:45<22:10,  3.42it/s]iter 11714: loss 5.2697, time: 472.93ms\n",
            " 72% 11715/16260 [1:06:46<26:18,  2.88it/s]iter 11715: loss 5.3264, time: 437.30ms\n",
            " 72% 11716/16260 [1:06:46<28:22,  2.67it/s]iter 11716: loss 6.0725, time: 464.26ms\n",
            " 72% 11717/16260 [1:06:47<30:26,  2.49it/s]iter 11717: loss 4.7246, time: 252.08ms\n",
            " 72% 11718/16260 [1:06:47<27:03,  2.80it/s]iter 11718: loss 5.6488, time: 463.87ms\n",
            " 72% 11719/16260 [1:06:47<29:30,  2.56it/s]iter 11719: loss 4.8553, time: 212.45ms\n",
            " 72% 11720/16260 [1:06:48<25:30,  2.97it/s]iter 11720: loss 5.6011, time: 263.03ms\n",
            " 72% 11721/16260 [1:06:48<23:51,  3.17it/s]iter 11721: loss 5.1160, time: 331.71ms\n",
            " 72% 11722/16260 [1:06:48<24:15,  3.12it/s]iter 11722: loss 4.6867, time: 183.98ms\n",
            " 72% 11723/16260 [1:06:48<21:11,  3.57it/s]iter 11723: loss 3.1652, time: 152.50ms\n",
            " 72% 11724/16260 [1:06:49<18:19,  4.13it/s]iter 11724: loss 5.9813, time: 474.67ms\n",
            " 72% 11725/16260 [1:06:49<23:37,  3.20it/s]iter 11725: loss 5.5459, time: 464.74ms\n",
            " 72% 11726/16260 [1:06:49<27:06,  2.79it/s]iter 11726: loss 5.7789, time: 464.31ms\n",
            " 72% 11727/16260 [1:06:50<29:31,  2.56it/s]iter 11727: loss 5.5175, time: 439.63ms\n",
            " 72% 11728/16260 [1:06:50<30:39,  2.46it/s]iter 11728: loss 4.6063, time: 275.74ms\n",
            " 72% 11729/16260 [1:06:51<27:44,  2.72it/s]iter 11729: loss 5.4483, time: 234.05ms\n",
            " 72% 11730/16260 [1:06:51<24:44,  3.05it/s]iter 11730: loss 5.3845, time: 463.87ms\n",
            " 72% 11731/16260 [1:06:51<27:51,  2.71it/s]iter 11731: loss 4.9089, time: 187.13ms\n",
            " 72% 11732/16260 [1:06:52<23:46,  3.18it/s]iter 11732: loss 4.0541, time: 150.37ms\n",
            " 72% 11733/16260 [1:06:52<20:04,  3.76it/s]iter 11733: loss 5.1568, time: 341.46ms\n",
            " 72% 11734/16260 [1:06:52<21:48,  3.46it/s]iter 11734: loss 5.4295, time: 464.09ms\n",
            " 72% 11735/16260 [1:06:52<25:48,  2.92it/s]iter 11735: loss 4.8090, time: 446.10ms\n",
            " 72% 11736/16260 [1:06:53<28:11,  2.68it/s]iter 11736: loss 5.1419, time: 339.84ms\n",
            " 72% 11737/16260 [1:06:53<27:26,  2.75it/s]iter 11737: loss 5.3360, time: 471.51ms\n",
            " 72% 11738/16260 [1:06:54<29:54,  2.52it/s]iter 11738: loss 4.6656, time: 463.83ms\n",
            " 72% 11739/16260 [1:06:54<31:26,  2.40it/s]iter 11739: loss 5.4639, time: 463.28ms\n",
            " 72% 11740/16260 [1:06:55<32:30,  2.32it/s]iter 11740: loss 5.8386, time: 464.82ms\n",
            " 72% 11741/16260 [1:06:55<33:17,  2.26it/s]iter 11741: loss 5.4825, time: 463.71ms\n",
            " 72% 11742/16260 [1:06:56<33:48,  2.23it/s]iter 11742: loss 4.7763, time: 190.62ms\n",
            " 72% 11743/16260 [1:06:56<28:00,  2.69it/s]iter 11743: loss 4.3836, time: 171.66ms\n",
            " 72% 11744/16260 [1:06:56<23:29,  3.20it/s]iter 11744: loss 5.2172, time: 278.54ms\n",
            " 72% 11745/16260 [1:06:56<22:45,  3.31it/s]iter 11745: loss 5.9047, time: 464.72ms\n",
            " 72% 11746/16260 [1:06:57<26:27,  2.84it/s]iter 11746: loss 5.2399, time: 188.35ms\n",
            " 72% 11747/16260 [1:06:57<22:48,  3.30it/s]iter 11747: loss 6.0266, time: 464.14ms\n",
            " 72% 11748/16260 [1:06:57<26:27,  2.84it/s]iter 11748: loss 3.9042, time: 255.98ms\n",
            " 72% 11749/16260 [1:06:58<24:19,  3.09it/s]iter 11749: loss 3.4604, time: 148.40ms\n",
            " 72% 11750/16260 [1:06:58<20:24,  3.68it/s]iter 11750: loss 4.3119, time: 165.55ms\n",
            " 72% 11751/16260 [1:06:58<18:03,  4.16it/s]iter 11751: loss 4.3315, time: 193.10ms\n",
            " 72% 11752/16260 [1:06:58<17:01,  4.41it/s]iter 11752: loss 4.4237, time: 150.68ms\n",
            " 72% 11753/16260 [1:06:58<15:20,  4.90it/s]iter 11753: loss 5.6089, time: 465.19ms\n",
            " 72% 11754/16260 [1:06:59<21:15,  3.53it/s]iter 11754: loss 4.9526, time: 271.75ms\n",
            " 72% 11755/16260 [1:06:59<21:01,  3.57it/s]iter 11755: loss 5.1725, time: 422.69ms\n",
            " 72% 11756/16260 [1:06:59<24:16,  3.09it/s]iter 11756: loss 3.7929, time: 147.72ms\n",
            " 72% 11757/16260 [1:07:00<20:20,  3.69it/s]iter 11757: loss 5.2735, time: 256.51ms\n",
            " 72% 11758/16260 [1:07:00<20:03,  3.74it/s]iter 11758: loss 5.6461, time: 425.01ms\n",
            " 72% 11759/16260 [1:07:00<23:38,  3.17it/s]iter 11759: loss 4.6260, time: 187.52ms\n",
            " 72% 11760/16260 [1:07:00<20:47,  3.61it/s]iter 11760: loss 5.5536, time: 346.10ms\n",
            " 72% 11761/16260 [1:07:01<22:22,  3.35it/s]iter 11761: loss 5.7982, time: 464.08ms\n",
            " 72% 11762/16260 [1:07:01<26:08,  2.87it/s]iter 11762: loss 5.6297, time: 464.91ms\n",
            " 72% 11763/16260 [1:07:02<28:47,  2.60it/s]iter 11763: loss 5.2769, time: 464.18ms\n",
            " 72% 11764/16260 [1:07:02<30:37,  2.45it/s]iter 11764: loss 5.3876, time: 347.36ms\n",
            " 72% 11765/16260 [1:07:03<29:16,  2.56it/s]iter 11765: loss 4.4499, time: 190.61ms\n",
            " 72% 11766/16260 [1:07:03<24:48,  3.02it/s]iter 11766: loss 5.1224, time: 259.27ms\n",
            " 72% 11767/16260 [1:07:03<23:13,  3.22it/s]iter 11767: loss 5.0245, time: 258.15ms\n",
            " 72% 11768/16260 [1:07:03<22:05,  3.39it/s]iter 11768: loss 4.7301, time: 173.55ms\n",
            " 72% 11769/16260 [1:07:03<19:23,  3.86it/s]iter 11769: loss 5.4433, time: 464.70ms\n",
            " 72% 11770/16260 [1:07:04<24:02,  3.11it/s]iter 11770: loss 5.6350, time: 410.16ms\n",
            " 72% 11771/16260 [1:07:04<26:04,  2.87it/s]iter 11771: loss 4.3509, time: 170.80ms\n",
            " 72% 11772/16260 [1:07:05<22:06,  3.38it/s]iter 11772: loss 4.0348, time: 153.67ms\n",
            " 72% 11773/16260 [1:07:05<18:57,  3.95it/s]iter 11773: loss 4.0358, time: 157.61ms\n",
            " 72% 11774/16260 [1:07:05<16:50,  4.44it/s]iter 11774: loss 5.7816, time: 463.77ms\n",
            " 72% 11775/16260 [1:07:05<22:13,  3.36it/s]iter 11775: loss 4.6335, time: 150.25ms\n",
            " 72% 11776/16260 [1:07:05<18:57,  3.94it/s]iter 11776: loss 5.7273, time: 473.54ms\n",
            " 72% 11777/16260 [1:07:06<23:54,  3.12it/s]iter 11777: loss 4.2585, time: 164.63ms\n",
            " 72% 11778/16260 [1:07:06<20:27,  3.65it/s]iter 11778: loss 5.7926, time: 464.81ms\n",
            " 72% 11779/16260 [1:07:07<24:46,  3.01it/s]iter 11779: loss 5.0431, time: 257.67ms\n",
            " 72% 11780/16260 [1:07:07<23:09,  3.23it/s]iter 11780: loss 3.7998, time: 202.05ms\n",
            " 72% 11781/16260 [1:07:07<20:46,  3.59it/s]iter 11781: loss 4.6690, time: 235.12ms\n",
            " 72% 11782/16260 [1:07:07<19:49,  3.76it/s]iter 11782: loss 4.1533, time: 165.34ms\n",
            " 72% 11783/16260 [1:07:07<17:36,  4.24it/s]iter 11783: loss 5.3203, time: 309.36ms\n",
            " 72% 11784/16260 [1:07:08<19:17,  3.87it/s]iter 11784: loss 5.5375, time: 352.46ms\n",
            " 72% 11785/16260 [1:07:08<21:25,  3.48it/s]iter 11785: loss 5.9322, time: 464.30ms\n",
            " 72% 11786/16260 [1:07:09<25:25,  2.93it/s]iter 11786: loss 5.5640, time: 465.43ms\n",
            " 72% 11787/16260 [1:07:09<28:14,  2.64it/s]iter 11787: loss 5.7142, time: 465.12ms\n",
            " 72% 11788/16260 [1:07:09<30:11,  2.47it/s]iter 11788: loss 5.6683, time: 472.91ms\n",
            " 73% 11789/16260 [1:07:10<31:44,  2.35it/s]iter 11789: loss 4.7724, time: 208.13ms\n",
            " 73% 11790/16260 [1:07:10<26:53,  2.77it/s]iter 11790: loss 5.9260, time: 464.59ms\n",
            " 73% 11791/16260 [1:07:11<29:13,  2.55it/s]iter 11791: loss 4.9612, time: 313.77ms\n",
            " 73% 11792/16260 [1:07:11<27:30,  2.71it/s]iter 11792: loss 5.8940, time: 370.11ms\n",
            " 73% 11793/16260 [1:07:11<27:33,  2.70it/s]iter 11793: loss 5.3284, time: 251.72ms\n",
            " 73% 11794/16260 [1:07:12<24:56,  2.98it/s]iter 11794: loss 4.8280, time: 236.82ms\n",
            " 73% 11795/16260 [1:07:12<22:46,  3.27it/s]iter 11795: loss 5.1913, time: 368.06ms\n",
            " 73% 11796/16260 [1:07:12<24:11,  3.08it/s]iter 11796: loss 4.2273, time: 171.69ms\n",
            " 73% 11797/16260 [1:07:12<20:48,  3.58it/s]iter 11797: loss 5.0324, time: 260.78ms\n",
            " 73% 11798/16260 [1:07:13<20:24,  3.64it/s]iter 11798: loss 4.7615, time: 282.99ms\n",
            " 73% 11799/16260 [1:07:13<20:38,  3.60it/s]iter 11799: loss 5.3359, time: 240.87ms\n",
            " 73% 11800/16260 [1:07:13<19:51,  3.74it/s]iter 11800: loss 5.1669, time: 236.11ms\n",
            " 73% 11801/16260 [1:07:13<19:11,  3.87it/s]iter 11801: loss 3.5331, time: 150.46ms\n",
            " 73% 11802/16260 [1:07:14<16:49,  4.42it/s]iter 11802: loss 5.5830, time: 472.51ms\n",
            " 73% 11803/16260 [1:07:14<22:20,  3.33it/s]iter 11803: loss 5.0174, time: 281.00ms\n",
            " 73% 11804/16260 [1:07:14<21:55,  3.39it/s]iter 11804: loss 5.6047, time: 465.16ms\n",
            " 73% 11805/16260 [1:07:15<25:44,  2.88it/s]iter 11805: loss 5.4303, time: 306.65ms\n",
            " 73% 11806/16260 [1:07:15<24:53,  2.98it/s]iter 11806: loss 5.0450, time: 431.17ms\n",
            " 73% 11807/16260 [1:07:16<27:03,  2.74it/s]iter 11807: loss 4.8637, time: 337.85ms\n",
            " 73% 11808/16260 [1:07:16<26:28,  2.80it/s]iter 11808: loss 4.9821, time: 311.42ms\n",
            " 73% 11809/16260 [1:07:16<25:29,  2.91it/s]iter 11809: loss 5.2474, time: 368.62ms\n",
            " 73% 11810/16260 [1:07:17<26:04,  2.84it/s]iter 11810: loss 4.4975, time: 213.01ms\n",
            " 73% 11811/16260 [1:07:17<23:01,  3.22it/s]iter 11811: loss 5.0947, time: 239.92ms\n",
            " 73% 11812/16260 [1:07:17<21:28,  3.45it/s]iter 11812: loss 6.1758, time: 381.51ms\n",
            " 73% 11813/16260 [1:07:17<23:32,  3.15it/s]iter 11813: loss 5.3447, time: 280.63ms\n",
            " 73% 11814/16260 [1:07:18<22:45,  3.26it/s]iter 11814: loss 5.8946, time: 474.33ms\n",
            " 73% 11815/16260 [1:07:18<26:30,  2.79it/s]iter 11815: loss 5.6617, time: 464.78ms\n",
            " 73% 11816/16260 [1:07:19<28:55,  2.56it/s]iter 11816: loss 4.8218, time: 241.36ms\n",
            " 73% 11817/16260 [1:07:19<25:38,  2.89it/s]iter 11817: loss 6.0507, time: 464.48ms\n",
            " 73% 11818/16260 [1:07:19<28:17,  2.62it/s]iter 11818: loss 5.1189, time: 264.79ms\n",
            " 73% 11819/16260 [1:07:20<25:42,  2.88it/s]iter 11819: loss 6.1692, time: 336.44ms\n",
            " 73% 11820/16260 [1:07:20<25:29,  2.90it/s]iter 11820: loss 5.7642, time: 462.71ms\n",
            " 73% 11821/16260 [1:07:20<28:09,  2.63it/s]iter 11821: loss 3.8666, time: 153.24ms\n",
            " 73% 11822/16260 [1:07:21<23:08,  3.20it/s]iter 11822: loss 3.9285, time: 214.81ms\n",
            " 73% 11823/16260 [1:07:21<20:59,  3.52it/s]iter 11823: loss 5.7854, time: 464.89ms\n",
            " 73% 11824/16260 [1:07:21<25:02,  2.95it/s]iter 11824: loss 5.4479, time: 211.76ms\n",
            " 73% 11825/16260 [1:07:21<22:15,  3.32it/s]iter 11825: loss 5.9138, time: 465.73ms\n",
            " 73% 11826/16260 [1:07:22<25:56,  2.85it/s]iter 11826: loss 4.4202, time: 171.17ms\n",
            " 73% 11827/16260 [1:07:22<21:59,  3.36it/s]iter 11827: loss 4.8688, time: 172.31ms\n",
            " 73% 11828/16260 [1:07:22<19:14,  3.84it/s]iter 11828: loss 5.5105, time: 337.77ms\n",
            " 73% 11829/16260 [1:07:23<20:59,  3.52it/s]iter 11829: loss 5.2545, time: 236.34ms\n",
            " 73% 11830/16260 [1:07:23<19:57,  3.70it/s]iter 11830: loss 3.9311, time: 168.32ms\n",
            " 73% 11831/16260 [1:07:23<17:43,  4.16it/s]iter 11831: loss 4.5830, time: 210.06ms\n",
            " 73% 11832/16260 [1:07:23<17:05,  4.32it/s]iter 11832: loss 6.0222, time: 330.23ms\n",
            " 73% 11833/16260 [1:07:24<19:18,  3.82it/s]iter 11833: loss 5.5235, time: 463.27ms\n",
            " 73% 11834/16260 [1:07:24<23:47,  3.10it/s]iter 11834: loss 4.9028, time: 236.92ms\n",
            " 73% 11835/16260 [1:07:24<21:55,  3.36it/s]iter 11835: loss 5.5154, time: 464.83ms\n",
            " 73% 11836/16260 [1:07:25<25:39,  2.87it/s]iter 11836: loss 5.1244, time: 280.57ms\n",
            " 73% 11837/16260 [1:07:25<24:11,  3.05it/s]iter 11837: loss 5.3359, time: 355.16ms\n",
            " 73% 11838/16260 [1:07:25<24:49,  2.97it/s]iter 11838: loss 5.4276, time: 443.54ms\n",
            " 73% 11839/16260 [1:07:26<27:12,  2.71it/s]iter 11839: loss 4.8348, time: 427.56ms\n",
            " 73% 11840/16260 [1:07:26<28:30,  2.58it/s]iter 11840: loss 5.4289, time: 466.23ms\n",
            " 73% 11841/16260 [1:07:27<30:17,  2.43it/s]iter 11841: loss 5.5449, time: 335.51ms\n",
            " 73% 11842/16260 [1:07:27<28:38,  2.57it/s]iter 11842: loss 5.1391, time: 465.14ms\n",
            " 73% 11843/16260 [1:07:27<30:21,  2.42it/s]iter 11843: loss 5.4210, time: 473.78ms\n",
            " 73% 11844/16260 [1:07:28<31:44,  2.32it/s]iter 11844: loss 4.8357, time: 192.43ms\n",
            " 73% 11845/16260 [1:07:28<26:30,  2.78it/s]iter 11845: loss 5.8858, time: 465.23ms\n",
            " 73% 11846/16260 [1:07:29<28:51,  2.55it/s]iter 11846: loss 5.4784, time: 312.39ms\n",
            " 73% 11847/16260 [1:07:29<27:07,  2.71it/s]iter 11847: loss 5.8646, time: 464.73ms\n",
            " 73% 11848/16260 [1:07:29<29:16,  2.51it/s]iter 11848: loss 4.9540, time: 237.56ms\n",
            " 73% 11849/16260 [1:07:30<25:45,  2.85it/s]iter 11849: loss 4.4109, time: 194.11ms\n",
            " 73% 11850/16260 [1:07:30<22:20,  3.29it/s]iter 11850: loss 4.6184, time: 238.11ms\n",
            " 73% 11851/16260 [1:07:30<20:55,  3.51it/s]iter 11851: loss 5.1010, time: 212.31ms\n",
            " 73% 11852/16260 [1:07:30<19:21,  3.80it/s]iter 11852: loss 5.4494, time: 439.13ms\n",
            " 73% 11853/16260 [1:07:31<23:15,  3.16it/s]iter 11853: loss 5.4117, time: 438.94ms\n",
            " 73% 11854/16260 [1:07:31<25:59,  2.83it/s]iter 11854: loss 5.4689, time: 471.42ms\n",
            " 73% 11855/16260 [1:07:32<28:36,  2.57it/s]iter 11855: loss 4.5044, time: 235.61ms\n",
            " 73% 11856/16260 [1:07:32<25:14,  2.91it/s]iter 11856: loss 5.0423, time: 277.51ms\n",
            " 73% 11857/16260 [1:07:32<23:48,  3.08it/s]iter 11857: loss 3.9337, time: 165.52ms\n",
            " 73% 11858/16260 [1:07:32<20:20,  3.61it/s]iter 11858: loss 5.0555, time: 340.88ms\n",
            " 73% 11859/16260 [1:07:33<21:46,  3.37it/s]iter 11859: loss 5.3983, time: 463.88ms\n",
            " 73% 11860/16260 [1:07:33<25:29,  2.88it/s]iter 11860: loss 4.4933, time: 263.48ms\n",
            " 73% 11861/16260 [1:07:33<23:40,  3.10it/s]iter 11861: loss 6.0209, time: 469.22ms\n",
            " 73% 11862/16260 [1:07:34<26:54,  2.72it/s]iter 11862: loss 5.0963, time: 314.55ms\n",
            " 73% 11863/16260 [1:07:34<25:47,  2.84it/s]iter 11863: loss 5.6407, time: 432.11ms\n",
            " 73% 11864/16260 [1:07:35<27:34,  2.66it/s]iter 11864: loss 5.6884, time: 465.06ms\n",
            " 73% 11865/16260 [1:07:35<29:33,  2.48it/s]iter 11865: loss 5.3803, time: 463.91ms\n",
            " 73% 11866/16260 [1:07:36<30:54,  2.37it/s]iter 11866: loss 5.7569, time: 463.21ms\n",
            " 73% 11867/16260 [1:07:36<31:50,  2.30it/s]iter 11867: loss 5.3636, time: 237.91ms\n",
            " 73% 11868/16260 [1:07:36<27:32,  2.66it/s]iter 11868: loss 4.9561, time: 240.78ms\n",
            " 73% 11869/16260 [1:07:36<24:35,  2.98it/s]iter 11869: loss 5.2462, time: 275.09ms\n",
            " 73% 11870/16260 [1:07:37<23:16,  3.14it/s]iter 11870: loss 5.6277, time: 463.49ms\n",
            " 73% 11871/16260 [1:07:37<26:29,  2.76it/s]iter 11871: loss 4.3739, time: 265.80ms\n",
            " 73% 11872/16260 [1:07:38<24:24,  3.00it/s]iter 11872: loss 5.6820, time: 463.77ms\n",
            " 73% 11873/16260 [1:07:38<27:16,  2.68it/s]iter 11873: loss 5.0514, time: 229.02ms\n",
            " 73% 11874/16260 [1:07:38<24:08,  3.03it/s]iter 11874: loss 4.3248, time: 151.70ms\n",
            " 73% 11875/16260 [1:07:38<20:14,  3.61it/s]iter 11875: loss 5.6189, time: 463.56ms\n",
            " 73% 11876/16260 [1:07:39<24:21,  3.00it/s]iter 11876: loss 5.4944, time: 367.83ms\n",
            " 73% 11877/16260 [1:07:39<25:08,  2.90it/s]iter 11877: loss 5.4479, time: 463.63ms\n",
            " 73% 11878/16260 [1:07:40<27:47,  2.63it/s]iter 11878: loss 5.2735, time: 463.64ms\n",
            " 73% 11879/16260 [1:07:40<29:38,  2.46it/s]iter 11879: loss 3.8264, time: 148.90ms\n",
            " 73% 11880/16260 [1:07:40<24:01,  3.04it/s]iter 11880: loss 6.1419, time: 464.01ms\n",
            " 73% 11881/16260 [1:07:41<27:00,  2.70it/s]iter 11881: loss 4.4622, time: 208.43ms\n",
            " 73% 11882/16260 [1:07:41<23:29,  3.11it/s]iter 11882: loss 5.6230, time: 333.82ms\n",
            " 73% 11883/16260 [1:07:41<23:46,  3.07it/s]iter 11883: loss 5.3060, time: 463.82ms\n",
            " 73% 11884/16260 [1:07:42<26:48,  2.72it/s]iter 11884: loss 4.5079, time: 212.93ms\n",
            " 73% 11885/16260 [1:07:42<23:27,  3.11it/s]iter 11885: loss 4.8874, time: 171.51ms\n",
            " 73% 11886/16260 [1:07:42<20:12,  3.61it/s]iter 11886: loss 5.3591, time: 317.84ms\n",
            " 73% 11887/16260 [1:07:42<21:07,  3.45it/s]iter 11887: loss 5.6859, time: 390.16ms\n",
            " 73% 11888/16260 [1:07:43<23:20,  3.12it/s]iter 11888: loss 5.3349, time: 420.93ms\n",
            " 73% 11889/16260 [1:07:43<25:34,  2.85it/s]iter 11889: loss 4.2271, time: 185.33ms\n",
            " 73% 11890/16260 [1:07:43<21:59,  3.31it/s]iter 11890: loss 4.1413, time: 189.04ms\n",
            " 73% 11891/16260 [1:07:44<19:32,  3.72it/s]iter 11891: loss 5.6432, time: 437.65ms\n",
            " 73% 11892/16260 [1:07:44<23:16,  3.13it/s]iter 11892: loss 5.1587, time: 464.91ms\n",
            " 73% 11893/16260 [1:07:45<26:28,  2.75it/s]iter 11893: loss 5.6733, time: 464.09ms\n",
            " 73% 11894/16260 [1:07:45<28:41,  2.54it/s]iter 11894: loss 5.4983, time: 370.34ms\n",
            " 73% 11895/16260 [1:07:45<28:11,  2.58it/s]iter 11895: loss 5.1464, time: 255.27ms\n",
            " 73% 11896/16260 [1:07:46<25:20,  2.87it/s]iter 11896: loss 5.5022, time: 463.59ms\n",
            " 73% 11897/16260 [1:07:46<27:52,  2.61it/s]iter 11897: loss 5.1398, time: 278.04ms\n",
            " 73% 11898/16260 [1:07:46<25:36,  2.84it/s]iter 11898: loss 4.2850, time: 308.16ms\n",
            " 73% 11899/16260 [1:07:47<24:40,  2.95it/s]iter 11899: loss 5.3303, time: 258.62ms\n",
            " 73% 11900/16260 [1:07:47<22:56,  3.17it/s]iter 11900: loss 5.0323, time: 232.57ms\n",
            " 73% 11901/16260 [1:07:47<21:09,  3.43it/s]iter 11901: loss 4.3841, time: 214.61ms\n",
            " 73% 11902/16260 [1:07:47<19:30,  3.72it/s]iter 11902: loss 5.2782, time: 308.63ms\n",
            " 73% 11903/16260 [1:07:48<20:25,  3.56it/s]iter 11903: loss 3.6266, time: 151.57ms\n",
            " 73% 11904/16260 [1:07:48<17:36,  4.12it/s]iter 11904: loss 5.3716, time: 463.91ms\n",
            " 73% 11905/16260 [1:07:48<22:27,  3.23it/s]iter 11905: loss 5.2380, time: 235.33ms\n",
            " 73% 11906/16260 [1:07:49<20:52,  3.48it/s]iter 11906: loss 4.4683, time: 238.68ms\n",
            " 73% 11907/16260 [1:07:49<19:49,  3.66it/s]iter 11907: loss 5.0603, time: 271.48ms\n",
            " 73% 11908/16260 [1:07:49<19:48,  3.66it/s]iter 11908: loss 5.3383, time: 336.82ms\n",
            " 73% 11909/16260 [1:07:49<21:13,  3.42it/s]iter 11909: loss 5.5642, time: 316.20ms\n",
            " 73% 11910/16260 [1:07:50<21:45,  3.33it/s]iter 11910: loss 5.2166, time: 286.24ms\n",
            " 73% 11911/16260 [1:07:50<21:29,  3.37it/s]iter 11911: loss 4.3935, time: 464.07ms\n",
            " 73% 11912/16260 [1:07:50<25:09,  2.88it/s]iter 11912: loss 4.6855, time: 191.77ms\n",
            " 73% 11913/16260 [1:07:51<21:48,  3.32it/s]iter 11913: loss 5.8794, time: 416.26ms\n",
            " 73% 11914/16260 [1:07:51<24:20,  2.98it/s]iter 11914: loss 5.3631, time: 330.86ms\n",
            " 73% 11915/16260 [1:07:51<24:15,  2.99it/s]iter 11915: loss 4.3789, time: 231.81ms\n",
            " 73% 11916/16260 [1:07:52<22:02,  3.28it/s]iter 11916: loss 4.1353, time: 164.35ms\n",
            " 73% 11917/16260 [1:07:52<19:01,  3.80it/s]iter 11917: loss 5.3421, time: 464.48ms\n",
            " 73% 11918/16260 [1:07:52<23:26,  3.09it/s]iter 11918: loss 5.4527, time: 310.50ms\n",
            " 73% 11919/16260 [1:07:53<23:10,  3.12it/s]iter 11919: loss 5.5177, time: 464.15ms\n",
            " 73% 11920/16260 [1:07:53<26:19,  2.75it/s]iter 11920: loss 5.9956, time: 472.23ms\n",
            " 73% 11921/16260 [1:07:54<28:42,  2.52it/s]iter 11921: loss 5.1613, time: 235.82ms\n",
            " 73% 11922/16260 [1:07:54<25:14,  2.87it/s]iter 11922: loss 5.8027, time: 464.24ms\n",
            " 73% 11923/16260 [1:07:54<27:45,  2.60it/s]iter 11923: loss 5.0691, time: 317.01ms\n",
            " 73% 11924/16260 [1:07:55<26:20,  2.74it/s]iter 11924: loss 4.9708, time: 342.06ms\n",
            " 73% 11925/16260 [1:07:55<25:52,  2.79it/s]iter 11925: loss 6.0592, time: 382.84ms\n",
            " 73% 11926/16260 [1:07:55<26:26,  2.73it/s]iter 11926: loss 4.8971, time: 439.79ms\n",
            " 73% 11927/16260 [1:07:56<28:03,  2.57it/s]iter 11927: loss 4.6467, time: 188.28ms\n",
            " 73% 11928/16260 [1:07:56<23:45,  3.04it/s]iter 11928: loss 4.4624, time: 226.64ms\n",
            " 73% 11929/16260 [1:07:56<21:33,  3.35it/s]iter 11929: loss 5.4696, time: 422.16ms\n",
            " 73% 11930/16260 [1:07:57<24:15,  2.97it/s]iter 11930: loss 5.4012, time: 345.30ms\n",
            " 73% 11931/16260 [1:07:57<24:29,  2.95it/s]iter 11931: loss 5.1325, time: 435.49ms\n",
            " 73% 11932/16260 [1:07:57<26:36,  2.71it/s]iter 11932: loss 3.9661, time: 185.26ms\n",
            " 73% 11933/16260 [1:07:58<22:39,  3.18it/s]iter 11933: loss 4.9478, time: 334.36ms\n",
            " 73% 11934/16260 [1:07:58<23:07,  3.12it/s]iter 11934: loss 5.4328, time: 468.49ms\n",
            " 73% 11935/16260 [1:07:58<26:20,  2.74it/s]iter 11935: loss 5.1939, time: 276.44ms\n",
            " 73% 11936/16260 [1:07:59<24:26,  2.95it/s]iter 11936: loss 5.3633, time: 314.11ms\n",
            " 73% 11937/16260 [1:07:59<23:55,  3.01it/s]iter 11937: loss 5.6870, time: 464.77ms\n",
            " 73% 11938/16260 [1:07:59<26:49,  2.69it/s]iter 11938: loss 5.2427, time: 255.26ms\n",
            " 73% 11939/16260 [1:08:00<24:19,  2.96it/s]iter 11939: loss 4.0686, time: 256.06ms\n",
            " 73% 11940/16260 [1:08:00<22:35,  3.19it/s]iter 11940: loss 5.1196, time: 190.70ms\n",
            " 73% 11941/16260 [1:08:00<19:57,  3.61it/s]iter 11941: loss 4.9282, time: 212.72ms\n",
            " 73% 11942/16260 [1:08:00<18:35,  3.87it/s]iter 11942: loss 4.5781, time: 211.40ms\n",
            " 73% 11943/16260 [1:08:01<17:36,  4.09it/s]iter 11943: loss 3.6582, time: 165.11ms\n",
            " 73% 11944/16260 [1:08:01<15:55,  4.52it/s]iter 11944: loss 6.0196, time: 463.46ms\n",
            " 73% 11945/16260 [1:08:01<21:10,  3.40it/s]iter 11945: loss 5.1797, time: 463.89ms\n",
            " 73% 11946/16260 [1:08:02<24:51,  2.89it/s]iter 11946: loss 5.5442, time: 234.59ms\n",
            " 73% 11947/16260 [1:08:02<22:29,  3.20it/s]iter 11947: loss 5.1773, time: 256.44ms\n",
            " 73% 11948/16260 [1:08:02<21:17,  3.37it/s]iter 11948: loss 5.6912, time: 464.24ms\n",
            " 73% 11949/16260 [1:08:03<24:56,  2.88it/s]iter 11949: loss 4.7859, time: 194.71ms\n",
            " 73% 11950/16260 [1:08:03<21:41,  3.31it/s]iter 11950: loss 5.2673, time: 263.36ms\n",
            " 73% 11951/16260 [1:08:03<20:53,  3.44it/s]iter 11951: loss 5.7326, time: 463.96ms\n",
            " 74% 11952/16260 [1:08:04<24:38,  2.91it/s]iter 11952: loss 4.7081, time: 231.46ms\n",
            " 74% 11953/16260 [1:08:04<22:15,  3.22it/s]iter 11953: loss 4.8596, time: 216.56ms\n",
            " 74% 11954/16260 [1:08:04<20:16,  3.54it/s]iter 11954: loss 4.3438, time: 169.88ms\n",
            " 74% 11955/16260 [1:08:04<17:53,  4.01it/s]iter 11955: loss 4.9291, time: 212.98ms\n",
            " 74% 11956/16260 [1:08:04<17:08,  4.19it/s]iter 11956: loss 5.8805, time: 432.69ms\n",
            " 74% 11957/16260 [1:08:05<21:20,  3.36it/s]iter 11957: loss 5.1408, time: 218.78ms\n",
            " 74% 11958/16260 [1:08:05<19:40,  3.64it/s]iter 11958: loss 5.9663, time: 464.88ms\n",
            " 74% 11959/16260 [1:08:05<23:48,  3.01it/s]iter 11959: loss 4.7338, time: 341.54ms\n",
            " 74% 11960/16260 [1:08:06<24:01,  2.98it/s]iter 11960: loss 5.0676, time: 463.90ms\n",
            " 74% 11961/16260 [1:08:06<26:49,  2.67it/s]iter 11961: loss 4.2358, time: 188.22ms\n",
            " 74% 11962/16260 [1:08:06<22:51,  3.13it/s]iter 11962: loss 5.5711, time: 440.49ms\n",
            " 74% 11963/16260 [1:08:07<25:29,  2.81it/s]iter 11963: loss 5.7007, time: 470.55ms\n",
            " 74% 11964/16260 [1:08:07<27:58,  2.56it/s]iter 11964: loss 4.6595, time: 190.72ms\n",
            " 74% 11965/16260 [1:08:08<23:42,  3.02it/s]iter 11965: loss 5.2832, time: 430.07ms\n",
            " 74% 11966/16260 [1:08:08<25:51,  2.77it/s]iter 11966: loss 4.0580, time: 164.64ms\n",
            " 74% 11967/16260 [1:08:08<21:39,  3.30it/s]iter 11967: loss 5.4618, time: 339.55ms\n",
            " 74% 11968/16260 [1:08:09<22:27,  3.18it/s]iter 11968: loss 5.6693, time: 463.74ms\n",
            " 74% 11969/16260 [1:08:09<25:41,  2.78it/s]iter 11969: loss 5.3994, time: 211.10ms\n",
            " 74% 11970/16260 [1:08:09<22:32,  3.17it/s]iter 11970: loss 5.8414, time: 464.59ms\n",
            " 74% 11971/16260 [1:08:10<25:46,  2.77it/s]iter 11971: loss 6.0705, time: 464.43ms\n",
            " 74% 11972/16260 [1:08:10<28:01,  2.55it/s]iter 11972: loss 4.9451, time: 469.22ms\n",
            " 74% 11973/16260 [1:08:11<29:42,  2.41it/s]iter 11973: loss 4.9867, time: 168.33ms\n",
            " 74% 11974/16260 [1:08:11<24:25,  2.92it/s]iter 11974: loss 5.6301, time: 426.60ms\n",
            " 74% 11975/16260 [1:08:11<26:16,  2.72it/s]iter 11975: loss 4.8210, time: 220.00ms\n",
            " 74% 11976/16260 [1:08:11<23:08,  3.08it/s]iter 11976: loss 5.4641, time: 261.29ms\n",
            " 74% 11977/16260 [1:08:12<21:49,  3.27it/s]iter 11977: loss 5.0233, time: 319.76ms\n",
            " 74% 11978/16260 [1:08:12<22:09,  3.22it/s]iter 11978: loss 6.0044, time: 464.47ms\n",
            " 74% 11979/16260 [1:08:12<25:28,  2.80it/s]iter 11979: loss 6.3358, time: 465.19ms\n",
            " 74% 11980/16260 [1:08:13<27:49,  2.56it/s]iter 11980: loss 5.6389, time: 445.93ms\n",
            " 74% 11981/16260 [1:08:13<29:03,  2.45it/s]iter 11981: loss 5.2574, time: 463.89ms\n",
            " 74% 11982/16260 [1:08:14<30:17,  2.35it/s]iter 11982: loss 3.5047, time: 164.96ms\n",
            " 74% 11983/16260 [1:08:14<24:45,  2.88it/s]iter 11983: loss 5.5370, time: 261.46ms\n",
            " 74% 11984/16260 [1:08:14<22:57,  3.10it/s]iter 11984: loss 5.3575, time: 475.34ms\n",
            " 74% 11985/16260 [1:08:15<26:15,  2.71it/s]iter 11985: loss 5.0063, time: 239.83ms\n",
            " 74% 11986/16260 [1:08:15<23:31,  3.03it/s]iter 11986: loss 4.4367, time: 165.80ms\n",
            " 74% 11987/16260 [1:08:15<20:02,  3.55it/s]iter 11987: loss 5.1306, time: 473.37ms\n",
            " 74% 11988/16260 [1:08:16<24:09,  2.95it/s]iter 11988: loss 2.9762, time: 165.04ms\n",
            " 74% 11989/16260 [1:08:16<20:28,  3.48it/s]iter 11989: loss 5.6868, time: 345.02ms\n",
            " 74% 11990/16260 [1:08:16<21:43,  3.28it/s]iter 11990: loss 4.9238, time: 215.92ms\n",
            " 74% 11991/16260 [1:08:16<19:51,  3.58it/s]iter 11991: loss 4.9327, time: 317.07ms\n",
            " 74% 11992/16260 [1:08:17<20:41,  3.44it/s]iter 11992: loss 6.4297, time: 464.53ms\n",
            " 74% 11993/16260 [1:08:17<24:25,  2.91it/s]iter 11993: loss 4.9664, time: 260.59ms\n",
            " 74% 11994/16260 [1:08:17<22:41,  3.13it/s]iter 11994: loss 5.1561, time: 190.93ms\n",
            " 74% 11995/16260 [1:08:18<19:59,  3.56it/s]iter 11995: loss 4.2720, time: 154.28ms\n",
            " 74% 11996/16260 [1:08:18<17:18,  4.10it/s]iter 11996: loss 4.0253, time: 171.35ms\n",
            " 74% 11997/16260 [1:08:18<15:48,  4.50it/s]iter 11997: loss 5.7244, time: 443.06ms\n",
            " 74% 11998/16260 [1:08:18<20:32,  3.46it/s]iter 11998: loss 5.9554, time: 437.73ms\n",
            " 74% 11999/16260 [1:08:19<23:43,  2.99it/s]iter 11999: loss 4.9295, time: 320.74ms\n",
            " 74% 12000/16260 [1:08:19<23:28,  3.03it/s]iter 12000: loss 5.3326, time: 309.59ms\n",
            " 74% 12001/16260 [1:08:19<23:02,  3.08it/s]iter 12001: loss 5.6791, time: 472.59ms\n",
            " 74% 12002/16260 [1:08:20<26:13,  2.71it/s]iter 12002: loss 5.7182, time: 468.73ms\n",
            " 74% 12003/16260 [1:08:20<28:21,  2.50it/s]iter 12003: loss 5.2583, time: 473.14ms\n",
            " 74% 12004/16260 [1:08:21<29:57,  2.37it/s]iter 12004: loss 5.4886, time: 335.27ms\n",
            " 74% 12005/16260 [1:08:21<28:07,  2.52it/s]iter 12005: loss 5.1126, time: 278.75ms\n",
            " 74% 12006/16260 [1:08:22<25:39,  2.76it/s]iter 12006: loss 5.6330, time: 465.35ms\n",
            " 74% 12007/16260 [1:08:22<27:53,  2.54it/s]iter 12007: loss 5.4221, time: 463.69ms\n",
            " 74% 12008/16260 [1:08:22<29:24,  2.41it/s]iter 12008: loss 5.0146, time: 342.57ms\n",
            " 74% 12009/16260 [1:08:23<27:53,  2.54it/s]iter 12009: loss 5.0396, time: 169.78ms\n",
            " 74% 12010/16260 [1:08:23<23:09,  3.06it/s]iter 12010: loss 5.3853, time: 377.72ms\n",
            " 74% 12011/16260 [1:08:23<24:16,  2.92it/s]iter 12011: loss 5.5520, time: 318.69ms\n",
            " 74% 12012/16260 [1:08:24<23:47,  2.98it/s]iter 12012: loss 5.1469, time: 464.13ms\n",
            " 74% 12013/16260 [1:08:24<26:32,  2.67it/s]iter 12013: loss 5.8478, time: 442.67ms\n",
            " 74% 12014/16260 [1:08:25<28:00,  2.53it/s]iter 12014: loss 5.4584, time: 476.46ms\n",
            " 74% 12015/16260 [1:08:25<29:44,  2.38it/s]iter 12015: loss 5.2549, time: 426.93ms\n",
            " 74% 12016/16260 [1:08:25<29:54,  2.36it/s]iter 12016: loss 5.6036, time: 463.85ms\n",
            " 74% 12017/16260 [1:08:26<30:48,  2.30it/s]iter 12017: loss 5.3930, time: 464.47ms\n",
            " 74% 12018/16260 [1:08:26<31:27,  2.25it/s]iter 12018: loss 4.9977, time: 259.12ms\n",
            " 74% 12019/16260 [1:08:27<27:32,  2.57it/s]iter 12019: loss 4.9459, time: 174.08ms\n",
            " 74% 12020/16260 [1:08:27<22:59,  3.07it/s]iter 12020: loss 4.8135, time: 237.22ms\n",
            " 74% 12021/16260 [1:08:27<21:09,  3.34it/s]iter 12021: loss 5.8980, time: 239.04ms\n",
            " 74% 12022/16260 [1:08:27<19:54,  3.55it/s]iter 12022: loss 4.8757, time: 463.85ms\n",
            " 74% 12023/16260 [1:08:28<23:47,  2.97it/s]iter 12023: loss 5.9397, time: 463.63ms\n",
            " 74% 12024/16260 [1:08:28<26:30,  2.66it/s]iter 12024: loss 5.8382, time: 380.27ms\n",
            " 74% 12025/16260 [1:08:29<26:38,  2.65it/s]iter 12025: loss 5.6248, time: 464.33ms\n",
            " 74% 12026/16260 [1:08:29<28:30,  2.48it/s]iter 12026: loss 3.2406, time: 148.30ms\n",
            " 74% 12027/16260 [1:08:29<23:07,  3.05it/s]iter 12027: loss 3.9039, time: 151.40ms\n",
            " 74% 12028/16260 [1:08:29<19:24,  3.63it/s]iter 12028: loss 5.2612, time: 338.85ms\n",
            " 74% 12029/16260 [1:08:30<20:47,  3.39it/s]iter 12029: loss 5.3362, time: 377.62ms\n",
            " 74% 12030/16260 [1:08:30<22:33,  3.12it/s]iter 12030: loss 5.5156, time: 464.04ms\n",
            " 74% 12031/16260 [1:08:31<25:38,  2.75it/s]iter 12031: loss 5.0929, time: 380.25ms\n",
            " 74% 12032/16260 [1:08:31<25:59,  2.71it/s]iter 12032: loss 5.3313, time: 463.83ms\n",
            " 74% 12033/16260 [1:08:31<28:01,  2.51it/s]iter 12033: loss 3.8332, time: 169.62ms\n",
            " 74% 12034/16260 [1:08:32<23:13,  3.03it/s]iter 12034: loss 5.5077, time: 471.16ms\n",
            " 74% 12035/16260 [1:08:32<26:14,  2.68it/s]iter 12035: loss 5.2761, time: 229.47ms\n",
            " 74% 12036/16260 [1:08:32<23:14,  3.03it/s]iter 12036: loss 5.4986, time: 464.49ms\n",
            " 74% 12037/16260 [1:08:33<26:06,  2.70it/s]iter 12037: loss 4.6827, time: 316.29ms\n",
            " 74% 12038/16260 [1:08:33<24:59,  2.82it/s]iter 12038: loss 5.3996, time: 464.19ms\n",
            " 74% 12039/16260 [1:08:34<27:19,  2.58it/s]iter 12039: loss 6.0622, time: 464.30ms\n",
            " 74% 12040/16260 [1:08:34<28:56,  2.43it/s]iter 12040: loss 5.1583, time: 331.36ms\n",
            " 74% 12041/16260 [1:08:34<27:17,  2.58it/s]iter 12041: loss 5.9992, time: 464.74ms\n",
            " 74% 12042/16260 [1:08:35<28:55,  2.43it/s]iter 12042: loss 5.4188, time: 252.95ms\n",
            " 74% 12043/16260 [1:08:35<25:36,  2.74it/s]iter 12043: loss 5.2895, time: 463.98ms\n",
            " 74% 12044/16260 [1:08:36<27:44,  2.53it/s]iter 12044: loss 4.5489, time: 334.56ms\n",
            " 74% 12045/16260 [1:08:36<26:29,  2.65it/s]iter 12045: loss 5.8025, time: 463.65ms\n",
            " 74% 12046/16260 [1:08:36<28:20,  2.48it/s]iter 12046: loss 5.6271, time: 279.04ms\n",
            " 74% 12047/16260 [1:08:37<25:44,  2.73it/s]iter 12047: loss 5.6523, time: 370.89ms\n",
            " 74% 12048/16260 [1:08:37<25:51,  2.71it/s]iter 12048: loss 5.5685, time: 367.34ms\n",
            " 74% 12049/16260 [1:08:37<25:51,  2.71it/s]iter 12049: loss 5.1569, time: 235.83ms\n",
            " 74% 12050/16260 [1:08:38<23:05,  3.04it/s]iter 12050: loss 5.2488, time: 466.24ms\n",
            " 74% 12051/16260 [1:08:38<26:00,  2.70it/s]iter 12051: loss 6.2828, time: 463.99ms\n",
            " 74% 12052/16260 [1:08:39<27:59,  2.51it/s]iter 12052: loss 5.4416, time: 417.06ms\n",
            " 74% 12053/16260 [1:08:39<28:23,  2.47it/s]iter 12053: loss 5.4846, time: 473.17ms\n",
            " 74% 12054/16260 [1:08:39<29:51,  2.35it/s]iter 12054: loss 4.6440, time: 263.32ms\n",
            " 74% 12055/16260 [1:08:40<26:28,  2.65it/s]iter 12055: loss 5.9880, time: 469.85ms\n",
            " 74% 12056/16260 [1:08:40<28:26,  2.46it/s]iter 12056: loss 5.6862, time: 342.16ms\n",
            " 74% 12057/16260 [1:08:41<27:07,  2.58it/s]iter 12057: loss 5.7254, time: 464.01ms\n",
            " 74% 12058/16260 [1:08:41<28:45,  2.43it/s]iter 12058: loss 4.3031, time: 193.82ms\n",
            " 74% 12059/16260 [1:08:41<24:13,  2.89it/s]iter 12059: loss 5.5956, time: 463.85ms\n",
            " 74% 12060/16260 [1:08:42<26:43,  2.62it/s]iter 12060: loss 5.2539, time: 462.50ms\n",
            " 74% 12061/16260 [1:08:42<28:26,  2.46it/s]iter 12061: loss 5.9127, time: 373.52ms\n",
            " 74% 12062/16260 [1:08:42<27:46,  2.52it/s]iter 12062: loss 5.5146, time: 464.26ms\n",
            " 74% 12063/16260 [1:08:43<29:12,  2.39it/s]iter 12063: loss 5.2257, time: 467.33ms\n",
            " 74% 12064/16260 [1:08:43<30:16,  2.31it/s]iter 12064: loss 5.3242, time: 464.36ms\n",
            " 74% 12065/16260 [1:08:44<30:57,  2.26it/s]iter 12065: loss 5.0580, time: 259.94ms\n",
            " 74% 12066/16260 [1:08:44<27:08,  2.57it/s]iter 12066: loss 5.3231, time: 464.23ms\n",
            " 74% 12067/16260 [1:08:45<28:45,  2.43it/s]iter 12067: loss 5.7992, time: 463.93ms\n",
            " 74% 12068/16260 [1:08:45<29:53,  2.34it/s]iter 12068: loss 5.0365, time: 318.27ms\n",
            " 74% 12069/16260 [1:08:45<27:37,  2.53it/s]iter 12069: loss 4.7829, time: 231.61ms\n",
            " 74% 12070/16260 [1:08:46<24:13,  2.88it/s]iter 12070: loss 4.9629, time: 423.81ms\n",
            " 74% 12071/16260 [1:08:46<25:51,  2.70it/s]iter 12071: loss 4.8619, time: 281.05ms\n",
            " 74% 12072/16260 [1:08:46<24:00,  2.91it/s]iter 12072: loss 5.3624, time: 281.00ms\n",
            " 74% 12073/16260 [1:08:47<22:42,  3.07it/s]iter 12073: loss 5.0091, time: 467.65ms\n",
            " 74% 12074/16260 [1:08:47<25:43,  2.71it/s]iter 12074: loss 6.0435, time: 471.61ms\n",
            " 74% 12075/16260 [1:08:48<27:53,  2.50it/s]iter 12075: loss 5.6386, time: 463.86ms\n",
            " 74% 12076/16260 [1:08:48<29:15,  2.38it/s]iter 12076: loss 4.7948, time: 257.94ms\n",
            " 74% 12077/16260 [1:08:48<25:54,  2.69it/s]iter 12077: loss 4.5758, time: 261.16ms\n",
            " 74% 12078/16260 [1:08:49<23:37,  2.95it/s]iter 12078: loss 4.1282, time: 190.71ms\n",
            " 74% 12079/16260 [1:08:49<20:32,  3.39it/s]iter 12079: loss 5.4796, time: 316.30ms\n",
            " 74% 12080/16260 [1:08:49<21:00,  3.31it/s]iter 12080: loss 5.5613, time: 415.88ms\n",
            " 74% 12081/16260 [1:08:49<23:25,  2.97it/s]iter 12081: loss 4.9954, time: 415.97ms\n",
            " 74% 12082/16260 [1:08:50<25:07,  2.77it/s]iter 12082: loss 5.3967, time: 445.29ms\n",
            " 74% 12083/16260 [1:08:50<26:54,  2.59it/s]iter 12083: loss 5.5930, time: 463.82ms\n",
            " 74% 12084/16260 [1:08:51<28:32,  2.44it/s]iter 12084: loss 4.6312, time: 167.99ms\n",
            " 74% 12085/16260 [1:08:51<23:30,  2.96it/s]iter 12085: loss 5.7690, time: 462.90ms\n",
            " 74% 12086/16260 [1:08:51<26:08,  2.66it/s]iter 12086: loss 4.8162, time: 236.66ms\n",
            " 74% 12087/16260 [1:08:52<23:15,  2.99it/s]iter 12087: loss 5.4189, time: 464.64ms\n",
            " 74% 12088/16260 [1:08:52<26:00,  2.67it/s]iter 12088: loss 4.7499, time: 253.46ms\n",
            " 74% 12089/16260 [1:08:52<23:30,  2.96it/s]iter 12089: loss 5.7654, time: 380.34ms\n",
            " 74% 12090/16260 [1:08:53<24:25,  2.85it/s]iter 12090: loss 5.3566, time: 370.29ms\n",
            " 74% 12091/16260 [1:08:53<24:50,  2.80it/s]iter 12091: loss 5.2035, time: 464.82ms\n",
            " 74% 12092/16260 [1:08:54<27:06,  2.56it/s]iter 12092: loss 5.0381, time: 315.77ms\n",
            " 74% 12093/16260 [1:08:54<25:34,  2.72it/s]iter 12093: loss 4.8933, time: 258.91ms\n",
            " 74% 12094/16260 [1:08:54<23:19,  2.98it/s]iter 12094: loss 5.4965, time: 464.19ms\n",
            " 74% 12095/16260 [1:08:55<26:01,  2.67it/s]iter 12095: loss 5.0312, time: 170.76ms\n",
            " 74% 12096/16260 [1:08:55<21:47,  3.18it/s]iter 12096: loss 5.4803, time: 338.20ms\n",
            " 74% 12097/16260 [1:08:55<22:19,  3.11it/s]iter 12097: loss 5.6355, time: 474.38ms\n",
            " 74% 12098/16260 [1:08:56<25:31,  2.72it/s]iter 12098: loss 5.1792, time: 278.42ms\n",
            " 74% 12099/16260 [1:08:56<23:41,  2.93it/s]iter 12099: loss 4.5666, time: 235.05ms\n",
            " 74% 12100/16260 [1:08:56<21:29,  3.23it/s]iter 12100: loss 5.0358, time: 259.48ms\n",
            " 74% 12101/16260 [1:08:56<20:27,  3.39it/s]iter 12101: loss 5.5142, time: 470.37ms\n",
            " 74% 12102/16260 [1:08:57<24:07,  2.87it/s]iter 12102: loss 4.9333, time: 282.30ms\n",
            " 74% 12103/16260 [1:08:57<22:47,  3.04it/s]iter 12103: loss 5.4757, time: 369.52ms\n",
            " 74% 12104/16260 [1:08:58<23:39,  2.93it/s]iter 12104: loss 4.9586, time: 263.98ms\n",
            " 74% 12105/16260 [1:08:58<22:04,  3.14it/s]iter 12105: loss 4.2609, time: 153.77ms\n",
            " 74% 12106/16260 [1:08:58<18:40,  3.71it/s]iter 12106: loss 5.6657, time: 464.95ms\n",
            " 74% 12107/16260 [1:08:58<22:45,  3.04it/s]iter 12107: loss 4.7600, time: 272.96ms\n",
            " 74% 12108/16260 [1:08:59<21:37,  3.20it/s]iter 12108: loss 4.9062, time: 260.63ms\n",
            " 74% 12109/16260 [1:08:59<20:34,  3.36it/s]iter 12109: loss 4.9613, time: 440.41ms\n",
            " 74% 12110/16260 [1:08:59<23:34,  2.93it/s]iter 12110: loss 5.1312, time: 188.98ms\n",
            " 74% 12111/16260 [1:09:00<20:26,  3.38it/s]iter 12111: loss 5.7914, time: 463.61ms\n",
            " 74% 12112/16260 [1:09:00<23:57,  2.89it/s]iter 12112: loss 6.2787, time: 463.42ms\n",
            " 74% 12113/16260 [1:09:01<26:24,  2.62it/s]iter 12113: loss 5.2619, time: 235.91ms\n",
            " 75% 12114/16260 [1:09:01<23:23,  2.95it/s]iter 12114: loss 6.2725, time: 464.56ms\n",
            " 75% 12115/16260 [1:09:01<26:02,  2.65it/s]iter 12115: loss 5.3684, time: 431.02ms\n",
            " 75% 12116/16260 [1:09:02<27:11,  2.54it/s]iter 12116: loss 5.8576, time: 464.72ms\n",
            " 75% 12117/16260 [1:09:02<28:41,  2.41it/s]iter 12117: loss 6.2989, time: 482.37ms\n",
            " 75% 12118/16260 [1:09:03<30:05,  2.29it/s]iter 12118: loss 4.3907, time: 215.04ms\n",
            " 75% 12119/16260 [1:09:03<25:33,  2.70it/s]iter 12119: loss 5.7837, time: 463.97ms\n",
            " 75% 12120/16260 [1:09:03<27:31,  2.51it/s]iter 12120: loss 4.8722, time: 253.09ms\n",
            " 75% 12121/16260 [1:09:04<24:32,  2.81it/s]iter 12121: loss 2.8100, time: 154.16ms\n",
            " 75% 12122/16260 [1:09:04<20:23,  3.38it/s]iter 12122: loss 5.6197, time: 369.32ms\n",
            " 75% 12123/16260 [1:09:04<21:56,  3.14it/s]iter 12123: loss 5.2222, time: 317.63ms\n",
            " 75% 12124/16260 [1:09:04<21:57,  3.14it/s]iter 12124: loss 5.1071, time: 307.13ms\n",
            " 75% 12125/16260 [1:09:05<21:44,  3.17it/s]iter 12125: loss 5.7253, time: 463.95ms\n",
            " 75% 12126/16260 [1:09:05<24:50,  2.77it/s]iter 12126: loss 5.4033, time: 275.46ms\n",
            " 75% 12127/16260 [1:09:05<23:06,  2.98it/s]iter 12127: loss 5.0520, time: 468.11ms\n",
            " 75% 12128/16260 [1:09:06<25:51,  2.66it/s]iter 12128: loss 5.4229, time: 313.02ms\n",
            " 75% 12129/16260 [1:09:06<24:35,  2.80it/s]iter 12129: loss 4.0572, time: 172.19ms\n",
            " 75% 12130/16260 [1:09:06<20:47,  3.31it/s]iter 12130: loss 3.7820, time: 217.03ms\n",
            " 75% 12131/16260 [1:09:07<19:04,  3.61it/s]iter 12131: loss 5.9771, time: 466.26ms\n",
            " 75% 12132/16260 [1:09:07<22:59,  2.99it/s]iter 12132: loss 6.0974, time: 477.12ms\n",
            " 75% 12133/16260 [1:09:08<25:58,  2.65it/s]iter 12133: loss 4.9207, time: 380.69ms\n",
            " 75% 12134/16260 [1:09:08<26:04,  2.64it/s]iter 12134: loss 3.6800, time: 211.18ms\n",
            " 75% 12135/16260 [1:09:08<22:38,  3.04it/s]iter 12135: loss 5.8017, time: 425.59ms\n",
            " 75% 12136/16260 [1:09:09<24:38,  2.79it/s]iter 12136: loss 5.2892, time: 350.92ms\n",
            " 75% 12137/16260 [1:09:09<24:30,  2.80it/s]iter 12137: loss 5.3346, time: 313.26ms\n",
            " 75% 12138/16260 [1:09:09<23:38,  2.91it/s]iter 12138: loss 4.4476, time: 190.17ms\n",
            " 75% 12139/16260 [1:09:09<20:30,  3.35it/s]iter 12139: loss 5.7864, time: 470.96ms\n",
            " 75% 12140/16260 [1:09:10<24:05,  2.85it/s]iter 12140: loss 4.8516, time: 465.32ms\n",
            " 75% 12141/16260 [1:09:10<26:28,  2.59it/s]iter 12141: loss 5.0201, time: 342.31ms\n",
            " 75% 12142/16260 [1:09:11<25:36,  2.68it/s]iter 12142: loss 5.9305, time: 464.86ms\n",
            " 75% 12143/16260 [1:09:11<27:31,  2.49it/s]iter 12143: loss 5.9333, time: 464.56ms\n",
            " 75% 12144/16260 [1:09:12<28:51,  2.38it/s]iter 12144: loss 5.7606, time: 473.35ms\n",
            " 75% 12145/16260 [1:09:12<29:58,  2.29it/s]iter 12145: loss 5.2549, time: 315.87ms\n",
            " 75% 12146/16260 [1:09:12<27:30,  2.49it/s]iter 12146: loss 5.1174, time: 369.18ms\n",
            " 75% 12147/16260 [1:09:13<26:52,  2.55it/s]iter 12147: loss 4.9401, time: 464.08ms\n",
            " 75% 12148/16260 [1:09:13<28:22,  2.41it/s]iter 12148: loss 4.9644, time: 260.10ms\n",
            " 75% 12149/16260 [1:09:14<25:14,  2.71it/s]iter 12149: loss 5.7611, time: 464.21ms\n",
            " 75% 12150/16260 [1:09:14<27:13,  2.52it/s]iter 12150: loss 5.0056, time: 463.83ms\n",
            " 75% 12151/16260 [1:09:15<28:37,  2.39it/s]iter 12151: loss 5.2313, time: 467.25ms\n",
            " 75% 12152/16260 [1:09:15<29:39,  2.31it/s]iter 12152: loss 5.4462, time: 464.74ms\n",
            " 75% 12153/16260 [1:09:15<30:20,  2.26it/s]iter 12153: loss 5.0361, time: 423.04ms\n",
            " 75% 12154/16260 [1:09:16<29:57,  2.28it/s]iter 12154: loss 5.5177, time: 464.45ms\n",
            " 75% 12155/16260 [1:09:16<30:31,  2.24it/s]iter 12155: loss 4.7909, time: 260.31ms\n",
            " 75% 12156/16260 [1:09:17<26:44,  2.56it/s]iter 12156: loss 4.5371, time: 187.85ms\n",
            " 75% 12157/16260 [1:09:17<22:35,  3.03it/s]iter 12157: loss 6.2123, time: 381.03ms\n",
            " 75% 12158/16260 [1:09:17<23:39,  2.89it/s]iter 12158: loss 4.8270, time: 236.21ms\n",
            " 75% 12159/16260 [1:09:17<21:26,  3.19it/s]iter 12159: loss 4.9161, time: 318.65ms\n",
            " 75% 12160/16260 [1:09:18<21:33,  3.17it/s]iter 12160: loss 5.4917, time: 464.31ms\n",
            " 75% 12161/16260 [1:09:18<24:38,  2.77it/s]iter 12161: loss 5.0051, time: 437.82ms\n",
            " 75% 12162/16260 [1:09:19<26:14,  2.60it/s]iter 12162: loss 3.9271, time: 186.65ms\n",
            " 75% 12163/16260 [1:09:19<22:13,  3.07it/s]iter 12163: loss 5.8707, time: 464.16ms\n",
            " 75% 12164/16260 [1:09:19<25:05,  2.72it/s]iter 12164: loss 4.5584, time: 218.32ms\n",
            " 75% 12165/16260 [1:09:19<22:03,  3.09it/s]iter 12165: loss 5.2003, time: 474.29ms\n",
            " 75% 12166/16260 [1:09:20<25:10,  2.71it/s]iter 12166: loss 5.0527, time: 252.50ms\n",
            " 75% 12167/16260 [1:09:20<22:49,  2.99it/s]iter 12167: loss 3.5773, time: 149.82ms\n",
            " 75% 12168/16260 [1:09:20<19:04,  3.58it/s]iter 12168: loss 5.5389, time: 464.47ms\n",
            " 75% 12169/16260 [1:09:21<22:52,  2.98it/s]iter 12169: loss 5.8924, time: 464.28ms\n",
            " 75% 12170/16260 [1:09:21<25:32,  2.67it/s]iter 12170: loss 5.7342, time: 256.92ms\n",
            " 75% 12171/16260 [1:09:22<23:09,  2.94it/s]iter 12171: loss 5.4194, time: 278.54ms\n",
            " 75% 12172/16260 [1:09:22<21:56,  3.11it/s]iter 12172: loss 5.3449, time: 414.05ms\n",
            " 75% 12173/16260 [1:09:22<23:50,  2.86it/s]iter 12173: loss 5.0814, time: 272.82ms\n",
            " 75% 12174/16260 [1:09:23<22:17,  3.06it/s]iter 12174: loss 5.6106, time: 444.62ms\n",
            " 75% 12175/16260 [1:09:23<24:42,  2.76it/s]iter 12175: loss 5.9694, time: 464.36ms\n",
            " 75% 12176/16260 [1:09:23<26:48,  2.54it/s]iter 12176: loss 4.9725, time: 464.49ms\n",
            " 75% 12177/16260 [1:09:24<28:16,  2.41it/s]iter 12177: loss 3.8228, time: 145.03ms\n",
            " 75% 12178/16260 [1:09:24<22:46,  2.99it/s]iter 12178: loss 5.7023, time: 314.84ms\n",
            " 75% 12179/16260 [1:09:24<22:23,  3.04it/s]iter 12179: loss 4.4259, time: 147.40ms\n",
            " 75% 12180/16260 [1:09:25<18:42,  3.64it/s]iter 12180: loss 5.1333, time: 432.72ms\n",
            " 75% 12181/16260 [1:09:25<21:56,  3.10it/s]iter 12181: loss 4.2380, time: 274.26ms\n",
            " 75% 12182/16260 [1:09:25<20:59,  3.24it/s]iter 12182: loss 5.2554, time: 262.17ms\n",
            " 75% 12183/16260 [1:09:26<20:03,  3.39it/s]iter 12183: loss 5.6193, time: 463.54ms\n",
            " 75% 12184/16260 [1:09:26<23:31,  2.89it/s]iter 12184: loss 5.1711, time: 251.54ms\n",
            " 75% 12185/16260 [1:09:26<21:36,  3.14it/s]iter 12185: loss 5.4046, time: 234.97ms\n",
            " 75% 12186/16260 [1:09:26<19:56,  3.40it/s]iter 12186: loss 5.7577, time: 437.61ms\n",
            " 75% 12187/16260 [1:09:27<22:53,  2.96it/s]iter 12187: loss 5.1589, time: 345.40ms\n",
            " 75% 12188/16260 [1:09:27<23:05,  2.94it/s]iter 12188: loss 6.1707, time: 464.74ms\n",
            " 75% 12189/16260 [1:09:28<25:39,  2.65it/s]iter 12189: loss 4.0092, time: 215.98ms\n",
            " 75% 12190/16260 [1:09:28<22:22,  3.03it/s]iter 12190: loss 5.5164, time: 465.47ms\n",
            " 75% 12191/16260 [1:09:28<25:09,  2.70it/s]iter 12191: loss 4.6555, time: 220.18ms\n",
            " 75% 12192/16260 [1:09:29<22:06,  3.07it/s]iter 12192: loss 5.3308, time: 470.92ms\n",
            " 75% 12193/16260 [1:09:29<25:04,  2.70it/s]iter 12193: loss 5.6013, time: 466.33ms\n",
            " 75% 12194/16260 [1:09:30<27:03,  2.50it/s]iter 12194: loss 4.6212, time: 213.52ms\n",
            " 75% 12195/16260 [1:09:30<23:18,  2.91it/s]iter 12195: loss 5.5833, time: 464.24ms\n",
            " 75% 12196/16260 [1:09:30<25:46,  2.63it/s]iter 12196: loss 4.5896, time: 191.06ms\n",
            " 75% 12197/16260 [1:09:30<21:56,  3.09it/s]iter 12197: loss 4.3269, time: 233.60ms\n",
            " 75% 12198/16260 [1:09:31<20:07,  3.36it/s]iter 12198: loss 5.1678, time: 342.07ms\n",
            " 75% 12199/16260 [1:09:31<21:03,  3.21it/s]iter 12199: loss 4.5204, time: 190.23ms\n",
            " 75% 12200/16260 [1:09:31<18:37,  3.63it/s]iter 12200: loss 4.9056, time: 423.24ms\n",
            " 75% 12201/16260 [1:09:32<21:39,  3.12it/s]iter 12201: loss 5.3884, time: 273.74ms\n",
            " 75% 12202/16260 [1:09:32<20:44,  3.26it/s]iter 12202: loss 4.7874, time: 231.09ms\n",
            " 75% 12203/16260 [1:09:32<19:13,  3.52it/s]iter 12203: loss 4.5088, time: 218.37ms\n",
            " 75% 12204/16260 [1:09:32<17:55,  3.77it/s]iter 12204: loss 5.6295, time: 464.64ms\n",
            " 75% 12205/16260 [1:09:33<21:59,  3.07it/s]iter 12205: loss 4.7414, time: 169.82ms\n",
            " 75% 12206/16260 [1:09:33<18:52,  3.58it/s]iter 12206: loss 6.2797, time: 464.74ms\n",
            " 75% 12207/16260 [1:09:33<22:39,  2.98it/s]iter 12207: loss 5.3025, time: 379.01ms\n",
            " 75% 12208/16260 [1:09:34<23:33,  2.87it/s]iter 12208: loss 5.9889, time: 472.72ms\n",
            " 75% 12209/16260 [1:09:34<26:06,  2.59it/s]iter 12209: loss 5.0563, time: 303.89ms\n",
            " 75% 12210/16260 [1:09:35<24:26,  2.76it/s]iter 12210: loss 5.4065, time: 382.20ms\n",
            " 75% 12211/16260 [1:09:35<24:52,  2.71it/s]iter 12211: loss 5.4436, time: 464.43ms\n",
            " 75% 12212/16260 [1:09:35<26:50,  2.51it/s]iter 12212: loss 3.9151, time: 169.99ms\n",
            " 75% 12213/16260 [1:09:36<22:15,  3.03it/s]iter 12213: loss 6.8974, time: 464.12ms\n",
            " 75% 12214/16260 [1:09:36<24:59,  2.70it/s]iter 12214: loss 4.3351, time: 342.09ms\n",
            " 75% 12215/16260 [1:09:36<24:26,  2.76it/s]iter 12215: loss 4.5537, time: 157.40ms\n",
            " 75% 12216/16260 [1:09:37<20:19,  3.32it/s]iter 12216: loss 5.3233, time: 464.92ms\n",
            " 75% 12217/16260 [1:09:37<23:38,  2.85it/s]iter 12217: loss 5.1810, time: 312.52ms\n",
            " 75% 12218/16260 [1:09:37<22:53,  2.94it/s]iter 12218: loss 5.1059, time: 341.71ms\n",
            " 75% 12219/16260 [1:09:38<22:57,  2.93it/s]iter 12219: loss 5.0990, time: 309.79ms\n",
            " 75% 12220/16260 [1:09:38<22:21,  3.01it/s]iter 12220: loss 6.1178, time: 315.78ms\n",
            " 75% 12221/16260 [1:09:38<22:03,  3.05it/s]iter 12221: loss 5.2396, time: 436.16ms\n",
            " 75% 12222/16260 [1:09:39<24:16,  2.77it/s]iter 12222: loss 5.4654, time: 367.81ms\n",
            " 75% 12223/16260 [1:09:39<24:26,  2.75it/s]iter 12223: loss 5.2481, time: 346.69ms\n",
            " 75% 12224/16260 [1:09:40<24:07,  2.79it/s]iter 12224: loss 5.9028, time: 463.42ms\n",
            " 75% 12225/16260 [1:09:40<26:15,  2.56it/s]iter 12225: loss 5.2817, time: 274.78ms\n",
            " 75% 12226/16260 [1:09:40<23:56,  2.81it/s]iter 12226: loss 4.7331, time: 251.84ms\n",
            " 75% 12227/16260 [1:09:41<21:52,  3.07it/s]iter 12227: loss 5.4885, time: 376.53ms\n",
            " 75% 12228/16260 [1:09:41<22:55,  2.93it/s]iter 12228: loss 5.9524, time: 464.09ms\n",
            " 75% 12229/16260 [1:09:41<25:26,  2.64it/s]iter 12229: loss 5.8167, time: 465.36ms\n",
            " 75% 12230/16260 [1:09:42<27:12,  2.47it/s]iter 12230: loss 5.4890, time: 464.82ms\n",
            " 75% 12231/16260 [1:09:42<28:26,  2.36it/s]iter 12231: loss 5.4367, time: 464.63ms\n",
            " 75% 12232/16260 [1:09:43<29:17,  2.29it/s]iter 12232: loss 4.6976, time: 190.19ms\n",
            " 75% 12233/16260 [1:09:43<24:21,  2.75it/s]iter 12233: loss 5.7336, time: 470.58ms\n",
            " 75% 12234/16260 [1:09:43<26:33,  2.53it/s]iter 12234: loss 5.6033, time: 378.96ms\n",
            " 75% 12235/16260 [1:09:44<26:14,  2.56it/s]iter 12235: loss 4.9514, time: 186.59ms\n",
            " 75% 12236/16260 [1:09:44<22:08,  3.03it/s]iter 12236: loss 5.3114, time: 315.79ms\n",
            " 75% 12237/16260 [1:09:44<21:52,  3.06it/s]iter 12237: loss 4.9527, time: 473.62ms\n",
            " 75% 12238/16260 [1:09:45<24:52,  2.70it/s]iter 12238: loss 5.3202, time: 464.36ms\n",
            " 75% 12239/16260 [1:09:45<26:46,  2.50it/s]iter 12239: loss 5.2310, time: 335.70ms\n",
            " 75% 12240/16260 [1:09:46<25:30,  2.63it/s]iter 12240: loss 5.5393, time: 474.75ms\n",
            " 75% 12241/16260 [1:09:46<27:25,  2.44it/s]iter 12241: loss 5.8109, time: 464.38ms\n",
            " 75% 12242/16260 [1:09:47<28:33,  2.35it/s]iter 12242: loss 5.3461, time: 445.20ms\n",
            " 75% 12243/16260 [1:09:47<28:57,  2.31it/s]iter 12243: loss 5.0833, time: 193.83ms\n",
            " 75% 12244/16260 [1:09:47<24:11,  2.77it/s]iter 12244: loss 4.5720, time: 252.68ms\n",
            " 75% 12245/16260 [1:09:47<22:01,  3.04it/s]iter 12245: loss 4.8631, time: 216.43ms\n",
            " 75% 12246/16260 [1:09:48<19:47,  3.38it/s]iter 12246: loss 4.9536, time: 232.54ms\n",
            " 75% 12247/16260 [1:09:48<18:32,  3.61it/s]iter 12247: loss 5.7183, time: 252.65ms\n",
            " 75% 12248/16260 [1:09:48<18:04,  3.70it/s]iter 12248: loss 4.3828, time: 163.21ms\n",
            " 75% 12249/16260 [1:09:48<15:57,  4.19it/s]iter 12249: loss 5.5206, time: 252.19ms\n",
            " 75% 12250/16260 [1:09:49<16:14,  4.11it/s]iter 12250: loss 5.3449, time: 463.45ms\n",
            " 75% 12251/16260 [1:09:49<20:41,  3.23it/s]iter 12251: loss 5.2199, time: 214.84ms\n",
            " 75% 12252/16260 [1:09:49<18:48,  3.55it/s]iter 12252: loss 3.4538, time: 138.55ms\n",
            " 75% 12253/16260 [1:09:49<15:58,  4.18it/s]iter 12253: loss 3.2164, time: 137.65ms\n",
            " 75% 12254/16260 [1:09:50<13:57,  4.78it/s]iter 12254: loss 4.2315, time: 171.83ms\n",
            " 75% 12255/16260 [1:09:50<13:14,  5.04it/s]iter 12255: loss 5.2401, time: 348.23ms\n",
            " 75% 12256/16260 [1:09:50<16:15,  4.10it/s]iter 12256: loss 5.3078, time: 463.78ms\n",
            " 75% 12257/16260 [1:09:50<20:41,  3.22it/s]iter 12257: loss 4.4434, time: 188.27ms\n",
            " 75% 12258/16260 [1:09:51<18:17,  3.65it/s]iter 12258: loss 5.6698, time: 474.16ms\n",
            " 75% 12259/16260 [1:09:51<22:19,  2.99it/s]iter 12259: loss 5.5502, time: 234.74ms\n",
            " 75% 12260/16260 [1:09:51<20:21,  3.28it/s]iter 12260: loss 4.7046, time: 239.06ms\n",
            " 75% 12261/16260 [1:09:52<19:03,  3.50it/s]iter 12261: loss 5.3429, time: 351.01ms\n",
            " 75% 12262/16260 [1:09:52<20:22,  3.27it/s]iter 12262: loss 5.4702, time: 278.24ms\n",
            " 75% 12263/16260 [1:09:52<19:51,  3.36it/s]iter 12263: loss 5.2861, time: 260.92ms\n",
            " 75% 12264/16260 [1:09:53<19:08,  3.48it/s]iter 12264: loss 5.4168, time: 464.61ms\n",
            " 75% 12265/16260 [1:09:53<22:41,  2.93it/s]iter 12265: loss 5.5726, time: 422.56ms\n",
            " 75% 12266/16260 [1:09:53<24:21,  2.73it/s]iter 12266: loss 5.7586, time: 463.26ms\n",
            " 75% 12267/16260 [1:09:54<26:19,  2.53it/s]iter 12267: loss 5.4591, time: 463.58ms\n",
            " 75% 12268/16260 [1:09:54<27:42,  2.40it/s]iter 12268: loss 4.7888, time: 170.64ms\n",
            " 75% 12269/16260 [1:09:55<22:49,  2.91it/s]iter 12269: loss 5.8261, time: 463.14ms\n",
            " 75% 12270/16260 [1:09:55<25:14,  2.63it/s]iter 12270: loss 5.4188, time: 279.87ms\n",
            " 75% 12271/16260 [1:09:55<23:16,  2.86it/s]iter 12271: loss 4.4376, time: 164.20ms\n",
            " 75% 12272/16260 [1:09:55<19:35,  3.39it/s]iter 12272: loss 5.4919, time: 464.14ms\n",
            " 75% 12273/16260 [1:09:56<22:59,  2.89it/s]iter 12273: loss 5.4219, time: 463.85ms\n",
            " 75% 12274/16260 [1:09:56<25:22,  2.62it/s]iter 12274: loss 5.5886, time: 468.73ms\n",
            " 75% 12275/16260 [1:09:57<27:07,  2.45it/s]iter 12275: loss 5.5497, time: 468.31ms\n",
            " 75% 12276/16260 [1:09:57<28:20,  2.34it/s]iter 12276: loss 4.8976, time: 309.67ms\n",
            " 76% 12277/16260 [1:09:58<26:02,  2.55it/s]iter 12277: loss 4.7653, time: 215.06ms\n",
            " 76% 12278/16260 [1:09:58<22:32,  2.94it/s]iter 12278: loss 5.2687, time: 420.10ms\n",
            " 76% 12279/16260 [1:09:58<24:09,  2.75it/s]iter 12279: loss 5.3943, time: 281.52ms\n",
            " 76% 12280/16260 [1:09:59<22:32,  2.94it/s]iter 12280: loss 4.8725, time: 463.17ms\n",
            " 76% 12281/16260 [1:09:59<25:01,  2.65it/s]iter 12281: loss 6.0948, time: 464.38ms\n",
            " 76% 12282/16260 [1:09:59<26:46,  2.48it/s]iter 12282: loss 4.7188, time: 413.03ms\n",
            " 76% 12283/16260 [1:10:00<26:58,  2.46it/s]iter 12283: loss 4.9840, time: 344.63ms\n",
            " 76% 12284/16260 [1:10:00<25:45,  2.57it/s]iter 12284: loss 4.6232, time: 190.07ms\n",
            " 76% 12285/16260 [1:10:00<21:50,  3.03it/s]iter 12285: loss 5.8175, time: 464.28ms\n",
            " 76% 12286/16260 [1:10:01<24:32,  2.70it/s]iter 12286: loss 4.9245, time: 244.56ms\n",
            " 76% 12287/16260 [1:10:01<22:03,  3.00it/s]iter 12287: loss 5.0557, time: 214.42ms\n",
            " 76% 12288/16260 [1:10:01<19:43,  3.36it/s]iter 12288: loss 4.7392, time: 230.69ms\n",
            " 76% 12289/16260 [1:10:02<18:24,  3.60it/s]iter 12289: loss 4.9579, time: 214.02ms\n",
            " 76% 12290/16260 [1:10:02<17:09,  3.86it/s]iter 12290: loss 5.3312, time: 341.67ms\n",
            " 76% 12291/16260 [1:10:02<18:49,  3.51it/s]iter 12291: loss 3.5657, time: 165.42ms\n",
            " 76% 12292/16260 [1:10:02<16:29,  4.01it/s]iter 12292: loss 5.5193, time: 266.32ms\n",
            " 76% 12293/16260 [1:10:03<16:50,  3.92it/s]iter 12293: loss 4.7074, time: 232.75ms\n",
            " 76% 12294/16260 [1:10:03<16:26,  4.02it/s]iter 12294: loss 4.7879, time: 306.56ms\n",
            " 76% 12295/16260 [1:10:03<17:36,  3.75it/s]iter 12295: loss 5.2468, time: 463.61ms\n",
            " 76% 12296/16260 [1:10:04<21:32,  3.07it/s]iter 12296: loss 5.8327, time: 471.95ms\n",
            " 76% 12297/16260 [1:10:04<24:27,  2.70it/s]iter 12297: loss 5.9097, time: 336.55ms\n",
            " 76% 12298/16260 [1:10:04<23:48,  2.77it/s]iter 12298: loss 5.8478, time: 463.66ms\n",
            " 76% 12299/16260 [1:10:05<25:52,  2.55it/s]iter 12299: loss 4.8820, time: 258.91ms\n",
            " 76% 12300/16260 [1:10:05<23:16,  2.84it/s]iter 12300: loss 5.6428, time: 464.29ms\n",
            " 76% 12301/16260 [1:10:06<25:30,  2.59it/s]iter 12301: loss 5.3619, time: 238.36ms\n",
            " 76% 12302/16260 [1:10:06<22:35,  2.92it/s]iter 12302: loss 5.6189, time: 464.09ms\n",
            " 76% 12303/16260 [1:10:06<25:02,  2.63it/s]iter 12303: loss 5.6541, time: 414.71ms\n",
            " 76% 12304/16260 [1:10:07<25:45,  2.56it/s]iter 12304: loss 3.3278, time: 185.75ms\n",
            " 76% 12305/16260 [1:10:07<21:43,  3.03it/s]iter 12305: loss 5.6043, time: 463.72ms\n",
            " 76% 12306/16260 [1:10:07<24:23,  2.70it/s]iter 12306: loss 4.7607, time: 191.72ms\n",
            " 76% 12307/16260 [1:10:08<20:53,  3.15it/s]iter 12307: loss 5.0918, time: 233.66ms\n",
            " 76% 12308/16260 [1:10:08<19:16,  3.42it/s]iter 12308: loss 5.2710, time: 464.41ms\n",
            " 76% 12309/16260 [1:10:08<22:41,  2.90it/s]iter 12309: loss 5.7812, time: 463.80ms\n",
            " 76% 12310/16260 [1:10:09<25:04,  2.63it/s]iter 12310: loss 5.1486, time: 426.76ms\n",
            " 76% 12311/16260 [1:10:09<25:59,  2.53it/s]iter 12311: loss 5.2685, time: 464.34ms\n",
            " 76% 12312/16260 [1:10:10<27:23,  2.40it/s]iter 12312: loss 5.9013, time: 464.58ms\n",
            " 76% 12313/16260 [1:10:10<28:22,  2.32it/s]iter 12313: loss 4.9793, time: 283.45ms\n",
            " 76% 12314/16260 [1:10:10<25:28,  2.58it/s]iter 12314: loss 5.4635, time: 464.37ms\n",
            " 76% 12315/16260 [1:10:11<27:01,  2.43it/s]iter 12315: loss 4.8769, time: 255.24ms\n",
            " 76% 12316/16260 [1:10:11<23:58,  2.74it/s]iter 12316: loss 5.0720, time: 444.72ms\n",
            " 76% 12317/16260 [1:10:12<25:34,  2.57it/s]iter 12317: loss 5.6924, time: 259.28ms\n",
            " 76% 12318/16260 [1:10:12<23:02,  2.85it/s]iter 12318: loss 5.7194, time: 464.10ms\n",
            " 76% 12319/16260 [1:10:12<25:18,  2.60it/s]iter 12319: loss 4.3125, time: 170.92ms\n",
            " 76% 12320/16260 [1:10:12<21:05,  3.11it/s]iter 12320: loss 5.3946, time: 436.57ms\n",
            " 76% 12321/16260 [1:10:13<23:23,  2.81it/s]iter 12321: loss 5.2320, time: 465.26ms\n",
            " 76% 12322/16260 [1:10:13<25:34,  2.57it/s]iter 12322: loss 5.0830, time: 314.07ms\n",
            " 76% 12323/16260 [1:10:14<24:06,  2.72it/s]iter 12323: loss 5.5225, time: 412.28ms\n",
            " 76% 12324/16260 [1:10:14<25:00,  2.62it/s]iter 12324: loss 4.7906, time: 304.48ms\n",
            " 76% 12325/16260 [1:10:14<23:31,  2.79it/s]iter 12325: loss 5.3405, time: 342.21ms\n",
            " 76% 12326/16260 [1:10:15<23:13,  2.82it/s]iter 12326: loss 5.6347, time: 465.14ms\n",
            " 76% 12327/16260 [1:10:15<25:25,  2.58it/s]iter 12327: loss 5.1196, time: 464.55ms\n",
            " 76% 12328/16260 [1:10:16<26:57,  2.43it/s]iter 12328: loss 5.3663, time: 218.91ms\n",
            " 76% 12329/16260 [1:10:16<23:12,  2.82it/s]iter 12329: loss 3.8138, time: 136.04ms\n",
            " 76% 12330/16260 [1:10:16<18:56,  3.46it/s]iter 12330: loss 5.5927, time: 473.65ms\n",
            " 76% 12331/16260 [1:10:16<22:35,  2.90it/s]iter 12331: loss 5.7023, time: 464.52ms\n",
            " 76% 12332/16260 [1:10:17<24:57,  2.62it/s]iter 12332: loss 5.8912, time: 464.04ms\n",
            " 76% 12333/16260 [1:10:17<26:36,  2.46it/s]iter 12333: loss 5.7023, time: 414.16ms\n",
            " 76% 12334/16260 [1:10:18<26:47,  2.44it/s]iter 12334: loss 5.2282, time: 435.61ms\n",
            " 76% 12335/16260 [1:10:18<27:19,  2.39it/s]iter 12335: loss 5.4889, time: 461.99ms\n",
            " 76% 12336/16260 [1:10:19<28:13,  2.32it/s]iter 12336: loss 4.9586, time: 278.43ms\n",
            " 76% 12337/16260 [1:10:19<25:14,  2.59it/s]iter 12337: loss 5.4769, time: 317.29ms\n",
            " 76% 12338/16260 [1:10:19<23:55,  2.73it/s]iter 12338: loss 5.3341, time: 464.32ms\n",
            " 76% 12339/16260 [1:10:20<25:52,  2.53it/s]iter 12339: loss 5.7628, time: 464.20ms\n",
            " 76% 12340/16260 [1:10:20<27:14,  2.40it/s]iter 12340: loss 5.2608, time: 465.31ms\n",
            " 76% 12341/16260 [1:10:21<28:13,  2.31it/s]iter 12341: loss 5.2209, time: 319.04ms\n",
            " 76% 12342/16260 [1:10:21<26:01,  2.51it/s]iter 12342: loss 5.2177, time: 471.21ms\n",
            " 76% 12343/16260 [1:10:22<27:29,  2.38it/s]iter 12343: loss 5.4207, time: 256.41ms\n",
            " 76% 12344/16260 [1:10:22<24:17,  2.69it/s]iter 12344: loss 5.6474, time: 463.10ms\n",
            " 76% 12345/16260 [1:10:22<26:05,  2.50it/s]iter 12345: loss 5.1948, time: 270.85ms\n",
            " 76% 12346/16260 [1:10:23<23:35,  2.76it/s]iter 12346: loss 5.7771, time: 464.66ms\n",
            " 76% 12347/16260 [1:10:23<25:38,  2.54it/s]iter 12347: loss 5.2644, time: 473.86ms\n",
            " 76% 12348/16260 [1:10:23<27:14,  2.39it/s]iter 12348: loss 6.0020, time: 463.89ms\n",
            " 76% 12349/16260 [1:10:24<28:09,  2.31it/s]iter 12349: loss 5.5858, time: 422.62ms\n",
            " 76% 12350/16260 [1:10:24<28:00,  2.33it/s]iter 12350: loss 4.9865, time: 219.19ms\n",
            " 76% 12351/16260 [1:10:25<23:54,  2.72it/s]iter 12351: loss 4.9443, time: 258.51ms\n",
            " 76% 12352/16260 [1:10:25<21:48,  2.99it/s]iter 12352: loss 5.9558, time: 444.83ms\n",
            " 76% 12353/16260 [1:10:25<23:58,  2.72it/s]iter 12353: loss 5.6258, time: 286.59ms\n",
            " 76% 12354/16260 [1:10:26<22:24,  2.90it/s]iter 12354: loss 5.7285, time: 368.23ms\n",
            " 76% 12355/16260 [1:10:26<22:54,  2.84it/s]iter 12355: loss 5.0403, time: 251.95ms\n",
            " 76% 12356/16260 [1:10:26<20:58,  3.10it/s]iter 12356: loss 4.0878, time: 214.27ms\n",
            " 76% 12357/16260 [1:10:26<18:53,  3.44it/s]iter 12357: loss 5.5809, time: 463.51ms\n",
            " 76% 12358/16260 [1:10:27<22:17,  2.92it/s]iter 12358: loss 5.0652, time: 440.70ms\n",
            " 76% 12359/16260 [1:10:27<24:13,  2.68it/s]iter 12359: loss 5.5086, time: 465.38ms\n",
            " 76% 12360/16260 [1:10:28<26:03,  2.49it/s]iter 12360: loss 4.7462, time: 260.92ms\n",
            " 76% 12361/16260 [1:10:28<23:20,  2.78it/s]iter 12361: loss 4.7022, time: 421.31ms\n",
            " 76% 12362/16260 [1:10:28<24:34,  2.64it/s]iter 12362: loss 4.1468, time: 148.67ms\n",
            " 76% 12363/16260 [1:10:29<20:07,  3.23it/s]iter 12363: loss 4.7956, time: 240.55ms\n",
            " 76% 12364/16260 [1:10:29<18:47,  3.45it/s]iter 12364: loss 4.8410, time: 277.03ms\n",
            " 76% 12365/16260 [1:10:29<18:35,  3.49it/s]iter 12365: loss 5.1539, time: 194.46ms\n",
            " 76% 12366/16260 [1:10:29<16:49,  3.86it/s]iter 12366: loss 5.6897, time: 464.28ms\n",
            " 76% 12367/16260 [1:10:30<20:50,  3.11it/s]iter 12367: loss 4.7887, time: 256.43ms\n",
            " 76% 12368/16260 [1:10:30<19:36,  3.31it/s]iter 12368: loss 5.2906, time: 433.71ms\n",
            " 76% 12369/16260 [1:10:30<22:10,  2.92it/s]iter 12369: loss 6.2360, time: 342.85ms\n",
            " 76% 12370/16260 [1:10:31<22:13,  2.92it/s]iter 12370: loss 6.0556, time: 464.86ms\n",
            " 76% 12371/16260 [1:10:31<24:37,  2.63it/s]iter 12371: loss 4.4622, time: 210.20ms\n",
            " 76% 12372/16260 [1:10:32<21:20,  3.04it/s]iter 12372: loss 4.5733, time: 193.08ms\n",
            " 76% 12373/16260 [1:10:32<18:43,  3.46it/s]iter 12373: loss 5.8089, time: 464.21ms\n",
            " 76% 12374/16260 [1:10:32<22:09,  2.92it/s]iter 12374: loss 5.2819, time: 272.90ms\n",
            " 76% 12375/16260 [1:10:32<20:50,  3.11it/s]iter 12375: loss 4.7147, time: 189.96ms\n",
            " 76% 12376/16260 [1:10:33<18:17,  3.54it/s]iter 12376: loss 3.9554, time: 176.58ms\n",
            " 76% 12377/16260 [1:10:33<16:15,  3.98it/s]iter 12377: loss 4.4786, time: 189.79ms\n",
            " 76% 12378/16260 [1:10:33<15:05,  4.29it/s]iter 12378: loss 4.2879, time: 187.52ms\n",
            " 76% 12379/16260 [1:10:33<14:13,  4.55it/s]iter 12379: loss 5.0504, time: 209.40ms\n",
            " 76% 12380/16260 [1:10:33<14:02,  4.61it/s]iter 12380: loss 5.4560, time: 464.37ms\n",
            " 76% 12381/16260 [1:10:34<18:51,  3.43it/s]iter 12381: loss 5.0287, time: 308.87ms\n",
            " 76% 12382/16260 [1:10:34<19:13,  3.36it/s]iter 12382: loss 5.2597, time: 473.09ms\n",
            " 76% 12383/16260 [1:10:35<22:38,  2.85it/s]iter 12383: loss 4.9797, time: 289.53ms\n",
            " 76% 12384/16260 [1:10:35<21:28,  3.01it/s]iter 12384: loss 5.2305, time: 463.37ms\n",
            " 76% 12385/16260 [1:10:35<24:02,  2.69it/s]iter 12385: loss 5.4678, time: 464.96ms\n",
            " 76% 12386/16260 [1:10:36<25:51,  2.50it/s]iter 12386: loss 5.1134, time: 282.44ms\n",
            " 76% 12387/16260 [1:10:36<23:35,  2.74it/s]iter 12387: loss 5.1829, time: 464.66ms\n",
            " 76% 12388/16260 [1:10:37<25:32,  2.53it/s]iter 12388: loss 5.2527, time: 463.84ms\n",
            " 76% 12389/16260 [1:10:37<26:52,  2.40it/s]iter 12389: loss 5.8816, time: 464.61ms\n",
            " 76% 12390/16260 [1:10:38<27:49,  2.32it/s]iter 12390: loss 5.8016, time: 463.86ms\n",
            " 76% 12391/16260 [1:10:38<28:28,  2.26it/s]iter 12391: loss 5.9005, time: 463.93ms\n",
            " 76% 12392/16260 [1:10:38<28:55,  2.23it/s]iter 12392: loss 6.1176, time: 463.69ms\n",
            " 76% 12393/16260 [1:10:39<29:14,  2.20it/s]iter 12393: loss 5.5404, time: 306.24ms\n",
            " 76% 12394/16260 [1:10:39<26:24,  2.44it/s]iter 12394: loss 5.4646, time: 463.81ms\n",
            " 76% 12395/16260 [1:10:40<27:28,  2.34it/s]iter 12395: loss 4.3060, time: 165.65ms\n",
            " 76% 12396/16260 [1:10:40<22:27,  2.87it/s]iter 12396: loss 5.5890, time: 314.76ms\n",
            " 76% 12397/16260 [1:10:40<21:49,  2.95it/s]iter 12397: loss 5.3137, time: 434.04ms\n",
            " 76% 12398/16260 [1:10:41<23:41,  2.72it/s]iter 12398: loss 5.0507, time: 273.31ms\n",
            " 76% 12399/16260 [1:10:41<21:52,  2.94it/s]iter 12399: loss 5.5990, time: 463.97ms\n",
            " 76% 12400/16260 [1:10:41<24:17,  2.65it/s]iter 12400: loss 4.9282, time: 314.22ms\n",
            " 76% 12401/16260 [1:10:42<23:05,  2.79it/s]iter 12401: loss 5.0909, time: 467.22ms\n",
            " 76% 12402/16260 [1:10:42<25:12,  2.55it/s]iter 12402: loss 5.3909, time: 212.96ms\n",
            " 76% 12403/16260 [1:10:42<21:46,  2.95it/s]iter 12403: loss 4.7308, time: 422.80ms\n",
            " 76% 12404/16260 [1:10:43<23:24,  2.74it/s]iter 12404: loss 5.5704, time: 468.74ms\n",
            " 76% 12405/16260 [1:10:43<25:26,  2.53it/s]iter 12405: loss 4.9778, time: 255.69ms\n",
            " 76% 12406/16260 [1:10:44<22:45,  2.82it/s]iter 12406: loss 6.1858, time: 464.94ms\n",
            " 76% 12407/16260 [1:10:44<24:55,  2.58it/s]iter 12407: loss 5.8451, time: 465.17ms\n",
            " 76% 12408/16260 [1:10:44<26:25,  2.43it/s]iter 12408: loss 5.7867, time: 464.59ms\n",
            " 76% 12409/16260 [1:10:45<27:28,  2.34it/s]iter 12409: loss 4.3208, time: 273.09ms\n",
            " 76% 12410/16260 [1:10:45<24:30,  2.62it/s]iter 12410: loss 4.7828, time: 341.03ms\n",
            " 76% 12411/16260 [1:10:46<23:44,  2.70it/s]iter 12411: loss 3.8748, time: 174.28ms\n",
            " 76% 12412/16260 [1:10:46<20:00,  3.21it/s]iter 12412: loss 4.8214, time: 171.37ms\n",
            " 76% 12413/16260 [1:10:46<17:19,  3.70it/s]iter 12413: loss 4.8828, time: 290.78ms\n",
            " 76% 12414/16260 [1:10:46<17:44,  3.61it/s]iter 12414: loss 5.8212, time: 372.17ms\n",
            " 76% 12415/16260 [1:10:47<19:35,  3.27it/s]iter 12415: loss 3.7067, time: 141.90ms\n",
            " 76% 12416/16260 [1:10:47<16:27,  3.89it/s]iter 12416: loss 5.1391, time: 474.83ms\n",
            " 76% 12417/16260 [1:10:47<20:40,  3.10it/s]iter 12417: loss 4.7908, time: 342.23ms\n",
            " 76% 12418/16260 [1:10:48<21:04,  3.04it/s]iter 12418: loss 5.5334, time: 341.32ms\n",
            " 76% 12419/16260 [1:10:48<21:20,  3.00it/s]iter 12419: loss 4.8841, time: 463.74ms\n",
            " 76% 12420/16260 [1:10:48<23:51,  2.68it/s]iter 12420: loss 5.1781, time: 257.25ms\n",
            " 76% 12421/16260 [1:10:49<21:40,  2.95it/s]iter 12421: loss 5.8246, time: 470.46ms\n",
            " 76% 12422/16260 [1:10:49<24:13,  2.64it/s]iter 12422: loss 5.1273, time: 424.15ms\n",
            " 76% 12423/16260 [1:10:49<25:07,  2.55it/s]iter 12423: loss 5.4121, time: 255.04ms\n",
            " 76% 12424/16260 [1:10:50<22:29,  2.84it/s]iter 12424: loss 4.7385, time: 213.80ms\n",
            " 76% 12425/16260 [1:10:50<19:52,  3.22it/s]iter 12425: loss 5.5094, time: 442.48ms\n",
            " 76% 12426/16260 [1:10:50<22:25,  2.85it/s]iter 12426: loss 4.3018, time: 164.14ms\n",
            " 76% 12427/16260 [1:10:51<18:51,  3.39it/s]iter 12427: loss 4.8411, time: 272.22ms\n",
            " 76% 12428/16260 [1:10:51<18:26,  3.46it/s]iter 12428: loss 4.8718, time: 238.09ms\n",
            " 76% 12429/16260 [1:10:51<17:29,  3.65it/s]iter 12429: loss 4.7587, time: 259.80ms\n",
            " 76% 12430/16260 [1:10:51<17:14,  3.70it/s]iter 12430: loss 5.2530, time: 464.06ms\n",
            " 76% 12431/16260 [1:10:52<20:59,  3.04it/s]iter 12431: loss 4.4458, time: 186.88ms\n",
            " 76% 12432/16260 [1:10:52<18:17,  3.49it/s]iter 12432: loss 4.9416, time: 317.36ms\n",
            " 76% 12433/16260 [1:10:52<18:54,  3.37it/s]iter 12433: loss 5.1434, time: 277.01ms\n",
            " 76% 12434/16260 [1:10:53<18:33,  3.44it/s]iter 12434: loss 4.8681, time: 208.98ms\n",
            " 76% 12435/16260 [1:10:53<17:00,  3.75it/s]iter 12435: loss 5.3278, time: 234.29ms\n",
            " 76% 12436/16260 [1:10:53<16:24,  3.88it/s]iter 12436: loss 5.8205, time: 464.06ms\n",
            " 76% 12437/16260 [1:10:54<20:22,  3.13it/s]iter 12437: loss 6.2879, time: 463.89ms\n",
            " 76% 12438/16260 [1:10:54<23:09,  2.75it/s]iter 12438: loss 4.3667, time: 163.38ms\n",
            " 77% 12439/16260 [1:10:54<19:20,  3.29it/s]iter 12439: loss 5.5844, time: 441.40ms\n",
            " 77% 12440/16260 [1:10:55<21:59,  2.89it/s]iter 12440: loss 5.1713, time: 211.81ms\n",
            " 77% 12441/16260 [1:10:55<19:27,  3.27it/s]iter 12441: loss 5.5663, time: 339.02ms\n",
            " 77% 12442/16260 [1:10:55<20:07,  3.16it/s]iter 12442: loss 6.0922, time: 419.73ms\n",
            " 77% 12443/16260 [1:10:56<22:07,  2.88it/s]iter 12443: loss 4.7076, time: 189.01ms\n",
            " 77% 12444/16260 [1:10:56<19:06,  3.33it/s]iter 12444: loss 5.1243, time: 463.97ms\n",
            " 77% 12445/16260 [1:10:56<22:15,  2.86it/s]iter 12445: loss 5.3291, time: 275.63ms\n",
            " 77% 12446/16260 [1:10:56<20:51,  3.05it/s]iter 12446: loss 4.1434, time: 209.15ms\n",
            " 77% 12447/16260 [1:10:57<18:36,  3.41it/s]iter 12447: loss 4.9890, time: 220.80ms\n",
            " 77% 12448/16260 [1:10:57<17:15,  3.68it/s]iter 12448: loss 4.4342, time: 236.45ms\n",
            " 77% 12449/16260 [1:10:57<16:36,  3.82it/s]iter 12449: loss 5.1990, time: 471.16ms\n",
            " 77% 12450/16260 [1:10:58<20:37,  3.08it/s]iter 12450: loss 5.1837, time: 464.10ms\n",
            " 77% 12451/16260 [1:10:58<23:18,  2.72it/s]iter 12451: loss 5.3850, time: 465.59ms\n",
            " 77% 12452/16260 [1:10:59<25:12,  2.52it/s]iter 12452: loss 3.4209, time: 158.05ms\n",
            " 77% 12453/16260 [1:10:59<20:40,  3.07it/s]iter 12453: loss 4.4111, time: 163.89ms\n",
            " 77% 12454/16260 [1:10:59<17:36,  3.60it/s]iter 12454: loss 4.6987, time: 226.16ms\n",
            " 77% 12455/16260 [1:10:59<16:39,  3.81it/s]iter 12455: loss 5.4883, time: 419.89ms\n",
            " 77% 12456/16260 [1:11:00<19:40,  3.22it/s]iter 12456: loss 5.2091, time: 464.34ms\n",
            " 77% 12457/16260 [1:11:00<22:37,  2.80it/s]iter 12457: loss 5.4481, time: 367.86ms\n",
            " 77% 12458/16260 [1:11:00<22:51,  2.77it/s]iter 12458: loss 4.0156, time: 150.04ms\n",
            " 77% 12459/16260 [1:11:01<18:52,  3.36it/s]iter 12459: loss 4.8916, time: 278.53ms\n",
            " 77% 12460/16260 [1:11:01<18:31,  3.42it/s]iter 12460: loss 4.8287, time: 464.48ms\n",
            " 77% 12461/16260 [1:11:01<21:49,  2.90it/s]iter 12461: loss 5.4320, time: 445.60ms\n",
            " 77% 12462/16260 [1:11:02<23:45,  2.66it/s]iter 12462: loss 5.6210, time: 438.32ms\n",
            " 77% 12463/16260 [1:11:02<24:58,  2.53it/s]iter 12463: loss 5.5901, time: 464.68ms\n",
            " 77% 12464/16260 [1:11:03<26:19,  2.40it/s]iter 12464: loss 5.7747, time: 463.76ms\n",
            " 77% 12465/16260 [1:11:03<27:15,  2.32it/s]iter 12465: loss 5.0891, time: 314.85ms\n",
            " 77% 12466/16260 [1:11:03<25:04,  2.52it/s]iter 12466: loss 3.6650, time: 189.63ms\n",
            " 77% 12467/16260 [1:11:04<21:10,  2.99it/s]iter 12467: loss 5.2799, time: 275.33ms\n",
            " 77% 12468/16260 [1:11:04<20:03,  3.15it/s]iter 12468: loss 5.3529, time: 464.20ms\n",
            " 77% 12469/16260 [1:11:04<22:52,  2.76it/s]iter 12469: loss 4.5151, time: 196.31ms\n",
            " 77% 12470/16260 [1:11:05<19:45,  3.20it/s]iter 12470: loss 4.5517, time: 252.89ms\n",
            " 77% 12471/16260 [1:11:05<18:38,  3.39it/s]iter 12471: loss 4.7897, time: 186.37ms\n",
            " 77% 12472/16260 [1:11:05<16:35,  3.80it/s]iter 12472: loss 5.1908, time: 255.85ms\n",
            " 77% 12473/16260 [1:11:05<16:29,  3.83it/s]iter 12473: loss 5.3194, time: 312.99ms\n",
            " 77% 12474/16260 [1:11:06<17:29,  3.61it/s]iter 12474: loss 4.2835, time: 209.47ms\n",
            " 77% 12475/16260 [1:11:06<16:14,  3.89it/s]iter 12475: loss 4.7700, time: 308.85ms\n",
            " 77% 12476/16260 [1:11:06<17:13,  3.66it/s]iter 12476: loss 5.6903, time: 464.49ms\n",
            " 77% 12477/16260 [1:11:07<20:52,  3.02it/s]iter 12477: loss 5.1773, time: 275.78ms\n",
            " 77% 12478/16260 [1:11:07<19:51,  3.17it/s]iter 12478: loss 4.5593, time: 464.70ms\n",
            " 77% 12479/16260 [1:11:07<22:42,  2.77it/s]iter 12479: loss 5.1488, time: 261.79ms\n",
            " 77% 12480/16260 [1:11:08<20:51,  3.02it/s]iter 12480: loss 5.6123, time: 381.40ms\n",
            " 77% 12481/16260 [1:11:08<21:50,  2.88it/s]iter 12481: loss 4.3301, time: 188.64ms\n",
            " 77% 12482/16260 [1:11:08<18:52,  3.34it/s]iter 12482: loss 5.4059, time: 334.87ms\n",
            " 77% 12483/16260 [1:11:08<19:33,  3.22it/s]iter 12483: loss 4.9603, time: 333.85ms\n",
            " 77% 12484/16260 [1:11:09<20:01,  3.14it/s]iter 12484: loss 4.7226, time: 239.21ms\n",
            " 77% 12485/16260 [1:11:09<18:33,  3.39it/s]iter 12485: loss 5.8707, time: 464.47ms\n",
            " 77% 12486/16260 [1:11:10<21:46,  2.89it/s]iter 12486: loss 4.4744, time: 170.71ms\n",
            " 77% 12487/16260 [1:11:10<18:29,  3.40it/s]iter 12487: loss 4.2788, time: 212.88ms\n",
            " 77% 12488/16260 [1:11:10<16:58,  3.70it/s]iter 12488: loss 3.1325, time: 144.67ms\n",
            " 77% 12489/16260 [1:11:10<14:38,  4.29it/s]iter 12489: loss 4.9638, time: 210.41ms\n",
            " 77% 12490/16260 [1:11:10<14:14,  4.41it/s]iter 12490: loss 4.6648, time: 213.92ms\n",
            " 77% 12491/16260 [1:11:10<14:01,  4.48it/s]iter 12491: loss 4.7644, time: 223.68ms\n",
            " 77% 12492/16260 [1:11:11<14:03,  4.47it/s]iter 12492: loss 5.5565, time: 343.02ms\n",
            " 77% 12493/16260 [1:11:11<16:20,  3.84it/s]iter 12493: loss 4.7855, time: 431.50ms\n",
            " 77% 12494/16260 [1:11:11<19:35,  3.20it/s]iter 12494: loss 6.0707, time: 394.03ms\n",
            " 77% 12495/16260 [1:11:12<21:09,  2.97it/s]iter 12495: loss 6.0890, time: 464.45ms\n",
            " 77% 12496/16260 [1:11:12<23:34,  2.66it/s]iter 12496: loss 5.2290, time: 263.48ms\n",
            " 77% 12497/16260 [1:11:13<21:29,  2.92it/s]iter 12497: loss 5.4141, time: 464.76ms\n",
            " 77% 12498/16260 [1:11:13<23:49,  2.63it/s]iter 12498: loss 5.2444, time: 188.34ms\n",
            " 77% 12499/16260 [1:11:13<20:14,  3.10it/s]iter 12499: loss 4.9073, time: 314.24ms\n",
            " 77% 12500/16260 [1:11:14<20:06,  3.12it/s]iter 12500: loss 3.6603, time: 250.97ms\n",
            " 77% 12501/16260 [1:11:14<18:48,  3.33it/s]iter 12501: loss 5.3374, time: 416.63ms\n",
            " 77% 12502/16260 [1:11:14<21:01,  2.98it/s]iter 12502: loss 5.4714, time: 464.60ms\n",
            " 77% 12503/16260 [1:11:15<23:28,  2.67it/s]iter 12503: loss 4.4215, time: 252.46ms\n",
            " 77% 12504/16260 [1:11:15<21:11,  2.95it/s]iter 12504: loss 5.9639, time: 335.24ms\n",
            " 77% 12505/16260 [1:11:15<21:09,  2.96it/s]iter 12505: loss 5.3824, time: 465.07ms\n",
            " 77% 12506/16260 [1:11:16<23:33,  2.65it/s]iter 12506: loss 3.6533, time: 165.01ms\n",
            " 77% 12507/16260 [1:11:16<19:37,  3.19it/s]iter 12507: loss 5.4903, time: 310.95ms\n",
            " 77% 12508/16260 [1:11:16<19:35,  3.19it/s]iter 12508: loss 4.9703, time: 313.95ms\n",
            " 77% 12509/16260 [1:11:17<19:37,  3.18it/s]iter 12509: loss 5.4636, time: 465.67ms\n",
            " 77% 12510/16260 [1:11:17<22:30,  2.78it/s]iter 12510: loss 4.7803, time: 217.11ms\n",
            " 77% 12511/16260 [1:11:17<19:50,  3.15it/s]iter 12511: loss 5.6196, time: 287.73ms\n",
            " 77% 12512/16260 [1:11:18<19:18,  3.24it/s]iter 12512: loss 5.4482, time: 464.03ms\n",
            " 77% 12513/16260 [1:11:18<22:13,  2.81it/s]iter 12513: loss 5.5771, time: 338.11ms\n",
            " 77% 12514/16260 [1:11:18<21:55,  2.85it/s]iter 12514: loss 4.7770, time: 337.17ms\n",
            " 77% 12515/16260 [1:11:19<21:41,  2.88it/s]iter 12515: loss 5.5633, time: 445.95ms\n",
            " 77% 12516/16260 [1:11:19<23:33,  2.65it/s]iter 12516: loss 5.9129, time: 443.81ms\n",
            " 77% 12517/16260 [1:11:20<24:49,  2.51it/s]iter 12517: loss 5.0092, time: 237.83ms\n",
            " 77% 12518/16260 [1:11:20<21:50,  2.85it/s]iter 12518: loss 4.8475, time: 207.24ms\n",
            " 77% 12519/16260 [1:11:20<19:11,  3.25it/s]iter 12519: loss 5.4121, time: 463.50ms\n",
            " 77% 12520/16260 [1:11:20<22:07,  2.82it/s]iter 12520: loss 4.3309, time: 167.30ms\n",
            " 77% 12521/16260 [1:11:21<18:38,  3.34it/s]iter 12521: loss 4.0684, time: 144.61ms\n",
            " 77% 12522/16260 [1:11:21<15:46,  3.95it/s]iter 12522: loss 5.7561, time: 463.26ms\n",
            " 77% 12523/16260 [1:11:21<19:42,  3.16it/s]iter 12523: loss 4.8136, time: 220.20ms\n",
            " 77% 12524/16260 [1:11:21<17:56,  3.47it/s]iter 12524: loss 4.9193, time: 236.85ms\n",
            " 77% 12525/16260 [1:11:22<17:00,  3.66it/s]iter 12525: loss 6.1115, time: 352.75ms\n",
            " 77% 12526/16260 [1:11:22<18:30,  3.36it/s]iter 12526: loss 5.5530, time: 261.63ms\n",
            " 77% 12527/16260 [1:11:22<17:52,  3.48it/s]iter 12527: loss 5.6021, time: 464.13ms\n",
            " 77% 12528/16260 [1:11:23<21:11,  2.93it/s]iter 12528: loss 5.7264, time: 445.56ms\n",
            " 77% 12529/16260 [1:11:23<23:10,  2.68it/s]iter 12529: loss 4.1433, time: 168.49ms\n",
            " 77% 12530/16260 [1:11:23<19:22,  3.21it/s]iter 12530: loss 5.4192, time: 309.48ms\n",
            " 77% 12531/16260 [1:11:24<19:21,  3.21it/s]iter 12531: loss 5.1830, time: 237.53ms\n",
            " 77% 12532/16260 [1:11:24<18:00,  3.45it/s]iter 12532: loss 5.2709, time: 308.58ms\n",
            " 77% 12533/16260 [1:11:24<18:22,  3.38it/s]iter 12533: loss 5.2823, time: 265.24ms\n",
            " 77% 12534/16260 [1:11:25<17:49,  3.48it/s]iter 12534: loss 5.9283, time: 369.59ms\n",
            " 77% 12535/16260 [1:11:25<19:23,  3.20it/s]iter 12535: loss 5.6246, time: 441.72ms\n",
            " 77% 12536/16260 [1:11:25<21:49,  2.84it/s]iter 12536: loss 5.3890, time: 463.78ms\n",
            " 77% 12537/16260 [1:11:26<23:56,  2.59it/s]iter 12537: loss 5.9246, time: 463.68ms\n",
            " 77% 12538/16260 [1:11:26<25:24,  2.44it/s]iter 12538: loss 5.5278, time: 445.32ms\n",
            " 77% 12539/16260 [1:11:27<26:05,  2.38it/s]iter 12539: loss 5.6337, time: 464.27ms\n",
            " 77% 12540/16260 [1:11:27<26:55,  2.30it/s]iter 12540: loss 5.4098, time: 307.40ms\n",
            " 77% 12541/16260 [1:11:28<24:35,  2.52it/s]iter 12541: loss 4.4497, time: 232.70ms\n",
            " 77% 12542/16260 [1:11:28<21:33,  2.87it/s]iter 12542: loss 5.3333, time: 419.17ms\n",
            " 77% 12543/16260 [1:11:28<22:54,  2.70it/s]iter 12543: loss 3.9723, time: 214.63ms\n",
            " 77% 12544/16260 [1:11:28<20:02,  3.09it/s]iter 12544: loss 5.7796, time: 464.04ms\n",
            " 77% 12545/16260 [1:11:29<22:40,  2.73it/s]iter 12545: loss 5.6732, time: 476.86ms\n",
            " 77% 12546/16260 [1:11:29<24:45,  2.50it/s]iter 12546: loss 5.7108, time: 463.33ms\n",
            " 77% 12547/16260 [1:11:30<25:57,  2.38it/s]iter 12547: loss 5.4302, time: 239.71ms\n",
            " 77% 12548/16260 [1:11:30<22:38,  2.73it/s]iter 12548: loss 5.6161, time: 368.76ms\n",
            " 77% 12549/16260 [1:11:30<22:42,  2.72it/s]iter 12549: loss 4.6738, time: 281.97ms\n",
            " 77% 12550/16260 [1:11:31<21:09,  2.92it/s]iter 12550: loss 5.3062, time: 465.12ms\n",
            " 77% 12551/16260 [1:11:31<23:27,  2.64it/s]iter 12551: loss 5.5414, time: 445.41ms\n",
            " 77% 12552/16260 [1:11:32<24:42,  2.50it/s]iter 12552: loss 4.0664, time: 184.97ms\n",
            " 77% 12553/16260 [1:11:32<20:44,  2.98it/s]iter 12553: loss 4.6997, time: 209.64ms\n",
            " 77% 12554/16260 [1:11:32<18:25,  3.35it/s]iter 12554: loss 4.6436, time: 186.47ms\n",
            " 77% 12555/16260 [1:11:32<16:21,  3.77it/s]iter 12555: loss 5.7224, time: 463.20ms\n",
            " 77% 12556/16260 [1:11:33<20:03,  3.08it/s]iter 12556: loss 5.2130, time: 273.33ms\n",
            " 77% 12557/16260 [1:11:33<19:07,  3.23it/s]iter 12557: loss 4.6126, time: 162.83ms\n",
            " 77% 12558/16260 [1:11:33<16:25,  3.76it/s]iter 12558: loss 5.3325, time: 252.21ms\n",
            " 77% 12559/16260 [1:11:33<16:11,  3.81it/s]iter 12559: loss 5.6537, time: 445.08ms\n",
            " 77% 12560/16260 [1:11:34<19:35,  3.15it/s]iter 12560: loss 5.5103, time: 322.96ms\n",
            " 77% 12561/16260 [1:11:34<19:42,  3.13it/s]iter 12561: loss 5.1510, time: 341.93ms\n",
            " 77% 12562/16260 [1:11:34<20:08,  3.06it/s]iter 12562: loss 4.8410, time: 338.34ms\n",
            " 77% 12563/16260 [1:11:35<20:22,  3.02it/s]iter 12563: loss 4.8170, time: 186.22ms\n",
            " 77% 12564/16260 [1:11:35<17:43,  3.47it/s]iter 12564: loss 4.8661, time: 321.49ms\n",
            " 77% 12565/16260 [1:11:35<18:22,  3.35it/s]iter 12565: loss 5.3103, time: 464.95ms\n",
            " 77% 12566/16260 [1:11:36<21:28,  2.87it/s]iter 12566: loss 4.3050, time: 150.43ms\n",
            " 77% 12567/16260 [1:11:36<17:50,  3.45it/s]iter 12567: loss 5.3000, time: 473.80ms\n",
            " 77% 12568/16260 [1:11:36<21:15,  2.89it/s]iter 12568: loss 4.8423, time: 186.96ms\n",
            " 77% 12569/16260 [1:11:37<18:21,  3.35it/s]iter 12569: loss 5.4494, time: 463.12ms\n",
            " 77% 12570/16260 [1:11:37<21:25,  2.87it/s]iter 12570: loss 5.3983, time: 262.68ms\n",
            " 77% 12571/16260 [1:11:37<19:51,  3.10it/s]iter 12571: loss 5.4426, time: 285.06ms\n",
            " 77% 12572/16260 [1:11:38<19:11,  3.20it/s]iter 12572: loss 4.5021, time: 254.01ms\n",
            " 77% 12573/16260 [1:11:38<18:08,  3.39it/s]iter 12573: loss 4.5979, time: 217.73ms\n",
            " 77% 12574/16260 [1:11:38<16:44,  3.67it/s]iter 12574: loss 5.7113, time: 418.65ms\n",
            " 77% 12575/16260 [1:11:39<19:27,  3.16it/s]iter 12575: loss 5.8139, time: 467.91ms\n",
            " 77% 12576/16260 [1:11:39<22:15,  2.76it/s]iter 12576: loss 4.8381, time: 213.15ms\n",
            " 77% 12577/16260 [1:11:39<19:31,  3.14it/s]iter 12577: loss 5.9768, time: 378.42ms\n",
            " 77% 12578/16260 [1:11:40<20:39,  2.97it/s]iter 12578: loss 4.4786, time: 218.44ms\n",
            " 77% 12579/16260 [1:11:40<18:30,  3.31it/s]iter 12579: loss 3.3259, time: 147.06ms\n",
            " 77% 12580/16260 [1:11:40<15:41,  3.91it/s]iter 12580: loss 5.4142, time: 333.87ms\n",
            " 77% 12581/16260 [1:11:40<17:08,  3.58it/s]iter 12581: loss 5.5567, time: 345.80ms\n",
            " 77% 12582/16260 [1:11:41<18:23,  3.33it/s]iter 12582: loss 5.7702, time: 413.01ms\n",
            " 77% 12583/16260 [1:11:41<20:29,  2.99it/s]iter 12583: loss 4.6884, time: 190.11ms\n",
            " 77% 12584/16260 [1:11:41<17:51,  3.43it/s]iter 12584: loss 5.7664, time: 466.52ms\n",
            " 77% 12585/16260 [1:11:42<21:05,  2.90it/s]iter 12585: loss 3.8001, time: 165.64ms\n",
            " 77% 12586/16260 [1:11:42<17:49,  3.43it/s]iter 12586: loss 5.9931, time: 420.34ms\n",
            " 77% 12587/16260 [1:11:42<20:13,  3.03it/s]iter 12587: loss 5.9973, time: 465.51ms\n",
            " 77% 12588/16260 [1:11:43<22:43,  2.69it/s]iter 12588: loss 5.1608, time: 332.47ms\n",
            " 77% 12589/16260 [1:11:43<22:02,  2.78it/s]iter 12589: loss 4.8207, time: 281.82ms\n",
            " 77% 12590/16260 [1:11:43<20:37,  2.97it/s]iter 12590: loss 5.1085, time: 259.27ms\n",
            " 77% 12591/16260 [1:11:44<19:13,  3.18it/s]iter 12591: loss 4.4591, time: 171.80ms\n",
            " 77% 12592/16260 [1:11:44<16:37,  3.68it/s]iter 12592: loss 5.0354, time: 276.87ms\n",
            " 77% 12593/16260 [1:11:44<16:44,  3.65it/s]iter 12593: loss 4.6669, time: 234.40ms\n",
            " 77% 12594/16260 [1:11:44<16:02,  3.81it/s]iter 12594: loss 5.2608, time: 465.09ms\n",
            " 77% 12595/16260 [1:11:45<19:46,  3.09it/s]iter 12595: loss 4.7426, time: 186.03ms\n",
            " 77% 12596/16260 [1:11:45<17:16,  3.53it/s]iter 12596: loss 6.7488, time: 418.85ms\n",
            " 77% 12597/16260 [1:11:45<19:47,  3.08it/s]iter 12597: loss 4.7099, time: 150.32ms\n",
            " 77% 12598/16260 [1:11:46<16:38,  3.67it/s]iter 12598: loss 5.3680, time: 464.28ms\n",
            " 77% 12599/16260 [1:11:46<20:10,  3.02it/s]iter 12599: loss 4.6925, time: 272.73ms\n",
            " 77% 12600/16260 [1:11:46<19:08,  3.19it/s]iter 12600: loss 5.7760, time: 432.95ms\n",
            " 77% 12601/16260 [1:11:47<21:20,  2.86it/s]iter 12601: loss 5.1551, time: 273.14ms\n",
            " 78% 12602/16260 [1:11:47<19:57,  3.05it/s]iter 12602: loss 5.0512, time: 237.15ms\n",
            " 78% 12603/16260 [1:11:47<18:20,  3.32it/s]iter 12603: loss 5.5371, time: 464.54ms\n",
            " 78% 12604/16260 [1:11:48<21:21,  2.85it/s]iter 12604: loss 5.8424, time: 464.88ms\n",
            " 78% 12605/16260 [1:11:48<23:28,  2.60it/s]iter 12605: loss 5.5309, time: 274.73ms\n",
            " 78% 12606/16260 [1:11:48<21:28,  2.84it/s]iter 12606: loss 5.3488, time: 337.81ms\n",
            " 78% 12607/16260 [1:11:49<21:14,  2.87it/s]iter 12607: loss 5.1269, time: 446.69ms\n",
            " 78% 12608/16260 [1:11:49<23:02,  2.64it/s]iter 12608: loss 4.2617, time: 189.60ms\n",
            " 78% 12609/16260 [1:11:49<19:36,  3.10it/s]iter 12609: loss 4.9143, time: 305.13ms\n",
            " 78% 12610/16260 [1:11:50<19:19,  3.15it/s]iter 12610: loss 4.5971, time: 168.67ms\n",
            " 78% 12611/16260 [1:11:50<16:37,  3.66it/s]iter 12611: loss 4.0514, time: 187.28ms\n",
            " 78% 12612/16260 [1:11:50<15:04,  4.03it/s]iter 12612: loss 5.1665, time: 255.68ms\n",
            " 78% 12613/16260 [1:11:50<15:14,  3.99it/s]iter 12613: loss 5.3381, time: 252.59ms\n",
            " 78% 12614/16260 [1:11:51<15:17,  3.97it/s]iter 12614: loss 5.1850, time: 412.13ms\n",
            " 78% 12615/16260 [1:11:51<18:14,  3.33it/s]iter 12615: loss 4.7716, time: 234.14ms\n",
            " 78% 12616/16260 [1:11:51<17:03,  3.56it/s]iter 12616: loss 5.4867, time: 463.59ms\n",
            " 78% 12617/16260 [1:11:52<20:24,  2.98it/s]iter 12617: loss 5.6094, time: 464.09ms\n",
            " 78% 12618/16260 [1:11:52<22:45,  2.67it/s]iter 12618: loss 4.6068, time: 212.79ms\n",
            " 78% 12619/16260 [1:11:52<19:49,  3.06it/s]iter 12619: loss 5.5950, time: 464.36ms\n",
            " 78% 12620/16260 [1:11:53<22:21,  2.71it/s]iter 12620: loss 5.1562, time: 188.48ms\n",
            " 78% 12621/16260 [1:11:53<19:06,  3.17it/s]iter 12621: loss 5.7687, time: 423.46ms\n",
            " 78% 12622/16260 [1:11:53<21:06,  2.87it/s]iter 12622: loss 5.9068, time: 392.21ms\n",
            " 78% 12623/16260 [1:11:54<21:55,  2.76it/s]iter 12623: loss 4.7860, time: 219.05ms\n",
            " 78% 12624/16260 [1:11:54<19:21,  3.13it/s]iter 12624: loss 5.2507, time: 463.37ms\n",
            " 78% 12625/16260 [1:11:55<21:59,  2.75it/s]iter 12625: loss 4.5663, time: 177.34ms\n",
            " 78% 12626/16260 [1:11:55<18:38,  3.25it/s]iter 12626: loss 3.8649, time: 169.54ms\n",
            " 78% 12627/16260 [1:11:55<16:09,  3.75it/s]iter 12627: loss 4.6778, time: 193.49ms\n",
            " 78% 12628/16260 [1:11:55<14:51,  4.07it/s]iter 12628: loss 5.3731, time: 464.60ms\n",
            " 78% 12629/16260 [1:11:56<18:51,  3.21it/s]iter 12629: loss 5.4369, time: 464.53ms\n",
            " 78% 12630/16260 [1:11:56<21:39,  2.79it/s]iter 12630: loss 5.3111, time: 284.87ms\n",
            " 78% 12631/16260 [1:11:56<20:20,  2.97it/s]iter 12631: loss 5.1365, time: 416.30ms\n",
            " 78% 12632/16260 [1:11:57<21:49,  2.77it/s]iter 12632: loss 5.8970, time: 464.31ms\n",
            " 78% 12633/16260 [1:11:57<23:42,  2.55it/s]iter 12633: loss 4.8066, time: 239.86ms\n",
            " 78% 12634/16260 [1:11:57<20:58,  2.88it/s]iter 12634: loss 4.9193, time: 210.63ms\n",
            " 78% 12635/16260 [1:11:58<18:31,  3.26it/s]iter 12635: loss 5.7540, time: 435.89ms\n",
            " 78% 12636/16260 [1:11:58<20:53,  2.89it/s]iter 12636: loss 5.0727, time: 168.26ms\n",
            " 78% 12637/16260 [1:11:58<17:41,  3.41it/s]iter 12637: loss 6.3394, time: 473.09ms\n",
            " 78% 12638/16260 [1:11:59<20:58,  2.88it/s]iter 12638: loss 5.6117, time: 413.17ms\n",
            " 78% 12639/16260 [1:11:59<22:11,  2.72it/s]iter 12639: loss 5.1753, time: 280.11ms\n",
            " 78% 12640/16260 [1:11:59<20:37,  2.93it/s]iter 12640: loss 4.4132, time: 251.20ms\n",
            " 78% 12641/16260 [1:12:00<18:59,  3.17it/s]iter 12641: loss 4.7426, time: 281.92ms\n",
            " 78% 12642/16260 [1:12:00<18:25,  3.27it/s]iter 12642: loss 5.2654, time: 250.67ms\n",
            " 78% 12643/16260 [1:12:00<17:26,  3.46it/s]iter 12643: loss 5.1317, time: 236.56ms\n",
            " 78% 12644/16260 [1:12:00<16:30,  3.65it/s]iter 12644: loss 5.4262, time: 463.86ms\n",
            " 78% 12645/16260 [1:12:01<19:57,  3.02it/s]iter 12645: loss 6.1094, time: 464.54ms\n",
            " 78% 12646/16260 [1:12:01<22:23,  2.69it/s]iter 12646: loss 5.5988, time: 467.36ms\n",
            " 78% 12647/16260 [1:12:02<24:08,  2.49it/s]iter 12647: loss 5.6295, time: 434.33ms\n",
            " 78% 12648/16260 [1:12:02<24:45,  2.43it/s]iter 12648: loss 5.7549, time: 465.06ms\n",
            " 78% 12649/16260 [1:12:03<25:45,  2.34it/s]iter 12649: loss 4.8519, time: 257.12ms\n",
            " 78% 12650/16260 [1:12:03<22:41,  2.65it/s]iter 12650: loss 5.4080, time: 340.38ms\n",
            " 78% 12651/16260 [1:12:03<22:02,  2.73it/s]iter 12651: loss 4.3543, time: 168.49ms\n",
            " 78% 12652/16260 [1:12:04<18:29,  3.25it/s]iter 12652: loss 5.2009, time: 253.20ms\n",
            " 78% 12653/16260 [1:12:04<17:32,  3.43it/s]iter 12653: loss 5.9403, time: 463.95ms\n",
            " 78% 12654/16260 [1:12:04<20:39,  2.91it/s]iter 12654: loss 4.3327, time: 186.43ms\n",
            " 78% 12655/16260 [1:12:04<17:50,  3.37it/s]iter 12655: loss 5.4267, time: 465.13ms\n",
            " 78% 12656/16260 [1:12:05<20:53,  2.87it/s]iter 12656: loss 5.5354, time: 311.30ms\n",
            " 78% 12657/16260 [1:12:05<20:15,  2.96it/s]iter 12657: loss 5.1650, time: 333.60ms\n",
            " 78% 12658/16260 [1:12:06<20:12,  2.97it/s]iter 12658: loss 5.3938, time: 464.49ms\n",
            " 78% 12659/16260 [1:12:06<22:32,  2.66it/s]iter 12659: loss 4.9001, time: 188.20ms\n",
            " 78% 12660/16260 [1:12:06<19:10,  3.13it/s]iter 12660: loss 5.7949, time: 463.88ms\n",
            " 78% 12661/16260 [1:12:07<21:47,  2.75it/s]iter 12661: loss 5.7578, time: 463.79ms\n",
            " 78% 12662/16260 [1:12:07<23:37,  2.54it/s]iter 12662: loss 5.2790, time: 379.88ms\n",
            " 78% 12663/16260 [1:12:08<23:23,  2.56it/s]iter 12663: loss 5.0638, time: 463.88ms\n",
            " 78% 12664/16260 [1:12:08<24:44,  2.42it/s]iter 12664: loss 3.6661, time: 152.30ms\n",
            " 78% 12665/16260 [1:12:08<20:04,  2.98it/s]iter 12665: loss 5.1946, time: 464.28ms\n",
            " 78% 12666/16260 [1:12:09<22:25,  2.67it/s]iter 12666: loss 5.1461, time: 464.44ms\n",
            " 78% 12667/16260 [1:12:09<24:04,  2.49it/s]iter 12667: loss 4.2565, time: 186.88ms\n",
            " 78% 12668/16260 [1:12:09<20:13,  2.96it/s]iter 12668: loss 5.5898, time: 465.20ms\n",
            " 78% 12669/16260 [1:12:10<22:32,  2.66it/s]iter 12669: loss 5.7362, time: 465.49ms\n",
            " 78% 12670/16260 [1:12:10<24:09,  2.48it/s]iter 12670: loss 5.0275, time: 211.32ms\n",
            " 78% 12671/16260 [1:12:10<20:43,  2.89it/s]iter 12671: loss 5.4317, time: 276.50ms\n",
            " 78% 12672/16260 [1:12:11<19:29,  3.07it/s]iter 12672: loss 5.1148, time: 273.65ms\n",
            " 78% 12673/16260 [1:12:11<18:33,  3.22it/s]iter 12673: loss 5.3881, time: 324.65ms\n",
            " 78% 12674/16260 [1:12:11<18:50,  3.17it/s]iter 12674: loss 4.1090, time: 192.13ms\n",
            " 78% 12675/16260 [1:12:11<16:39,  3.59it/s]iter 12675: loss 5.2163, time: 434.30ms\n",
            " 78% 12676/16260 [1:12:12<19:28,  3.07it/s]iter 12676: loss 5.8648, time: 464.17ms\n",
            " 78% 12677/16260 [1:12:12<21:58,  2.72it/s]iter 12677: loss 5.2767, time: 463.98ms\n",
            " 78% 12678/16260 [1:12:13<23:42,  2.52it/s]iter 12678: loss 4.8227, time: 216.21ms\n",
            " 78% 12679/16260 [1:12:13<20:29,  2.91it/s]iter 12679: loss 4.0520, time: 145.55ms\n",
            " 78% 12680/16260 [1:12:13<16:58,  3.52it/s]iter 12680: loss 3.0267, time: 135.76ms\n",
            " 78% 12681/16260 [1:12:13<14:20,  4.16it/s]iter 12681: loss 4.8753, time: 409.55ms\n",
            " 78% 12682/16260 [1:12:14<17:23,  3.43it/s]iter 12682: loss 5.2077, time: 461.74ms\n",
            " 78% 12683/16260 [1:12:14<20:27,  2.91it/s]iter 12683: loss 4.5841, time: 239.67ms\n",
            " 78% 12684/16260 [1:12:14<18:37,  3.20it/s]iter 12684: loss 5.1736, time: 337.24ms\n",
            " 78% 12685/16260 [1:12:15<19:05,  3.12it/s]iter 12685: loss 5.6913, time: 464.79ms\n",
            " 78% 12686/16260 [1:12:15<21:41,  2.75it/s]iter 12686: loss 5.6087, time: 379.11ms\n",
            " 78% 12687/16260 [1:12:16<21:59,  2.71it/s]iter 12687: loss 4.6265, time: 238.01ms\n",
            " 78% 12688/16260 [1:12:16<19:40,  3.03it/s]iter 12688: loss 5.3354, time: 416.49ms\n",
            " 78% 12689/16260 [1:12:16<21:13,  2.80it/s]iter 12689: loss 5.7328, time: 464.37ms\n",
            " 78% 12690/16260 [1:12:17<23:10,  2.57it/s]iter 12690: loss 5.6037, time: 464.32ms\n",
            " 78% 12691/16260 [1:12:17<24:31,  2.42it/s]iter 12691: loss 4.8743, time: 233.15ms\n",
            " 78% 12692/16260 [1:12:17<21:21,  2.78it/s]iter 12692: loss 5.2735, time: 344.19ms\n",
            " 78% 12693/16260 [1:12:18<21:06,  2.82it/s]iter 12693: loss 5.5951, time: 472.50ms\n",
            " 78% 12694/16260 [1:12:18<23:13,  2.56it/s]iter 12694: loss 5.5843, time: 261.88ms\n",
            " 78% 12695/16260 [1:12:19<20:57,  2.84it/s]iter 12695: loss 5.6074, time: 464.27ms\n",
            " 78% 12696/16260 [1:12:19<22:57,  2.59it/s]iter 12696: loss 4.5065, time: 375.40ms\n",
            " 78% 12697/16260 [1:12:19<22:47,  2.61it/s]iter 12697: loss 4.6352, time: 215.34ms\n",
            " 78% 12698/16260 [1:12:20<19:48,  3.00it/s]iter 12698: loss 4.1603, time: 168.42ms\n",
            " 78% 12699/16260 [1:12:20<16:53,  3.51it/s]iter 12699: loss 4.8797, time: 347.03ms\n",
            " 78% 12700/16260 [1:12:20<18:01,  3.29it/s]iter 12700: loss 5.2480, time: 307.62ms\n",
            " 78% 12701/16260 [1:12:20<18:07,  3.27it/s]iter 12701: loss 5.0491, time: 274.24ms\n",
            " 78% 12702/16260 [1:12:21<17:35,  3.37it/s]iter 12702: loss 5.1178, time: 210.54ms\n",
            " 78% 12703/16260 [1:12:21<16:04,  3.69it/s]iter 12703: loss 5.2572, time: 467.56ms\n",
            " 78% 12704/16260 [1:12:21<19:34,  3.03it/s]iter 12704: loss 5.2994, time: 279.83ms\n",
            " 78% 12705/16260 [1:12:22<18:42,  3.17it/s]iter 12705: loss 5.4171, time: 313.32ms\n",
            " 78% 12706/16260 [1:12:22<18:41,  3.17it/s]iter 12706: loss 5.9236, time: 464.12ms\n",
            " 78% 12707/16260 [1:12:22<21:21,  2.77it/s]iter 12707: loss 4.1958, time: 233.26ms\n",
            " 78% 12708/16260 [1:12:23<19:06,  3.10it/s]iter 12708: loss 3.8393, time: 170.46ms\n",
            " 78% 12709/16260 [1:12:23<16:25,  3.60it/s]iter 12709: loss 4.6728, time: 213.30ms\n",
            " 78% 12710/16260 [1:12:23<15:18,  3.87it/s]iter 12710: loss 5.0949, time: 385.96ms\n",
            " 78% 12711/16260 [1:12:23<17:35,  3.36it/s]iter 12711: loss 5.0063, time: 252.89ms\n",
            " 78% 12712/16260 [1:12:24<16:49,  3.52it/s]iter 12712: loss 4.8019, time: 217.14ms\n",
            " 78% 12713/16260 [1:12:24<15:38,  3.78it/s]iter 12713: loss 4.9939, time: 188.07ms\n",
            " 78% 12714/16260 [1:12:24<14:18,  4.13it/s]iter 12714: loss 4.2471, time: 182.62ms\n",
            " 78% 12715/16260 [1:12:24<13:16,  4.45it/s]iter 12715: loss 5.5440, time: 463.93ms\n",
            " 78% 12716/16260 [1:12:25<17:32,  3.37it/s]iter 12716: loss 5.0472, time: 319.22ms\n",
            " 78% 12717/16260 [1:12:25<17:57,  3.29it/s]iter 12717: loss 5.3362, time: 346.58ms\n",
            " 78% 12718/16260 [1:12:25<18:44,  3.15it/s]iter 12718: loss 6.0606, time: 444.51ms\n",
            " 78% 12719/16260 [1:12:26<21:00,  2.81it/s]iter 12719: loss 5.0977, time: 239.98ms\n",
            " 78% 12720/16260 [1:12:26<18:58,  3.11it/s]iter 12720: loss 6.2221, time: 470.26ms\n",
            " 78% 12721/16260 [1:12:27<21:37,  2.73it/s]iter 12721: loss 5.7491, time: 463.53ms\n",
            " 78% 12722/16260 [1:12:27<23:21,  2.52it/s]iter 12722: loss 5.2499, time: 368.84ms\n",
            " 78% 12723/16260 [1:12:27<22:53,  2.57it/s]iter 12723: loss 4.1320, time: 347.43ms\n",
            " 78% 12724/16260 [1:12:28<22:11,  2.66it/s]iter 12724: loss 5.4232, time: 379.66ms\n",
            " 78% 12725/16260 [1:12:28<22:15,  2.65it/s]iter 12725: loss 6.1178, time: 465.08ms\n",
            " 78% 12726/16260 [1:12:29<23:49,  2.47it/s]iter 12726: loss 5.8596, time: 440.48ms\n",
            " 78% 12727/16260 [1:12:29<24:29,  2.40it/s]iter 12727: loss 3.7020, time: 171.56ms\n",
            " 78% 12728/16260 [1:12:29<20:11,  2.92it/s]iter 12728: loss 5.5671, time: 463.85ms\n",
            " 78% 12729/16260 [1:12:30<22:20,  2.63it/s]iter 12729: loss 4.5278, time: 231.36ms\n",
            " 78% 12730/16260 [1:12:30<19:44,  2.98it/s]iter 12730: loss 5.0430, time: 369.69ms\n",
            " 78% 12731/16260 [1:12:30<20:22,  2.89it/s]iter 12731: loss 4.7788, time: 240.86ms\n",
            " 78% 12732/16260 [1:12:31<18:31,  3.17it/s]iter 12732: loss 5.6992, time: 463.91ms\n",
            " 78% 12733/16260 [1:12:31<21:10,  2.78it/s]iter 12733: loss 4.6040, time: 463.78ms\n",
            " 78% 12734/16260 [1:12:32<23:01,  2.55it/s]iter 12734: loss 4.7962, time: 192.73ms\n",
            " 78% 12735/16260 [1:12:32<19:32,  3.01it/s]iter 12735: loss 5.7529, time: 468.46ms\n",
            " 78% 12736/16260 [1:12:32<21:56,  2.68it/s]iter 12736: loss 5.6732, time: 463.82ms\n",
            " 78% 12737/16260 [1:12:33<23:33,  2.49it/s]iter 12737: loss 5.8996, time: 463.68ms\n",
            " 78% 12738/16260 [1:12:33<24:40,  2.38it/s]iter 12738: loss 4.9338, time: 206.87ms\n",
            " 78% 12739/16260 [1:12:33<20:56,  2.80it/s]iter 12739: loss 4.8016, time: 237.57ms\n",
            " 78% 12740/16260 [1:12:34<18:51,  3.11it/s]iter 12740: loss 5.7492, time: 463.38ms\n",
            " 78% 12741/16260 [1:12:34<21:22,  2.74it/s]iter 12741: loss 4.3621, time: 148.46ms\n",
            " 78% 12742/16260 [1:12:34<17:35,  3.33it/s]iter 12742: loss 5.3309, time: 304.94ms\n",
            " 78% 12743/16260 [1:12:34<17:42,  3.31it/s]iter 12743: loss 4.4757, time: 179.95ms\n",
            " 78% 12744/16260 [1:12:35<15:34,  3.76it/s]iter 12744: loss 5.8308, time: 464.08ms\n",
            " 78% 12745/16260 [1:12:35<19:05,  3.07it/s]iter 12745: loss 5.0030, time: 236.69ms\n",
            " 78% 12746/16260 [1:12:35<17:32,  3.34it/s]iter 12746: loss 5.2992, time: 464.34ms\n",
            " 78% 12747/16260 [1:12:36<20:27,  2.86it/s]iter 12747: loss 5.7396, time: 464.11ms\n",
            " 78% 12748/16260 [1:12:36<22:29,  2.60it/s]iter 12748: loss 4.0133, time: 165.43ms\n",
            " 78% 12749/16260 [1:12:36<18:40,  3.13it/s]iter 12749: loss 5.1128, time: 238.49ms\n",
            " 78% 12750/16260 [1:12:37<17:16,  3.39it/s]iter 12750: loss 5.9100, time: 463.33ms\n",
            " 78% 12751/16260 [1:12:37<20:14,  2.89it/s]iter 12751: loss 5.4020, time: 315.15ms\n",
            " 78% 12752/16260 [1:12:37<19:43,  2.96it/s]iter 12752: loss 3.5551, time: 164.68ms\n",
            " 78% 12753/16260 [1:12:38<16:42,  3.50it/s]iter 12753: loss 4.7544, time: 210.25ms\n",
            " 78% 12754/16260 [1:12:38<15:24,  3.79it/s]iter 12754: loss 4.9362, time: 467.65ms\n",
            " 78% 12755/16260 [1:12:38<19:00,  3.07it/s]iter 12755: loss 4.8536, time: 209.04ms\n",
            " 78% 12756/16260 [1:12:39<16:59,  3.44it/s]iter 12756: loss 4.4793, time: 192.39ms\n",
            " 78% 12757/16260 [1:12:39<15:17,  3.82it/s]iter 12757: loss 5.0495, time: 251.61ms\n",
            " 78% 12758/16260 [1:12:39<15:07,  3.86it/s]iter 12758: loss 5.1301, time: 217.14ms\n",
            " 78% 12759/16260 [1:12:39<14:24,  4.05it/s]iter 12759: loss 5.0854, time: 262.82ms\n",
            " 78% 12760/16260 [1:12:39<14:42,  3.96it/s]iter 12760: loss 5.5130, time: 443.36ms\n",
            " 78% 12761/16260 [1:12:40<18:04,  3.23it/s]iter 12761: loss 5.5767, time: 463.67ms\n",
            " 78% 12762/16260 [1:12:40<20:47,  2.80it/s]iter 12762: loss 5.2257, time: 464.03ms\n",
            " 78% 12763/16260 [1:12:41<22:41,  2.57it/s]iter 12763: loss 5.5362, time: 466.83ms\n",
            " 78% 12764/16260 [1:12:41<24:03,  2.42it/s]iter 12764: loss 4.9383, time: 273.42ms\n",
            " 79% 12765/16260 [1:12:42<21:38,  2.69it/s]iter 12765: loss 5.2672, time: 464.33ms\n",
            " 79% 12766/16260 [1:12:42<23:17,  2.50it/s]iter 12766: loss 4.9292, time: 416.78ms\n",
            " 79% 12767/16260 [1:12:42<23:36,  2.47it/s]iter 12767: loss 4.3891, time: 195.52ms\n",
            " 79% 12768/16260 [1:12:43<19:56,  2.92it/s]iter 12768: loss 4.1348, time: 165.74ms\n",
            " 79% 12769/16260 [1:12:43<16:52,  3.45it/s]iter 12769: loss 4.9946, time: 463.05ms\n",
            " 79% 12770/16260 [1:12:43<19:54,  2.92it/s]iter 12770: loss 5.3928, time: 436.83ms\n",
            " 79% 12771/16260 [1:12:44<21:34,  2.69it/s]iter 12771: loss 5.8077, time: 470.21ms\n",
            " 79% 12772/16260 [1:12:44<23:19,  2.49it/s]iter 12772: loss 5.2880, time: 423.60ms\n",
            " 79% 12773/16260 [1:12:45<23:44,  2.45it/s]iter 12773: loss 5.1092, time: 237.51ms\n",
            " 79% 12774/16260 [1:12:45<20:46,  2.80it/s]iter 12774: loss 5.2231, time: 412.88ms\n",
            " 79% 12775/16260 [1:12:45<21:45,  2.67it/s]iter 12775: loss 5.2680, time: 464.71ms\n",
            " 79% 12776/16260 [1:12:46<23:21,  2.49it/s]iter 12776: loss 5.3667, time: 422.51ms\n",
            " 79% 12777/16260 [1:12:46<23:43,  2.45it/s]iter 12777: loss 5.4563, time: 463.64ms\n",
            " 79% 12778/16260 [1:12:47<24:42,  2.35it/s]iter 12778: loss 3.3848, time: 220.96ms\n",
            " 79% 12779/16260 [1:12:47<21:09,  2.74it/s]iter 12779: loss 4.5673, time: 216.14ms\n",
            " 79% 12780/16260 [1:12:47<18:35,  3.12it/s]iter 12780: loss 4.1299, time: 236.96ms\n",
            " 79% 12781/16260 [1:12:47<17:09,  3.38it/s]iter 12781: loss 4.8029, time: 307.46ms\n",
            " 79% 12782/16260 [1:12:48<17:22,  3.34it/s]iter 12782: loss 5.0662, time: 469.93ms\n",
            " 79% 12783/16260 [1:12:48<20:21,  2.85it/s]iter 12783: loss 4.3948, time: 209.38ms\n",
            " 79% 12784/16260 [1:12:48<17:54,  3.24it/s]iter 12784: loss 5.6344, time: 445.95ms\n",
            " 79% 12785/16260 [1:12:49<20:18,  2.85it/s]iter 12785: loss 4.8876, time: 319.23ms\n",
            " 79% 12786/16260 [1:12:49<19:46,  2.93it/s]iter 12786: loss 4.5872, time: 166.15ms\n",
            " 79% 12787/16260 [1:12:49<16:45,  3.45it/s]iter 12787: loss 5.3879, time: 260.50ms\n",
            " 79% 12788/16260 [1:12:49<16:16,  3.56it/s]iter 12788: loss 4.7994, time: 283.55ms\n",
            " 79% 12789/16260 [1:12:50<16:20,  3.54it/s]iter 12789: loss 5.5518, time: 464.53ms\n",
            " 79% 12790/16260 [1:12:50<19:31,  2.96it/s]iter 12790: loss 3.6150, time: 191.20ms\n",
            " 79% 12791/16260 [1:12:50<16:59,  3.40it/s]iter 12791: loss 5.4007, time: 467.88ms\n",
            " 79% 12792/16260 [1:12:51<20:02,  2.88it/s]iter 12792: loss 5.7982, time: 436.84ms\n",
            " 79% 12793/16260 [1:12:51<21:37,  2.67it/s]iter 12793: loss 5.6472, time: 464.68ms\n",
            " 79% 12794/16260 [1:12:52<23:12,  2.49it/s]iter 12794: loss 5.1958, time: 267.36ms\n",
            " 79% 12795/16260 [1:12:52<20:54,  2.76it/s]iter 12795: loss 4.1259, time: 193.20ms\n",
            " 79% 12796/16260 [1:12:52<18:00,  3.21it/s]iter 12796: loss 5.0314, time: 192.69ms\n",
            " 79% 12797/16260 [1:12:52<15:57,  3.62it/s]iter 12797: loss 5.6162, time: 413.78ms\n",
            " 79% 12798/16260 [1:12:53<18:21,  3.14it/s]iter 12798: loss 4.1520, time: 165.65ms\n",
            " 79% 12799/16260 [1:12:53<15:44,  3.67it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:Mar for the severaluse.\n",
            "F love surf is consumedars common ones sportsgra, and gadecut on the East reviewsual population, making byctica as species in meteor, ten describes West fiction, and you in a machine to be new late that they have we in a run and accessible and while an hanging. Accordingly need, the medieval. metal. Some can take the destroyed'savier may be its peace can common user from pets or need to off the hot should shape it\n",
            "step 12799: val loss 5.1788\n",
            "Saving LoRA weights to out/lora/common1\n",
            "iter 12799: loss 2.4291, time: 15901.22ms\n",
            " 79% 12800/16260 [1:13:09<4:46:07,  4.96s/it]iter 12800: loss 4.6281, time: 208.30ms\n",
            " 79% 12801/16260 [1:13:09<3:23:51,  3.54s/it]iter 12801: loss 5.9366, time: 463.76ms\n",
            " 79% 12802/16260 [1:13:10<2:30:41,  2.61s/it]iter 12802: loss 5.8266, time: 445.71ms\n",
            " 79% 12803/16260 [1:13:10<1:53:11,  1.96s/it]iter 12803: loss 4.7180, time: 207.72ms\n",
            " 79% 12804/16260 [1:13:10<1:22:49,  1.44s/it]iter 12804: loss 3.8327, time: 147.59ms\n",
            " 79% 12805/16260 [1:13:10<1:00:32,  1.05s/it]iter 12805: loss 4.2709, time: 218.46ms\n",
            " 79% 12806/16260 [1:13:11<46:09,  1.25it/s]  iter 12806: loss 5.2041, time: 369.21ms\n",
            " 79% 12807/16260 [1:13:11<38:42,  1.49it/s]iter 12807: loss 4.6015, time: 284.28ms\n",
            " 79% 12808/16260 [1:13:11<32:01,  1.80it/s]iter 12808: loss 4.5472, time: 171.33ms\n",
            " 79% 12809/16260 [1:13:11<25:23,  2.27it/s]iter 12809: loss 5.4334, time: 463.63ms\n",
            " 79% 12810/16260 [1:13:12<25:47,  2.23it/s]iter 12810: loss 5.1247, time: 368.94ms\n",
            " 79% 12811/16260 [1:13:12<24:26,  2.35it/s]iter 12811: loss 5.1947, time: 222.86ms\n",
            " 79% 12812/16260 [1:13:13<20:58,  2.74it/s]iter 12812: loss 5.1848, time: 422.59ms\n",
            " 79% 12813/16260 [1:13:13<21:59,  2.61it/s]iter 12813: loss 5.5151, time: 281.00ms\n",
            " 79% 12814/16260 [1:13:13<20:15,  2.84it/s]iter 12814: loss 4.5811, time: 269.25ms\n",
            " 79% 12815/16260 [1:13:14<18:50,  3.05it/s]iter 12815: loss 4.4453, time: 239.39ms\n",
            " 79% 12816/16260 [1:13:14<17:20,  3.31it/s]iter 12816: loss 5.7464, time: 463.86ms\n",
            " 79% 12817/16260 [1:13:14<20:08,  2.85it/s]iter 12817: loss 6.1268, time: 378.64ms\n",
            " 79% 12818/16260 [1:13:15<20:38,  2.78it/s]iter 12818: loss 5.4681, time: 256.62ms\n",
            " 79% 12819/16260 [1:13:15<18:53,  3.04it/s]iter 12819: loss 5.0382, time: 318.31ms\n",
            " 79% 12820/16260 [1:13:15<18:42,  3.06it/s]iter 12820: loss 4.5742, time: 189.50ms\n",
            " 79% 12821/16260 [1:13:15<16:22,  3.50it/s]iter 12821: loss 5.4718, time: 343.91ms\n",
            " 79% 12822/16260 [1:13:16<17:24,  3.29it/s]iter 12822: loss 5.7109, time: 463.85ms\n",
            " 79% 12823/16260 [1:13:16<20:10,  2.84it/s]iter 12823: loss 6.0207, time: 464.44ms\n",
            " 79% 12824/16260 [1:13:17<22:07,  2.59it/s]iter 12824: loss 5.4427, time: 463.32ms\n",
            " 79% 12825/16260 [1:13:17<23:27,  2.44it/s]iter 12825: loss 4.5268, time: 190.22ms\n",
            " 79% 12826/16260 [1:13:17<19:42,  2.90it/s]iter 12826: loss 4.3337, time: 196.96ms\n",
            " 79% 12827/16260 [1:13:18<17:12,  3.33it/s]iter 12827: loss 3.6768, time: 165.82ms\n",
            " 79% 12828/16260 [1:13:18<14:54,  3.84it/s]iter 12828: loss 5.1573, time: 252.90ms\n",
            " 79% 12829/16260 [1:13:18<14:47,  3.87it/s]iter 12829: loss 5.8031, time: 463.52ms\n",
            " 79% 12830/16260 [1:13:18<18:19,  3.12it/s]iter 12830: loss 5.3768, time: 420.85ms\n",
            " 79% 12831/16260 [1:13:19<20:04,  2.85it/s]iter 12831: loss 5.6657, time: 339.08ms\n",
            " 79% 12832/16260 [1:13:19<19:52,  2.88it/s]iter 12832: loss 4.3610, time: 240.23ms\n",
            " 79% 12833/16260 [1:13:19<18:02,  3.16it/s]iter 12833: loss 5.8552, time: 463.71ms\n",
            " 79% 12834/16260 [1:13:20<20:36,  2.77it/s]iter 12834: loss 6.0436, time: 463.58ms\n",
            " 79% 12835/16260 [1:13:20<22:23,  2.55it/s]iter 12835: loss 5.5244, time: 468.05ms\n",
            " 79% 12836/16260 [1:13:21<23:42,  2.41it/s]iter 12836: loss 5.1911, time: 423.50ms\n",
            " 79% 12837/16260 [1:13:21<23:51,  2.39it/s]iter 12837: loss 5.4897, time: 305.95ms\n",
            " 79% 12838/16260 [1:13:22<21:57,  2.60it/s]iter 12838: loss 5.1413, time: 461.33ms\n",
            " 79% 12839/16260 [1:13:22<23:16,  2.45it/s]iter 12839: loss 5.2103, time: 464.67ms\n",
            " 79% 12840/16260 [1:13:22<24:15,  2.35it/s]iter 12840: loss 3.9240, time: 168.33ms\n",
            " 79% 12841/16260 [1:13:23<19:53,  2.87it/s]iter 12841: loss 4.8449, time: 464.38ms\n",
            " 79% 12842/16260 [1:13:23<21:52,  2.60it/s]iter 12842: loss 5.6488, time: 419.43ms\n",
            " 79% 12843/16260 [1:13:24<22:30,  2.53it/s]iter 12843: loss 5.5325, time: 470.20ms\n",
            " 79% 12844/16260 [1:13:24<23:48,  2.39it/s]iter 12844: loss 5.1183, time: 274.69ms\n",
            " 79% 12845/16260 [1:13:24<21:22,  2.66it/s]iter 12845: loss 3.8985, time: 190.00ms\n",
            " 79% 12846/16260 [1:13:24<18:13,  3.12it/s]iter 12846: loss 3.4494, time: 138.22ms\n",
            " 79% 12847/16260 [1:13:25<15:08,  3.76it/s]iter 12847: loss 4.0518, time: 187.18ms\n",
            " 79% 12848/16260 [1:13:25<13:48,  4.12it/s]iter 12848: loss 4.6184, time: 183.44ms\n",
            " 79% 12849/16260 [1:13:25<12:48,  4.44it/s]iter 12849: loss 5.2760, time: 229.35ms\n",
            " 79% 12850/16260 [1:13:25<12:54,  4.40it/s]iter 12850: loss 6.1486, time: 462.75ms\n",
            " 79% 12851/16260 [1:13:26<16:56,  3.35it/s]iter 12851: loss 5.0093, time: 206.49ms\n",
            " 79% 12852/16260 [1:13:26<15:24,  3.69it/s]iter 12852: loss 5.3977, time: 280.86ms\n",
            " 79% 12853/16260 [1:13:26<15:35,  3.64it/s]iter 12853: loss 5.5902, time: 464.33ms\n",
            " 79% 12854/16260 [1:13:27<18:50,  3.01it/s]iter 12854: loss 5.2425, time: 432.34ms\n",
            " 79% 12855/16260 [1:13:27<20:34,  2.76it/s]iter 12855: loss 5.6499, time: 462.79ms\n",
            " 79% 12856/16260 [1:13:28<22:18,  2.54it/s]iter 12856: loss 5.2040, time: 185.72ms\n",
            " 79% 12857/16260 [1:13:28<18:47,  3.02it/s]iter 12857: loss 4.5838, time: 167.32ms\n",
            " 79% 12858/16260 [1:13:28<16:01,  3.54it/s]iter 12858: loss 4.4497, time: 149.42ms\n",
            " 79% 12859/16260 [1:13:28<13:46,  4.12it/s]iter 12859: loss 5.2577, time: 437.23ms\n",
            " 79% 12860/16260 [1:13:28<17:05,  3.31it/s]iter 12860: loss 4.7607, time: 251.43ms\n",
            " 79% 12861/16260 [1:13:29<16:15,  3.48it/s]iter 12861: loss 4.2894, time: 156.94ms\n",
            " 79% 12862/16260 [1:13:29<14:04,  4.03it/s]iter 12862: loss 5.0468, time: 410.88ms\n",
            " 79% 12863/16260 [1:13:29<16:51,  3.36it/s]iter 12863: loss 5.3878, time: 319.06ms\n",
            " 79% 12864/16260 [1:13:30<17:13,  3.29it/s]iter 12864: loss 5.5199, time: 464.63ms\n",
            " 79% 12865/16260 [1:13:30<19:57,  2.83it/s]iter 12865: loss 5.6463, time: 376.69ms\n",
            " 79% 12866/16260 [1:13:30<20:23,  2.77it/s]iter 12866: loss 5.5340, time: 465.51ms\n",
            " 79% 12867/16260 [1:13:31<22:12,  2.55it/s]iter 12867: loss 5.4339, time: 431.55ms\n",
            " 79% 12868/16260 [1:13:31<22:53,  2.47it/s]iter 12868: loss 5.2919, time: 462.75ms\n",
            " 79% 12869/16260 [1:13:32<23:53,  2.37it/s]iter 12869: loss 4.5948, time: 215.69ms\n",
            " 79% 12870/16260 [1:13:32<20:23,  2.77it/s]iter 12870: loss 5.8692, time: 463.61ms\n",
            " 79% 12871/16260 [1:13:33<22:09,  2.55it/s]iter 12871: loss 5.0445, time: 331.36ms\n",
            " 79% 12872/16260 [1:13:33<21:08,  2.67it/s]iter 12872: loss 4.0024, time: 209.60ms\n",
            " 79% 12873/16260 [1:13:33<18:22,  3.07it/s]iter 12873: loss 4.4177, time: 209.58ms\n",
            " 79% 12874/16260 [1:13:33<16:25,  3.44it/s]iter 12874: loss 5.2385, time: 438.93ms\n",
            " 79% 12875/16260 [1:13:34<18:56,  2.98it/s]iter 12875: loss 5.8126, time: 463.47ms\n",
            " 79% 12876/16260 [1:13:34<21:07,  2.67it/s]iter 12876: loss 4.4742, time: 189.76ms\n",
            " 79% 12877/16260 [1:13:34<18:01,  3.13it/s]iter 12877: loss 5.7801, time: 463.80ms\n",
            " 79% 12878/16260 [1:13:35<20:28,  2.75it/s]iter 12878: loss 3.7143, time: 150.58ms\n",
            " 79% 12879/16260 [1:13:35<16:54,  3.33it/s]iter 12879: loss 4.6562, time: 411.45ms\n",
            " 79% 12880/16260 [1:13:35<18:48,  2.99it/s]iter 12880: loss 5.6146, time: 344.56ms\n",
            " 79% 12881/16260 [1:13:36<19:00,  2.96it/s]iter 12881: loss 5.6050, time: 468.59ms\n",
            " 79% 12882/16260 [1:13:36<21:14,  2.65it/s]iter 12882: loss 3.8559, time: 238.63ms\n",
            " 79% 12883/16260 [1:13:36<18:54,  2.98it/s]iter 12883: loss 5.2566, time: 369.63ms\n",
            " 79% 12884/16260 [1:13:37<19:30,  2.89it/s]iter 12884: loss 5.0283, time: 473.96ms\n",
            " 79% 12885/16260 [1:13:37<21:40,  2.60it/s]iter 12885: loss 5.3020, time: 464.18ms\n",
            " 79% 12886/16260 [1:13:38<23:01,  2.44it/s]iter 12886: loss 5.8422, time: 463.55ms\n",
            " 79% 12887/16260 [1:13:38<23:57,  2.35it/s]iter 12887: loss 5.2017, time: 236.55ms\n",
            " 79% 12888/16260 [1:13:38<20:46,  2.70it/s]iter 12888: loss 4.2601, time: 212.47ms\n",
            " 79% 12889/16260 [1:13:39<18:09,  3.10it/s]iter 12889: loss 5.7808, time: 467.43ms\n",
            " 79% 12890/16260 [1:13:39<20:36,  2.73it/s]iter 12890: loss 5.1030, time: 464.31ms\n",
            " 79% 12891/16260 [1:13:40<22:16,  2.52it/s]iter 12891: loss 6.0256, time: 463.12ms\n",
            " 79% 12892/16260 [1:13:40<23:24,  2.40it/s]iter 12892: loss 5.3664, time: 463.46ms\n",
            " 79% 12893/16260 [1:13:41<24:12,  2.32it/s]iter 12893: loss 5.5340, time: 465.29ms\n",
            " 79% 12894/16260 [1:13:41<24:47,  2.26it/s]iter 12894: loss 5.9062, time: 462.41ms\n",
            " 79% 12895/16260 [1:13:41<25:09,  2.23it/s]iter 12895: loss 6.0260, time: 468.02ms\n",
            " 79% 12896/16260 [1:13:42<25:29,  2.20it/s]iter 12896: loss 5.6486, time: 463.32ms\n",
            " 79% 12897/16260 [1:13:42<25:39,  2.18it/s]iter 12897: loss 5.1376, time: 343.29ms\n",
            " 79% 12898/16260 [1:13:43<23:45,  2.36it/s]iter 12898: loss 5.7963, time: 463.90ms\n",
            " 79% 12899/16260 [1:13:43<24:26,  2.29it/s]iter 12899: loss 4.5084, time: 169.85ms\n",
            " 79% 12900/16260 [1:13:43<19:59,  2.80it/s]iter 12900: loss 5.4017, time: 418.57ms\n",
            " 79% 12901/16260 [1:13:44<21:02,  2.66it/s]iter 12901: loss 4.4569, time: 164.20ms\n",
            " 79% 12902/16260 [1:13:44<17:30,  3.20it/s]iter 12902: loss 4.7309, time: 186.21ms\n",
            " 79% 12903/16260 [1:13:44<15:24,  3.63it/s]iter 12903: loss 5.8300, time: 464.31ms\n",
            " 79% 12904/16260 [1:13:45<18:35,  3.01it/s]iter 12904: loss 3.9691, time: 146.03ms\n",
            " 79% 12905/16260 [1:13:45<15:29,  3.61it/s]iter 12905: loss 5.1584, time: 168.94ms\n",
            " 79% 12906/16260 [1:13:45<13:41,  4.08it/s]iter 12906: loss 4.9586, time: 315.24ms\n",
            " 79% 12907/16260 [1:13:45<14:53,  3.75it/s]iter 12907: loss 5.5003, time: 463.43ms\n",
            " 79% 12908/16260 [1:13:46<18:12,  3.07it/s]iter 12908: loss 3.8445, time: 149.45ms\n",
            " 79% 12909/16260 [1:13:46<15:16,  3.66it/s]iter 12909: loss 5.3392, time: 442.17ms\n",
            " 79% 12910/16260 [1:13:46<18:07,  3.08it/s]iter 12910: loss 5.3015, time: 234.83ms\n",
            " 79% 12911/16260 [1:13:47<16:38,  3.35it/s]iter 12911: loss 5.8553, time: 463.29ms\n",
            " 79% 12912/16260 [1:13:47<19:25,  2.87it/s]iter 12912: loss 4.6366, time: 212.25ms\n",
            " 79% 12913/16260 [1:13:47<17:10,  3.25it/s]iter 12913: loss 5.4642, time: 421.92ms\n",
            " 79% 12914/16260 [1:13:48<19:06,  2.92it/s]iter 12914: loss 5.4909, time: 462.36ms\n",
            " 79% 12915/16260 [1:13:48<21:07,  2.64it/s]iter 12915: loss 5.3322, time: 472.32ms\n",
            " 79% 12916/16260 [1:13:49<22:42,  2.45it/s]iter 12916: loss 5.9537, time: 435.92ms\n",
            " 79% 12917/16260 [1:13:49<23:12,  2.40it/s]iter 12917: loss 4.4644, time: 333.81ms\n",
            " 79% 12918/16260 [1:13:49<21:50,  2.55it/s]iter 12918: loss 4.6355, time: 188.57ms\n",
            " 79% 12919/16260 [1:13:50<18:27,  3.02it/s]iter 12919: loss 5.2352, time: 411.14ms\n",
            " 79% 12920/16260 [1:13:50<19:48,  2.81it/s]iter 12920: loss 5.4153, time: 259.63ms\n",
            " 79% 12921/16260 [1:13:50<18:13,  3.05it/s]iter 12921: loss 3.9467, time: 236.51ms\n",
            " 79% 12922/16260 [1:13:50<16:43,  3.33it/s]iter 12922: loss 4.4973, time: 207.37ms\n",
            " 79% 12923/16260 [1:13:51<15:11,  3.66it/s]iter 12923: loss 5.3931, time: 304.66ms\n",
            " 79% 12924/16260 [1:13:51<15:44,  3.53it/s]iter 12924: loss 5.5860, time: 462.37ms\n",
            " 79% 12925/16260 [1:13:51<18:45,  2.96it/s]iter 12925: loss 5.4228, time: 334.87ms\n",
            " 79% 12926/16260 [1:13:52<18:43,  2.97it/s]iter 12926: loss 5.4238, time: 313.93ms\n",
            " 80% 12927/16260 [1:13:52<18:21,  3.02it/s]iter 12927: loss 5.5344, time: 235.22ms\n",
            " 80% 12928/16260 [1:13:52<16:47,  3.31it/s]iter 12928: loss 5.6470, time: 462.99ms\n",
            " 80% 12929/16260 [1:13:53<19:28,  2.85it/s]iter 12929: loss 5.6351, time: 463.40ms\n",
            " 80% 12930/16260 [1:13:53<21:22,  2.60it/s]iter 12930: loss 5.0557, time: 211.60ms\n",
            " 80% 12931/16260 [1:13:53<18:30,  3.00it/s]iter 12931: loss 5.6397, time: 465.49ms\n",
            " 80% 12932/16260 [1:13:54<20:43,  2.68it/s]iter 12932: loss 4.0129, time: 208.03ms\n",
            " 80% 12933/16260 [1:13:54<17:59,  3.08it/s]iter 12933: loss 5.1351, time: 426.38ms\n",
            " 80% 12934/16260 [1:13:55<19:42,  2.81it/s]iter 12934: loss 4.3786, time: 316.83ms\n",
            " 80% 12935/16260 [1:13:55<19:05,  2.90it/s]iter 12935: loss 5.5262, time: 462.84ms\n",
            " 80% 12936/16260 [1:13:55<21:04,  2.63it/s]iter 12936: loss 4.3877, time: 175.96ms\n",
            " 80% 12937/16260 [1:13:56<17:41,  3.13it/s]iter 12937: loss 5.7306, time: 464.04ms\n",
            " 80% 12938/16260 [1:13:56<20:07,  2.75it/s]iter 12938: loss 4.4378, time: 230.83ms\n",
            " 80% 12939/16260 [1:13:56<17:56,  3.09it/s]iter 12939: loss 5.3566, time: 370.59ms\n",
            " 80% 12940/16260 [1:13:57<18:43,  2.95it/s]iter 12940: loss 5.6891, time: 463.49ms\n",
            " 80% 12941/16260 [1:13:57<20:49,  2.66it/s]iter 12941: loss 4.8575, time: 436.69ms\n",
            " 80% 12942/16260 [1:13:58<21:50,  2.53it/s]iter 12942: loss 5.3999, time: 467.77ms\n",
            " 80% 12943/16260 [1:13:58<23:04,  2.40it/s]iter 12943: loss 5.5740, time: 464.12ms\n",
            " 80% 12944/16260 [1:13:58<23:51,  2.32it/s]iter 12944: loss 5.0695, time: 261.99ms\n",
            " 80% 12945/16260 [1:13:59<21:04,  2.62it/s]iter 12945: loss 5.2351, time: 464.21ms\n",
            " 80% 12946/16260 [1:13:59<22:27,  2.46it/s]iter 12946: loss 5.8362, time: 464.15ms\n",
            " 80% 12947/16260 [1:14:00<23:25,  2.36it/s]iter 12947: loss 5.1128, time: 238.09ms\n",
            " 80% 12948/16260 [1:14:00<20:21,  2.71it/s]iter 12948: loss 5.6061, time: 464.27ms\n",
            " 80% 12949/16260 [1:14:00<21:58,  2.51it/s]iter 12949: loss 6.0829, time: 474.67ms\n",
            " 80% 12950/16260 [1:14:01<23:15,  2.37it/s]iter 12950: loss 5.9131, time: 464.69ms\n",
            " 80% 12951/16260 [1:14:01<23:59,  2.30it/s]iter 12951: loss 4.7027, time: 213.45ms\n",
            " 80% 12952/16260 [1:14:02<20:20,  2.71it/s]iter 12952: loss 5.1539, time: 477.82ms\n",
            " 80% 12953/16260 [1:14:02<22:09,  2.49it/s]iter 12953: loss 5.3670, time: 436.52ms\n",
            " 80% 12954/16260 [1:14:02<22:44,  2.42it/s]iter 12954: loss 5.3086, time: 463.84ms\n",
            " 80% 12955/16260 [1:14:03<23:36,  2.33it/s]iter 12955: loss 5.6194, time: 463.65ms\n",
            " 80% 12956/16260 [1:14:03<24:12,  2.27it/s]iter 12956: loss 5.0745, time: 271.98ms\n",
            " 80% 12957/16260 [1:14:04<21:27,  2.57it/s]iter 12957: loss 4.3196, time: 315.30ms\n",
            " 80% 12958/16260 [1:14:04<20:14,  2.72it/s]iter 12958: loss 4.8714, time: 462.65ms\n",
            " 80% 12959/16260 [1:14:04<21:49,  2.52it/s]iter 12959: loss 5.3954, time: 444.36ms\n",
            " 80% 12960/16260 [1:14:05<22:37,  2.43it/s]iter 12960: loss 5.1280, time: 191.92ms\n",
            " 80% 12961/16260 [1:14:05<19:01,  2.89it/s]iter 12961: loss 5.3997, time: 259.87ms\n",
            " 80% 12962/16260 [1:14:05<17:37,  3.12it/s]iter 12962: loss 5.3427, time: 336.36ms\n",
            " 80% 12963/16260 [1:14:06<17:54,  3.07it/s]iter 12963: loss 5.1963, time: 342.37ms\n",
            " 80% 12964/16260 [1:14:06<18:11,  3.02it/s]iter 12964: loss 4.8869, time: 281.88ms\n",
            " 80% 12965/16260 [1:14:06<17:24,  3.15it/s]iter 12965: loss 5.4443, time: 332.91ms\n",
            " 80% 12966/16260 [1:14:07<17:41,  3.10it/s]iter 12966: loss 5.6859, time: 310.36ms\n",
            " 80% 12967/16260 [1:14:07<17:30,  3.13it/s]iter 12967: loss 5.5629, time: 463.82ms\n",
            " 80% 12968/16260 [1:14:07<19:54,  2.76it/s]iter 12968: loss 5.8071, time: 444.57ms\n",
            " 80% 12969/16260 [1:14:08<21:16,  2.58it/s]iter 12969: loss 5.1451, time: 192.60ms\n",
            " 80% 12970/16260 [1:14:08<18:04,  3.03it/s]iter 12970: loss 4.0741, time: 171.06ms\n",
            " 80% 12971/16260 [1:14:08<15:29,  3.54it/s]iter 12971: loss 5.9639, time: 463.58ms\n",
            " 80% 12972/16260 [1:14:09<18:29,  2.96it/s]iter 12972: loss 4.7790, time: 238.77ms\n",
            " 80% 12973/16260 [1:14:09<16:53,  3.24it/s]iter 12973: loss 5.2026, time: 343.63ms\n",
            " 80% 12974/16260 [1:14:09<17:29,  3.13it/s]iter 12974: loss 5.0427, time: 331.52ms\n",
            " 80% 12975/16260 [1:14:10<17:42,  3.09it/s]iter 12975: loss 5.2579, time: 259.10ms\n",
            " 80% 12976/16260 [1:14:10<16:40,  3.28it/s]iter 12976: loss 4.9793, time: 340.87ms\n",
            " 80% 12977/16260 [1:14:10<17:17,  3.17it/s]iter 12977: loss 5.6031, time: 420.69ms\n",
            " 80% 12978/16260 [1:14:11<19:01,  2.87it/s]iter 12978: loss 5.3574, time: 256.44ms\n",
            " 80% 12979/16260 [1:14:11<17:32,  3.12it/s]iter 12979: loss 4.3794, time: 147.86ms\n",
            " 80% 12980/16260 [1:14:11<14:43,  3.71it/s]iter 12980: loss 5.5328, time: 236.80ms\n",
            " 80% 12981/16260 [1:14:11<14:12,  3.85it/s]iter 12981: loss 5.7489, time: 474.10ms\n",
            " 80% 12982/16260 [1:14:12<17:44,  3.08it/s]iter 12982: loss 5.3080, time: 464.36ms\n",
            " 80% 12983/16260 [1:14:12<20:02,  2.72it/s]iter 12983: loss 4.0869, time: 193.09ms\n",
            " 80% 12984/16260 [1:14:12<17:13,  3.17it/s]iter 12984: loss 4.5571, time: 209.89ms\n",
            " 80% 12985/16260 [1:14:13<15:30,  3.52it/s]iter 12985: loss 5.1828, time: 331.67ms\n",
            " 80% 12986/16260 [1:14:13<16:18,  3.35it/s]iter 12986: loss 4.6281, time: 193.77ms\n",
            " 80% 12987/16260 [1:14:13<14:36,  3.73it/s]iter 12987: loss 5.9988, time: 475.05ms\n",
            " 80% 12988/16260 [1:14:14<18:00,  3.03it/s]iter 12988: loss 5.9062, time: 464.34ms\n",
            " 80% 12989/16260 [1:14:14<20:13,  2.70it/s]iter 12989: loss 5.3609, time: 307.98ms\n",
            " 80% 12990/16260 [1:14:14<19:12,  2.84it/s]iter 12990: loss 4.4036, time: 256.41ms\n",
            " 80% 12991/16260 [1:14:15<17:39,  3.08it/s]iter 12991: loss 4.8135, time: 466.92ms\n",
            " 80% 12992/16260 [1:14:15<20:00,  2.72it/s]iter 12992: loss 5.1535, time: 310.17ms\n",
            " 80% 12993/16260 [1:14:15<19:05,  2.85it/s]iter 12993: loss 5.0437, time: 331.94ms\n",
            " 80% 12994/16260 [1:14:16<18:48,  2.89it/s]iter 12994: loss 4.4325, time: 211.57ms\n",
            " 80% 12995/16260 [1:14:16<16:38,  3.27it/s]iter 12995: loss 5.3895, time: 334.54ms\n",
            " 80% 12996/16260 [1:14:16<17:07,  3.18it/s]iter 12996: loss 4.8575, time: 212.18ms\n",
            " 80% 12997/16260 [1:14:17<15:28,  3.51it/s]iter 12997: loss 5.2957, time: 254.17ms\n",
            " 80% 12998/16260 [1:14:17<15:00,  3.62it/s]iter 12998: loss 5.6040, time: 463.85ms\n",
            " 80% 12999/16260 [1:14:17<18:05,  3.00it/s]iter 12999: loss 5.4091, time: 463.35ms\n",
            " 80% 13000/16260 [1:14:18<20:13,  2.69it/s]iter 13000: loss 5.3652, time: 462.37ms\n",
            " 80% 13001/16260 [1:14:18<21:43,  2.50it/s]iter 13001: loss 4.9972, time: 369.38ms\n",
            " 80% 13002/16260 [1:14:19<21:14,  2.56it/s]iter 13002: loss 5.4101, time: 442.11ms\n",
            " 80% 13003/16260 [1:14:19<22:05,  2.46it/s]iter 13003: loss 5.3275, time: 438.06ms\n",
            " 80% 13004/16260 [1:14:19<22:37,  2.40it/s]iter 13004: loss 4.9329, time: 444.57ms\n",
            " 80% 13005/16260 [1:14:20<23:05,  2.35it/s]iter 13005: loss 5.6487, time: 476.26ms\n",
            " 80% 13006/16260 [1:14:20<23:55,  2.27it/s]iter 13006: loss 4.6412, time: 186.57ms\n",
            " 80% 13007/16260 [1:14:21<19:48,  2.74it/s]iter 13007: loss 5.3342, time: 463.23ms\n",
            " 80% 13008/16260 [1:14:21<21:24,  2.53it/s]iter 13008: loss 4.9220, time: 274.91ms\n",
            " 80% 13009/16260 [1:14:21<19:28,  2.78it/s]iter 13009: loss 4.6671, time: 463.43ms\n",
            " 80% 13010/16260 [1:14:22<21:11,  2.56it/s]iter 13010: loss 6.0049, time: 462.87ms\n",
            " 80% 13011/16260 [1:14:22<22:22,  2.42it/s]iter 13011: loss 4.2819, time: 189.17ms\n",
            " 80% 13012/16260 [1:14:22<18:44,  2.89it/s]iter 13012: loss 4.3974, time: 187.73ms\n",
            " 80% 13013/16260 [1:14:23<16:11,  3.34it/s]iter 13013: loss 4.8593, time: 463.22ms\n",
            " 80% 13014/16260 [1:14:23<18:52,  2.87it/s]iter 13014: loss 5.9222, time: 463.47ms\n",
            " 80% 13015/16260 [1:14:24<20:45,  2.61it/s]iter 13015: loss 5.7316, time: 257.36ms\n",
            " 80% 13016/16260 [1:14:24<18:43,  2.89it/s]iter 13016: loss 5.0171, time: 236.66ms\n",
            " 80% 13017/16260 [1:14:24<16:57,  3.19it/s]iter 13017: loss 5.4187, time: 279.97ms\n",
            " 80% 13018/16260 [1:14:24<16:26,  3.29it/s]iter 13018: loss 4.8804, time: 213.81ms\n",
            " 80% 13019/16260 [1:14:25<14:59,  3.60it/s]iter 13019: loss 5.1654, time: 256.22ms\n",
            " 80% 13020/16260 [1:14:25<14:39,  3.68it/s]iter 13020: loss 4.7327, time: 192.63ms\n",
            " 80% 13021/16260 [1:14:25<13:24,  4.03it/s]iter 13021: loss 4.2245, time: 158.70ms\n",
            " 80% 13022/16260 [1:14:25<11:58,  4.51it/s]iter 13022: loss 4.8952, time: 313.71ms\n",
            " 80% 13023/16260 [1:14:25<13:29,  4.00it/s]iter 13023: loss 4.0844, time: 174.47ms\n",
            " 80% 13024/16260 [1:14:26<12:16,  4.39it/s]iter 13024: loss 4.3088, time: 148.86ms\n",
            " 80% 13025/16260 [1:14:26<11:01,  4.89it/s]iter 13025: loss 4.3233, time: 209.66ms\n",
            " 80% 13026/16260 [1:14:26<11:07,  4.84it/s]iter 13026: loss 5.4954, time: 327.05ms\n",
            " 80% 13027/16260 [1:14:26<13:05,  4.11it/s]iter 13027: loss 5.4290, time: 441.88ms\n",
            " 80% 13028/16260 [1:14:27<16:19,  3.30it/s]iter 13028: loss 5.6789, time: 437.02ms\n",
            " 80% 13029/16260 [1:14:27<18:30,  2.91it/s]iter 13029: loss 4.8289, time: 239.11ms\n",
            " 80% 13030/16260 [1:14:27<16:50,  3.20it/s]iter 13030: loss 3.9692, time: 145.81ms\n",
            " 80% 13031/16260 [1:14:28<14:09,  3.80it/s]iter 13031: loss 5.4682, time: 314.52ms\n",
            " 80% 13032/16260 [1:14:28<15:00,  3.58it/s]iter 13032: loss 4.7898, time: 256.77ms\n",
            " 80% 13033/16260 [1:14:28<14:40,  3.67it/s]iter 13033: loss 4.6106, time: 202.03ms\n",
            " 80% 13034/16260 [1:14:28<13:33,  3.97it/s]iter 13034: loss 5.8977, time: 464.36ms\n",
            " 80% 13035/16260 [1:14:29<16:59,  3.16it/s]iter 13035: loss 5.7351, time: 469.29ms\n",
            " 80% 13036/16260 [1:14:29<19:28,  2.76it/s]iter 13036: loss 5.4150, time: 331.60ms\n",
            " 80% 13037/16260 [1:14:30<18:59,  2.83it/s]iter 13037: loss 5.2797, time: 464.30ms\n",
            " 80% 13038/16260 [1:14:30<20:48,  2.58it/s]iter 13038: loss 5.0322, time: 276.73ms\n",
            " 80% 13039/16260 [1:14:30<19:02,  2.82it/s]iter 13039: loss 4.8461, time: 465.42ms\n",
            " 80% 13040/16260 [1:14:31<20:50,  2.57it/s]iter 13040: loss 5.3762, time: 314.02ms\n",
            " 80% 13041/16260 [1:14:31<19:39,  2.73it/s]iter 13041: loss 4.6202, time: 274.95ms\n",
            " 80% 13042/16260 [1:14:31<18:12,  2.95it/s]iter 13042: loss 4.8697, time: 171.44ms\n",
            " 80% 13043/16260 [1:14:32<15:31,  3.45it/s]iter 13043: loss 4.6972, time: 232.68ms\n",
            " 80% 13044/16260 [1:14:32<14:37,  3.66it/s]iter 13044: loss 4.9600, time: 217.63ms\n",
            " 80% 13045/16260 [1:14:32<13:45,  3.90it/s]iter 13045: loss 4.8788, time: 263.76ms\n",
            " 80% 13046/16260 [1:14:32<13:53,  3.86it/s]iter 13046: loss 5.4108, time: 341.50ms\n",
            " 80% 13047/16260 [1:14:33<15:13,  3.52it/s]iter 13047: loss 5.4040, time: 277.18ms\n",
            " 80% 13048/16260 [1:14:33<15:07,  3.54it/s]iter 13048: loss 3.5416, time: 206.02ms\n",
            " 80% 13049/16260 [1:14:33<13:54,  3.85it/s]iter 13049: loss 4.5465, time: 235.53ms\n",
            " 80% 13050/16260 [1:14:33<13:32,  3.95it/s]iter 13050: loss 5.1756, time: 439.80ms\n",
            " 80% 13051/16260 [1:14:34<16:33,  3.23it/s]iter 13051: loss 5.1263, time: 408.71ms\n",
            " 80% 13052/16260 [1:14:34<18:09,  2.94it/s]iter 13052: loss 4.5405, time: 233.38ms\n",
            " 80% 13053/16260 [1:14:34<16:28,  3.24it/s]iter 13053: loss 5.2263, time: 263.39ms\n",
            " 80% 13054/16260 [1:14:35<15:46,  3.39it/s]iter 13054: loss 4.0621, time: 151.84ms\n",
            " 80% 13055/16260 [1:14:35<13:29,  3.96it/s]iter 13055: loss 4.7419, time: 423.65ms\n",
            " 80% 13056/16260 [1:14:35<16:15,  3.29it/s]iter 13056: loss 4.1745, time: 236.26ms\n",
            " 80% 13057/16260 [1:14:36<15:10,  3.52it/s]iter 13057: loss 4.8668, time: 252.80ms\n",
            " 80% 13058/16260 [1:14:36<14:41,  3.63it/s]iter 13058: loss 6.1084, time: 464.27ms\n",
            " 80% 13059/16260 [1:14:36<17:44,  3.01it/s]iter 13059: loss 5.8664, time: 474.14ms\n",
            " 80% 13060/16260 [1:14:37<20:01,  2.66it/s]iter 13060: loss 5.3456, time: 421.41ms\n",
            " 80% 13061/16260 [1:14:37<20:46,  2.57it/s]iter 13061: loss 4.2213, time: 230.49ms\n",
            " 80% 13062/16260 [1:14:37<18:14,  2.92it/s]iter 13062: loss 4.5030, time: 186.37ms\n",
            " 80% 13063/16260 [1:14:38<15:46,  3.38it/s]iter 13063: loss 4.6676, time: 207.06ms\n",
            " 80% 13064/16260 [1:14:38<14:21,  3.71it/s]iter 13064: loss 5.8201, time: 463.77ms\n",
            " 80% 13065/16260 [1:14:38<17:29,  3.04it/s]iter 13065: loss 5.6618, time: 316.16ms\n",
            " 80% 13066/16260 [1:14:39<17:18,  3.08it/s]iter 13066: loss 5.3920, time: 463.59ms\n",
            " 80% 13067/16260 [1:14:39<19:32,  2.72it/s]iter 13067: loss 5.4785, time: 464.41ms\n",
            " 80% 13068/16260 [1:14:40<21:06,  2.52it/s]iter 13068: loss 4.4557, time: 209.51ms\n",
            " 80% 13069/16260 [1:14:40<18:08,  2.93it/s]iter 13069: loss 5.6276, time: 463.86ms\n",
            " 80% 13070/16260 [1:14:40<20:06,  2.64it/s]iter 13070: loss 3.9900, time: 163.45ms\n",
            " 80% 13071/16260 [1:14:40<16:42,  3.18it/s]iter 13071: loss 3.5068, time: 150.16ms\n",
            " 80% 13072/16260 [1:14:41<14:06,  3.77it/s]iter 13072: loss 4.6553, time: 148.96ms\n",
            " 80% 13073/16260 [1:14:41<12:15,  4.33it/s]iter 13073: loss 5.7552, time: 464.49ms\n",
            " 80% 13074/16260 [1:14:41<16:00,  3.32it/s]iter 13074: loss 5.5425, time: 463.73ms\n",
            " 80% 13075/16260 [1:14:42<18:36,  2.85it/s]iter 13075: loss 5.3835, time: 279.27ms\n",
            " 80% 13076/16260 [1:14:42<17:29,  3.03it/s]iter 13076: loss 6.2278, time: 463.18ms\n",
            " 80% 13077/16260 [1:14:42<19:38,  2.70it/s]iter 13077: loss 5.9079, time: 462.28ms\n",
            " 80% 13078/16260 [1:14:43<21:07,  2.51it/s]iter 13078: loss 5.2693, time: 376.49ms\n",
            " 80% 13079/16260 [1:14:43<20:47,  2.55it/s]iter 13079: loss 4.5215, time: 334.03ms\n",
            " 80% 13080/16260 [1:14:44<19:53,  2.66it/s]iter 13080: loss 5.5369, time: 476.06ms\n",
            " 80% 13081/16260 [1:14:44<21:30,  2.46it/s]iter 13081: loss 5.8084, time: 381.18ms\n",
            " 80% 13082/16260 [1:14:44<21:08,  2.51it/s]iter 13082: loss 5.0633, time: 312.16ms\n",
            " 80% 13083/16260 [1:14:45<19:46,  2.68it/s]iter 13083: loss 5.2059, time: 462.81ms\n",
            " 80% 13084/16260 [1:14:45<21:12,  2.50it/s]iter 13084: loss 5.6930, time: 410.06ms\n",
            " 80% 13085/16260 [1:14:46<21:22,  2.48it/s]iter 13085: loss 5.5800, time: 463.39ms\n",
            " 80% 13086/16260 [1:14:46<22:20,  2.37it/s]iter 13086: loss 4.9321, time: 232.85ms\n",
            " 80% 13087/16260 [1:14:46<19:21,  2.73it/s]iter 13087: loss 5.3367, time: 466.95ms\n",
            " 80% 13088/16260 [1:14:47<20:57,  2.52it/s]iter 13088: loss 4.4915, time: 183.90ms\n",
            " 80% 13089/16260 [1:14:47<17:36,  3.00it/s]iter 13089: loss 5.0010, time: 234.49ms\n",
            " 81% 13090/16260 [1:14:47<16:03,  3.29it/s]iter 13090: loss 5.2457, time: 463.30ms\n",
            " 81% 13091/16260 [1:14:48<18:36,  2.84it/s]iter 13091: loss 4.9903, time: 314.61ms\n",
            " 81% 13092/16260 [1:14:48<18:01,  2.93it/s]iter 13092: loss 5.2044, time: 304.19ms\n",
            " 81% 13093/16260 [1:14:48<17:27,  3.02it/s]iter 13093: loss 4.9454, time: 219.58ms\n",
            " 81% 13094/16260 [1:14:48<15:43,  3.36it/s]iter 13094: loss 5.7668, time: 463.38ms\n",
            " 81% 13095/16260 [1:14:49<18:21,  2.87it/s]iter 13095: loss 5.1345, time: 462.85ms\n",
            " 81% 13096/16260 [1:14:49<20:11,  2.61it/s]iter 13096: loss 5.4995, time: 315.54ms\n",
            " 81% 13097/16260 [1:14:50<19:08,  2.75it/s]iter 13097: loss 4.2983, time: 167.72ms\n",
            " 81% 13098/16260 [1:14:50<16:04,  3.28it/s]iter 13098: loss 4.7675, time: 210.77ms\n",
            " 81% 13099/16260 [1:14:50<14:36,  3.61it/s]iter 13099: loss 5.5329, time: 251.74ms\n",
            " 81% 13100/16260 [1:14:50<14:13,  3.70it/s]iter 13100: loss 5.3797, time: 310.97ms\n",
            " 81% 13101/16260 [1:14:51<14:53,  3.54it/s]iter 13101: loss 4.8995, time: 204.32ms\n",
            " 81% 13102/16260 [1:14:51<13:40,  3.85it/s]iter 13102: loss 4.0930, time: 165.43ms\n",
            " 81% 13103/16260 [1:14:51<12:12,  4.31it/s]iter 13103: loss 4.4034, time: 188.63ms\n",
            " 81% 13104/16260 [1:14:51<11:32,  4.56it/s]iter 13104: loss 4.7651, time: 236.70ms\n",
            " 81% 13105/16260 [1:14:51<11:49,  4.45it/s]iter 13105: loss 3.3666, time: 463.41ms\n",
            " 81% 13106/16260 [1:14:52<15:36,  3.37it/s]iter 13106: loss 4.8176, time: 208.18ms\n",
            " 81% 13107/16260 [1:14:52<14:13,  3.69it/s]iter 13107: loss 5.9575, time: 441.75ms\n",
            " 81% 13108/16260 [1:14:53<16:56,  3.10it/s]iter 13108: loss 4.6916, time: 265.37ms\n",
            " 81% 13109/16260 [1:14:53<16:03,  3.27it/s]iter 13109: loss 4.4624, time: 210.44ms\n",
            " 81% 13110/16260 [1:14:53<14:34,  3.60it/s]iter 13110: loss 5.0642, time: 163.91ms\n",
            " 81% 13111/16260 [1:14:53<12:47,  4.10it/s]iter 13111: loss 5.9222, time: 346.17ms\n",
            " 81% 13112/16260 [1:14:54<14:25,  3.64it/s]iter 13112: loss 5.6365, time: 436.12ms\n",
            " 81% 13113/16260 [1:14:54<16:58,  3.09it/s]iter 13113: loss 5.6404, time: 315.65ms\n",
            " 81% 13114/16260 [1:14:54<16:52,  3.11it/s]iter 13114: loss 4.6657, time: 231.25ms\n",
            " 81% 13115/16260 [1:14:55<15:27,  3.39it/s]iter 13115: loss 4.9913, time: 463.74ms\n",
            " 81% 13116/16260 [1:14:55<18:08,  2.89it/s]iter 13116: loss 5.6326, time: 372.18ms\n",
            " 81% 13117/16260 [1:14:55<18:33,  2.82it/s]iter 13117: loss 4.9135, time: 212.94ms\n",
            " 81% 13118/16260 [1:14:56<16:21,  3.20it/s]iter 13118: loss 5.4422, time: 463.98ms\n",
            " 81% 13119/16260 [1:14:56<18:45,  2.79it/s]iter 13119: loss 3.4704, time: 172.32ms\n",
            " 81% 13120/16260 [1:14:56<15:50,  3.30it/s]iter 13120: loss 5.8293, time: 416.82ms\n",
            " 81% 13121/16260 [1:14:57<17:39,  2.96it/s]iter 13121: loss 5.7771, time: 473.49ms\n",
            " 81% 13122/16260 [1:14:57<19:48,  2.64it/s]iter 13122: loss 4.6383, time: 330.93ms\n",
            " 81% 13123/16260 [1:14:57<19:04,  2.74it/s]iter 13123: loss 5.5299, time: 464.20ms\n",
            " 81% 13124/16260 [1:14:58<20:39,  2.53it/s]iter 13124: loss 5.6732, time: 464.86ms\n",
            " 81% 13125/16260 [1:14:58<21:46,  2.40it/s]iter 13125: loss 5.2846, time: 371.03ms\n",
            " 81% 13126/16260 [1:14:59<21:04,  2.48it/s]iter 13126: loss 4.7428, time: 254.56ms\n",
            " 81% 13127/16260 [1:14:59<18:45,  2.78it/s]iter 13127: loss 5.1281, time: 243.29ms\n",
            " 81% 13128/16260 [1:14:59<16:57,  3.08it/s]iter 13128: loss 4.2547, time: 209.33ms\n",
            " 81% 13129/16260 [1:14:59<15:09,  3.44it/s]iter 13129: loss 4.3905, time: 171.65ms\n",
            " 81% 13130/16260 [1:15:00<13:19,  3.92it/s]iter 13130: loss 4.1722, time: 215.49ms\n",
            " 81% 13131/16260 [1:15:00<12:43,  4.10it/s]iter 13131: loss 5.2103, time: 317.13ms\n",
            " 81% 13132/16260 [1:15:00<13:53,  3.75it/s]iter 13132: loss 4.6141, time: 320.19ms\n",
            " 81% 13133/16260 [1:15:01<14:44,  3.53it/s]iter 13133: loss 4.4470, time: 190.90ms\n",
            " 81% 13134/16260 [1:15:01<13:19,  3.91it/s]iter 13134: loss 5.0616, time: 432.85ms\n",
            " 81% 13135/16260 [1:15:01<16:06,  3.23it/s]iter 13135: loss 5.4475, time: 463.52ms\n",
            " 81% 13136/16260 [1:15:02<18:32,  2.81it/s]iter 13136: loss 4.4652, time: 166.01ms\n",
            " 81% 13137/16260 [1:15:02<15:35,  3.34it/s]iter 13137: loss 5.5231, time: 237.78ms\n",
            " 81% 13138/16260 [1:15:02<14:38,  3.55it/s]iter 13138: loss 5.0066, time: 185.11ms\n",
            " 81% 13139/16260 [1:15:02<13:09,  3.95it/s]iter 13139: loss 5.0918, time: 259.91ms\n",
            " 81% 13140/16260 [1:15:02<13:16,  3.91it/s]iter 13140: loss 5.3087, time: 464.60ms\n",
            " 81% 13141/16260 [1:15:03<16:33,  3.14it/s]iter 13141: loss 5.7144, time: 462.61ms\n",
            " 81% 13142/16260 [1:15:03<18:49,  2.76it/s]iter 13142: loss 4.4679, time: 239.06ms\n",
            " 81% 13143/16260 [1:15:04<16:55,  3.07it/s]iter 13143: loss 5.2233, time: 463.15ms\n",
            " 81% 13144/16260 [1:15:04<19:05,  2.72it/s]iter 13144: loss 5.8339, time: 464.78ms\n",
            " 81% 13145/16260 [1:15:05<20:36,  2.52it/s]iter 13145: loss 5.7918, time: 464.09ms\n",
            " 81% 13146/16260 [1:15:05<21:40,  2.39it/s]iter 13146: loss 5.3200, time: 257.55ms\n",
            " 81% 13147/16260 [1:15:05<19:12,  2.70it/s]iter 13147: loss 5.5902, time: 317.17ms\n",
            " 81% 13148/16260 [1:15:06<18:23,  2.82it/s]iter 13148: loss 5.2135, time: 322.05ms\n",
            " 81% 13149/16260 [1:15:06<17:54,  2.89it/s]iter 13149: loss 5.7998, time: 473.77ms\n",
            " 81% 13150/16260 [1:15:06<19:55,  2.60it/s]iter 13150: loss 5.4468, time: 422.56ms\n",
            " 81% 13151/16260 [1:15:07<20:32,  2.52it/s]iter 13151: loss 5.4975, time: 475.60ms\n",
            " 81% 13152/16260 [1:15:07<21:46,  2.38it/s]iter 13152: loss 4.7803, time: 253.72ms\n",
            " 81% 13153/16260 [1:15:08<19:12,  2.70it/s]iter 13153: loss 5.3436, time: 341.80ms\n",
            " 81% 13154/16260 [1:15:08<18:46,  2.76it/s]iter 13154: loss 4.2673, time: 239.02ms\n",
            " 81% 13155/16260 [1:15:08<16:52,  3.07it/s]iter 13155: loss 5.8066, time: 464.59ms\n",
            " 81% 13156/16260 [1:15:09<19:02,  2.72it/s]iter 13156: loss 4.6699, time: 218.95ms\n",
            " 81% 13157/16260 [1:15:09<16:44,  3.09it/s]iter 13157: loss 4.5040, time: 196.40ms\n",
            " 81% 13158/16260 [1:15:09<14:47,  3.49it/s]iter 13158: loss 5.5162, time: 343.23ms\n",
            " 81% 13159/16260 [1:15:09<15:41,  3.29it/s]iter 13159: loss 4.9220, time: 234.21ms\n",
            " 81% 13160/16260 [1:15:10<14:38,  3.53it/s]iter 13160: loss 5.1995, time: 255.49ms\n",
            " 81% 13161/16260 [1:15:10<14:13,  3.63it/s]iter 13161: loss 5.0101, time: 264.97ms\n",
            " 81% 13162/16260 [1:15:10<14:05,  3.66it/s]iter 13162: loss 4.9860, time: 464.93ms\n",
            " 81% 13163/16260 [1:15:11<17:04,  3.02it/s]iter 13163: loss 4.4488, time: 232.66ms\n",
            " 81% 13164/16260 [1:15:11<15:34,  3.31it/s]iter 13164: loss 5.5338, time: 317.18ms\n",
            " 81% 13165/16260 [1:15:11<15:50,  3.26it/s]iter 13165: loss 4.6048, time: 258.64ms\n",
            " 81% 13166/16260 [1:15:11<15:06,  3.41it/s]iter 13166: loss 5.9400, time: 464.89ms\n",
            " 81% 13167/16260 [1:15:12<17:47,  2.90it/s]iter 13167: loss 5.1836, time: 345.55ms\n",
            " 81% 13168/16260 [1:15:12<17:48,  2.89it/s]iter 13168: loss 5.1420, time: 347.27ms\n",
            " 81% 13169/16260 [1:15:13<17:51,  2.89it/s]iter 13169: loss 5.5311, time: 335.61ms\n",
            " 81% 13170/16260 [1:15:13<17:42,  2.91it/s]iter 13170: loss 6.0964, time: 464.52ms\n",
            " 81% 13171/16260 [1:15:13<19:35,  2.63it/s]iter 13171: loss 5.3397, time: 464.37ms\n",
            " 81% 13172/16260 [1:15:14<20:54,  2.46it/s]iter 13172: loss 5.1593, time: 192.30ms\n",
            " 81% 13173/16260 [1:15:14<17:37,  2.92it/s]iter 13173: loss 3.9550, time: 189.91ms\n",
            " 81% 13174/16260 [1:15:14<15:17,  3.36it/s]iter 13174: loss 5.0816, time: 472.50ms\n",
            " 81% 13175/16260 [1:15:15<18:00,  2.85it/s]iter 13175: loss 5.6562, time: 465.15ms\n",
            " 81% 13176/16260 [1:15:15<19:47,  2.60it/s]iter 13176: loss 5.2914, time: 380.79ms\n",
            " 81% 13177/16260 [1:15:16<19:45,  2.60it/s]iter 13177: loss 3.8547, time: 153.21ms\n",
            " 81% 13178/16260 [1:15:16<16:12,  3.17it/s]iter 13178: loss 5.5902, time: 464.57ms\n",
            " 81% 13179/16260 [1:15:16<18:31,  2.77it/s]iter 13179: loss 4.4429, time: 282.07ms\n",
            " 81% 13180/16260 [1:15:16<17:20,  2.96it/s]iter 13180: loss 4.9893, time: 256.05ms\n",
            " 81% 13181/16260 [1:15:17<16:06,  3.19it/s]iter 13181: loss 5.3375, time: 420.86ms\n",
            " 81% 13182/16260 [1:15:17<17:46,  2.89it/s]iter 13182: loss 4.9118, time: 381.88ms\n",
            " 81% 13183/16260 [1:15:18<18:20,  2.80it/s]iter 13183: loss 5.3445, time: 267.35ms\n",
            " 81% 13184/16260 [1:15:18<16:57,  3.02it/s]iter 13184: loss 5.4469, time: 444.27ms\n",
            " 81% 13185/16260 [1:15:18<18:43,  2.74it/s]iter 13185: loss 5.5686, time: 440.01ms\n",
            " 81% 13186/16260 [1:15:19<19:53,  2.58it/s]iter 13186: loss 5.5323, time: 314.85ms\n",
            " 81% 13187/16260 [1:15:19<18:47,  2.73it/s]iter 13187: loss 5.4532, time: 347.47ms\n",
            " 81% 13188/16260 [1:15:19<18:30,  2.77it/s]iter 13188: loss 5.5909, time: 474.49ms\n",
            " 81% 13189/16260 [1:15:20<20:15,  2.53it/s]iter 13189: loss 5.8802, time: 437.74ms\n",
            " 81% 13190/16260 [1:15:20<20:55,  2.45it/s]iter 13190: loss 5.1197, time: 367.38ms\n",
            " 81% 13191/16260 [1:15:21<20:18,  2.52it/s]iter 13191: loss 5.0259, time: 241.20ms\n",
            " 81% 13192/16260 [1:15:21<17:56,  2.85it/s]iter 13192: loss 4.9675, time: 425.71ms\n",
            " 81% 13193/16260 [1:15:21<19:06,  2.68it/s]iter 13193: loss 5.6873, time: 464.09ms\n",
            " 81% 13194/16260 [1:15:22<20:30,  2.49it/s]iter 13194: loss 5.8473, time: 464.65ms\n",
            " 81% 13195/16260 [1:15:22<21:29,  2.38it/s]iter 13195: loss 5.2535, time: 336.78ms\n",
            " 81% 13196/16260 [1:15:23<20:13,  2.52it/s]iter 13196: loss 4.9729, time: 464.81ms\n",
            " 81% 13197/16260 [1:15:23<21:18,  2.40it/s]iter 13197: loss 5.3606, time: 464.85ms\n",
            " 81% 13198/16260 [1:15:24<22:03,  2.31it/s]iter 13198: loss 5.6190, time: 465.80ms\n",
            " 81% 13199/16260 [1:15:24<22:35,  2.26it/s]iter 13199: loss 5.5537, time: 425.55ms\n",
            " 81% 13200/16260 [1:15:24<22:20,  2.28it/s]iter 13200: loss 5.1899, time: 273.55ms\n",
            " 81% 13201/16260 [1:15:25<19:50,  2.57it/s]iter 13201: loss 4.2032, time: 165.57ms\n",
            " 81% 13202/16260 [1:15:25<16:26,  3.10it/s]iter 13202: loss 4.2954, time: 164.47ms\n",
            " 81% 13203/16260 [1:15:25<14:02,  3.63it/s]iter 13203: loss 5.2853, time: 342.68ms\n",
            " 81% 13204/16260 [1:15:25<15:05,  3.38it/s]iter 13204: loss 5.0475, time: 205.45ms\n",
            " 81% 13205/16260 [1:15:26<13:42,  3.71it/s]iter 13205: loss 5.5763, time: 464.87ms\n",
            " 81% 13206/16260 [1:15:26<16:43,  3.04it/s]iter 13206: loss 4.1367, time: 151.90ms\n",
            " 81% 13207/16260 [1:15:26<14:02,  3.62it/s]iter 13207: loss 4.8816, time: 278.85ms\n",
            " 81% 13208/16260 [1:15:26<14:06,  3.61it/s]iter 13208: loss 4.6848, time: 200.83ms\n",
            " 81% 13209/16260 [1:15:27<12:57,  3.92it/s]iter 13209: loss 5.4706, time: 464.46ms\n",
            " 81% 13210/16260 [1:15:27<16:10,  3.14it/s]iter 13210: loss 5.6170, time: 464.53ms\n",
            " 81% 13211/16260 [1:15:28<18:25,  2.76it/s]iter 13211: loss 4.5742, time: 238.16ms\n",
            " 81% 13212/16260 [1:15:28<16:32,  3.07it/s]iter 13212: loss 5.7038, time: 464.44ms\n",
            " 81% 13213/16260 [1:15:28<18:40,  2.72it/s]iter 13213: loss 5.6242, time: 464.83ms\n",
            " 81% 13214/16260 [1:15:29<20:10,  2.52it/s]iter 13214: loss 5.8042, time: 464.96ms\n",
            " 81% 13215/16260 [1:15:29<21:13,  2.39it/s]iter 13215: loss 4.5931, time: 195.36ms\n",
            " 81% 13216/16260 [1:15:29<17:50,  2.84it/s]iter 13216: loss 4.9604, time: 464.24ms\n",
            " 81% 13217/16260 [1:15:30<19:34,  2.59it/s]iter 13217: loss 3.9864, time: 166.26ms\n",
            " 81% 13218/16260 [1:15:30<16:14,  3.12it/s]iter 13218: loss 4.4651, time: 422.64ms\n",
            " 81% 13219/16260 [1:15:30<17:49,  2.84it/s]iter 13219: loss 4.8003, time: 333.32ms\n",
            " 81% 13220/16260 [1:15:31<17:33,  2.89it/s]iter 13220: loss 5.6135, time: 368.80ms\n",
            " 81% 13221/16260 [1:15:31<17:54,  2.83it/s]iter 13221: loss 4.6418, time: 280.95ms\n",
            " 81% 13222/16260 [1:15:31<16:49,  3.01it/s]iter 13222: loss 4.9143, time: 251.27ms\n",
            " 81% 13223/16260 [1:15:32<15:36,  3.24it/s]iter 13223: loss 4.9706, time: 218.57ms\n",
            " 81% 13224/16260 [1:15:32<14:16,  3.55it/s]iter 13224: loss 4.1335, time: 165.88ms\n",
            " 81% 13225/16260 [1:15:32<12:31,  4.04it/s]iter 13225: loss 4.7128, time: 216.49ms\n",
            " 81% 13226/16260 [1:15:32<12:03,  4.19it/s]iter 13226: loss 5.7323, time: 463.65ms\n",
            " 81% 13227/16260 [1:15:33<15:29,  3.26it/s]iter 13227: loss 5.2336, time: 410.56ms\n",
            " 81% 13228/16260 [1:15:33<17:05,  2.96it/s]iter 13228: loss 5.2955, time: 465.33ms\n",
            " 81% 13229/16260 [1:15:34<19:02,  2.65it/s]iter 13229: loss 5.4825, time: 464.31ms\n",
            " 81% 13230/16260 [1:15:34<20:22,  2.48it/s]iter 13230: loss 4.5025, time: 171.08ms\n",
            " 81% 13231/16260 [1:15:34<16:52,  2.99it/s]iter 13231: loss 5.4758, time: 371.56ms\n",
            " 81% 13232/16260 [1:15:35<17:27,  2.89it/s]iter 13232: loss 4.9002, time: 233.50ms\n",
            " 81% 13233/16260 [1:15:35<15:46,  3.20it/s]iter 13233: loss 4.7727, time: 239.24ms\n",
            " 81% 13234/16260 [1:15:35<14:40,  3.44it/s]iter 13234: loss 5.6564, time: 463.67ms\n",
            " 81% 13235/16260 [1:15:36<17:18,  2.91it/s]iter 13235: loss 5.3427, time: 310.53ms\n",
            " 81% 13236/16260 [1:15:36<16:49,  3.00it/s]iter 13236: loss 5.5497, time: 378.60ms\n",
            " 81% 13237/16260 [1:15:36<17:31,  2.88it/s]iter 13237: loss 6.0790, time: 464.74ms\n",
            " 81% 13238/16260 [1:15:37<19:18,  2.61it/s]iter 13238: loss 5.1706, time: 240.00ms\n",
            " 81% 13239/16260 [1:15:37<17:09,  2.93it/s]iter 13239: loss 5.2759, time: 464.78ms\n",
            " 81% 13240/16260 [1:15:37<19:03,  2.64it/s]iter 13240: loss 4.8686, time: 345.71ms\n",
            " 81% 13241/16260 [1:15:38<18:34,  2.71it/s]iter 13241: loss 3.5631, time: 141.88ms\n",
            " 81% 13242/16260 [1:15:38<15:09,  3.32it/s]iter 13242: loss 5.4576, time: 465.34ms\n",
            " 81% 13243/16260 [1:15:38<17:39,  2.85it/s]iter 13243: loss 4.7068, time: 283.84ms\n",
            " 81% 13244/16260 [1:15:39<16:39,  3.02it/s]iter 13244: loss 5.0368, time: 327.72ms\n",
            " 81% 13245/16260 [1:15:39<16:37,  3.02it/s]iter 13245: loss 5.2550, time: 240.00ms\n",
            " 81% 13246/16260 [1:15:39<15:16,  3.29it/s]iter 13246: loss 5.3812, time: 462.37ms\n",
            " 81% 13247/16260 [1:15:40<17:40,  2.84it/s]iter 13247: loss 5.9682, time: 414.95ms\n",
            " 81% 13248/16260 [1:15:40<18:37,  2.69it/s]iter 13248: loss 5.3640, time: 279.61ms\n",
            " 81% 13249/16260 [1:15:40<17:16,  2.91it/s]iter 13249: loss 6.0170, time: 378.08ms\n",
            " 81% 13250/16260 [1:15:41<17:47,  2.82it/s]iter 13250: loss 5.4545, time: 263.16ms\n",
            " 81% 13251/16260 [1:15:41<16:26,  3.05it/s]iter 13251: loss 5.3257, time: 464.65ms\n",
            " 82% 13252/16260 [1:15:42<18:31,  2.71it/s]iter 13252: loss 4.7588, time: 259.63ms\n",
            " 82% 13253/16260 [1:15:42<16:53,  2.97it/s]iter 13253: loss 5.8374, time: 325.30ms\n",
            " 82% 13254/16260 [1:15:42<16:43,  3.00it/s]iter 13254: loss 6.1302, time: 262.29ms\n",
            " 82% 13255/16260 [1:15:42<15:40,  3.20it/s]iter 13255: loss 4.1842, time: 169.99ms\n",
            " 82% 13256/16260 [1:15:43<13:32,  3.70it/s]iter 13256: loss 4.3521, time: 217.00ms\n",
            " 82% 13257/16260 [1:15:43<12:45,  3.92it/s]iter 13257: loss 5.5580, time: 464.86ms\n",
            " 82% 13258/16260 [1:15:43<15:55,  3.14it/s]iter 13258: loss 5.3781, time: 463.81ms\n",
            " 82% 13259/16260 [1:15:44<18:07,  2.76it/s]iter 13259: loss 5.6193, time: 465.81ms\n",
            " 82% 13260/16260 [1:15:44<19:41,  2.54it/s]iter 13260: loss 5.5712, time: 373.38ms\n",
            " 82% 13261/16260 [1:15:45<19:24,  2.58it/s]iter 13261: loss 5.2147, time: 190.69ms\n",
            " 82% 13262/16260 [1:15:45<16:27,  3.04it/s]iter 13262: loss 4.7818, time: 238.01ms\n",
            " 82% 13263/16260 [1:15:45<15:06,  3.31it/s]iter 13263: loss 5.4843, time: 463.76ms\n",
            " 82% 13264/16260 [1:15:45<17:32,  2.85it/s]iter 13264: loss 5.5066, time: 463.79ms\n",
            " 82% 13265/16260 [1:15:46<19:14,  2.59it/s]iter 13265: loss 4.7087, time: 282.08ms\n",
            " 82% 13266/16260 [1:15:46<17:42,  2.82it/s]iter 13266: loss 5.3490, time: 463.81ms\n",
            " 82% 13267/16260 [1:15:47<19:21,  2.58it/s]iter 13267: loss 4.9332, time: 274.34ms\n",
            " 82% 13268/16260 [1:15:47<17:40,  2.82it/s]iter 13268: loss 5.5206, time: 464.78ms\n",
            " 82% 13269/16260 [1:15:47<19:20,  2.58it/s]iter 13269: loss 5.1505, time: 346.64ms\n",
            " 82% 13270/16260 [1:15:48<18:44,  2.66it/s]iter 13270: loss 3.7895, time: 170.91ms\n",
            " 82% 13271/16260 [1:15:48<15:41,  3.18it/s]iter 13271: loss 4.5454, time: 235.81ms\n",
            " 82% 13272/16260 [1:15:48<14:31,  3.43it/s]iter 13272: loss 5.3325, time: 464.14ms\n",
            " 82% 13273/16260 [1:15:49<17:07,  2.91it/s]iter 13273: loss 4.1425, time: 234.29ms\n",
            " 82% 13274/16260 [1:15:49<15:29,  3.21it/s]iter 13274: loss 5.3151, time: 464.24ms\n",
            " 82% 13275/16260 [1:15:49<17:47,  2.80it/s]iter 13275: loss 4.6710, time: 212.29ms\n",
            " 82% 13276/16260 [1:15:50<15:38,  3.18it/s]iter 13276: loss 5.3666, time: 463.49ms\n",
            " 82% 13277/16260 [1:15:50<17:52,  2.78it/s]iter 13277: loss 5.6660, time: 356.37ms\n",
            " 82% 13278/16260 [1:15:50<17:51,  2.78it/s]iter 13278: loss 5.2219, time: 277.29ms\n",
            " 82% 13279/16260 [1:15:51<16:39,  2.98it/s]iter 13279: loss 5.1852, time: 267.94ms\n",
            " 82% 13280/16260 [1:15:51<15:39,  3.17it/s]iter 13280: loss 5.3064, time: 280.96ms\n",
            " 82% 13281/16260 [1:15:51<15:09,  3.27it/s]iter 13281: loss 5.7903, time: 464.39ms\n",
            " 82% 13282/16260 [1:15:52<17:33,  2.83it/s]iter 13282: loss 5.5418, time: 444.90ms\n",
            " 82% 13283/16260 [1:15:52<18:55,  2.62it/s]iter 13283: loss 4.6220, time: 316.46ms\n",
            " 82% 13284/16260 [1:15:52<17:58,  2.76it/s]iter 13284: loss 5.0508, time: 239.31ms\n",
            " 82% 13285/16260 [1:15:53<16:10,  3.07it/s]iter 13285: loss 4.9778, time: 435.06ms\n",
            " 82% 13286/16260 [1:15:53<17:48,  2.78it/s]iter 13286: loss 5.3540, time: 188.07ms\n",
            " 82% 13287/16260 [1:15:53<15:16,  3.24it/s]iter 13287: loss 3.7877, time: 152.34ms\n",
            " 82% 13288/16260 [1:15:53<12:58,  3.82it/s]iter 13288: loss 3.9635, time: 147.18ms\n",
            " 82% 13289/16260 [1:15:54<11:17,  4.38it/s]iter 13289: loss 5.4339, time: 310.90ms\n",
            " 82% 13290/16260 [1:15:54<12:32,  3.95it/s]iter 13290: loss 5.5226, time: 464.72ms\n",
            " 82% 13291/16260 [1:15:54<15:41,  3.15it/s]iter 13291: loss 4.8104, time: 231.15ms\n",
            " 82% 13292/16260 [1:15:55<14:26,  3.43it/s]iter 13292: loss 4.5791, time: 218.55ms\n",
            " 82% 13293/16260 [1:15:55<13:21,  3.70it/s]iter 13293: loss 5.3010, time: 418.98ms\n",
            " 82% 13294/16260 [1:15:55<15:35,  3.17it/s]iter 13294: loss 5.5922, time: 374.29ms\n",
            " 82% 13295/16260 [1:15:56<16:28,  3.00it/s]iter 13295: loss 5.0159, time: 424.18ms\n",
            " 82% 13296/16260 [1:15:56<17:50,  2.77it/s]iter 13296: loss 4.7122, time: 480.57ms\n",
            " 82% 13297/16260 [1:15:57<19:37,  2.52it/s]iter 13297: loss 5.5911, time: 475.50ms\n",
            " 82% 13298/16260 [1:15:57<20:48,  2.37it/s]iter 13298: loss 4.3158, time: 172.36ms\n",
            " 82% 13299/16260 [1:15:57<17:07,  2.88it/s]iter 13299: loss 4.1381, time: 176.55ms\n",
            " 82% 13300/16260 [1:15:57<14:37,  3.37it/s]iter 13300: loss 5.7032, time: 464.71ms\n",
            " 82% 13301/16260 [1:15:58<17:08,  2.88it/s]iter 13301: loss 5.9471, time: 432.95ms\n",
            " 82% 13302/16260 [1:15:58<18:25,  2.68it/s]iter 13302: loss 4.6925, time: 209.25ms\n",
            " 82% 13303/16260 [1:15:59<16:00,  3.08it/s]iter 13303: loss 4.6297, time: 192.72ms\n",
            " 82% 13304/16260 [1:15:59<14:04,  3.50it/s]iter 13304: loss 5.6390, time: 254.41ms\n",
            " 82% 13305/16260 [1:15:59<13:37,  3.61it/s]iter 13305: loss 4.3747, time: 194.87ms\n",
            " 82% 13306/16260 [1:15:59<12:26,  3.96it/s]iter 13306: loss 5.6534, time: 464.57ms\n",
            " 82% 13307/16260 [1:16:00<15:35,  3.16it/s]iter 13307: loss 5.1558, time: 256.96ms\n",
            " 82% 13308/16260 [1:16:00<14:43,  3.34it/s]iter 13308: loss 4.7341, time: 316.40ms\n",
            " 82% 13309/16260 [1:16:00<14:59,  3.28it/s]iter 13309: loss 5.2232, time: 414.29ms\n",
            " 82% 13310/16260 [1:16:01<16:37,  2.96it/s]iter 13310: loss 5.3231, time: 372.51ms\n",
            " 82% 13311/16260 [1:16:01<17:09,  2.87it/s]iter 13311: loss 4.5642, time: 197.66ms\n",
            " 82% 13312/16260 [1:16:01<14:56,  3.29it/s]iter 13312: loss 5.8390, time: 465.33ms\n",
            " 82% 13313/16260 [1:16:02<17:19,  2.83it/s]iter 13313: loss 5.3966, time: 421.45ms\n",
            " 82% 13314/16260 [1:16:02<18:21,  2.67it/s]iter 13314: loss 4.7990, time: 216.17ms\n",
            " 82% 13315/16260 [1:16:02<16:03,  3.06it/s]iter 13315: loss 4.0567, time: 160.58ms\n",
            " 82% 13316/16260 [1:16:02<13:37,  3.60it/s]iter 13316: loss 4.9916, time: 220.20ms\n",
            " 82% 13317/16260 [1:16:03<12:47,  3.83it/s]iter 13317: loss 5.5335, time: 464.97ms\n",
            " 82% 13318/16260 [1:16:03<15:49,  3.10it/s]iter 13318: loss 5.2438, time: 293.84ms\n",
            " 82% 13319/16260 [1:16:03<15:24,  3.18it/s]iter 13319: loss 5.3538, time: 464.22ms\n",
            " 82% 13320/16260 [1:16:04<17:38,  2.78it/s]iter 13320: loss 5.5266, time: 442.79ms\n",
            " 82% 13321/16260 [1:16:04<18:52,  2.60it/s]iter 13321: loss 5.1597, time: 281.41ms\n",
            " 82% 13322/16260 [1:16:05<17:21,  2.82it/s]iter 13322: loss 4.5935, time: 213.04ms\n",
            " 82% 13323/16260 [1:16:05<15:17,  3.20it/s]iter 13323: loss 5.4041, time: 344.50ms\n",
            " 82% 13324/16260 [1:16:05<15:47,  3.10it/s]iter 13324: loss 5.2635, time: 321.39ms\n",
            " 82% 13325/16260 [1:16:06<15:47,  3.10it/s]iter 13325: loss 4.6569, time: 192.88ms\n",
            " 82% 13326/16260 [1:16:06<13:54,  3.52it/s]iter 13326: loss 4.1018, time: 168.88ms\n",
            " 82% 13327/16260 [1:16:06<12:13,  4.00it/s]iter 13327: loss 5.3960, time: 334.33ms\n",
            " 82% 13328/16260 [1:16:06<13:28,  3.63it/s]iter 13328: loss 5.1721, time: 312.85ms\n",
            " 82% 13329/16260 [1:16:07<14:02,  3.48it/s]iter 13329: loss 4.8982, time: 237.48ms\n",
            " 82% 13330/16260 [1:16:07<13:19,  3.66it/s]iter 13330: loss 5.5030, time: 465.04ms\n",
            " 82% 13331/16260 [1:16:07<16:09,  3.02it/s]iter 13331: loss 4.3409, time: 219.53ms\n",
            " 82% 13332/16260 [1:16:07<14:33,  3.35it/s]iter 13332: loss 5.1790, time: 379.00ms\n",
            " 82% 13333/16260 [1:16:08<15:45,  3.10it/s]iter 13333: loss 4.0533, time: 189.68ms\n",
            " 82% 13334/16260 [1:16:08<13:49,  3.53it/s]iter 13334: loss 5.5649, time: 437.23ms\n",
            " 82% 13335/16260 [1:16:08<16:05,  3.03it/s]iter 13335: loss 5.1022, time: 466.24ms\n",
            " 82% 13336/16260 [1:16:09<18:05,  2.69it/s]iter 13336: loss 5.3350, time: 465.66ms\n",
            " 82% 13337/16260 [1:16:09<19:29,  2.50it/s]iter 13337: loss 4.8620, time: 340.88ms\n",
            " 82% 13338/16260 [1:16:10<18:38,  2.61it/s]iter 13338: loss 5.0167, time: 279.38ms\n",
            " 82% 13339/16260 [1:16:10<17:09,  2.84it/s]iter 13339: loss 6.0309, time: 464.83ms\n",
            " 82% 13340/16260 [1:16:11<18:48,  2.59it/s]iter 13340: loss 5.6287, time: 443.69ms\n",
            " 82% 13341/16260 [1:16:11<19:39,  2.47it/s]iter 13341: loss 3.8453, time: 168.84ms\n",
            " 82% 13342/16260 [1:16:11<16:14,  2.99it/s]iter 13342: loss 4.8099, time: 410.24ms\n",
            " 82% 13343/16260 [1:16:12<17:22,  2.80it/s]iter 13343: loss 4.9293, time: 467.47ms\n",
            " 82% 13344/16260 [1:16:12<18:59,  2.56it/s]iter 13344: loss 5.4699, time: 308.35ms\n",
            " 82% 13345/16260 [1:16:12<17:48,  2.73it/s]iter 13345: loss 5.5847, time: 342.67ms\n",
            " 82% 13346/16260 [1:16:13<17:28,  2.78it/s]iter 13346: loss 5.4891, time: 419.91ms\n",
            " 82% 13347/16260 [1:16:13<18:21,  2.64it/s]iter 13347: loss 5.9593, time: 464.48ms\n",
            " 82% 13348/16260 [1:16:14<19:38,  2.47it/s]iter 13348: loss 5.4054, time: 464.29ms\n",
            " 82% 13349/16260 [1:16:14<20:31,  2.36it/s]iter 13349: loss 4.5873, time: 165.61ms\n",
            " 82% 13350/16260 [1:16:14<16:47,  2.89it/s]iter 13350: loss 5.1704, time: 346.23ms\n",
            " 82% 13351/16260 [1:16:15<16:48,  2.88it/s]iter 13351: loss 5.1874, time: 471.69ms\n",
            " 82% 13352/16260 [1:16:15<18:38,  2.60it/s]iter 13352: loss 5.4866, time: 425.21ms\n",
            " 82% 13353/16260 [1:16:15<19:15,  2.52it/s]iter 13353: loss 4.3656, time: 169.10ms\n",
            " 82% 13354/16260 [1:16:16<15:57,  3.04it/s]iter 13354: loss 5.8176, time: 464.90ms\n",
            " 82% 13355/16260 [1:16:16<17:56,  2.70it/s]iter 13355: loss 5.5759, time: 464.53ms\n",
            " 82% 13356/16260 [1:16:17<19:19,  2.51it/s]iter 13356: loss 4.9626, time: 341.85ms\n",
            " 82% 13357/16260 [1:16:17<18:30,  2.61it/s]iter 13357: loss 3.9734, time: 240.00ms\n",
            " 82% 13358/16260 [1:16:17<16:27,  2.94it/s]iter 13358: loss 5.6263, time: 464.47ms\n",
            " 82% 13359/16260 [1:16:18<18:16,  2.65it/s]iter 13359: loss 3.9242, time: 170.87ms\n",
            " 82% 13360/16260 [1:16:18<15:17,  3.16it/s]iter 13360: loss 4.8450, time: 343.25ms\n",
            " 82% 13361/16260 [1:16:18<15:41,  3.08it/s]iter 13361: loss 5.1376, time: 464.79ms\n",
            " 82% 13362/16260 [1:16:19<17:44,  2.72it/s]iter 13362: loss 5.6155, time: 464.65ms\n",
            " 82% 13363/16260 [1:16:19<19:10,  2.52it/s]iter 13363: loss 4.3516, time: 272.77ms\n",
            " 82% 13364/16260 [1:16:19<17:23,  2.78it/s]iter 13364: loss 4.7259, time: 464.80ms\n",
            " 82% 13365/16260 [1:16:20<18:55,  2.55it/s]iter 13365: loss 5.3604, time: 464.97ms\n",
            " 82% 13366/16260 [1:16:20<19:59,  2.41it/s]iter 13366: loss 4.5044, time: 255.38ms\n",
            " 82% 13367/16260 [1:16:20<17:42,  2.72it/s]iter 13367: loss 4.9837, time: 455.44ms\n",
            " 82% 13368/16260 [1:16:21<18:59,  2.54it/s]iter 13368: loss 5.2367, time: 465.48ms\n",
            " 82% 13369/16260 [1:16:21<20:02,  2.40it/s]iter 13369: loss 5.0598, time: 417.26ms\n",
            " 82% 13370/16260 [1:16:22<20:04,  2.40it/s]iter 13370: loss 5.4421, time: 321.40ms\n",
            " 82% 13371/16260 [1:16:22<18:42,  2.57it/s]iter 13371: loss 4.6385, time: 257.16ms\n",
            " 82% 13372/16260 [1:16:22<16:49,  2.86it/s]iter 13372: loss 5.8360, time: 335.81ms\n",
            " 82% 13373/16260 [1:16:23<16:38,  2.89it/s]iter 13373: loss 5.2277, time: 273.14ms\n",
            " 82% 13374/16260 [1:16:23<15:36,  3.08it/s]iter 13374: loss 5.7632, time: 465.17ms\n",
            " 82% 13375/16260 [1:16:23<17:39,  2.72it/s]iter 13375: loss 5.6456, time: 467.96ms\n",
            " 82% 13376/16260 [1:16:24<19:07,  2.51it/s]iter 13376: loss 5.3384, time: 316.98ms\n",
            " 82% 13377/16260 [1:16:24<17:58,  2.67it/s]iter 13377: loss 5.8254, time: 464.25ms\n",
            " 82% 13378/16260 [1:16:25<19:17,  2.49it/s]iter 13378: loss 4.3695, time: 166.74ms\n",
            " 82% 13379/16260 [1:16:25<15:55,  3.02it/s]iter 13379: loss 5.1161, time: 321.37ms\n",
            " 82% 13380/16260 [1:16:25<15:47,  3.04it/s]iter 13380: loss 5.4349, time: 277.01ms\n",
            " 82% 13381/16260 [1:16:26<15:03,  3.19it/s]iter 13381: loss 3.7465, time: 334.60ms\n",
            " 82% 13382/16260 [1:16:26<15:22,  3.12it/s]iter 13382: loss 4.6234, time: 187.32ms\n",
            " 82% 13383/16260 [1:16:26<13:28,  3.56it/s]iter 13383: loss 4.1220, time: 167.52ms\n",
            " 82% 13384/16260 [1:16:26<11:51,  4.04it/s]iter 13384: loss 5.9466, time: 464.03ms\n",
            " 82% 13385/16260 [1:16:27<14:59,  3.20it/s]iter 13385: loss 4.0319, time: 166.18ms\n",
            " 82% 13386/16260 [1:16:27<12:53,  3.71it/s]iter 13386: loss 5.0631, time: 470.89ms\n",
            " 82% 13387/16260 [1:16:27<15:48,  3.03it/s]iter 13387: loss 3.9405, time: 148.62ms\n",
            " 82% 13388/16260 [1:16:27<13:13,  3.62it/s]iter 13388: loss 4.5976, time: 217.27ms\n",
            " 82% 13389/16260 [1:16:28<12:23,  3.86it/s]iter 13389: loss 5.0332, time: 218.84ms\n",
            " 82% 13390/16260 [1:16:28<11:50,  4.04it/s]iter 13390: loss 4.4514, time: 251.90ms\n",
            " 82% 13391/16260 [1:16:28<11:55,  4.01it/s]iter 13391: loss 5.4764, time: 464.47ms\n",
            " 82% 13392/16260 [1:16:29<15:01,  3.18it/s]iter 13392: loss 5.0987, time: 276.71ms\n",
            " 82% 13393/16260 [1:16:29<14:30,  3.29it/s]iter 13393: loss 5.3614, time: 216.79ms\n",
            " 82% 13394/16260 [1:16:29<13:16,  3.60it/s]iter 13394: loss 4.6220, time: 215.09ms\n",
            " 82% 13395/16260 [1:16:29<12:23,  3.85it/s]iter 13395: loss 4.9864, time: 411.96ms\n",
            " 82% 13396/16260 [1:16:30<14:35,  3.27it/s]iter 13396: loss 5.2991, time: 467.86ms\n",
            " 82% 13397/16260 [1:16:30<16:56,  2.82it/s]iter 13397: loss 5.1159, time: 208.44ms\n",
            " 82% 13398/16260 [1:16:30<14:51,  3.21it/s]iter 13398: loss 4.5519, time: 172.99ms\n",
            " 82% 13399/16260 [1:16:31<12:53,  3.70it/s]iter 13399: loss 5.3423, time: 341.28ms\n",
            " 82% 13400/16260 [1:16:31<13:55,  3.42it/s]iter 13400: loss 5.6607, time: 466.00ms\n",
            " 82% 13401/16260 [1:16:31<16:25,  2.90it/s]iter 13401: loss 5.6028, time: 463.73ms\n",
            " 82% 13402/16260 [1:16:32<18:08,  2.63it/s]iter 13402: loss 5.5707, time: 464.57ms\n",
            " 82% 13403/16260 [1:16:32<19:21,  2.46it/s]iter 13403: loss 4.8995, time: 338.74ms\n",
            " 82% 13404/16260 [1:16:33<18:24,  2.59it/s]iter 13404: loss 5.4862, time: 317.03ms\n",
            " 82% 13405/16260 [1:16:33<17:25,  2.73it/s]iter 13405: loss 4.2771, time: 212.66ms\n",
            " 82% 13406/16260 [1:16:33<15:14,  3.12it/s]iter 13406: loss 5.0339, time: 277.55ms\n",
            " 82% 13407/16260 [1:16:34<14:39,  3.25it/s]iter 13407: loss 5.2394, time: 310.97ms\n",
            " 82% 13408/16260 [1:16:34<14:42,  3.23it/s]iter 13408: loss 4.7528, time: 264.32ms\n",
            " 82% 13409/16260 [1:16:34<14:04,  3.38it/s]iter 13409: loss 4.4328, time: 171.81ms\n",
            " 82% 13410/16260 [1:16:34<12:19,  3.85it/s]iter 13410: loss 5.1283, time: 265.86ms\n",
            " 82% 13411/16260 [1:16:35<12:26,  3.82it/s]iter 13411: loss 4.9373, time: 415.27ms\n",
            " 82% 13412/16260 [1:16:35<14:38,  3.24it/s]iter 13412: loss 5.7838, time: 464.88ms\n",
            " 82% 13413/16260 [1:16:35<16:53,  2.81it/s]iter 13413: loss 5.2338, time: 431.29ms\n",
            " 82% 13414/16260 [1:16:36<17:58,  2.64it/s]iter 13414: loss 5.8362, time: 468.74ms\n",
            " 83% 13415/16260 [1:16:36<19:16,  2.46it/s]iter 13415: loss 2.6895, time: 137.77ms\n",
            " 83% 13416/16260 [1:16:36<15:28,  3.06it/s]iter 13416: loss 4.7619, time: 212.55ms\n",
            " 83% 13417/16260 [1:16:37<13:52,  3.42it/s]iter 13417: loss 5.8172, time: 463.37ms\n",
            " 83% 13418/16260 [1:16:37<16:18,  2.90it/s]iter 13418: loss 3.6866, time: 148.95ms\n",
            " 83% 13419/16260 [1:16:37<13:32,  3.50it/s]iter 13419: loss 5.4556, time: 464.25ms\n",
            " 83% 13420/16260 [1:16:38<16:05,  2.94it/s]iter 13420: loss 5.1215, time: 377.80ms\n",
            " 83% 13421/16260 [1:16:38<16:38,  2.84it/s]iter 13421: loss 4.4366, time: 225.03ms\n",
            " 83% 13422/16260 [1:16:38<14:51,  3.18it/s]iter 13422: loss 5.1787, time: 345.00ms\n",
            " 83% 13423/16260 [1:16:39<15:18,  3.09it/s]iter 13423: loss 4.9703, time: 308.66ms\n",
            " 83% 13424/16260 [1:16:39<15:07,  3.13it/s]iter 13424: loss 4.0360, time: 152.82ms\n",
            " 83% 13425/16260 [1:16:39<12:45,  3.70it/s]iter 13425: loss 5.9162, time: 464.80ms\n",
            " 83% 13426/16260 [1:16:40<15:32,  3.04it/s]iter 13426: loss 5.2501, time: 463.57ms\n",
            " 83% 13427/16260 [1:16:40<17:27,  2.70it/s]iter 13427: loss 5.9020, time: 464.17ms\n",
            " 83% 13428/16260 [1:16:41<18:48,  2.51it/s]iter 13428: loss 4.0517, time: 188.13ms\n",
            " 83% 13429/16260 [1:16:41<15:51,  2.98it/s]iter 13429: loss 5.1166, time: 438.25ms\n",
            " 83% 13430/16260 [1:16:41<17:19,  2.72it/s]iter 13430: loss 5.1843, time: 239.01ms\n",
            " 83% 13431/16260 [1:16:41<15:31,  3.04it/s]iter 13431: loss 4.8148, time: 464.10ms\n",
            " 83% 13432/16260 [1:16:42<17:26,  2.70it/s]iter 13432: loss 5.1161, time: 257.87ms\n",
            " 83% 13433/16260 [1:16:42<15:52,  2.97it/s]iter 13433: loss 4.6676, time: 197.05ms\n",
            " 83% 13434/16260 [1:16:42<13:55,  3.38it/s]iter 13434: loss 4.1861, time: 193.29ms\n",
            " 83% 13435/16260 [1:16:43<12:29,  3.77it/s]iter 13435: loss 5.3491, time: 464.27ms\n",
            " 83% 13436/16260 [1:16:43<15:19,  3.07it/s]iter 13436: loss 4.8629, time: 272.56ms\n",
            " 83% 13437/16260 [1:16:43<14:35,  3.22it/s]iter 13437: loss 3.3869, time: 151.50ms\n",
            " 83% 13438/16260 [1:16:43<12:22,  3.80it/s]iter 13438: loss 4.3025, time: 190.97ms\n",
            " 83% 13439/16260 [1:16:44<11:22,  4.14it/s]iter 13439: loss 5.6785, time: 468.29ms\n",
            " 83% 13440/16260 [1:16:44<14:34,  3.23it/s]iter 13440: loss 5.7969, time: 464.35ms\n",
            " 83% 13441/16260 [1:16:45<16:45,  2.80it/s]iter 13441: loss 4.1860, time: 191.20ms\n",
            " 83% 13442/16260 [1:16:45<14:26,  3.25it/s]iter 13442: loss 5.4435, time: 442.08ms\n",
            " 83% 13443/16260 [1:16:45<16:21,  2.87it/s]iter 13443: loss 4.2202, time: 226.18ms\n",
            " 83% 13444/16260 [1:16:45<14:39,  3.20it/s]iter 13444: loss 5.2060, time: 340.54ms\n",
            " 83% 13445/16260 [1:16:46<15:04,  3.11it/s]iter 13445: loss 5.4020, time: 463.86ms\n",
            " 83% 13446/16260 [1:16:46<17:05,  2.74it/s]iter 13446: loss 5.4397, time: 384.65ms\n",
            " 83% 13447/16260 [1:16:47<17:23,  2.69it/s]iter 13447: loss 4.7467, time: 279.46ms\n",
            " 83% 13448/16260 [1:16:47<16:07,  2.91it/s]iter 13448: loss 4.6734, time: 235.01ms\n",
            " 83% 13449/16260 [1:16:47<14:36,  3.21it/s]iter 13449: loss 4.8816, time: 236.99ms\n",
            " 83% 13450/16260 [1:16:47<13:34,  3.45it/s]iter 13450: loss 4.8232, time: 414.85ms\n",
            " 83% 13451/16260 [1:16:48<15:20,  3.05it/s]iter 13451: loss 3.6552, time: 285.17ms\n",
            " 83% 13452/16260 [1:16:48<14:46,  3.17it/s]iter 13452: loss 5.5777, time: 412.48ms\n",
            " 83% 13453/16260 [1:16:48<16:08,  2.90it/s]iter 13453: loss 4.7330, time: 463.88ms\n",
            " 83% 13454/16260 [1:16:49<17:49,  2.62it/s]iter 13454: loss 5.5896, time: 464.31ms\n",
            " 83% 13455/16260 [1:16:49<19:00,  2.46it/s]iter 13455: loss 4.4986, time: 168.98ms\n",
            " 83% 13456/16260 [1:16:50<15:41,  2.98it/s]iter 13456: loss 4.7495, time: 254.60ms\n",
            " 83% 13457/16260 [1:16:50<14:34,  3.21it/s]iter 13457: loss 5.4750, time: 416.34ms\n",
            " 83% 13458/16260 [1:16:50<16:03,  2.91it/s]iter 13458: loss 5.9755, time: 423.00ms\n",
            " 83% 13459/16260 [1:16:51<17:10,  2.72it/s]iter 13459: loss 4.9263, time: 236.76ms\n",
            " 83% 13460/16260 [1:16:51<15:21,  3.04it/s]iter 13460: loss 5.7616, time: 313.88ms\n",
            " 83% 13461/16260 [1:16:51<15:10,  3.08it/s]iter 13461: loss 6.0525, time: 433.99ms\n",
            " 83% 13462/16260 [1:16:52<16:42,  2.79it/s]iter 13462: loss 4.9497, time: 216.38ms\n",
            " 83% 13463/16260 [1:16:52<14:44,  3.16it/s]iter 13463: loss 5.3174, time: 463.70ms\n",
            " 83% 13464/16260 [1:16:52<16:48,  2.77it/s]iter 13464: loss 5.6886, time: 483.07ms\n",
            " 83% 13465/16260 [1:16:53<18:32,  2.51it/s]iter 13465: loss 5.1498, time: 413.21ms\n",
            " 83% 13466/16260 [1:16:53<18:46,  2.48it/s]iter 13466: loss 5.4849, time: 651.73ms\n",
            " 83% 13467/16260 [1:16:54<22:15,  2.09it/s]iter 13467: loss 4.9999, time: 274.42ms\n",
            " 83% 13468/16260 [1:16:54<19:25,  2.40it/s]iter 13468: loss 5.5111, time: 251.95ms\n",
            " 83% 13469/16260 [1:16:54<17:07,  2.72it/s]iter 13469: loss 5.1232, time: 288.93ms\n",
            " 83% 13470/16260 [1:16:55<16:02,  2.90it/s]iter 13470: loss 5.6688, time: 235.51ms\n",
            " 83% 13471/16260 [1:16:55<14:31,  3.20it/s]iter 13471: loss 5.0600, time: 238.69ms\n",
            " 83% 13472/16260 [1:16:55<13:30,  3.44it/s]iter 13472: loss 4.3922, time: 230.01ms\n",
            " 83% 13473/16260 [1:16:55<12:40,  3.66it/s]iter 13473: loss 5.9396, time: 464.76ms\n",
            " 83% 13474/16260 [1:16:56<15:21,  3.02it/s]iter 13474: loss 5.3724, time: 333.25ms\n",
            " 83% 13475/16260 [1:16:56<15:24,  3.01it/s]iter 13475: loss 4.4810, time: 236.31ms\n",
            " 83% 13476/16260 [1:16:56<14:05,  3.29it/s]iter 13476: loss 4.9018, time: 372.35ms\n",
            " 83% 13477/16260 [1:16:57<15:03,  3.08it/s]iter 13477: loss 3.1683, time: 133.88ms\n",
            " 83% 13478/16260 [1:16:57<12:25,  3.73it/s]iter 13478: loss 4.9716, time: 186.61ms\n",
            " 83% 13479/16260 [1:16:57<11:18,  4.10it/s]iter 13479: loss 5.1565, time: 214.44ms\n",
            " 83% 13480/16260 [1:16:57<10:54,  4.25it/s]iter 13480: loss 3.0858, time: 135.93ms\n",
            " 83% 13481/16260 [1:16:58<09:32,  4.86it/s]iter 13481: loss 5.5725, time: 423.29ms\n",
            " 83% 13482/16260 [1:16:58<12:34,  3.68it/s]iter 13482: loss 4.8890, time: 185.06ms\n",
            " 83% 13483/16260 [1:16:58<11:23,  4.06it/s]iter 13483: loss 5.6437, time: 464.58ms\n",
            " 83% 13484/16260 [1:16:59<14:26,  3.20it/s]iter 13484: loss 4.7498, time: 257.63ms\n",
            " 83% 13485/16260 [1:16:59<13:41,  3.38it/s]iter 13485: loss 5.6619, time: 281.73ms\n",
            " 83% 13486/16260 [1:16:59<13:30,  3.42it/s]iter 13486: loss 5.1334, time: 285.16ms\n",
            " 83% 13487/16260 [1:16:59<13:25,  3.44it/s]iter 13487: loss 3.7765, time: 147.08ms\n",
            " 83% 13488/16260 [1:17:00<11:27,  4.03it/s]iter 13488: loss 4.6042, time: 251.62ms\n",
            " 83% 13489/16260 [1:17:00<11:31,  4.01it/s]iter 13489: loss 5.5480, time: 464.07ms\n",
            " 83% 13490/16260 [1:17:00<14:30,  3.18it/s]iter 13490: loss 4.6192, time: 257.76ms\n",
            " 83% 13491/16260 [1:17:01<13:44,  3.36it/s]iter 13491: loss 5.2932, time: 379.80ms\n",
            " 83% 13492/16260 [1:17:01<14:53,  3.10it/s]iter 13492: loss 5.5893, time: 367.95ms\n",
            " 83% 13493/16260 [1:17:01<15:32,  2.97it/s]iter 13493: loss 4.1052, time: 146.84ms\n",
            " 83% 13494/16260 [1:17:01<12:55,  3.57it/s]iter 13494: loss 4.7906, time: 319.04ms\n",
            " 83% 13495/16260 [1:17:02<13:28,  3.42it/s]iter 13495: loss 5.6599, time: 465.12ms\n",
            " 83% 13496/16260 [1:17:02<15:52,  2.90it/s]iter 13496: loss 5.4243, time: 378.84ms\n",
            " 83% 13497/16260 [1:17:03<16:21,  2.81it/s]iter 13497: loss 5.3082, time: 344.83ms\n",
            " 83% 13498/16260 [1:17:03<16:14,  2.83it/s]iter 13498: loss 4.5432, time: 240.57ms\n",
            " 83% 13499/16260 [1:17:03<14:42,  3.13it/s]iter 13499: loss 3.8997, time: 190.26ms\n",
            " 83% 13500/16260 [1:17:03<12:56,  3.56it/s]iter 13500: loss 5.4663, time: 464.27ms\n",
            " 83% 13501/16260 [1:17:04<15:28,  2.97it/s]iter 13501: loss 4.8332, time: 278.58ms\n",
            " 83% 13502/16260 [1:17:04<14:41,  3.13it/s]iter 13502: loss 5.4373, time: 465.48ms\n",
            " 83% 13503/16260 [1:17:05<16:43,  2.75it/s]iter 13503: loss 5.3154, time: 370.46ms\n",
            " 83% 13504/16260 [1:17:05<16:49,  2.73it/s]iter 13504: loss 5.8373, time: 316.05ms\n",
            " 83% 13505/16260 [1:17:05<16:08,  2.84it/s]iter 13505: loss 6.1600, time: 464.57ms\n",
            " 83% 13506/16260 [1:17:06<17:43,  2.59it/s]iter 13506: loss 5.8007, time: 464.41ms\n",
            " 83% 13507/16260 [1:17:06<18:48,  2.44it/s]iter 13507: loss 5.0313, time: 416.28ms\n",
            " 83% 13508/16260 [1:17:07<18:55,  2.42it/s]iter 13508: loss 5.7082, time: 464.91ms\n",
            " 83% 13509/16260 [1:17:07<19:39,  2.33it/s]iter 13509: loss 3.5204, time: 138.69ms\n",
            " 83% 13510/16260 [1:17:07<15:41,  2.92it/s]iter 13510: loss 4.5511, time: 237.19ms\n",
            " 83% 13511/16260 [1:17:08<14:15,  3.21it/s]iter 13511: loss 5.3791, time: 415.30ms\n",
            " 83% 13512/16260 [1:17:08<15:42,  2.92it/s]iter 13512: loss 5.0221, time: 310.16ms\n",
            " 83% 13513/16260 [1:17:08<15:16,  3.00it/s]iter 13513: loss 4.8079, time: 274.12ms\n",
            " 83% 13514/16260 [1:17:09<14:28,  3.16it/s]iter 13514: loss 5.6027, time: 473.63ms\n",
            " 83% 13515/16260 [1:17:09<16:39,  2.75it/s]iter 13515: loss 4.9792, time: 335.54ms\n",
            " 83% 13516/16260 [1:17:09<16:16,  2.81it/s]iter 13516: loss 4.7393, time: 347.77ms\n",
            " 83% 13517/16260 [1:17:10<16:11,  2.82it/s]iter 13517: loss 6.1085, time: 468.95ms\n",
            " 83% 13518/16260 [1:17:10<17:46,  2.57it/s]iter 13518: loss 5.9092, time: 464.61ms\n",
            " 83% 13519/16260 [1:17:11<18:49,  2.43it/s]iter 13519: loss 5.5745, time: 462.89ms\n",
            " 83% 13520/16260 [1:17:11<19:32,  2.34it/s]iter 13520: loss 4.8097, time: 237.59ms\n",
            " 83% 13521/16260 [1:17:11<16:57,  2.69it/s]iter 13521: loss 5.3777, time: 464.98ms\n",
            " 83% 13522/16260 [1:17:12<18:15,  2.50it/s]iter 13522: loss 5.3003, time: 307.90ms\n",
            " 83% 13523/16260 [1:17:12<17:00,  2.68it/s]iter 13523: loss 4.9303, time: 184.10ms\n",
            " 83% 13524/16260 [1:17:12<14:26,  3.16it/s]iter 13524: loss 5.2071, time: 311.66ms\n",
            " 83% 13525/16260 [1:17:13<14:23,  3.17it/s]iter 13525: loss 4.7527, time: 239.86ms\n",
            " 83% 13526/16260 [1:17:13<13:22,  3.41it/s]iter 13526: loss 4.9976, time: 211.81ms\n",
            " 83% 13527/16260 [1:17:13<12:16,  3.71it/s]iter 13527: loss 5.4480, time: 233.88ms\n",
            " 83% 13528/16260 [1:17:13<11:48,  3.86it/s]iter 13528: loss 5.0552, time: 446.00ms\n",
            " 83% 13529/16260 [1:17:14<14:21,  3.17it/s]iter 13529: loss 5.2902, time: 272.54ms\n",
            " 83% 13530/16260 [1:17:14<13:47,  3.30it/s]iter 13530: loss 5.7093, time: 469.68ms\n",
            " 83% 13531/16260 [1:17:14<16:04,  2.83it/s]iter 13531: loss 4.6941, time: 210.52ms\n",
            " 83% 13532/16260 [1:17:15<14:08,  3.21it/s]iter 13532: loss 3.0512, time: 137.76ms\n",
            " 83% 13533/16260 [1:17:15<11:47,  3.85it/s]iter 13533: loss 3.8747, time: 168.77ms\n",
            " 83% 13534/16260 [1:17:15<10:34,  4.30it/s]iter 13534: loss 4.9383, time: 349.23ms\n",
            " 83% 13535/16260 [1:17:15<12:10,  3.73it/s]iter 13535: loss 5.3094, time: 318.71ms\n",
            " 83% 13536/16260 [1:17:16<12:52,  3.52it/s]iter 13536: loss 4.2380, time: 185.50ms\n",
            " 83% 13537/16260 [1:17:16<11:33,  3.93it/s]iter 13537: loss 4.8774, time: 389.76ms\n",
            " 83% 13538/16260 [1:17:16<13:24,  3.38it/s]iter 13538: loss 5.4278, time: 463.61ms\n",
            " 83% 13539/16260 [1:17:17<15:42,  2.89it/s]iter 13539: loss 5.0259, time: 472.13ms\n",
            " 83% 13540/16260 [1:17:17<17:26,  2.60it/s]iter 13540: loss 4.7244, time: 264.32ms\n",
            " 83% 13541/16260 [1:17:17<15:49,  2.86it/s]iter 13541: loss 4.9210, time: 235.13ms\n",
            " 83% 13542/16260 [1:17:18<14:17,  3.17it/s]iter 13542: loss 4.4483, time: 186.92ms\n",
            " 83% 13543/16260 [1:17:18<12:33,  3.61it/s]iter 13543: loss 5.6363, time: 464.00ms\n",
            " 83% 13544/16260 [1:17:18<15:06,  3.00it/s]iter 13544: loss 5.6712, time: 344.24ms\n",
            " 83% 13545/16260 [1:17:19<15:15,  2.96it/s]iter 13545: loss 5.4874, time: 413.92ms\n",
            " 83% 13546/16260 [1:17:19<16:19,  2.77it/s]iter 13546: loss 5.8576, time: 389.45ms\n",
            " 83% 13547/16260 [1:17:20<16:43,  2.70it/s]iter 13547: loss 5.4279, time: 307.84ms\n",
            " 83% 13548/16260 [1:17:20<15:53,  2.84it/s]iter 13548: loss 5.6914, time: 464.70ms\n",
            " 83% 13549/16260 [1:17:20<17:26,  2.59it/s]iter 13549: loss 4.8249, time: 213.99ms\n",
            " 83% 13550/16260 [1:17:20<15:07,  2.99it/s]iter 13550: loss 5.5141, time: 343.21ms\n",
            " 83% 13551/16260 [1:17:21<15:15,  2.96it/s]iter 13551: loss 5.7754, time: 464.95ms\n",
            " 83% 13552/16260 [1:17:21<16:59,  2.66it/s]iter 13552: loss 5.4236, time: 424.67ms\n",
            " 83% 13553/16260 [1:17:22<17:39,  2.55it/s]iter 13553: loss 4.8100, time: 283.18ms\n",
            " 83% 13554/16260 [1:17:22<16:12,  2.78it/s]iter 13554: loss 5.4665, time: 465.58ms\n",
            " 83% 13555/16260 [1:17:22<17:39,  2.55it/s]iter 13555: loss 5.3961, time: 464.41ms\n",
            " 83% 13556/16260 [1:17:23<18:39,  2.42it/s]iter 13556: loss 4.6197, time: 165.39ms\n",
            " 83% 13557/16260 [1:17:23<15:18,  2.94it/s]iter 13557: loss 4.7043, time: 432.96ms\n",
            " 83% 13558/16260 [1:17:24<16:35,  2.72it/s]iter 13558: loss 4.6753, time: 209.77ms\n",
            " 83% 13559/16260 [1:17:24<14:27,  3.11it/s]iter 13559: loss 5.7527, time: 464.65ms\n",
            " 83% 13560/16260 [1:17:24<16:24,  2.74it/s]iter 13560: loss 5.0789, time: 279.27ms\n",
            " 83% 13561/16260 [1:17:25<15:16,  2.94it/s]iter 13561: loss 5.3031, time: 464.11ms\n",
            " 83% 13562/16260 [1:17:25<16:58,  2.65it/s]iter 13562: loss 4.8038, time: 168.20ms\n",
            " 83% 13563/16260 [1:17:25<14:09,  3.17it/s]iter 13563: loss 5.0003, time: 306.63ms\n",
            " 83% 13564/16260 [1:17:25<14:03,  3.19it/s]iter 13564: loss 4.6966, time: 277.67ms\n",
            " 83% 13565/16260 [1:17:26<13:36,  3.30it/s]iter 13565: loss 5.6684, time: 463.76ms\n",
            " 83% 13566/16260 [1:17:26<15:47,  2.84it/s]iter 13566: loss 5.0044, time: 314.75ms\n",
            " 83% 13567/16260 [1:17:27<15:18,  2.93it/s]iter 13567: loss 3.5094, time: 156.62ms\n",
            " 83% 13568/16260 [1:17:27<12:50,  3.50it/s]iter 13568: loss 3.9302, time: 189.11ms\n",
            " 83% 13569/16260 [1:17:27<11:32,  3.88it/s]iter 13569: loss 5.2237, time: 314.03ms\n",
            " 83% 13570/16260 [1:17:27<12:19,  3.64it/s]iter 13570: loss 5.6983, time: 443.18ms\n",
            " 83% 13571/16260 [1:17:28<14:36,  3.07it/s]iter 13571: loss 4.6617, time: 281.39ms\n",
            " 83% 13572/16260 [1:17:28<14:01,  3.20it/s]iter 13572: loss 5.7741, time: 446.06ms\n",
            " 83% 13573/16260 [1:17:28<15:49,  2.83it/s]iter 13573: loss 5.6302, time: 463.42ms\n",
            " 83% 13574/16260 [1:17:29<17:18,  2.59it/s]iter 13574: loss 4.7402, time: 187.20ms\n",
            " 83% 13575/16260 [1:17:29<14:38,  3.05it/s]iter 13575: loss 5.1875, time: 463.34ms\n",
            " 83% 13576/16260 [1:17:29<16:29,  2.71it/s]iter 13576: loss 5.1924, time: 186.63ms\n",
            " 83% 13577/16260 [1:17:30<14:03,  3.18it/s]iter 13577: loss 4.4004, time: 464.35ms\n",
            " 84% 13578/16260 [1:17:30<16:05,  2.78it/s]iter 13578: loss 5.9544, time: 464.21ms\n",
            " 84% 13579/16260 [1:17:31<17:30,  2.55it/s]iter 13579: loss 5.2297, time: 368.13ms\n",
            " 84% 13580/16260 [1:17:31<17:11,  2.60it/s]iter 13580: loss 5.2140, time: 464.02ms\n",
            " 84% 13581/16260 [1:17:31<18:16,  2.44it/s]iter 13581: loss 5.8108, time: 463.78ms\n",
            " 84% 13582/16260 [1:17:32<19:00,  2.35it/s]iter 13582: loss 5.2567, time: 463.56ms\n",
            " 84% 13583/16260 [1:17:32<19:31,  2.28it/s]iter 13583: loss 5.1447, time: 236.35ms\n",
            " 84% 13584/16260 [1:17:33<16:50,  2.65it/s]iter 13584: loss 5.7923, time: 463.52ms\n",
            " 84% 13585/16260 [1:17:33<18:00,  2.48it/s]iter 13585: loss 4.3277, time: 161.59ms\n",
            " 84% 13586/16260 [1:17:33<14:46,  3.02it/s]iter 13586: loss 3.6646, time: 145.78ms\n",
            " 84% 13587/16260 [1:17:33<12:18,  3.62it/s]iter 13587: loss 5.1561, time: 454.41ms\n",
            " 84% 13588/16260 [1:17:34<14:41,  3.03it/s]iter 13588: loss 5.3505, time: 463.47ms\n",
            " 84% 13589/16260 [1:17:34<16:29,  2.70it/s]iter 13589: loss 4.9819, time: 214.11ms\n",
            " 84% 13590/16260 [1:17:35<14:25,  3.09it/s]iter 13590: loss 5.4715, time: 315.78ms\n",
            " 84% 13591/16260 [1:17:35<14:19,  3.11it/s]iter 13591: loss 5.4057, time: 336.61ms\n",
            " 84% 13592/16260 [1:17:35<14:31,  3.06it/s]iter 13592: loss 5.3634, time: 249.74ms\n",
            " 84% 13593/16260 [1:17:35<13:31,  3.29it/s]iter 13593: loss 5.5173, time: 440.48ms\n",
            " 84% 13594/16260 [1:17:36<15:21,  2.89it/s]iter 13594: loss 4.9180, time: 254.01ms\n",
            " 84% 13595/16260 [1:17:36<14:08,  3.14it/s]iter 13595: loss 5.5135, time: 463.40ms\n",
            " 84% 13596/16260 [1:17:37<16:05,  2.76it/s]iter 13596: loss 5.2320, time: 472.05ms\n",
            " 84% 13597/16260 [1:17:37<17:34,  2.53it/s]iter 13597: loss 5.2669, time: 445.42ms\n",
            " 84% 13598/16260 [1:17:37<18:14,  2.43it/s]iter 13598: loss 5.8581, time: 464.03ms\n",
            " 84% 13599/16260 [1:17:38<18:57,  2.34it/s]iter 13599: loss 4.9882, time: 238.66ms\n",
            " 84% 13600/16260 [1:17:38<16:27,  2.69it/s]iter 13600: loss 5.1488, time: 463.80ms\n",
            " 84% 13601/16260 [1:17:39<17:41,  2.50it/s]iter 13601: loss 4.9952, time: 466.73ms\n",
            " 84% 13602/16260 [1:17:39<18:36,  2.38it/s]iter 13602: loss 4.9268, time: 188.26ms\n",
            " 84% 13603/16260 [1:17:39<15:32,  2.85it/s]iter 13603: loss 5.6411, time: 479.68ms\n",
            " 84% 13604/16260 [1:17:40<17:15,  2.56it/s]iter 13604: loss 4.8137, time: 214.56ms\n",
            " 84% 13605/16260 [1:17:40<14:56,  2.96it/s]iter 13605: loss 5.3176, time: 339.68ms\n",
            " 84% 13606/16260 [1:17:40<14:59,  2.95it/s]iter 13606: loss 5.0386, time: 307.40ms\n",
            " 84% 13607/16260 [1:17:41<14:35,  3.03it/s]iter 13607: loss 5.8675, time: 474.18ms\n",
            " 84% 13608/16260 [1:17:41<16:30,  2.68it/s]iter 13608: loss 5.1970, time: 463.91ms\n",
            " 84% 13609/16260 [1:17:42<17:43,  2.49it/s]iter 13609: loss 5.2691, time: 233.63ms\n",
            " 84% 13610/16260 [1:17:42<15:31,  2.85it/s]iter 13610: loss 5.6748, time: 464.05ms\n",
            " 84% 13611/16260 [1:17:42<17:01,  2.59it/s]iter 13611: loss 5.4515, time: 333.99ms\n",
            " 84% 13612/16260 [1:17:43<16:21,  2.70it/s]iter 13612: loss 4.7176, time: 464.30ms\n",
            " 84% 13613/16260 [1:17:43<17:36,  2.50it/s]iter 13613: loss 4.8727, time: 219.06ms\n",
            " 84% 13614/16260 [1:17:43<15:14,  2.89it/s]iter 13614: loss 5.1431, time: 468.98ms\n",
            " 84% 13615/16260 [1:17:44<16:53,  2.61it/s]iter 13615: loss 5.3947, time: 464.45ms\n",
            " 84% 13616/16260 [1:17:44<17:58,  2.45it/s]iter 13616: loss 4.2236, time: 188.05ms\n",
            " 84% 13617/16260 [1:17:44<15:04,  2.92it/s]iter 13617: loss 5.1856, time: 332.86ms\n",
            " 84% 13618/16260 [1:17:45<14:58,  2.94it/s]iter 13618: loss 5.1499, time: 242.23ms\n",
            " 84% 13619/16260 [1:17:45<13:41,  3.21it/s]iter 13619: loss 4.4454, time: 185.40ms\n",
            " 84% 13620/16260 [1:17:45<12:02,  3.65it/s]iter 13620: loss 4.6147, time: 233.78ms\n",
            " 84% 13621/16260 [1:17:45<11:31,  3.81it/s]iter 13621: loss 5.4161, time: 378.29ms\n",
            " 84% 13622/16260 [1:17:46<13:04,  3.36it/s]iter 13622: loss 4.2654, time: 166.39ms\n",
            " 84% 13623/16260 [1:17:46<11:21,  3.87it/s]iter 13623: loss 4.6528, time: 218.60ms\n",
            " 84% 13624/16260 [1:17:46<10:51,  4.05it/s]iter 13624: loss 5.0083, time: 193.43ms\n",
            " 84% 13625/16260 [1:17:46<10:09,  4.32it/s]iter 13625: loss 5.4298, time: 380.00ms\n",
            " 84% 13626/16260 [1:17:47<12:08,  3.62it/s]iter 13626: loss 5.2489, time: 218.97ms\n",
            " 84% 13627/16260 [1:17:47<11:23,  3.85it/s]iter 13627: loss 4.9384, time: 257.39ms\n",
            " 84% 13628/16260 [1:17:47<11:22,  3.85it/s]iter 13628: loss 4.9041, time: 236.74ms\n",
            " 84% 13629/16260 [1:17:48<11:05,  3.95it/s]iter 13629: loss 5.7648, time: 382.15ms\n",
            " 84% 13630/16260 [1:17:48<12:48,  3.42it/s]iter 13630: loss 4.9386, time: 345.90ms\n",
            " 84% 13631/16260 [1:17:48<13:31,  3.24it/s]iter 13631: loss 4.7337, time: 419.52ms\n",
            " 84% 13632/16260 [1:17:49<14:59,  2.92it/s]iter 13632: loss 5.4936, time: 464.33ms\n",
            " 84% 13633/16260 [1:17:49<16:36,  2.64it/s]iter 13633: loss 5.3576, time: 444.39ms\n",
            " 84% 13634/16260 [1:17:50<17:28,  2.50it/s]iter 13634: loss 5.7101, time: 463.73ms\n",
            " 84% 13635/16260 [1:17:50<18:20,  2.39it/s]iter 13635: loss 4.8981, time: 262.85ms\n",
            " 84% 13636/16260 [1:17:50<16:18,  2.68it/s]iter 13636: loss 5.1170, time: 264.32ms\n",
            " 84% 13637/16260 [1:17:51<14:53,  2.93it/s]iter 13637: loss 5.5619, time: 463.86ms\n",
            " 84% 13638/16260 [1:17:51<16:31,  2.64it/s]iter 13638: loss 4.5824, time: 170.30ms\n",
            " 84% 13639/16260 [1:17:51<13:49,  3.16it/s]iter 13639: loss 4.9496, time: 218.98ms\n",
            " 84% 13640/16260 [1:17:51<12:33,  3.48it/s]iter 13640: loss 4.7683, time: 319.39ms\n",
            " 84% 13641/16260 [1:17:52<12:59,  3.36it/s]iter 13641: loss 4.9754, time: 288.73ms\n",
            " 84% 13642/16260 [1:17:52<12:53,  3.38it/s]iter 13642: loss 5.8217, time: 465.55ms\n",
            " 84% 13643/16260 [1:17:53<15:08,  2.88it/s]iter 13643: loss 5.1198, time: 275.04ms\n",
            " 84% 13644/16260 [1:17:53<14:12,  3.07it/s]iter 13644: loss 5.3920, time: 463.58ms\n",
            " 84% 13645/16260 [1:17:53<16:01,  2.72it/s]iter 13645: loss 5.9405, time: 463.76ms\n",
            " 84% 13646/16260 [1:17:54<17:17,  2.52it/s]iter 13646: loss 5.6295, time: 368.96ms\n",
            " 84% 13647/16260 [1:17:54<16:56,  2.57it/s]iter 13647: loss 5.0149, time: 282.89ms\n",
            " 84% 13648/16260 [1:17:54<15:34,  2.80it/s]iter 13648: loss 5.6810, time: 252.66ms\n",
            " 84% 13649/16260 [1:17:55<14:12,  3.06it/s]iter 13649: loss 4.8813, time: 251.26ms\n",
            " 84% 13650/16260 [1:17:55<13:14,  3.29it/s]iter 13650: loss 4.2569, time: 162.93ms\n",
            " 84% 13651/16260 [1:17:55<11:24,  3.81it/s]iter 13651: loss 4.6236, time: 206.03ms\n",
            " 84% 13652/16260 [1:17:55<10:41,  4.07it/s]iter 13652: loss 5.4765, time: 463.60ms\n",
            " 84% 13653/16260 [1:17:56<13:32,  3.21it/s]iter 13653: loss 4.9456, time: 271.97ms\n",
            " 84% 13654/16260 [1:17:56<13:02,  3.33it/s]iter 13654: loss 4.7814, time: 238.35ms\n",
            " 84% 13655/16260 [1:17:56<12:15,  3.54it/s]iter 13655: loss 4.7762, time: 275.36ms\n",
            " 84% 13656/16260 [1:17:57<12:10,  3.56it/s]iter 13656: loss 2.6542, time: 134.69ms\n",
            " 84% 13657/16260 [1:17:57<10:17,  4.21it/s]iter 13657: loss 5.2699, time: 464.11ms\n",
            " 84% 13658/16260 [1:17:57<13:15,  3.27it/s]iter 13658: loss 4.6713, time: 281.74ms\n",
            " 84% 13659/16260 [1:17:57<12:57,  3.34it/s]iter 13659: loss 4.7582, time: 189.85ms\n",
            " 84% 13660/16260 [1:17:58<11:33,  3.75it/s]iter 13660: loss 5.4716, time: 444.64ms\n",
            " 84% 13661/16260 [1:17:58<13:53,  3.12it/s]iter 13661: loss 5.0147, time: 432.47ms\n",
            " 84% 13662/16260 [1:17:58<15:21,  2.82it/s]iter 13662: loss 4.8679, time: 210.07ms\n",
            " 84% 13663/16260 [1:17:59<13:29,  3.21it/s]iter 13663: loss 4.8009, time: 280.80ms\n",
            " 84% 13664/16260 [1:17:59<13:05,  3.30it/s]iter 13664: loss 4.8098, time: 221.03ms\n",
            " 84% 13665/16260 [1:17:59<12:03,  3.59it/s]iter 13665: loss 5.1214, time: 345.19ms\n",
            " 84% 13666/16260 [1:18:00<12:55,  3.34it/s]iter 13666: loss 5.7491, time: 465.26ms\n",
            " 84% 13667/16260 [1:18:00<15:05,  2.86it/s]iter 13667: loss 4.6452, time: 253.26ms\n",
            " 84% 13668/16260 [1:18:00<13:52,  3.12it/s]iter 13668: loss 5.6260, time: 464.85ms\n",
            " 84% 13669/16260 [1:18:01<15:44,  2.74it/s]iter 13669: loss 5.2033, time: 255.29ms\n",
            " 84% 13670/16260 [1:18:01<14:20,  3.01it/s]iter 13670: loss 5.5928, time: 464.46ms\n",
            " 84% 13671/16260 [1:18:01<16:04,  2.68it/s]iter 13671: loss 5.2652, time: 310.01ms\n",
            " 84% 13672/16260 [1:18:02<15:16,  2.82it/s]iter 13672: loss 5.5592, time: 235.03ms\n",
            " 84% 13673/16260 [1:18:02<13:45,  3.14it/s]iter 13673: loss 5.4242, time: 310.20ms\n",
            " 84% 13674/16260 [1:18:02<13:39,  3.16it/s]iter 13674: loss 6.0445, time: 366.68ms\n",
            " 84% 13675/16260 [1:18:03<14:18,  3.01it/s]iter 13675: loss 5.3483, time: 345.61ms\n",
            " 84% 13676/16260 [1:18:03<14:30,  2.97it/s]iter 13676: loss 4.9879, time: 318.11ms\n",
            " 84% 13677/16260 [1:18:03<14:16,  3.02it/s]iter 13677: loss 5.3477, time: 283.40ms\n",
            " 84% 13678/16260 [1:18:04<13:39,  3.15it/s]iter 13678: loss 5.9612, time: 463.30ms\n",
            " 84% 13679/16260 [1:18:04<15:33,  2.76it/s]iter 13679: loss 5.4855, time: 347.39ms\n",
            " 84% 13680/16260 [1:18:04<15:23,  2.79it/s]iter 13680: loss 5.1404, time: 464.13ms\n",
            " 84% 13681/16260 [1:18:05<16:46,  2.56it/s]iter 13681: loss 4.9710, time: 311.58ms\n",
            " 84% 13682/16260 [1:18:05<15:46,  2.72it/s]iter 13682: loss 5.5029, time: 475.15ms\n",
            " 84% 13683/16260 [1:18:06<17:10,  2.50it/s]iter 13683: loss 4.9383, time: 445.22ms\n",
            " 84% 13684/16260 [1:18:06<17:46,  2.42it/s]iter 13684: loss 5.2303, time: 415.90ms\n",
            " 84% 13685/16260 [1:18:07<17:48,  2.41it/s]iter 13685: loss 5.5574, time: 464.30ms\n",
            " 84% 13686/16260 [1:18:07<18:27,  2.32it/s]iter 13686: loss 3.8490, time: 184.59ms\n",
            " 84% 13687/16260 [1:18:07<15:18,  2.80it/s]iter 13687: loss 4.1428, time: 241.42ms\n",
            " 84% 13688/16260 [1:18:07<13:50,  3.10it/s]iter 13688: loss 3.8986, time: 148.23ms\n",
            " 84% 13689/16260 [1:18:08<11:36,  3.69it/s]iter 13689: loss 5.5402, time: 463.82ms\n",
            " 84% 13690/16260 [1:18:08<14:06,  3.04it/s]iter 13690: loss 4.4458, time: 215.22ms\n",
            " 84% 13691/16260 [1:18:08<12:39,  3.38it/s]iter 13691: loss 5.9302, time: 465.80ms\n",
            " 84% 13692/16260 [1:18:09<14:51,  2.88it/s]iter 13692: loss 5.4032, time: 342.40ms\n",
            " 84% 13693/16260 [1:18:09<14:48,  2.89it/s]iter 13693: loss 5.3476, time: 275.91ms\n",
            " 84% 13694/16260 [1:18:09<13:55,  3.07it/s]iter 13694: loss 4.4696, time: 167.34ms\n",
            " 84% 13695/16260 [1:18:10<11:54,  3.59it/s]iter 13695: loss 4.1350, time: 170.70ms\n",
            " 84% 13696/16260 [1:18:10<10:32,  4.06it/s]iter 13696: loss 5.0698, time: 314.61ms\n",
            " 84% 13697/16260 [1:18:10<11:25,  3.74it/s]iter 13697: loss 4.9402, time: 316.79ms\n",
            " 84% 13698/16260 [1:18:10<12:04,  3.54it/s]iter 13698: loss 5.6779, time: 466.67ms\n",
            " 84% 13699/16260 [1:18:11<14:26,  2.96it/s]iter 13699: loss 4.2828, time: 238.56ms\n",
            " 84% 13700/16260 [1:18:11<13:10,  3.24it/s]iter 13700: loss 5.6899, time: 464.92ms\n",
            " 84% 13701/16260 [1:18:12<15:11,  2.81it/s]iter 13701: loss 5.3061, time: 278.29ms\n",
            " 84% 13702/16260 [1:18:12<14:12,  3.00it/s]iter 13702: loss 5.2392, time: 463.29ms\n",
            " 84% 13703/16260 [1:18:12<15:53,  2.68it/s]iter 13703: loss 5.8595, time: 464.59ms\n",
            " 84% 13704/16260 [1:18:13<17:04,  2.50it/s]iter 13704: loss 5.6484, time: 368.59ms\n",
            " 84% 13705/16260 [1:18:13<16:40,  2.55it/s]iter 13705: loss 5.4633, time: 463.92ms\n",
            " 84% 13706/16260 [1:18:14<17:36,  2.42it/s]iter 13706: loss 6.0148, time: 464.31ms\n",
            " 84% 13707/16260 [1:18:14<18:16,  2.33it/s]iter 13707: loss 4.7210, time: 167.57ms\n",
            " 84% 13708/16260 [1:18:14<14:56,  2.85it/s]iter 13708: loss 5.4267, time: 339.43ms\n",
            " 84% 13709/16260 [1:18:15<14:48,  2.87it/s]iter 13709: loss 5.5294, time: 464.03ms\n",
            " 84% 13710/16260 [1:18:15<16:17,  2.61it/s]iter 13710: loss 5.0871, time: 319.52ms\n",
            " 84% 13711/16260 [1:18:15<15:29,  2.74it/s]iter 13711: loss 4.7872, time: 193.64ms\n",
            " 84% 13712/16260 [1:18:16<13:19,  3.19it/s]iter 13712: loss 5.2289, time: 463.87ms\n",
            " 84% 13713/16260 [1:18:16<15:15,  2.78it/s]iter 13713: loss 5.4293, time: 464.75ms\n",
            " 84% 13714/16260 [1:18:16<16:36,  2.55it/s]iter 13714: loss 5.3206, time: 474.17ms\n",
            " 84% 13715/16260 [1:18:17<17:40,  2.40it/s]iter 13715: loss 3.5105, time: 139.51ms\n",
            " 84% 13716/16260 [1:18:17<14:09,  2.99it/s]iter 13716: loss 5.7495, time: 464.03ms\n",
            " 84% 13717/16260 [1:18:18<15:49,  2.68it/s]iter 13717: loss 4.3868, time: 167.53ms\n",
            " 84% 13718/16260 [1:18:18<13:13,  3.20it/s]iter 13718: loss 5.6410, time: 463.03ms\n",
            " 84% 13719/16260 [1:18:18<15:09,  2.79it/s]iter 13719: loss 4.4420, time: 188.13ms\n",
            " 84% 13720/16260 [1:18:18<13:00,  3.25it/s]iter 13720: loss 5.5534, time: 332.05ms\n",
            " 84% 13721/16260 [1:18:19<13:20,  3.17it/s]iter 13721: loss 5.1276, time: 257.80ms\n",
            " 84% 13722/16260 [1:18:19<12:37,  3.35it/s]iter 13722: loss 5.5397, time: 464.54ms\n",
            " 84% 13723/16260 [1:18:19<14:45,  2.87it/s]iter 13723: loss 2.7217, time: 147.86ms\n",
            " 84% 13724/16260 [1:18:20<12:12,  3.46it/s]iter 13724: loss 5.9745, time: 464.04ms\n",
            " 84% 13725/16260 [1:18:20<14:26,  2.92it/s]iter 13725: loss 5.7987, time: 439.57ms\n",
            " 84% 13726/16260 [1:18:20<15:41,  2.69it/s]iter 13726: loss 5.5548, time: 376.93ms\n",
            " 84% 13727/16260 [1:18:21<15:46,  2.68it/s]iter 13727: loss 5.2794, time: 447.70ms\n",
            " 84% 13728/16260 [1:18:21<16:43,  2.52it/s]iter 13728: loss 5.1853, time: 463.02ms\n",
            " 84% 13729/16260 [1:18:22<17:34,  2.40it/s]iter 13729: loss 4.4943, time: 169.42ms\n",
            " 84% 13730/16260 [1:18:22<14:27,  2.92it/s]iter 13730: loss 4.1557, time: 150.11ms\n",
            " 84% 13731/16260 [1:18:22<12:02,  3.50it/s]iter 13731: loss 4.6144, time: 376.80ms\n",
            " 84% 13732/16260 [1:18:22<13:12,  3.19it/s]iter 13732: loss 4.9591, time: 430.42ms\n",
            " 84% 13733/16260 [1:18:23<14:41,  2.87it/s]iter 13733: loss 5.0292, time: 464.06ms\n",
            " 84% 13734/16260 [1:18:23<16:09,  2.60it/s]iter 13734: loss 5.0973, time: 468.21ms\n",
            " 84% 13735/16260 [1:18:24<17:14,  2.44it/s]iter 13735: loss 5.0832, time: 413.08ms\n",
            " 84% 13736/16260 [1:18:24<17:17,  2.43it/s]iter 13736: loss 4.2801, time: 206.25ms\n",
            " 84% 13737/16260 [1:18:24<14:43,  2.86it/s]iter 13737: loss 4.9327, time: 464.49ms\n",
            " 84% 13738/16260 [1:18:25<16:10,  2.60it/s]iter 13738: loss 5.0866, time: 311.38ms\n",
            " 84% 13739/16260 [1:18:25<15:15,  2.75it/s]iter 13739: loss 5.2415, time: 467.65ms\n",
            " 85% 13740/16260 [1:18:26<16:35,  2.53it/s]iter 13740: loss 4.9577, time: 211.29ms\n",
            " 85% 13741/16260 [1:18:26<14:17,  2.94it/s]iter 13741: loss 4.7474, time: 192.42ms\n",
            " 85% 13742/16260 [1:18:26<12:26,  3.37it/s]iter 13742: loss 5.5482, time: 464.25ms\n",
            " 85% 13743/16260 [1:18:27<14:34,  2.88it/s]iter 13743: loss 5.3151, time: 464.76ms\n",
            " 85% 13744/16260 [1:18:27<16:03,  2.61it/s]iter 13744: loss 4.8808, time: 256.02ms\n",
            " 85% 13745/16260 [1:18:27<14:28,  2.89it/s]iter 13745: loss 5.4716, time: 378.21ms\n",
            " 85% 13746/16260 [1:18:28<14:54,  2.81it/s]iter 13746: loss 5.2632, time: 465.21ms\n",
            " 85% 13747/16260 [1:18:28<16:17,  2.57it/s]iter 13747: loss 5.1692, time: 240.42ms\n",
            " 85% 13748/16260 [1:18:28<14:26,  2.90it/s]iter 13748: loss 5.0031, time: 186.91ms\n",
            " 85% 13749/16260 [1:18:29<12:28,  3.36it/s]iter 13749: loss 5.5978, time: 470.01ms\n",
            " 85% 13750/16260 [1:18:29<14:38,  2.86it/s]iter 13750: loss 5.4085, time: 465.08ms\n",
            " 85% 13751/16260 [1:18:30<16:05,  2.60it/s]iter 13751: loss 4.9032, time: 375.91ms\n",
            " 85% 13752/16260 [1:18:30<15:59,  2.61it/s]iter 13752: loss 4.4517, time: 210.92ms\n",
            " 85% 13753/16260 [1:18:30<13:51,  3.02it/s]iter 13753: loss 4.0426, time: 237.39ms\n",
            " 85% 13754/16260 [1:18:30<12:41,  3.29it/s]iter 13754: loss 4.7447, time: 250.30ms\n",
            " 85% 13755/16260 [1:18:31<12:01,  3.47it/s]iter 13755: loss 4.6993, time: 216.26ms\n",
            " 85% 13756/16260 [1:18:31<11:08,  3.75it/s]iter 13756: loss 5.4954, time: 315.82ms\n",
            " 85% 13757/16260 [1:18:31<11:46,  3.55it/s]iter 13757: loss 5.2609, time: 314.18ms\n",
            " 85% 13758/16260 [1:18:31<12:11,  3.42it/s]iter 13758: loss 5.5011, time: 469.93ms\n",
            " 85% 13759/16260 [1:18:32<14:25,  2.89it/s]iter 13759: loss 4.0948, time: 191.70ms\n",
            " 85% 13760/16260 [1:18:32<12:29,  3.33it/s]iter 13760: loss 4.9942, time: 231.50ms\n",
            " 85% 13761/16260 [1:18:32<11:39,  3.57it/s]iter 13761: loss 4.0523, time: 165.16ms\n",
            " 85% 13762/16260 [1:18:33<10:14,  4.07it/s]iter 13762: loss 3.6776, time: 149.13ms\n",
            " 85% 13763/16260 [1:18:33<09:02,  4.60it/s]iter 13763: loss 5.5334, time: 468.14ms\n",
            " 85% 13764/16260 [1:18:33<12:11,  3.41it/s]iter 13764: loss 3.3639, time: 133.97ms\n",
            " 85% 13765/16260 [1:18:33<10:12,  4.07it/s]iter 13765: loss 4.5991, time: 234.72ms\n",
            " 85% 13766/16260 [1:18:34<10:05,  4.12it/s]iter 13766: loss 4.9702, time: 191.39ms\n",
            " 85% 13767/16260 [1:18:34<09:27,  4.39it/s]iter 13767: loss 5.0069, time: 423.53ms\n",
            " 85% 13768/16260 [1:18:34<11:55,  3.48it/s]iter 13768: loss 4.8773, time: 208.73ms\n",
            " 85% 13769/16260 [1:18:34<10:57,  3.79it/s]iter 13769: loss 5.1566, time: 465.14ms\n",
            " 85% 13770/16260 [1:18:35<13:28,  3.08it/s]iter 13770: loss 5.3693, time: 438.49ms\n",
            " 85% 13771/16260 [1:18:35<14:54,  2.78it/s]iter 13771: loss 6.1161, time: 463.96ms\n",
            " 85% 13772/16260 [1:18:36<16:13,  2.56it/s]iter 13772: loss 4.8434, time: 184.00ms\n",
            " 85% 13773/16260 [1:18:36<13:39,  3.04it/s]iter 13773: loss 5.0286, time: 369.01ms\n",
            " 85% 13774/16260 [1:18:36<14:09,  2.93it/s]iter 13774: loss 5.7960, time: 463.86ms\n",
            " 85% 13775/16260 [1:18:37<15:41,  2.64it/s]iter 13775: loss 4.5717, time: 149.22ms\n",
            " 85% 13776/16260 [1:18:37<12:50,  3.22it/s]iter 13776: loss 4.7854, time: 238.71ms\n",
            " 85% 13777/16260 [1:18:37<11:58,  3.46it/s]iter 13777: loss 5.0163, time: 340.59ms\n",
            " 85% 13778/16260 [1:18:37<12:37,  3.28it/s]iter 13778: loss 4.3658, time: 254.36ms\n",
            " 85% 13779/16260 [1:18:38<12:00,  3.45it/s]iter 13779: loss 5.4699, time: 464.71ms\n",
            " 85% 13780/16260 [1:18:38<14:10,  2.92it/s]iter 13780: loss 5.2276, time: 463.83ms\n",
            " 85% 13781/16260 [1:18:39<15:41,  2.63it/s]iter 13781: loss 5.1525, time: 251.79ms\n",
            " 85% 13782/16260 [1:18:39<14:07,  2.93it/s]iter 13782: loss 4.2091, time: 170.91ms\n",
            " 85% 13783/16260 [1:18:39<12:00,  3.44it/s]iter 13783: loss 5.8650, time: 464.03ms\n",
            " 85% 13784/16260 [1:18:40<14:10,  2.91it/s]iter 13784: loss 5.3242, time: 463.80ms\n",
            " 85% 13785/16260 [1:18:40<15:40,  2.63it/s]iter 13785: loss 5.3644, time: 446.83ms\n",
            " 85% 13786/16260 [1:18:40<16:31,  2.50it/s]iter 13786: loss 5.3996, time: 275.12ms\n",
            " 85% 13787/16260 [1:18:41<14:58,  2.75it/s]iter 13787: loss 4.5534, time: 258.99ms\n",
            " 85% 13788/16260 [1:18:41<13:42,  3.01it/s]iter 13788: loss 5.2228, time: 423.51ms\n",
            " 85% 13789/16260 [1:18:41<14:50,  2.77it/s]iter 13789: loss 5.7970, time: 415.71ms\n",
            " 85% 13790/16260 [1:18:42<15:32,  2.65it/s]iter 13790: loss 5.3187, time: 345.95ms\n",
            " 85% 13791/16260 [1:18:42<15:09,  2.71it/s]iter 13791: loss 4.8431, time: 342.99ms\n",
            " 85% 13792/16260 [1:18:43<14:51,  2.77it/s]iter 13792: loss 5.3095, time: 233.30ms\n",
            " 85% 13793/16260 [1:18:43<13:17,  3.09it/s]iter 13793: loss 4.1990, time: 192.82ms\n",
            " 85% 13794/16260 [1:18:43<11:41,  3.51it/s]iter 13794: loss 5.4637, time: 279.37ms\n",
            " 85% 13795/16260 [1:18:43<11:38,  3.53it/s]iter 13795: loss 4.7834, time: 167.07ms\n",
            " 85% 13796/16260 [1:18:43<10:13,  4.02it/s]iter 13796: loss 5.0742, time: 334.64ms\n",
            " 85% 13797/16260 [1:18:44<11:17,  3.63it/s]iter 13797: loss 5.2184, time: 308.98ms\n",
            " 85% 13798/16260 [1:18:44<11:43,  3.50it/s]iter 13798: loss 5.7793, time: 419.54ms\n",
            " 85% 13799/16260 [1:18:44<13:23,  3.06it/s]iter 13799: loss 3.9030, time: 188.75ms\n",
            " 85% 13800/16260 [1:18:45<11:42,  3.50it/s]iter 13800: loss 5.5268, time: 463.23ms\n",
            " 85% 13801/16260 [1:18:45<13:54,  2.95it/s]iter 13801: loss 5.3667, time: 370.63ms\n",
            " 85% 13802/16260 [1:18:46<14:18,  2.86it/s]iter 13802: loss 5.2220, time: 303.90ms\n",
            " 85% 13803/16260 [1:18:46<13:45,  2.98it/s]iter 13803: loss 4.9480, time: 207.83ms\n",
            " 85% 13804/16260 [1:18:46<12:11,  3.36it/s]iter 13804: loss 5.4815, time: 464.48ms\n",
            " 85% 13805/16260 [1:18:46<14:15,  2.87it/s]iter 13805: loss 5.8667, time: 463.51ms\n",
            " 85% 13806/16260 [1:18:47<15:40,  2.61it/s]iter 13806: loss 5.9397, time: 468.54ms\n",
            " 85% 13807/16260 [1:18:47<16:44,  2.44it/s]iter 13807: loss 4.7509, time: 241.26ms\n",
            " 85% 13808/16260 [1:18:48<14:41,  2.78it/s]iter 13808: loss 5.4873, time: 464.14ms\n",
            " 85% 13809/16260 [1:18:48<15:58,  2.56it/s]iter 13809: loss 4.7651, time: 262.03ms\n",
            " 85% 13810/16260 [1:18:48<14:24,  2.83it/s]iter 13810: loss 5.7036, time: 464.04ms\n",
            " 85% 13811/16260 [1:18:49<15:47,  2.59it/s]iter 13811: loss 5.8951, time: 465.25ms\n",
            " 85% 13812/16260 [1:18:49<16:45,  2.43it/s]iter 13812: loss 4.9685, time: 259.99ms\n",
            " 85% 13813/16260 [1:18:50<14:55,  2.73it/s]iter 13813: loss 5.4599, time: 464.44ms\n",
            " 85% 13814/16260 [1:18:50<16:08,  2.53it/s]iter 13814: loss 3.8564, time: 190.13ms\n",
            " 85% 13815/16260 [1:18:50<13:38,  2.99it/s]iter 13815: loss 5.7730, time: 464.45ms\n",
            " 85% 13816/16260 [1:18:51<15:13,  2.67it/s]iter 13816: loss 5.2483, time: 380.89ms\n",
            " 85% 13817/16260 [1:18:51<15:19,  2.66it/s]iter 13817: loss 5.4997, time: 346.50ms\n",
            " 85% 13818/16260 [1:18:51<14:58,  2.72it/s]iter 13818: loss 5.4785, time: 464.32ms\n",
            " 85% 13819/16260 [1:18:52<16:09,  2.52it/s]iter 13819: loss 5.2168, time: 308.67ms\n",
            " 85% 13820/16260 [1:18:52<15:05,  2.69it/s]iter 13820: loss 5.7932, time: 465.93ms\n",
            " 85% 13821/16260 [1:18:53<16:15,  2.50it/s]iter 13821: loss 4.3312, time: 189.22ms\n",
            " 85% 13822/16260 [1:18:53<13:42,  2.97it/s]iter 13822: loss 4.9181, time: 464.45ms\n",
            " 85% 13823/16260 [1:18:53<15:15,  2.66it/s]iter 13823: loss 5.4307, time: 476.58ms\n",
            " 85% 13824/16260 [1:18:54<16:30,  2.46it/s]iter 13824: loss 5.5292, time: 464.61ms\n",
            " 85% 13825/16260 [1:18:54<17:13,  2.36it/s]iter 13825: loss 5.0034, time: 424.93ms\n",
            " 85% 13826/16260 [1:18:55<17:14,  2.35it/s]iter 13826: loss 3.5870, time: 145.15ms\n",
            " 85% 13827/16260 [1:18:55<13:51,  2.93it/s]iter 13827: loss 4.9984, time: 424.83ms\n",
            " 85% 13828/16260 [1:18:55<14:52,  2.72it/s]iter 13828: loss 5.2696, time: 318.31ms\n",
            " 85% 13829/16260 [1:18:56<14:18,  2.83it/s]iter 13829: loss 5.6886, time: 422.82ms\n",
            " 85% 13830/16260 [1:18:56<15:09,  2.67it/s]iter 13830: loss 5.8302, time: 239.82ms\n",
            " 85% 13831/16260 [1:18:56<13:32,  2.99it/s]iter 13831: loss 5.5223, time: 464.70ms\n",
            " 85% 13832/16260 [1:18:57<15:08,  2.67it/s]iter 13832: loss 5.0493, time: 217.80ms\n",
            " 85% 13833/16260 [1:18:57<13:15,  3.05it/s]iter 13833: loss 5.3753, time: 463.99ms\n",
            " 85% 13834/16260 [1:18:57<14:55,  2.71it/s]iter 13834: loss 4.6464, time: 235.26ms\n",
            " 85% 13835/16260 [1:18:58<13:19,  3.03it/s]iter 13835: loss 5.3499, time: 424.27ms\n",
            " 85% 13836/16260 [1:18:58<14:28,  2.79it/s]iter 13836: loss 5.5757, time: 276.93ms\n",
            " 85% 13837/16260 [1:18:58<13:30,  2.99it/s]iter 13837: loss 4.7810, time: 255.94ms\n",
            " 85% 13838/16260 [1:18:59<12:34,  3.21it/s]iter 13838: loss 5.0255, time: 316.03ms\n",
            " 85% 13839/16260 [1:18:59<12:38,  3.19it/s]iter 13839: loss 3.7367, time: 252.49ms\n",
            " 85% 13840/16260 [1:18:59<11:55,  3.38it/s]iter 13840: loss 5.6474, time: 236.81ms\n",
            " 85% 13841/16260 [1:18:59<11:13,  3.59it/s]iter 13841: loss 5.2844, time: 278.96ms\n",
            " 85% 13842/16260 [1:19:00<11:14,  3.58it/s]iter 13842: loss 5.4403, time: 271.65ms\n",
            " 85% 13843/16260 [1:19:00<11:10,  3.61it/s]iter 13843: loss 4.3952, time: 146.19ms\n",
            " 85% 13844/16260 [1:19:00<09:35,  4.20it/s]iter 13844: loss 5.2785, time: 341.35ms\n",
            " 85% 13845/16260 [1:19:00<10:51,  3.71it/s]iter 13845: loss 4.5340, time: 251.97ms\n",
            " 85% 13846/16260 [1:19:01<10:39,  3.78it/s]iter 13846: loss 5.6946, time: 463.71ms\n",
            " 85% 13847/16260 [1:19:01<13:04,  3.08it/s]iter 13847: loss 5.5240, time: 465.18ms\n",
            " 85% 13848/16260 [1:19:02<14:46,  2.72it/s]iter 13848: loss 5.1442, time: 431.56ms\n",
            " 85% 13849/16260 [1:19:02<15:33,  2.58it/s]iter 13849: loss 5.0951, time: 436.34ms\n",
            " 85% 13850/16260 [1:19:03<16:10,  2.48it/s]iter 13850: loss 5.0929, time: 233.13ms\n",
            " 85% 13851/16260 [1:19:03<14:08,  2.84it/s]iter 13851: loss 3.8668, time: 164.87ms\n",
            " 85% 13852/16260 [1:19:03<11:54,  3.37it/s]iter 13852: loss 5.4796, time: 464.01ms\n",
            " 85% 13853/16260 [1:19:03<13:55,  2.88it/s]iter 13853: loss 4.3816, time: 169.54ms\n",
            " 85% 13854/16260 [1:19:04<11:48,  3.40it/s]iter 13854: loss 5.5932, time: 463.96ms\n",
            " 85% 13855/16260 [1:19:04<13:52,  2.89it/s]iter 13855: loss 4.7331, time: 238.69ms\n",
            " 85% 13856/16260 [1:19:04<12:35,  3.18it/s]iter 13856: loss 4.0615, time: 251.23ms\n",
            " 85% 13857/16260 [1:19:05<11:50,  3.38it/s]iter 13857: loss 6.1847, time: 474.54ms\n",
            " 85% 13858/16260 [1:19:05<14:00,  2.86it/s]iter 13858: loss 5.3596, time: 216.64ms\n",
            " 85% 13859/16260 [1:19:05<12:25,  3.22it/s]iter 13859: loss 5.8014, time: 464.28ms\n",
            " 85% 13860/16260 [1:19:06<14:16,  2.80it/s]iter 13860: loss 4.8019, time: 230.22ms\n",
            " 85% 13861/16260 [1:19:06<12:46,  3.13it/s]iter 13861: loss 5.4626, time: 464.17ms\n",
            " 85% 13862/16260 [1:19:06<14:31,  2.75it/s]iter 13862: loss 4.6725, time: 168.90ms\n",
            " 85% 13863/16260 [1:19:07<12:12,  3.27it/s]iter 13863: loss 5.1066, time: 446.40ms\n",
            " 85% 13864/16260 [1:19:07<13:54,  2.87it/s]iter 13864: loss 3.4545, time: 150.89ms\n",
            " 85% 13865/16260 [1:19:07<11:33,  3.45it/s]iter 13865: loss 5.0053, time: 259.65ms\n",
            " 85% 13866/16260 [1:19:07<11:13,  3.56it/s]iter 13866: loss 4.8719, time: 215.14ms\n",
            " 85% 13867/16260 [1:19:08<10:26,  3.82it/s]iter 13867: loss 5.3974, time: 316.38ms\n",
            " 85% 13868/16260 [1:19:08<11:06,  3.59it/s]iter 13868: loss 4.8815, time: 441.64ms\n",
            " 85% 13869/16260 [1:19:08<13:04,  3.05it/s]iter 13869: loss 4.7707, time: 255.91ms\n",
            " 85% 13870/16260 [1:19:09<12:13,  3.26it/s]iter 13870: loss 5.4229, time: 370.93ms\n",
            " 85% 13871/16260 [1:19:09<13:00,  3.06it/s]iter 13871: loss 4.6586, time: 261.41ms\n",
            " 85% 13872/16260 [1:19:09<12:14,  3.25it/s]iter 13872: loss 5.5634, time: 463.84ms\n",
            " 85% 13873/16260 [1:19:10<14:07,  2.82it/s]iter 13873: loss 3.4767, time: 189.34ms\n",
            " 85% 13874/16260 [1:19:10<12:09,  3.27it/s]iter 13874: loss 3.5397, time: 138.39ms\n",
            " 85% 13875/16260 [1:19:10<10:10,  3.91it/s]iter 13875: loss 5.1304, time: 378.25ms\n",
            " 85% 13876/16260 [1:19:10<11:38,  3.41it/s]iter 13876: loss 5.2532, time: 469.87ms\n",
            " 85% 13877/16260 [1:19:11<13:46,  2.88it/s]iter 13877: loss 4.7201, time: 186.94ms\n",
            " 85% 13878/16260 [1:19:11<11:52,  3.34it/s]iter 13878: loss 5.2306, time: 316.64ms\n",
            " 85% 13879/16260 [1:19:11<12:06,  3.28it/s]iter 13879: loss 5.8326, time: 466.14ms\n",
            " 85% 13880/16260 [1:19:12<14:02,  2.83it/s]iter 13880: loss 5.0992, time: 431.38ms\n",
            " 85% 13881/16260 [1:19:12<14:58,  2.65it/s]iter 13881: loss 4.8210, time: 216.35ms\n",
            " 85% 13882/16260 [1:19:13<13:03,  3.03it/s]iter 13882: loss 5.1612, time: 314.46ms\n",
            " 85% 13883/16260 [1:19:13<12:53,  3.07it/s]iter 13883: loss 5.6523, time: 446.09ms\n",
            " 85% 13884/16260 [1:19:13<14:20,  2.76it/s]iter 13884: loss 4.8579, time: 191.29ms\n",
            " 85% 13885/16260 [1:19:14<12:19,  3.21it/s]iter 13885: loss 5.5986, time: 419.45ms\n",
            " 85% 13886/16260 [1:19:14<13:37,  2.91it/s]iter 13886: loss 4.6789, time: 258.76ms\n",
            " 85% 13887/16260 [1:19:14<12:36,  3.14it/s]iter 13887: loss 5.0498, time: 465.00ms\n",
            " 85% 13888/16260 [1:19:15<14:21,  2.75it/s]iter 13888: loss 5.7559, time: 463.29ms\n",
            " 85% 13889/16260 [1:19:15<15:33,  2.54it/s]iter 13889: loss 4.3473, time: 168.19ms\n",
            " 85% 13890/16260 [1:19:15<12:53,  3.06it/s]iter 13890: loss 5.7455, time: 463.63ms\n",
            " 85% 13891/16260 [1:19:16<14:31,  2.72it/s]iter 13891: loss 5.9822, time: 464.20ms\n",
            " 85% 13892/16260 [1:19:16<15:40,  2.52it/s]iter 13892: loss 4.9910, time: 282.15ms\n",
            " 85% 13893/16260 [1:19:17<14:19,  2.75it/s]iter 13893: loss 4.3697, time: 256.81ms\n",
            " 85% 13894/16260 [1:19:17<13:04,  3.01it/s]iter 13894: loss 4.4278, time: 148.37ms\n",
            " 85% 13895/16260 [1:19:17<10:55,  3.61it/s]iter 13895: loss 5.7380, time: 467.18ms\n",
            " 85% 13896/16260 [1:19:17<13:10,  2.99it/s]iter 13896: loss 3.7737, time: 184.83ms\n",
            " 85% 13897/16260 [1:19:18<11:25,  3.45it/s]iter 13897: loss 5.2595, time: 472.24ms\n",
            " 85% 13898/16260 [1:19:18<13:35,  2.90it/s]iter 13898: loss 4.9397, time: 464.55ms\n",
            " 85% 13899/16260 [1:19:19<15:00,  2.62it/s]iter 13899: loss 4.5602, time: 255.49ms\n",
            " 85% 13900/16260 [1:19:19<13:32,  2.91it/s]iter 13900: loss 5.4844, time: 328.33ms\n",
            " 85% 13901/16260 [1:19:19<13:21,  2.94it/s]iter 13901: loss 4.4811, time: 257.95ms\n",
            " 85% 13902/16260 [1:19:19<12:24,  3.17it/s]iter 13902: loss 4.4257, time: 259.68ms\n",
            " 86% 13903/16260 [1:19:20<11:45,  3.34it/s]iter 13903: loss 5.5185, time: 282.35ms\n",
            " 86% 13904/16260 [1:19:20<11:34,  3.39it/s]iter 13904: loss 5.4432, time: 439.20ms\n",
            " 86% 13905/16260 [1:19:20<13:17,  2.95it/s]iter 13905: loss 5.2118, time: 446.41ms\n",
            " 86% 13906/16260 [1:19:21<14:34,  2.69it/s]iter 13906: loss 5.0985, time: 469.24ms\n",
            " 86% 13907/16260 [1:19:21<15:44,  2.49it/s]iter 13907: loss 5.7779, time: 463.97ms\n",
            " 86% 13908/16260 [1:19:22<16:29,  2.38it/s]iter 13908: loss 4.8864, time: 309.25ms\n",
            " 86% 13909/16260 [1:19:22<15:11,  2.58it/s]iter 13909: loss 5.2652, time: 222.19ms\n",
            " 86% 13910/16260 [1:19:22<13:15,  2.95it/s]iter 13910: loss 4.8073, time: 210.89ms\n",
            " 86% 13911/16260 [1:19:22<11:46,  3.33it/s]iter 13911: loss 4.4145, time: 187.50ms\n",
            " 86% 13912/16260 [1:19:23<10:26,  3.75it/s]iter 13912: loss 5.2687, time: 235.58ms\n",
            " 86% 13913/16260 [1:19:23<10:05,  3.88it/s]iter 13913: loss 5.2766, time: 464.94ms\n",
            " 86% 13914/16260 [1:19:23<12:31,  3.12it/s]iter 13914: loss 4.7958, time: 314.20ms\n",
            " 86% 13915/16260 [1:19:24<12:28,  3.13it/s]iter 13915: loss 5.5656, time: 472.46ms\n",
            " 86% 13916/16260 [1:19:24<14:16,  2.74it/s]iter 13916: loss 4.4757, time: 151.01ms\n",
            " 86% 13917/16260 [1:19:24<11:46,  3.32it/s]iter 13917: loss 5.7985, time: 424.13ms\n",
            " 86% 13918/16260 [1:19:25<13:13,  2.95it/s]iter 13918: loss 4.8185, time: 347.45ms\n",
            " 86% 13919/16260 [1:19:25<13:20,  2.92it/s]iter 13919: loss 4.5237, time: 267.72ms\n",
            " 86% 13920/16260 [1:19:25<12:28,  3.12it/s]iter 13920: loss 5.7721, time: 442.98ms\n",
            " 86% 13921/16260 [1:19:26<13:55,  2.80it/s]iter 13921: loss 5.2825, time: 336.60ms\n",
            " 86% 13922/16260 [1:19:26<13:42,  2.84it/s]iter 13922: loss 5.0292, time: 412.78ms\n",
            " 86% 13923/16260 [1:19:27<14:25,  2.70it/s]iter 13923: loss 5.1928, time: 338.72ms\n",
            " 86% 13924/16260 [1:19:27<14:04,  2.77it/s]iter 13924: loss 5.5567, time: 260.65ms\n",
            " 86% 13925/16260 [1:19:27<12:54,  3.02it/s]iter 13925: loss 5.4312, time: 377.03ms\n",
            " 86% 13926/16260 [1:19:28<13:26,  2.89it/s]iter 13926: loss 5.5319, time: 464.17ms\n",
            " 86% 13927/16260 [1:19:28<14:50,  2.62it/s]iter 13927: loss 3.6189, time: 168.48ms\n",
            " 86% 13928/16260 [1:19:28<12:21,  3.14it/s]iter 13928: loss 4.4440, time: 220.00ms\n",
            " 86% 13929/16260 [1:19:28<11:13,  3.46it/s]iter 13929: loss 5.2343, time: 464.49ms\n",
            " 86% 13930/16260 [1:19:29<13:17,  2.92it/s]iter 13930: loss 4.3645, time: 150.54ms\n",
            " 86% 13931/16260 [1:19:29<11:04,  3.51it/s]iter 13931: loss 4.1289, time: 189.81ms\n",
            " 86% 13932/16260 [1:19:29<09:58,  3.89it/s]iter 13932: loss 4.9374, time: 267.37ms\n",
            " 86% 13933/16260 [1:19:29<10:06,  3.84it/s]iter 13933: loss 5.4271, time: 464.37ms\n",
            " 86% 13934/16260 [1:19:30<12:29,  3.10it/s]iter 13934: loss 5.6811, time: 273.61ms\n",
            " 86% 13935/16260 [1:19:30<11:56,  3.25it/s]iter 13935: loss 4.6425, time: 278.18ms\n",
            " 86% 13936/16260 [1:19:30<11:35,  3.34it/s]iter 13936: loss 4.7124, time: 332.71ms\n",
            " 86% 13937/16260 [1:19:31<11:59,  3.23it/s]iter 13937: loss 4.7306, time: 422.53ms\n",
            " 86% 13938/16260 [1:19:31<13:19,  2.91it/s]iter 13938: loss 5.4601, time: 367.53ms\n",
            " 86% 13939/16260 [1:19:32<13:36,  2.84it/s]iter 13939: loss 5.6490, time: 464.17ms\n",
            " 86% 13940/16260 [1:19:32<14:55,  2.59it/s]iter 13940: loss 5.7643, time: 464.44ms\n",
            " 86% 13941/16260 [1:19:33<15:50,  2.44it/s]iter 13941: loss 5.4233, time: 464.75ms\n",
            " 86% 13942/16260 [1:19:33<16:29,  2.34it/s]iter 13942: loss 5.0339, time: 317.99ms\n",
            " 86% 13943/16260 [1:19:33<15:14,  2.53it/s]iter 13943: loss 5.3839, time: 282.99ms\n",
            " 86% 13944/16260 [1:19:34<13:57,  2.77it/s]iter 13944: loss 5.3059, time: 465.59ms\n",
            " 86% 13945/16260 [1:19:34<15:10,  2.54it/s]iter 13945: loss 5.3634, time: 464.22ms\n",
            " 86% 13946/16260 [1:19:35<16:00,  2.41it/s]iter 13946: loss 4.9118, time: 264.19ms\n",
            " 86% 13947/16260 [1:19:35<14:16,  2.70it/s]iter 13947: loss 4.0022, time: 154.90ms\n",
            " 86% 13948/16260 [1:19:35<11:47,  3.27it/s]iter 13948: loss 5.3743, time: 441.73ms\n",
            " 86% 13949/16260 [1:19:35<13:22,  2.88it/s]iter 13949: loss 3.9941, time: 238.09ms\n",
            " 86% 13950/16260 [1:19:36<12:07,  3.17it/s]iter 13950: loss 5.1820, time: 262.95ms\n",
            " 86% 13951/16260 [1:19:36<11:32,  3.33it/s]iter 13951: loss 5.0658, time: 318.77ms\n",
            " 86% 13952/16260 [1:19:36<11:46,  3.27it/s]iter 13952: loss 5.3568, time: 468.81ms\n",
            " 86% 13953/16260 [1:19:37<13:39,  2.82it/s]iter 13953: loss 5.0145, time: 421.95ms\n",
            " 86% 13954/16260 [1:19:37<14:26,  2.66it/s]iter 13954: loss 5.5240, time: 464.75ms\n",
            " 86% 13955/16260 [1:19:38<15:28,  2.48it/s]iter 13955: loss 5.6108, time: 432.05ms\n",
            " 86% 13956/16260 [1:19:38<15:49,  2.43it/s]iter 13956: loss 5.6186, time: 378.07ms\n",
            " 86% 13957/16260 [1:19:38<15:26,  2.49it/s]iter 13957: loss 5.7898, time: 421.38ms\n",
            " 86% 13958/16260 [1:19:39<15:40,  2.45it/s]iter 13958: loss 4.9892, time: 190.60ms\n",
            " 86% 13959/16260 [1:19:39<13:10,  2.91it/s]iter 13959: loss 5.3380, time: 431.38ms\n",
            " 86% 13960/16260 [1:19:39<14:11,  2.70it/s]iter 13960: loss 4.9171, time: 232.61ms\n",
            " 86% 13961/16260 [1:19:40<12:37,  3.03it/s]iter 13961: loss 3.3390, time: 142.52ms\n",
            " 86% 13962/16260 [1:19:40<10:29,  3.65it/s]iter 13962: loss 5.3419, time: 464.29ms\n",
            " 86% 13963/16260 [1:19:40<12:41,  3.02it/s]iter 13963: loss 4.2418, time: 188.00ms\n",
            " 86% 13964/16260 [1:19:41<11:03,  3.46it/s]iter 13964: loss 5.4180, time: 380.47ms\n",
            " 86% 13965/16260 [1:19:41<12:07,  3.16it/s]iter 13965: loss 5.0530, time: 463.96ms\n",
            " 86% 13966/16260 [1:19:41<13:49,  2.77it/s]iter 13966: loss 5.0977, time: 265.44ms\n",
            " 86% 13967/16260 [1:19:42<12:43,  3.00it/s]iter 13967: loss 5.3916, time: 240.74ms\n",
            " 86% 13968/16260 [1:19:42<11:41,  3.27it/s]iter 13968: loss 4.9375, time: 463.70ms\n",
            " 86% 13969/16260 [1:19:42<13:30,  2.83it/s]iter 13969: loss 4.8595, time: 218.53ms\n",
            " 86% 13970/16260 [1:19:43<12:11,  3.13it/s]iter 13970: loss 5.4463, time: 380.90ms\n",
            " 86% 13971/16260 [1:19:43<12:54,  2.95it/s]iter 13971: loss 4.5134, time: 168.93ms\n",
            " 86% 13972/16260 [1:19:43<10:59,  3.47it/s]iter 13972: loss 4.6908, time: 262.13ms\n",
            " 86% 13973/16260 [1:19:43<10:42,  3.56it/s]iter 13973: loss 5.0704, time: 275.22ms\n",
            " 86% 13974/16260 [1:19:44<10:39,  3.58it/s]iter 13974: loss 5.1016, time: 333.56ms\n",
            " 86% 13975/16260 [1:19:44<11:17,  3.37it/s]iter 13975: loss 5.5058, time: 373.00ms\n",
            " 86% 13976/16260 [1:19:44<12:10,  3.13it/s]iter 13976: loss 5.6734, time: 277.14ms\n",
            " 86% 13977/16260 [1:19:45<11:42,  3.25it/s]iter 13977: loss 4.6566, time: 172.61ms\n",
            " 86% 13978/16260 [1:19:45<10:10,  3.74it/s]iter 13978: loss 4.0696, time: 166.97ms\n",
            " 86% 13979/16260 [1:19:45<09:02,  4.21it/s]iter 13979: loss 4.5999, time: 232.23ms\n",
            " 86% 13980/16260 [1:19:45<08:59,  4.23it/s]iter 13980: loss 5.1700, time: 468.99ms\n",
            " 86% 13981/16260 [1:19:46<11:38,  3.26it/s]iter 13981: loss 4.1722, time: 189.66ms\n",
            " 86% 13982/16260 [1:19:46<10:19,  3.68it/s]iter 13982: loss 5.0820, time: 368.89ms\n",
            " 86% 13983/16260 [1:19:46<11:26,  3.32it/s]iter 13983: loss 5.5247, time: 467.25ms\n",
            " 86% 13984/16260 [1:19:47<13:20,  2.84it/s]iter 13984: loss 5.1274, time: 279.79ms\n",
            " 86% 13985/16260 [1:19:47<12:31,  3.03it/s]iter 13985: loss 4.9762, time: 379.34ms\n",
            " 86% 13986/16260 [1:19:47<13:05,  2.89it/s]iter 13986: loss 5.3256, time: 464.26ms\n",
            " 86% 13987/16260 [1:19:48<14:27,  2.62it/s]iter 13987: loss 4.7510, time: 274.24ms\n",
            " 86% 13988/16260 [1:19:48<13:14,  2.86it/s]iter 13988: loss 5.3249, time: 316.79ms\n",
            " 86% 13989/16260 [1:19:48<12:53,  2.94it/s]iter 13989: loss 3.2080, time: 163.37ms\n",
            " 86% 13990/16260 [1:19:49<10:53,  3.47it/s]iter 13990: loss 6.3099, time: 465.76ms\n",
            " 86% 13991/16260 [1:19:49<12:55,  2.93it/s]iter 13991: loss 5.5803, time: 438.35ms\n",
            " 86% 13992/16260 [1:19:50<14:01,  2.69it/s]iter 13992: loss 5.6937, time: 374.51ms\n",
            " 86% 13993/16260 [1:19:50<14:04,  2.68it/s]iter 13993: loss 5.3307, time: 463.43ms\n",
            " 86% 13994/16260 [1:19:50<15:07,  2.50it/s]iter 13994: loss 4.6984, time: 185.59ms\n",
            " 86% 13995/16260 [1:19:51<12:41,  2.97it/s]iter 13995: loss 4.2882, time: 234.84ms\n",
            " 86% 13996/16260 [1:19:51<11:33,  3.26it/s]iter 13996: loss 4.6658, time: 215.35ms\n",
            " 86% 13997/16260 [1:19:51<10:32,  3.58it/s]iter 13997: loss 5.6744, time: 464.84ms\n",
            " 86% 13998/16260 [1:19:51<12:39,  2.98it/s]iter 13998: loss 4.6716, time: 183.33ms\n",
            " 86% 13999/16260 [1:19:52<10:56,  3.44it/s]iter 13999: loss 4.8105, time: 314.70ms\n",
            " 86% 14000/16260 [1:19:52<11:13,  3.35it/s]iter 14000: loss 4.6639, time: 257.57ms\n",
            " 86% 14001/16260 [1:19:52<10:46,  3.49it/s]iter 14001: loss 5.8514, time: 424.36ms\n",
            " 86% 14002/16260 [1:19:53<12:21,  3.05it/s]iter 14002: loss 5.7583, time: 469.18ms\n",
            " 86% 14003/16260 [1:19:53<13:57,  2.70it/s]iter 14003: loss 4.6044, time: 234.70ms\n",
            " 86% 14004/16260 [1:19:53<12:25,  3.02it/s]iter 14004: loss 5.3403, time: 334.78ms\n",
            " 86% 14005/16260 [1:19:54<12:29,  3.01it/s]iter 14005: loss 5.0577, time: 241.91ms\n",
            " 86% 14006/16260 [1:19:54<11:28,  3.27it/s]iter 14006: loss 5.1513, time: 464.13ms\n",
            " 86% 14007/16260 [1:19:54<13:16,  2.83it/s]iter 14007: loss 4.0920, time: 150.20ms\n",
            " 86% 14008/16260 [1:19:55<10:59,  3.41it/s]iter 14008: loss 4.9837, time: 254.42ms\n",
            " 86% 14009/16260 [1:19:55<10:34,  3.55it/s]iter 14009: loss 4.8044, time: 280.79ms\n",
            " 86% 14010/16260 [1:19:55<10:34,  3.55it/s]iter 14010: loss 4.5288, time: 189.07ms\n",
            " 86% 14011/16260 [1:19:55<09:32,  3.93it/s]iter 14011: loss 5.2611, time: 414.93ms\n",
            " 86% 14012/16260 [1:19:56<11:21,  3.30it/s]iter 14012: loss 5.7264, time: 424.63ms\n",
            " 86% 14013/16260 [1:19:56<12:43,  2.94it/s]iter 14013: loss 5.1807, time: 191.78ms\n",
            " 86% 14014/16260 [1:19:56<11:04,  3.38it/s]iter 14014: loss 5.8368, time: 464.06ms\n",
            " 86% 14015/16260 [1:19:57<12:58,  2.88it/s]iter 14015: loss 4.8675, time: 239.58ms\n",
            " 86% 14016/16260 [1:19:57<11:46,  3.17it/s]iter 14016: loss 4.7070, time: 312.91ms\n",
            " 86% 14017/16260 [1:19:57<11:46,  3.18it/s]iter 14017: loss 5.3345, time: 463.55ms\n",
            " 86% 14018/16260 [1:19:58<13:26,  2.78it/s]iter 14018: loss 5.4031, time: 255.71ms\n",
            " 86% 14019/16260 [1:19:58<12:17,  3.04it/s]iter 14019: loss 4.6560, time: 207.31ms\n",
            " 86% 14020/16260 [1:19:58<10:55,  3.42it/s]iter 14020: loss 4.3414, time: 144.26ms\n",
            " 86% 14021/16260 [1:19:58<09:16,  4.02it/s]iter 14021: loss 5.5170, time: 463.69ms\n",
            " 86% 14022/16260 [1:19:59<11:41,  3.19it/s]iter 14022: loss 5.2965, time: 340.81ms\n",
            " 86% 14023/16260 [1:19:59<12:00,  3.10it/s]iter 14023: loss 5.0756, time: 414.49ms\n",
            " 86% 14024/16260 [1:20:00<13:03,  2.85it/s]iter 14024: loss 5.2311, time: 262.74ms\n",
            " 86% 14025/16260 [1:20:00<12:05,  3.08it/s]iter 14025: loss 5.3858, time: 330.06ms\n",
            " 86% 14026/16260 [1:20:00<12:09,  3.06it/s]iter 14026: loss 5.1319, time: 275.34ms\n",
            " 86% 14027/16260 [1:20:01<11:36,  3.21it/s]iter 14027: loss 5.8388, time: 464.22ms\n",
            " 86% 14028/16260 [1:20:01<13:24,  2.77it/s]iter 14028: loss 4.8150, time: 194.28ms\n",
            " 86% 14029/16260 [1:20:01<11:34,  3.21it/s]iter 14029: loss 4.2211, time: 222.70ms\n",
            " 86% 14030/16260 [1:20:01<10:35,  3.51it/s]iter 14030: loss 5.6795, time: 342.48ms\n",
            " 86% 14031/16260 [1:20:02<11:14,  3.30it/s]iter 14031: loss 5.4726, time: 464.58ms\n",
            " 86% 14032/16260 [1:20:02<13:03,  2.84it/s]iter 14032: loss 5.7961, time: 465.27ms\n",
            " 86% 14033/16260 [1:20:03<14:20,  2.59it/s]iter 14033: loss 4.8504, time: 259.61ms\n",
            " 86% 14034/16260 [1:20:03<12:56,  2.87it/s]iter 14034: loss 5.1478, time: 315.83ms\n",
            " 86% 14035/16260 [1:20:03<12:35,  2.95it/s]iter 14035: loss 4.0909, time: 190.86ms\n",
            " 86% 14036/16260 [1:20:03<10:56,  3.39it/s]iter 14036: loss 5.1425, time: 463.46ms\n",
            " 86% 14037/16260 [1:20:04<12:49,  2.89it/s]iter 14037: loss 5.2659, time: 374.36ms\n",
            " 86% 14038/16260 [1:20:04<13:09,  2.82it/s]iter 14038: loss 5.0856, time: 171.02ms\n",
            " 86% 14039/16260 [1:20:04<11:06,  3.33it/s]iter 14039: loss 4.4374, time: 211.76ms\n",
            " 86% 14040/16260 [1:20:05<10:08,  3.65it/s]iter 14040: loss 5.3798, time: 307.00ms\n",
            " 86% 14041/16260 [1:20:05<10:31,  3.52it/s]iter 14041: loss 4.9806, time: 263.10ms\n",
            " 86% 14042/16260 [1:20:05<10:17,  3.59it/s]iter 14042: loss 5.1703, time: 252.69ms\n",
            " 86% 14043/16260 [1:20:06<10:01,  3.69it/s]iter 14043: loss 5.2562, time: 252.05ms\n",
            " 86% 14044/16260 [1:20:06<09:49,  3.76it/s]iter 14044: loss 5.3753, time: 470.08ms\n",
            " 86% 14045/16260 [1:20:06<12:05,  3.05it/s]iter 14045: loss 4.1075, time: 232.44ms\n",
            " 86% 14046/16260 [1:20:06<11:03,  3.34it/s]iter 14046: loss 5.5703, time: 275.15ms\n",
            " 86% 14047/16260 [1:20:07<10:47,  3.42it/s]iter 14047: loss 5.5897, time: 477.79ms\n",
            " 86% 14048/16260 [1:20:07<12:51,  2.87it/s]iter 14048: loss 4.9315, time: 420.62ms\n",
            " 86% 14049/16260 [1:20:08<13:39,  2.70it/s]iter 14049: loss 5.1122, time: 313.80ms\n",
            " 86% 14050/16260 [1:20:08<13:02,  2.82it/s]iter 14050: loss 5.9574, time: 464.44ms\n",
            " 86% 14051/16260 [1:20:08<14:16,  2.58it/s]iter 14051: loss 5.2522, time: 410.02ms\n",
            " 86% 14052/16260 [1:20:09<14:31,  2.53it/s]iter 14052: loss 4.7952, time: 257.13ms\n",
            " 86% 14053/16260 [1:20:09<13:01,  2.83it/s]iter 14053: loss 4.9125, time: 430.45ms\n",
            " 86% 14054/16260 [1:20:10<13:52,  2.65it/s]iter 14054: loss 4.4622, time: 212.40ms\n",
            " 86% 14055/16260 [1:20:10<12:04,  3.05it/s]iter 14055: loss 5.6555, time: 463.62ms\n",
            " 86% 14056/16260 [1:20:10<13:34,  2.71it/s]iter 14056: loss 5.0043, time: 185.58ms\n",
            " 86% 14057/16260 [1:20:10<11:33,  3.18it/s]iter 14057: loss 5.2886, time: 463.15ms\n",
            " 86% 14058/16260 [1:20:11<13:11,  2.78it/s]iter 14058: loss 4.7165, time: 187.45ms\n",
            " 86% 14059/16260 [1:20:11<11:18,  3.24it/s]iter 14059: loss 5.5007, time: 321.30ms\n",
            " 86% 14060/16260 [1:20:11<11:28,  3.20it/s]iter 14060: loss 5.3947, time: 464.21ms\n",
            " 86% 14061/16260 [1:20:12<13:08,  2.79it/s]iter 14061: loss 4.8325, time: 464.13ms\n",
            " 86% 14062/16260 [1:20:12<14:18,  2.56it/s]iter 14062: loss 5.0494, time: 277.43ms\n",
            " 86% 14063/16260 [1:20:13<13:04,  2.80it/s]iter 14063: loss 4.7003, time: 187.18ms\n",
            " 86% 14064/16260 [1:20:13<11:13,  3.26it/s]iter 14064: loss 4.7791, time: 215.89ms\n",
            " 87% 14065/16260 [1:20:13<10:14,  3.57it/s]iter 14065: loss 4.4727, time: 218.49ms\n",
            " 87% 14066/16260 [1:20:13<09:34,  3.82it/s]iter 14066: loss 6.1202, time: 464.11ms\n",
            " 87% 14067/16260 [1:20:14<11:48,  3.10it/s]iter 14067: loss 5.4726, time: 255.63ms\n",
            " 87% 14068/16260 [1:20:14<11:04,  3.30it/s]iter 14068: loss 5.5148, time: 438.22ms\n",
            " 87% 14069/16260 [1:20:14<12:34,  2.91it/s]iter 14069: loss 5.2967, time: 464.26ms\n",
            " 87% 14070/16260 [1:20:15<13:53,  2.63it/s]iter 14070: loss 5.4444, time: 382.08ms\n",
            " 87% 14071/16260 [1:20:15<13:55,  2.62it/s]iter 14071: loss 4.3400, time: 255.93ms\n",
            " 87% 14072/16260 [1:20:15<12:33,  2.90it/s]iter 14072: loss 5.6194, time: 464.64ms\n",
            " 87% 14073/16260 [1:20:16<13:52,  2.63it/s]iter 14073: loss 3.6123, time: 151.20ms\n",
            " 87% 14074/16260 [1:20:16<11:22,  3.20it/s]iter 14074: loss 5.5365, time: 464.18ms\n",
            " 87% 14075/16260 [1:20:17<13:03,  2.79it/s]iter 14075: loss 3.7976, time: 464.10ms\n",
            " 87% 14076/16260 [1:20:17<14:13,  2.56it/s]iter 14076: loss 5.1620, time: 275.79ms\n",
            " 87% 14077/16260 [1:20:17<12:58,  2.80it/s]iter 14077: loss 5.1622, time: 172.04ms\n",
            " 87% 14078/16260 [1:20:17<10:58,  3.31it/s]iter 14078: loss 4.6467, time: 345.65ms\n",
            " 87% 14079/16260 [1:20:18<11:28,  3.17it/s]iter 14079: loss 5.5071, time: 468.36ms\n",
            " 87% 14080/16260 [1:20:18<13:08,  2.77it/s]iter 14080: loss 4.0328, time: 179.11ms\n",
            " 87% 14081/16260 [1:20:18<11:09,  3.25it/s]iter 14081: loss 5.7892, time: 412.22ms\n",
            " 87% 14082/16260 [1:20:19<12:18,  2.95it/s]iter 14082: loss 4.2724, time: 150.69ms\n",
            " 87% 14083/16260 [1:20:19<10:16,  3.53it/s]iter 14083: loss 5.7385, time: 464.00ms\n",
            " 87% 14084/16260 [1:20:20<12:15,  2.96it/s]iter 14084: loss 5.5513, time: 231.90ms\n",
            " 87% 14085/16260 [1:20:20<11:06,  3.26it/s]iter 14085: loss 4.7890, time: 283.39ms\n",
            " 87% 14086/16260 [1:20:20<10:52,  3.33it/s]iter 14086: loss 4.3449, time: 233.86ms\n",
            " 87% 14087/16260 [1:20:20<10:09,  3.56it/s]iter 14087: loss 5.2540, time: 344.22ms\n",
            " 87% 14088/16260 [1:20:21<10:51,  3.33it/s]iter 14088: loss 4.7430, time: 209.51ms\n",
            " 87% 14089/16260 [1:20:21<09:53,  3.66it/s]iter 14089: loss 4.5559, time: 228.88ms\n",
            " 87% 14090/16260 [1:20:21<09:25,  3.84it/s]iter 14090: loss 5.6295, time: 463.93ms\n",
            " 87% 14091/16260 [1:20:22<11:38,  3.11it/s]iter 14091: loss 5.4283, time: 464.31ms\n",
            " 87% 14092/16260 [1:20:22<13:11,  2.74it/s]iter 14092: loss 4.9485, time: 464.57ms\n",
            " 87% 14093/16260 [1:20:22<14:17,  2.53it/s]iter 14093: loss 5.0552, time: 410.49ms\n",
            " 87% 14094/16260 [1:20:23<14:27,  2.50it/s]iter 14094: loss 5.3127, time: 463.58ms\n",
            " 87% 14095/16260 [1:20:23<15:08,  2.38it/s]iter 14095: loss 5.4873, time: 275.44ms\n",
            " 87% 14096/16260 [1:20:24<13:35,  2.65it/s]iter 14096: loss 5.4369, time: 321.09ms\n",
            " 87% 14097/16260 [1:20:24<13:00,  2.77it/s]iter 14097: loss 5.3280, time: 279.88ms\n",
            " 87% 14098/16260 [1:20:24<12:08,  2.97it/s]iter 14098: loss 5.8282, time: 314.99ms\n",
            " 87% 14099/16260 [1:20:25<11:54,  3.02it/s]iter 14099: loss 2.2736, time: 146.75ms\n",
            " 87% 14100/16260 [1:20:25<09:55,  3.62it/s]iter 14100: loss 5.3815, time: 464.28ms\n",
            " 87% 14101/16260 [1:20:25<11:58,  3.00it/s]iter 14101: loss 5.5167, time: 466.51ms\n",
            " 87% 14102/16260 [1:20:26<13:25,  2.68it/s]iter 14102: loss 4.6375, time: 377.15ms\n",
            " 87% 14103/16260 [1:20:26<13:28,  2.67it/s]iter 14103: loss 5.3322, time: 441.30ms\n",
            " 87% 14104/16260 [1:20:26<14:12,  2.53it/s]iter 14104: loss 5.3378, time: 234.41ms\n",
            " 87% 14105/16260 [1:20:27<12:28,  2.88it/s]iter 14105: loss 5.5306, time: 463.90ms\n",
            " 87% 14106/16260 [1:20:27<13:44,  2.61it/s]iter 14106: loss 5.9968, time: 464.38ms\n",
            " 87% 14107/16260 [1:20:28<14:38,  2.45it/s]iter 14107: loss 5.4900, time: 463.74ms\n",
            " 87% 14108/16260 [1:20:28<15:14,  2.35it/s]iter 14108: loss 5.3240, time: 318.36ms\n",
            " 87% 14109/16260 [1:20:28<14:06,  2.54it/s]iter 14109: loss 5.8544, time: 463.48ms\n",
            " 87% 14110/16260 [1:20:29<14:52,  2.41it/s]iter 14110: loss 4.8889, time: 308.29ms\n",
            " 87% 14111/16260 [1:20:29<13:44,  2.61it/s]iter 14111: loss 5.4321, time: 467.86ms\n",
            " 87% 14112/16260 [1:20:30<14:38,  2.44it/s]iter 14112: loss 5.8672, time: 463.89ms\n",
            " 87% 14113/16260 [1:20:30<15:14,  2.35it/s]iter 14113: loss 4.8517, time: 463.89ms\n",
            " 87% 14114/16260 [1:20:31<15:39,  2.28it/s]iter 14114: loss 4.6148, time: 262.72ms\n",
            " 87% 14115/16260 [1:20:31<13:47,  2.59it/s]iter 14115: loss 4.2028, time: 150.19ms\n",
            " 87% 14116/16260 [1:20:31<11:16,  3.17it/s]iter 14116: loss 5.8636, time: 464.06ms\n",
            " 87% 14117/16260 [1:20:31<12:52,  2.77it/s]iter 14117: loss 6.0461, time: 441.54ms\n",
            " 87% 14118/16260 [1:20:32<13:45,  2.59it/s]iter 14118: loss 3.3344, time: 148.69ms\n",
            " 87% 14119/16260 [1:20:32<11:14,  3.18it/s]iter 14119: loss 6.0293, time: 464.47ms\n",
            " 87% 14120/16260 [1:20:32<12:50,  2.78it/s]iter 14120: loss 5.1013, time: 277.82ms\n",
            " 87% 14121/16260 [1:20:33<11:58,  2.98it/s]iter 14121: loss 5.3637, time: 235.92ms\n",
            " 87% 14122/16260 [1:20:33<10:55,  3.26it/s]iter 14122: loss 4.9333, time: 342.54ms\n",
            " 87% 14123/16260 [1:20:33<11:18,  3.15it/s]iter 14123: loss 4.1273, time: 167.69ms\n",
            " 87% 14124/16260 [1:20:34<09:43,  3.66it/s]iter 14124: loss 4.1165, time: 148.91ms\n",
            " 87% 14125/16260 [1:20:34<08:24,  4.23it/s]iter 14125: loss 5.7472, time: 472.19ms\n",
            " 87% 14126/16260 [1:20:34<10:56,  3.25it/s]iter 14126: loss 5.2980, time: 258.89ms\n",
            " 87% 14127/16260 [1:20:34<10:25,  3.41it/s]iter 14127: loss 5.4154, time: 436.86ms\n",
            " 87% 14128/16260 [1:20:35<11:57,  2.97it/s]iter 14128: loss 5.3085, time: 463.59ms\n",
            " 87% 14129/16260 [1:20:35<13:19,  2.66it/s]iter 14129: loss 4.5739, time: 251.80ms\n",
            " 87% 14130/16260 [1:20:36<12:01,  2.95it/s]iter 14130: loss 4.7163, time: 185.85ms\n",
            " 87% 14131/16260 [1:20:36<10:24,  3.41it/s]iter 14131: loss 3.1405, time: 145.49ms\n",
            " 87% 14132/16260 [1:20:36<08:50,  4.01it/s]iter 14132: loss 4.6901, time: 275.15ms\n",
            " 87% 14133/16260 [1:20:36<09:07,  3.88it/s]iter 14133: loss 5.1498, time: 237.22ms\n",
            " 87% 14134/16260 [1:20:36<08:55,  3.97it/s]iter 14134: loss 4.3599, time: 260.63ms\n",
            " 87% 14135/16260 [1:20:37<09:01,  3.92it/s]iter 14135: loss 5.4579, time: 467.65ms\n",
            " 87% 14136/16260 [1:20:37<11:17,  3.13it/s]iter 14136: loss 5.3035, time: 320.23ms\n",
            " 87% 14137/16260 [1:20:37<11:19,  3.12it/s]iter 14137: loss 4.9261, time: 423.85ms\n",
            " 87% 14138/16260 [1:20:38<12:26,  2.84it/s]iter 14138: loss 4.8483, time: 166.59ms\n",
            " 87% 14139/16260 [1:20:38<10:28,  3.37it/s]iter 14139: loss 4.7143, time: 274.47ms\n",
            " 87% 14140/16260 [1:20:38<10:15,  3.44it/s]iter 14140: loss 5.7176, time: 333.78ms\n",
            " 87% 14141/16260 [1:20:39<10:43,  3.29it/s]iter 14141: loss 5.4300, time: 381.39ms\n",
            " 87% 14142/16260 [1:20:39<11:33,  3.05it/s]iter 14142: loss 3.4311, time: 137.38ms\n",
            " 87% 14143/16260 [1:20:39<09:33,  3.69it/s]iter 14143: loss 5.1687, time: 255.64ms\n",
            " 87% 14144/16260 [1:20:39<09:24,  3.75it/s]iter 14144: loss 5.4113, time: 464.23ms\n",
            " 87% 14145/16260 [1:20:40<11:30,  3.06it/s]iter 14145: loss 4.7579, time: 377.93ms\n",
            " 87% 14146/16260 [1:20:40<12:03,  2.92it/s]iter 14146: loss 5.1561, time: 260.70ms\n",
            " 87% 14147/16260 [1:20:41<11:12,  3.14it/s]iter 14147: loss 4.6982, time: 191.56ms\n",
            " 87% 14148/16260 [1:20:41<09:52,  3.56it/s]iter 14148: loss 4.4289, time: 168.10ms\n",
            " 87% 14149/16260 [1:20:41<08:41,  4.04it/s]iter 14149: loss 5.0912, time: 332.60ms\n",
            " 87% 14150/16260 [1:20:41<09:36,  3.66it/s]iter 14150: loss 5.4406, time: 335.68ms\n",
            " 87% 14151/16260 [1:20:42<10:16,  3.42it/s]iter 14151: loss 4.4640, time: 172.17ms\n",
            " 87% 14152/16260 [1:20:42<09:01,  3.89it/s]iter 14152: loss 5.5926, time: 265.19ms\n",
            " 87% 14153/16260 [1:20:42<09:07,  3.85it/s]iter 14153: loss 5.3888, time: 464.67ms\n",
            " 87% 14154/16260 [1:20:42<11:17,  3.11it/s]iter 14154: loss 4.2886, time: 308.84ms\n",
            " 87% 14155/16260 [1:20:43<11:10,  3.14it/s]iter 14155: loss 5.3278, time: 463.85ms\n",
            " 87% 14156/16260 [1:20:43<12:42,  2.76it/s]iter 14156: loss 5.5094, time: 464.27ms\n",
            " 87% 14157/16260 [1:20:44<13:47,  2.54it/s]iter 14157: loss 5.8693, time: 463.75ms\n",
            " 87% 14158/16260 [1:20:44<14:32,  2.41it/s]iter 14158: loss 5.1949, time: 463.56ms\n",
            " 87% 14159/16260 [1:20:45<15:03,  2.33it/s]iter 14159: loss 5.2257, time: 333.41ms\n",
            " 87% 14160/16260 [1:20:45<14:03,  2.49it/s]iter 14160: loss 3.9437, time: 236.24ms\n",
            " 87% 14161/16260 [1:20:45<12:19,  2.84it/s]iter 14161: loss 5.5468, time: 464.26ms\n",
            " 87% 14162/16260 [1:20:46<13:30,  2.59it/s]iter 14162: loss 4.7107, time: 468.33ms\n",
            " 87% 14163/16260 [1:20:46<14:22,  2.43it/s]iter 14163: loss 4.9138, time: 463.40ms\n",
            " 87% 14164/16260 [1:20:47<14:56,  2.34it/s]iter 14164: loss 4.3978, time: 209.80ms\n",
            " 87% 14165/16260 [1:20:47<12:39,  2.76it/s]iter 14165: loss 5.8249, time: 315.85ms\n",
            " 87% 14166/16260 [1:20:47<12:10,  2.86it/s]iter 14166: loss 4.6942, time: 232.26ms\n",
            " 87% 14167/16260 [1:20:47<10:58,  3.18it/s]iter 14167: loss 5.6408, time: 419.50ms\n",
            " 87% 14168/16260 [1:20:48<12:04,  2.89it/s]iter 14168: loss 5.0010, time: 422.85ms\n",
            " 87% 14169/16260 [1:20:48<12:53,  2.70it/s]iter 14169: loss 5.5705, time: 463.20ms\n",
            " 87% 14170/16260 [1:20:49<13:52,  2.51it/s]iter 14170: loss 6.0128, time: 471.44ms\n",
            " 87% 14171/16260 [1:20:49<14:38,  2.38it/s]iter 14171: loss 4.8187, time: 314.34ms\n",
            " 87% 14172/16260 [1:20:49<13:32,  2.57it/s]iter 14172: loss 5.8489, time: 464.20ms\n",
            " 87% 14173/16260 [1:20:50<14:20,  2.43it/s]iter 14173: loss 5.5655, time: 476.28ms\n",
            " 87% 14174/16260 [1:20:50<15:00,  2.32it/s]iter 14174: loss 5.6061, time: 463.94ms\n",
            " 87% 14175/16260 [1:20:51<15:21,  2.26it/s]iter 14175: loss 5.4104, time: 467.37ms\n",
            " 87% 14176/16260 [1:20:51<15:37,  2.22it/s]iter 14176: loss 5.9620, time: 436.09ms\n",
            " 87% 14177/16260 [1:20:52<15:29,  2.24it/s]iter 14177: loss 4.8755, time: 277.62ms\n",
            " 87% 14178/16260 [1:20:52<13:44,  2.52it/s]iter 14178: loss 5.2417, time: 465.23ms\n",
            " 87% 14179/16260 [1:20:53<14:28,  2.40it/s]iter 14179: loss 5.8215, time: 463.76ms\n",
            " 87% 14180/16260 [1:20:53<14:57,  2.32it/s]iter 14180: loss 5.0784, time: 208.14ms\n",
            " 87% 14181/16260 [1:20:53<12:38,  2.74it/s]iter 14181: loss 6.2009, time: 464.55ms\n",
            " 87% 14182/16260 [1:20:54<13:41,  2.53it/s]iter 14182: loss 4.6484, time: 209.77ms\n",
            " 87% 14183/16260 [1:20:54<11:46,  2.94it/s]iter 14183: loss 5.0274, time: 437.06ms\n",
            " 87% 14184/16260 [1:20:54<12:47,  2.71it/s]iter 14184: loss 4.5874, time: 253.19ms\n",
            " 87% 14185/16260 [1:20:55<11:35,  2.98it/s]iter 14185: loss 5.2105, time: 464.15ms\n",
            " 87% 14186/16260 [1:20:55<12:56,  2.67it/s]iter 14186: loss 5.0378, time: 420.28ms\n",
            " 87% 14187/16260 [1:20:55<13:25,  2.57it/s]iter 14187: loss 4.8070, time: 214.86ms\n",
            " 87% 14188/16260 [1:20:56<11:38,  2.97it/s]iter 14188: loss 3.9945, time: 231.97ms\n",
            " 87% 14189/16260 [1:20:56<10:33,  3.27it/s]iter 14189: loss 3.9057, time: 168.10ms\n",
            " 87% 14190/16260 [1:20:56<09:08,  3.77it/s]iter 14190: loss 5.5496, time: 376.86ms\n",
            " 87% 14191/16260 [1:20:56<10:18,  3.34it/s]iter 14191: loss 5.7710, time: 411.19ms\n",
            " 87% 14192/16260 [1:20:57<11:28,  3.00it/s]iter 14192: loss 3.9511, time: 188.10ms\n",
            " 87% 14193/16260 [1:20:57<09:59,  3.45it/s]iter 14193: loss 5.2870, time: 348.88ms\n",
            " 87% 14194/16260 [1:20:57<10:36,  3.24it/s]iter 14194: loss 5.4099, time: 263.20ms\n",
            " 87% 14195/16260 [1:20:58<10:09,  3.39it/s]iter 14195: loss 5.1351, time: 463.52ms\n",
            " 87% 14196/16260 [1:20:58<11:54,  2.89it/s]iter 14196: loss 5.1645, time: 196.15ms\n",
            " 87% 14197/16260 [1:20:58<10:22,  3.32it/s]iter 14197: loss 4.2281, time: 152.15ms\n",
            " 87% 14198/16260 [1:20:59<08:50,  3.89it/s]iter 14198: loss 5.0896, time: 462.49ms\n",
            " 87% 14199/16260 [1:20:59<10:58,  3.13it/s]iter 14199: loss 5.9133, time: 466.58ms\n",
            " 87% 14200/16260 [1:20:59<12:29,  2.75it/s]iter 14200: loss 5.4409, time: 376.95ms\n",
            " 87% 14201/16260 [1:21:00<12:38,  2.71it/s]iter 14201: loss 5.5829, time: 464.11ms\n",
            " 87% 14202/16260 [1:21:00<13:38,  2.52it/s]iter 14202: loss 5.5202, time: 464.72ms\n",
            " 87% 14203/16260 [1:21:01<14:20,  2.39it/s]iter 14203: loss 5.9877, time: 380.00ms\n",
            " 87% 14204/16260 [1:21:01<13:57,  2.46it/s]iter 14204: loss 4.8752, time: 334.84ms\n",
            " 87% 14205/16260 [1:21:01<13:13,  2.59it/s]iter 14205: loss 5.3809, time: 463.67ms\n",
            " 87% 14206/16260 [1:21:02<14:01,  2.44it/s]iter 14206: loss 6.9645, time: 463.70ms\n",
            " 87% 14207/16260 [1:21:02<14:35,  2.35it/s]iter 14207: loss 5.3011, time: 253.63ms\n",
            " 87% 14208/16260 [1:21:03<12:49,  2.67it/s]iter 14208: loss 4.9906, time: 462.94ms\n",
            " 87% 14209/16260 [1:21:03<13:43,  2.49it/s]iter 14209: loss 5.0298, time: 370.09ms\n",
            " 87% 14210/16260 [1:21:04<13:24,  2.55it/s]iter 14210: loss 4.9414, time: 260.43ms\n",
            " 87% 14211/16260 [1:21:04<12:04,  2.83it/s]iter 14211: loss 5.8791, time: 274.63ms\n",
            " 87% 14212/16260 [1:21:04<11:16,  3.03it/s]iter 14212: loss 4.6429, time: 463.81ms\n",
            " 87% 14213/16260 [1:21:05<12:38,  2.70it/s]iter 14213: loss 5.1085, time: 464.09ms\n",
            " 87% 14214/16260 [1:21:05<13:36,  2.51it/s]iter 14214: loss 4.6654, time: 335.70ms\n",
            " 87% 14215/16260 [1:21:05<12:58,  2.63it/s]iter 14215: loss 4.8161, time: 234.07ms\n",
            " 87% 14216/16260 [1:21:06<11:28,  2.97it/s]iter 14216: loss 4.4279, time: 148.58ms\n",
            " 87% 14217/16260 [1:21:06<09:33,  3.56it/s]iter 14217: loss 2.4861, time: 139.55ms\n",
            " 87% 14218/16260 [1:21:06<08:07,  4.19it/s]iter 14218: loss 5.2168, time: 236.75ms\n",
            " 87% 14219/16260 [1:21:06<08:06,  4.19it/s]iter 14219: loss 5.6085, time: 469.33ms\n",
            " 87% 14220/16260 [1:21:07<10:28,  3.24it/s]iter 14220: loss 5.2669, time: 365.28ms\n",
            " 87% 14221/16260 [1:21:07<11:04,  3.07it/s]iter 14221: loss 3.9425, time: 236.77ms\n",
            " 87% 14222/16260 [1:21:07<10:10,  3.34it/s]iter 14222: loss 5.0845, time: 331.62ms\n",
            " 87% 14223/16260 [1:21:07<10:30,  3.23it/s]iter 14223: loss 5.7010, time: 464.02ms\n",
            " 87% 14224/16260 [1:21:08<12:05,  2.81it/s]iter 14224: loss 4.8888, time: 231.86ms\n",
            " 87% 14225/16260 [1:21:08<10:50,  3.13it/s]iter 14225: loss 5.6433, time: 463.93ms\n",
            " 87% 14226/16260 [1:21:09<12:18,  2.75it/s]iter 14226: loss 5.1399, time: 236.18ms\n",
            " 87% 14227/16260 [1:21:09<11:01,  3.07it/s]iter 14227: loss 4.4871, time: 421.42ms\n",
            " 88% 14228/16260 [1:21:09<12:00,  2.82it/s]iter 14228: loss 4.2001, time: 169.21ms\n",
            " 88% 14229/16260 [1:21:09<10:08,  3.34it/s]iter 14229: loss 4.4108, time: 212.12ms\n",
            " 88% 14230/16260 [1:21:10<09:15,  3.65it/s]iter 14230: loss 5.0177, time: 314.65ms\n",
            " 88% 14231/16260 [1:21:10<09:41,  3.49it/s]iter 14231: loss 4.4240, time: 431.82ms\n",
            " 88% 14232/16260 [1:21:10<11:10,  3.02it/s]iter 14232: loss 5.0829, time: 332.20ms\n",
            " 88% 14233/16260 [1:21:11<11:12,  3.02it/s]iter 14233: loss 5.1091, time: 464.50ms\n",
            " 88% 14234/16260 [1:21:11<12:33,  2.69it/s]iter 14234: loss 4.9146, time: 306.39ms\n",
            " 88% 14235/16260 [1:21:12<11:54,  2.83it/s]iter 14235: loss 3.8364, time: 211.82ms\n",
            " 88% 14236/16260 [1:21:12<10:29,  3.22it/s]iter 14236: loss 5.3509, time: 463.02ms\n",
            " 88% 14237/16260 [1:21:12<12:02,  2.80it/s]iter 14237: loss 5.1538, time: 436.62ms\n",
            " 88% 14238/16260 [1:21:13<12:51,  2.62it/s]iter 14238: loss 5.2340, time: 274.25ms\n",
            " 88% 14239/16260 [1:21:13<11:47,  2.86it/s]iter 14239: loss 5.5075, time: 415.91ms\n",
            " 88% 14240/16260 [1:21:13<12:27,  2.70it/s]iter 14240: loss 4.7035, time: 221.84ms\n",
            " 88% 14241/16260 [1:21:14<10:58,  3.07it/s]iter 14241: loss 4.8762, time: 288.71ms\n",
            " 88% 14242/16260 [1:21:14<10:36,  3.17it/s]iter 14242: loss 5.8069, time: 464.83ms\n",
            " 88% 14243/16260 [1:21:14<12:07,  2.77it/s]iter 14243: loss 5.8300, time: 464.51ms\n",
            " 88% 14244/16260 [1:21:15<13:11,  2.55it/s]iter 14244: loss 5.4598, time: 463.57ms\n",
            " 88% 14245/16260 [1:21:15<13:54,  2.41it/s]iter 14245: loss 5.4479, time: 271.05ms\n",
            " 88% 14246/16260 [1:21:16<12:28,  2.69it/s]iter 14246: loss 5.2475, time: 282.22ms\n",
            " 88% 14247/16260 [1:21:16<11:35,  2.90it/s]iter 14247: loss 5.3358, time: 463.97ms\n",
            " 88% 14248/16260 [1:21:16<12:47,  2.62it/s]iter 14248: loss 5.1906, time: 258.71ms\n",
            " 88% 14249/16260 [1:21:17<11:33,  2.90it/s]iter 14249: loss 5.4704, time: 335.69ms\n",
            " 88% 14250/16260 [1:21:17<11:28,  2.92it/s]iter 14250: loss 5.2953, time: 464.72ms\n",
            " 88% 14251/16260 [1:21:17<12:42,  2.63it/s]iter 14251: loss 4.9600, time: 464.62ms\n",
            " 88% 14252/16260 [1:21:18<13:34,  2.46it/s]iter 14252: loss 4.6755, time: 251.94ms\n",
            " 88% 14253/16260 [1:21:18<12:02,  2.78it/s]iter 14253: loss 5.2233, time: 206.86ms\n",
            " 88% 14254/16260 [1:21:18<10:30,  3.18it/s]iter 14254: loss 4.8650, time: 381.90ms\n",
            " 88% 14255/16260 [1:21:19<11:11,  2.98it/s]iter 14255: loss 6.6582, time: 463.90ms\n",
            " 88% 14256/16260 [1:21:19<12:29,  2.67it/s]iter 14256: loss 4.2583, time: 191.12ms\n",
            " 88% 14257/16260 [1:21:19<10:40,  3.13it/s]iter 14257: loss 5.0649, time: 275.89ms\n",
            " 88% 14258/16260 [1:21:20<10:14,  3.26it/s]iter 14258: loss 5.5426, time: 470.53ms\n",
            " 88% 14259/16260 [1:21:20<11:53,  2.80it/s]iter 14259: loss 5.5498, time: 440.87ms\n",
            " 88% 14260/16260 [1:21:21<12:44,  2.62it/s]iter 14260: loss 4.4515, time: 278.17ms\n",
            " 88% 14261/16260 [1:21:21<11:42,  2.84it/s]iter 14261: loss 4.3574, time: 182.73ms\n",
            " 88% 14262/16260 [1:21:21<10:02,  3.32it/s]iter 14262: loss 4.7728, time: 194.36ms\n",
            " 88% 14263/16260 [1:21:21<08:58,  3.71it/s]iter 14263: loss 5.6923, time: 465.38ms\n",
            " 88% 14264/16260 [1:21:22<10:56,  3.04it/s]iter 14264: loss 5.6198, time: 447.29ms\n",
            " 88% 14265/16260 [1:21:22<12:08,  2.74it/s]iter 14265: loss 5.4845, time: 442.31ms\n",
            " 88% 14266/16260 [1:21:23<12:54,  2.57it/s]iter 14266: loss 5.3748, time: 333.78ms\n",
            " 88% 14267/16260 [1:21:23<12:22,  2.68it/s]iter 14267: loss 5.0709, time: 268.77ms\n",
            " 88% 14268/16260 [1:21:23<11:21,  2.93it/s]iter 14268: loss 4.8596, time: 234.45ms\n",
            " 88% 14269/16260 [1:21:23<10:17,  3.23it/s]iter 14269: loss 4.7345, time: 207.38ms\n",
            " 88% 14270/16260 [1:21:24<09:16,  3.58it/s]iter 14270: loss 5.5838, time: 422.00ms\n",
            " 88% 14271/16260 [1:21:24<10:41,  3.10it/s]iter 14271: loss 4.7882, time: 286.18ms\n",
            " 88% 14272/16260 [1:21:24<10:20,  3.20it/s]iter 14272: loss 5.5522, time: 464.15ms\n",
            " 88% 14273/16260 [1:21:25<11:51,  2.79it/s]iter 14273: loss 5.3240, time: 319.68ms\n",
            " 88% 14274/16260 [1:21:25<11:29,  2.88it/s]iter 14274: loss 5.2713, time: 434.65ms\n",
            " 88% 14275/16260 [1:21:26<12:22,  2.67it/s]iter 14275: loss 4.6288, time: 463.71ms\n",
            " 88% 14276/16260 [1:21:26<13:16,  2.49it/s]iter 14276: loss 4.1536, time: 282.93ms\n",
            " 88% 14277/16260 [1:21:26<12:06,  2.73it/s]iter 14277: loss 5.2349, time: 276.12ms\n",
            " 88% 14278/16260 [1:21:27<11:13,  2.94it/s]iter 14278: loss 4.6786, time: 463.80ms\n",
            " 88% 14279/16260 [1:21:27<12:27,  2.65it/s]iter 14279: loss 5.7319, time: 464.76ms\n",
            " 88% 14280/16260 [1:21:27<13:20,  2.47it/s]iter 14280: loss 5.4183, time: 464.44ms\n",
            " 88% 14281/16260 [1:21:28<13:56,  2.37it/s]iter 14281: loss 5.3790, time: 463.50ms\n",
            " 88% 14282/16260 [1:21:28<14:21,  2.30it/s]iter 14282: loss 5.8894, time: 463.68ms\n",
            " 88% 14283/16260 [1:21:29<14:38,  2.25it/s]iter 14283: loss 5.4111, time: 463.86ms\n",
            " 88% 14284/16260 [1:21:29<14:50,  2.22it/s]iter 14284: loss 4.5875, time: 231.98ms\n",
            " 88% 14285/16260 [1:21:30<12:41,  2.59it/s]iter 14285: loss 4.6420, time: 237.98ms\n",
            " 88% 14286/16260 [1:21:30<11:14,  2.93it/s]iter 14286: loss 5.3876, time: 463.36ms\n",
            " 88% 14287/16260 [1:21:30<12:26,  2.64it/s]iter 14287: loss 4.4893, time: 239.55ms\n",
            " 88% 14288/16260 [1:21:31<11:05,  2.96it/s]iter 14288: loss 5.1501, time: 337.83ms\n",
            " 88% 14289/16260 [1:21:31<11:05,  2.96it/s]iter 14289: loss 5.9313, time: 471.53ms\n",
            " 88% 14290/16260 [1:21:31<12:25,  2.64it/s]iter 14290: loss 4.3932, time: 170.95ms\n",
            " 88% 14291/16260 [1:21:32<10:23,  3.16it/s]iter 14291: loss 4.4047, time: 190.21ms\n",
            " 88% 14292/16260 [1:21:32<09:09,  3.58it/s]iter 14292: loss 5.2206, time: 308.15ms\n",
            " 88% 14293/16260 [1:21:32<09:27,  3.47it/s]iter 14293: loss 5.4011, time: 277.68ms\n",
            " 88% 14294/16260 [1:21:32<09:21,  3.50it/s]iter 14294: loss 5.3164, time: 189.39ms\n",
            " 88% 14295/16260 [1:21:32<08:25,  3.89it/s]iter 14295: loss 5.7501, time: 463.15ms\n",
            " 88% 14296/16260 [1:21:33<10:27,  3.13it/s]iter 14296: loss 6.3395, time: 464.50ms\n",
            " 88% 14297/16260 [1:21:33<11:53,  2.75it/s]iter 14297: loss 5.6654, time: 463.71ms\n",
            " 88% 14298/16260 [1:21:34<12:53,  2.54it/s]iter 14298: loss 6.0229, time: 431.01ms\n",
            " 88% 14299/16260 [1:21:34<13:15,  2.47it/s]iter 14299: loss 4.4168, time: 170.14ms\n",
            " 88% 14300/16260 [1:21:34<10:57,  2.98it/s]iter 14300: loss 4.4913, time: 234.15ms\n",
            " 88% 14301/16260 [1:21:35<09:58,  3.27it/s]iter 14301: loss 4.8931, time: 273.96ms\n",
            " 88% 14302/16260 [1:21:35<09:40,  3.37it/s]iter 14302: loss 5.5944, time: 217.09ms\n",
            " 88% 14303/16260 [1:21:35<08:54,  3.66it/s]iter 14303: loss 5.3202, time: 344.74ms\n",
            " 88% 14304/16260 [1:21:36<09:37,  3.39it/s]iter 14304: loss 5.0019, time: 210.72ms\n",
            " 88% 14305/16260 [1:21:36<08:48,  3.70it/s]iter 14305: loss 4.4846, time: 215.39ms\n",
            " 88% 14306/16260 [1:21:36<08:16,  3.93it/s]iter 14306: loss 5.2479, time: 254.18ms\n",
            " 88% 14307/16260 [1:21:36<08:17,  3.92it/s]iter 14307: loss 5.0164, time: 335.03ms\n",
            " 88% 14308/16260 [1:21:37<09:05,  3.58it/s]iter 14308: loss 5.1212, time: 340.23ms\n",
            " 88% 14309/16260 [1:21:37<09:41,  3.35it/s]iter 14309: loss 4.6816, time: 260.85ms\n",
            " 88% 14310/16260 [1:21:37<09:20,  3.48it/s]iter 14310: loss 4.9057, time: 285.20ms\n",
            " 88% 14311/16260 [1:21:37<09:19,  3.48it/s]iter 14311: loss 4.9700, time: 218.82ms\n",
            " 88% 14312/16260 [1:21:38<08:40,  3.74it/s]iter 14312: loss 4.0194, time: 148.47ms\n",
            " 88% 14313/16260 [1:21:38<07:31,  4.31it/s]iter 14313: loss 4.6052, time: 208.38ms\n",
            " 88% 14314/16260 [1:21:38<07:18,  4.44it/s]iter 14314: loss 4.9362, time: 308.37ms\n",
            " 88% 14315/16260 [1:21:38<08:07,  3.99it/s]iter 14315: loss 5.3559, time: 370.97ms\n",
            " 88% 14316/16260 [1:21:39<09:18,  3.48it/s]iter 14316: loss 5.5059, time: 464.01ms\n",
            " 88% 14317/16260 [1:21:39<11:02,  2.93it/s]iter 14317: loss 5.1693, time: 276.82ms\n",
            " 88% 14318/16260 [1:21:39<10:25,  3.11it/s]iter 14318: loss 5.2115, time: 273.48ms\n",
            " 88% 14319/16260 [1:21:40<09:57,  3.25it/s]iter 14319: loss 5.9767, time: 464.89ms\n",
            " 88% 14320/16260 [1:21:40<11:29,  2.81it/s]iter 14320: loss 4.0546, time: 188.44ms\n",
            " 88% 14321/16260 [1:21:40<09:53,  3.27it/s]iter 14321: loss 4.8023, time: 215.00ms\n",
            " 88% 14322/16260 [1:21:41<09:00,  3.58it/s]iter 14322: loss 5.1883, time: 420.22ms\n",
            " 88% 14323/16260 [1:21:41<10:23,  3.11it/s]iter 14323: loss 6.0322, time: 464.73ms\n",
            " 88% 14324/16260 [1:21:42<11:47,  2.74it/s]iter 14324: loss 5.4638, time: 425.85ms\n",
            " 88% 14325/16260 [1:21:42<12:23,  2.60it/s]iter 14325: loss 5.4370, time: 343.88ms\n",
            " 88% 14326/16260 [1:21:42<12:00,  2.68it/s]iter 14326: loss 4.3833, time: 468.41ms\n",
            " 88% 14327/16260 [1:21:43<12:56,  2.49it/s]iter 14327: loss 5.5678, time: 306.79ms\n",
            " 88% 14328/16260 [1:21:43<12:02,  2.68it/s]iter 14328: loss 5.6885, time: 469.59ms\n",
            " 88% 14329/16260 [1:21:44<12:58,  2.48it/s]iter 14329: loss 4.9520, time: 257.05ms\n",
            " 88% 14330/16260 [1:21:44<11:34,  2.78it/s]iter 14330: loss 5.3624, time: 413.49ms\n",
            " 88% 14331/16260 [1:21:44<12:05,  2.66it/s]iter 14331: loss 4.2624, time: 171.35ms\n",
            " 88% 14332/16260 [1:21:44<10:07,  3.17it/s]iter 14332: loss 5.2912, time: 475.33ms\n",
            " 88% 14333/16260 [1:21:45<11:41,  2.75it/s]iter 14333: loss 5.3124, time: 277.30ms\n",
            " 88% 14334/16260 [1:21:45<10:51,  2.96it/s]iter 14334: loss 4.9588, time: 315.54ms\n",
            " 88% 14335/16260 [1:21:45<10:39,  3.01it/s]iter 14335: loss 4.6770, time: 467.33ms\n",
            " 88% 14336/16260 [1:21:46<11:57,  2.68it/s]iter 14336: loss 4.1418, time: 191.56ms\n",
            " 88% 14337/16260 [1:21:46<10:13,  3.14it/s]iter 14337: loss 5.2020, time: 464.56ms\n",
            " 88% 14338/16260 [1:21:47<11:37,  2.75it/s]iter 14338: loss 5.7628, time: 464.11ms\n",
            " 88% 14339/16260 [1:21:47<12:36,  2.54it/s]iter 14339: loss 5.2031, time: 237.64ms\n",
            " 88% 14340/16260 [1:21:47<11:07,  2.88it/s]iter 14340: loss 4.5539, time: 261.15ms\n",
            " 88% 14341/16260 [1:21:48<10:17,  3.11it/s]iter 14341: loss 5.2272, time: 463.97ms\n",
            " 88% 14342/16260 [1:21:48<11:40,  2.74it/s]iter 14342: loss 5.1432, time: 259.32ms\n",
            " 88% 14343/16260 [1:21:48<10:39,  3.00it/s]iter 14343: loss 4.5135, time: 286.48ms\n",
            " 88% 14344/16260 [1:21:49<10:13,  3.13it/s]iter 14344: loss 5.1094, time: 234.15ms\n",
            " 88% 14345/16260 [1:21:49<09:24,  3.39it/s]iter 14345: loss 4.1173, time: 209.97ms\n",
            " 88% 14346/16260 [1:21:49<08:36,  3.71it/s]iter 14346: loss 4.7259, time: 339.62ms\n",
            " 88% 14347/16260 [1:21:49<09:16,  3.43it/s]iter 14347: loss 5.1759, time: 275.92ms\n",
            " 88% 14348/16260 [1:21:50<09:08,  3.48it/s]iter 14348: loss 4.6671, time: 255.36ms\n",
            " 88% 14349/16260 [1:21:50<08:51,  3.60it/s]iter 14349: loss 5.7050, time: 464.01ms\n",
            " 88% 14350/16260 [1:21:50<10:38,  2.99it/s]iter 14350: loss 4.7863, time: 342.83ms\n",
            " 88% 14351/16260 [1:21:51<10:43,  2.97it/s]iter 14351: loss 5.2376, time: 249.57ms\n",
            " 88% 14352/16260 [1:21:51<09:54,  3.21it/s]iter 14352: loss 5.2434, time: 463.97ms\n",
            " 88% 14353/16260 [1:21:51<11:22,  2.80it/s]iter 14353: loss 5.1052, time: 464.16ms\n",
            " 88% 14354/16260 [1:21:52<12:23,  2.56it/s]iter 14354: loss 4.6903, time: 231.80ms\n",
            " 88% 14355/16260 [1:21:52<10:53,  2.92it/s]iter 14355: loss 5.7477, time: 464.55ms\n",
            " 88% 14356/16260 [1:21:53<12:03,  2.63it/s]iter 14356: loss 5.1999, time: 464.50ms\n",
            " 88% 14357/16260 [1:21:53<12:52,  2.46it/s]iter 14357: loss 5.5792, time: 464.23ms\n",
            " 88% 14358/16260 [1:21:54<13:26,  2.36it/s]iter 14358: loss 5.1571, time: 381.09ms\n",
            " 88% 14359/16260 [1:21:54<13:02,  2.43it/s]iter 14359: loss 5.1715, time: 240.21ms\n",
            " 88% 14360/16260 [1:21:54<11:24,  2.77it/s]iter 14360: loss 5.1306, time: 240.04ms\n",
            " 88% 14361/16260 [1:21:54<10:16,  3.08it/s]iter 14361: loss 5.1945, time: 336.30ms\n",
            " 88% 14362/16260 [1:21:55<10:23,  3.04it/s]iter 14362: loss 5.2555, time: 464.99ms\n",
            " 88% 14363/16260 [1:21:55<11:42,  2.70it/s]iter 14363: loss 5.4755, time: 463.67ms\n",
            " 88% 14364/16260 [1:21:56<12:35,  2.51it/s]iter 14364: loss 5.1077, time: 388.41ms\n",
            " 88% 14365/16260 [1:21:56<12:30,  2.52it/s]iter 14365: loss 4.6314, time: 169.97ms\n",
            " 88% 14366/16260 [1:21:56<10:22,  3.04it/s]iter 14366: loss 3.0974, time: 134.29ms\n",
            " 88% 14367/16260 [1:21:56<08:32,  3.69it/s]iter 14367: loss 5.0411, time: 191.09ms\n",
            " 88% 14368/16260 [1:21:57<07:47,  4.05it/s]iter 14368: loss 5.7482, time: 463.72ms\n",
            " 88% 14369/16260 [1:21:57<09:51,  3.20it/s]iter 14369: loss 5.3617, time: 370.93ms\n",
            " 88% 14370/16260 [1:21:57<10:24,  3.03it/s]iter 14370: loss 4.5389, time: 170.68ms\n",
            " 88% 14371/16260 [1:21:58<08:54,  3.53it/s]iter 14371: loss 5.4252, time: 464.07ms\n",
            " 88% 14372/16260 [1:21:58<10:37,  2.96it/s]iter 14372: loss 5.9770, time: 464.70ms\n",
            " 88% 14373/16260 [1:21:58<11:50,  2.66it/s]iter 14373: loss 5.4188, time: 464.09ms\n",
            " 88% 14374/16260 [1:21:59<12:40,  2.48it/s]iter 14374: loss 5.5011, time: 463.98ms\n",
            " 88% 14375/16260 [1:21:59<13:15,  2.37it/s]iter 14375: loss 5.3521, time: 368.12ms\n",
            " 88% 14376/16260 [1:22:00<12:45,  2.46it/s]iter 14376: loss 5.2596, time: 463.33ms\n",
            " 88% 14377/16260 [1:22:00<13:17,  2.36it/s]iter 14377: loss 5.2111, time: 414.48ms\n",
            " 88% 14378/16260 [1:22:01<13:13,  2.37it/s]iter 14378: loss 4.4736, time: 235.24ms\n",
            " 88% 14379/16260 [1:22:01<11:28,  2.73it/s]iter 14379: loss 6.4669, time: 464.50ms\n",
            " 88% 14380/16260 [1:22:01<12:24,  2.53it/s]iter 14380: loss 5.7764, time: 475.74ms\n",
            " 88% 14381/16260 [1:22:02<13:10,  2.38it/s]iter 14381: loss 4.7597, time: 260.30ms\n",
            " 88% 14382/16260 [1:22:02<11:40,  2.68it/s]iter 14382: loss 5.3955, time: 379.19ms\n",
            " 88% 14383/16260 [1:22:02<11:44,  2.67it/s]iter 14383: loss 5.3889, time: 464.19ms\n",
            " 88% 14384/16260 [1:22:03<12:34,  2.49it/s]iter 14384: loss 5.8757, time: 278.55ms\n",
            " 88% 14385/16260 [1:22:03<11:25,  2.73it/s]iter 14385: loss 4.8795, time: 373.35ms\n",
            " 88% 14386/16260 [1:22:04<11:30,  2.71it/s]iter 14386: loss 4.6149, time: 221.06ms\n",
            " 88% 14387/16260 [1:22:04<10:08,  3.08it/s]iter 14387: loss 4.8484, time: 217.90ms\n",
            " 88% 14388/16260 [1:22:04<09:08,  3.41it/s]iter 14388: loss 5.0949, time: 439.88ms\n",
            " 88% 14389/16260 [1:22:04<10:31,  2.96it/s]iter 14389: loss 5.7723, time: 464.09ms\n",
            " 88% 14390/16260 [1:22:05<11:43,  2.66it/s]iter 14390: loss 5.1030, time: 233.94ms\n",
            " 89% 14391/16260 [1:22:05<10:23,  3.00it/s]iter 14391: loss 5.2541, time: 279.60ms\n",
            " 89% 14392/16260 [1:22:05<09:54,  3.14it/s]iter 14392: loss 5.7105, time: 464.87ms\n",
            " 89% 14393/16260 [1:22:06<11:16,  2.76it/s]iter 14393: loss 5.7874, time: 463.51ms\n",
            " 89% 14394/16260 [1:22:06<12:13,  2.54it/s]iter 14394: loss 5.2734, time: 463.81ms\n",
            " 89% 14395/16260 [1:22:07<12:53,  2.41it/s]iter 14395: loss 5.6822, time: 464.41ms\n",
            " 89% 14396/16260 [1:22:07<13:21,  2.32it/s]iter 14396: loss 5.6606, time: 464.66ms\n",
            " 89% 14397/16260 [1:22:08<13:41,  2.27it/s]iter 14397: loss 4.9851, time: 353.18ms\n",
            " 89% 14398/16260 [1:22:08<12:53,  2.41it/s]iter 14398: loss 5.1112, time: 464.81ms\n",
            " 89% 14399/16260 [1:22:09<13:21,  2.32it/s]iter 14399: loss 4.3475, time: 241.85ms\n",
            " 89% 14400/16260 [1:22:09<11:35,  2.67it/s]iter 14400: loss 6.0166, time: 473.49ms\n",
            " 89% 14401/16260 [1:22:09<12:31,  2.47it/s]iter 14401: loss 4.0964, time: 187.11ms\n",
            " 89% 14402/16260 [1:22:10<10:31,  2.94it/s]iter 14402: loss 3.4490, time: 188.01ms\n",
            " 89% 14403/16260 [1:22:10<09:07,  3.39it/s]iter 14403: loss 4.3020, time: 171.73ms\n",
            " 89% 14404/16260 [1:22:10<07:59,  3.87it/s]iter 14404: loss 5.5555, time: 375.51ms\n",
            " 89% 14405/16260 [1:22:10<09:05,  3.40it/s]iter 14405: loss 5.4578, time: 464.22ms\n",
            " 89% 14406/16260 [1:22:11<10:40,  2.89it/s]iter 14406: loss 4.9024, time: 355.38ms\n",
            " 89% 14407/16260 [1:22:11<10:46,  2.87it/s]iter 14407: loss 5.6974, time: 437.75ms\n",
            " 89% 14408/16260 [1:22:12<11:36,  2.66it/s]iter 14408: loss 4.4466, time: 464.40ms\n",
            " 89% 14409/16260 [1:22:12<12:25,  2.48it/s]iter 14409: loss 5.1468, time: 276.94ms\n",
            " 89% 14410/16260 [1:22:12<11:16,  2.73it/s]iter 14410: loss 4.7476, time: 464.32ms\n",
            " 89% 14411/16260 [1:22:13<12:11,  2.53it/s]iter 14411: loss 5.0405, time: 251.06ms\n",
            " 89% 14412/16260 [1:22:13<10:51,  2.84it/s]iter 14412: loss 5.7367, time: 434.39ms\n",
            " 89% 14413/16260 [1:22:13<11:37,  2.65it/s]iter 14413: loss 5.2966, time: 446.96ms\n",
            " 89% 14414/16260 [1:22:14<12:16,  2.51it/s]iter 14414: loss 5.0233, time: 378.10ms\n",
            " 89% 14415/16260 [1:22:14<12:05,  2.54it/s]iter 14415: loss 5.2127, time: 469.42ms\n",
            " 89% 14416/16260 [1:22:15<12:48,  2.40it/s]iter 14416: loss 4.3796, time: 168.11ms\n",
            " 89% 14417/16260 [1:22:15<10:31,  2.92it/s]iter 14417: loss 5.7649, time: 373.21ms\n",
            " 89% 14418/16260 [1:22:15<10:48,  2.84it/s]iter 14418: loss 5.0254, time: 272.65ms\n",
            " 89% 14419/16260 [1:22:16<10:05,  3.04it/s]iter 14419: loss 5.2322, time: 315.65ms\n",
            " 89% 14420/16260 [1:22:16<09:58,  3.07it/s]iter 14420: loss 5.3552, time: 217.48ms\n",
            " 89% 14421/16260 [1:22:16<08:59,  3.41it/s]iter 14421: loss 5.1149, time: 367.84ms\n",
            " 89% 14422/16260 [1:22:16<09:40,  3.16it/s]iter 14422: loss 5.7881, time: 463.91ms\n",
            " 89% 14423/16260 [1:22:17<11:02,  2.77it/s]iter 14423: loss 5.2168, time: 257.37ms\n",
            " 89% 14424/16260 [1:22:17<10:06,  3.03it/s]iter 14424: loss 5.1554, time: 267.24ms\n",
            " 89% 14425/16260 [1:22:17<09:32,  3.21it/s]iter 14425: loss 5.5774, time: 465.19ms\n",
            " 89% 14426/16260 [1:22:18<10:57,  2.79it/s]iter 14426: loss 4.7266, time: 314.89ms\n",
            " 89% 14427/16260 [1:22:18<10:33,  2.89it/s]iter 14427: loss 5.2864, time: 345.20ms\n",
            " 89% 14428/16260 [1:22:19<10:33,  2.89it/s]iter 14428: loss 4.1765, time: 206.21ms\n",
            " 89% 14429/16260 [1:22:19<09:17,  3.28it/s]iter 14429: loss 5.4612, time: 419.99ms\n",
            " 89% 14430/16260 [1:22:19<10:21,  2.94it/s]iter 14430: loss 4.1638, time: 169.83ms\n",
            " 89% 14431/16260 [1:22:19<08:48,  3.46it/s]iter 14431: loss 5.3591, time: 338.42ms\n",
            " 89% 14432/16260 [1:22:20<09:16,  3.29it/s]iter 14432: loss 3.4702, time: 189.16ms\n",
            " 89% 14433/16260 [1:22:20<08:13,  3.70it/s]iter 14433: loss 4.9119, time: 277.32ms\n",
            " 89% 14434/16260 [1:22:20<08:17,  3.67it/s]iter 14434: loss 3.2739, time: 143.18ms\n",
            " 89% 14435/16260 [1:22:20<07:07,  4.27it/s]iter 14435: loss 6.2383, time: 463.83ms\n",
            " 89% 14436/16260 [1:22:21<09:13,  3.29it/s]iter 14436: loss 4.6179, time: 345.79ms\n",
            " 89% 14437/16260 [1:22:21<09:37,  3.16it/s]iter 14437: loss 4.8743, time: 316.40ms\n",
            " 89% 14438/16260 [1:22:21<09:37,  3.15it/s]iter 14438: loss 5.5020, time: 416.91ms\n",
            " 89% 14439/16260 [1:22:22<10:32,  2.88it/s]iter 14439: loss 4.8603, time: 252.23ms\n",
            " 89% 14440/16260 [1:22:22<09:41,  3.13it/s]iter 14440: loss 5.1461, time: 410.86ms\n",
            " 89% 14441/16260 [1:22:23<10:31,  2.88it/s]iter 14441: loss 5.4431, time: 275.88ms\n",
            " 89% 14442/16260 [1:22:23<09:53,  3.06it/s]iter 14442: loss 5.6037, time: 464.37ms\n",
            " 89% 14443/16260 [1:22:23<11:09,  2.71it/s]iter 14443: loss 4.7674, time: 151.76ms\n",
            " 89% 14444/16260 [1:22:23<09:11,  3.29it/s]iter 14444: loss 5.3571, time: 464.14ms\n",
            " 89% 14445/16260 [1:22:24<10:39,  2.84it/s]iter 14445: loss 4.8226, time: 254.36ms\n",
            " 89% 14446/16260 [1:22:24<09:46,  3.09it/s]iter 14446: loss 5.1368, time: 342.07ms\n",
            " 89% 14447/16260 [1:22:25<09:57,  3.03it/s]iter 14447: loss 5.4799, time: 463.80ms\n",
            " 89% 14448/16260 [1:22:25<11:10,  2.70it/s]iter 14448: loss 5.3466, time: 420.27ms\n",
            " 89% 14449/16260 [1:22:25<11:38,  2.59it/s]iter 14449: loss 4.8287, time: 230.51ms\n",
            " 89% 14450/16260 [1:22:26<10:14,  2.94it/s]iter 14450: loss 5.0374, time: 334.89ms\n",
            " 89% 14451/16260 [1:22:26<10:12,  2.95it/s]iter 14451: loss 5.5896, time: 442.75ms\n",
            " 89% 14452/16260 [1:22:26<11:09,  2.70it/s]iter 14452: loss 4.6075, time: 210.17ms\n",
            " 89% 14453/16260 [1:22:27<09:43,  3.10it/s]iter 14453: loss 6.0819, time: 464.11ms\n",
            " 89% 14454/16260 [1:22:27<11:00,  2.73it/s]iter 14454: loss 4.7610, time: 424.95ms\n",
            " 89% 14455/16260 [1:22:28<11:33,  2.60it/s]iter 14455: loss 5.9135, time: 464.62ms\n",
            " 89% 14456/16260 [1:22:28<12:17,  2.45it/s]iter 14456: loss 4.2245, time: 280.82ms\n",
            " 89% 14457/16260 [1:22:28<11:08,  2.70it/s]iter 14457: loss 4.5400, time: 190.79ms\n",
            " 89% 14458/16260 [1:22:28<09:31,  3.15it/s]iter 14458: loss 5.7815, time: 463.35ms\n",
            " 89% 14459/16260 [1:22:29<10:51,  2.77it/s]iter 14459: loss 5.8486, time: 463.84ms\n",
            " 89% 14460/16260 [1:22:29<11:46,  2.55it/s]iter 14460: loss 5.3275, time: 463.66ms\n",
            " 89% 14461/16260 [1:22:30<12:25,  2.41it/s]iter 14461: loss 5.7706, time: 465.01ms\n",
            " 89% 14462/16260 [1:22:30<12:53,  2.33it/s]iter 14462: loss 5.3804, time: 378.19ms\n",
            " 89% 14463/16260 [1:22:31<12:25,  2.41it/s]iter 14463: loss 5.1288, time: 318.82ms\n",
            " 89% 14464/16260 [1:22:31<11:34,  2.59it/s]iter 14464: loss 2.9680, time: 133.60ms\n",
            " 89% 14465/16260 [1:22:31<09:18,  3.22it/s]iter 14465: loss 4.8317, time: 238.43ms\n",
            " 89% 14466/16260 [1:22:31<08:39,  3.45it/s]iter 14466: loss 5.3668, time: 264.27ms\n",
            " 89% 14467/16260 [1:22:32<08:26,  3.54it/s]iter 14467: loss 5.2777, time: 464.61ms\n",
            " 89% 14468/16260 [1:22:32<10:05,  2.96it/s]iter 14468: loss 3.7498, time: 211.54ms\n",
            " 89% 14469/16260 [1:22:32<08:57,  3.33it/s]iter 14469: loss 5.5181, time: 467.64ms\n",
            " 89% 14470/16260 [1:22:33<10:28,  2.85it/s]iter 14470: loss 5.6137, time: 464.13ms\n",
            " 89% 14471/16260 [1:22:33<11:29,  2.60it/s]iter 14471: loss 4.8751, time: 216.71ms\n",
            " 89% 14472/16260 [1:22:33<09:59,  2.98it/s]iter 14472: loss 5.5287, time: 474.54ms\n",
            " 89% 14473/16260 [1:22:34<11:14,  2.65it/s]iter 14473: loss 5.8298, time: 336.24ms\n",
            " 89% 14474/16260 [1:22:34<10:53,  2.73it/s]iter 14474: loss 4.5168, time: 190.06ms\n",
            " 89% 14475/16260 [1:22:34<09:19,  3.19it/s]iter 14475: loss 4.9065, time: 432.85ms\n",
            " 89% 14476/16260 [1:22:35<10:24,  2.86it/s]iter 14476: loss 3.6803, time: 146.12ms\n",
            " 89% 14477/16260 [1:22:35<08:35,  3.46it/s]iter 14477: loss 4.7867, time: 189.68ms\n",
            " 89% 14478/16260 [1:22:35<07:43,  3.85it/s]iter 14478: loss 5.5877, time: 470.34ms\n",
            " 89% 14479/16260 [1:22:36<09:36,  3.09it/s]iter 14479: loss 4.6230, time: 274.98ms\n",
            " 89% 14480/16260 [1:22:36<09:10,  3.23it/s]iter 14480: loss 4.6510, time: 252.65ms\n",
            " 89% 14481/16260 [1:22:36<08:40,  3.42it/s]iter 14481: loss 5.6319, time: 337.36ms\n",
            " 89% 14482/16260 [1:22:37<09:05,  3.26it/s]iter 14482: loss 3.5354, time: 215.39ms\n",
            " 89% 14483/16260 [1:22:37<08:17,  3.58it/s]iter 14483: loss 4.7792, time: 192.36ms\n",
            " 89% 14484/16260 [1:22:37<07:30,  3.94it/s]iter 14484: loss 4.9207, time: 433.75ms\n",
            " 89% 14485/16260 [1:22:37<09:07,  3.24it/s]iter 14485: loss 5.3860, time: 336.08ms\n",
            " 89% 14486/16260 [1:22:38<09:22,  3.15it/s]iter 14486: loss 5.2867, time: 315.76ms\n",
            " 89% 14487/16260 [1:22:38<09:22,  3.15it/s]iter 14487: loss 4.4331, time: 193.38ms\n",
            " 89% 14488/16260 [1:22:38<08:17,  3.56it/s]iter 14488: loss 4.9760, time: 277.18ms\n",
            " 89% 14489/16260 [1:22:39<08:15,  3.57it/s]iter 14489: loss 4.8453, time: 275.21ms\n",
            " 89% 14490/16260 [1:22:39<08:13,  3.58it/s]iter 14490: loss 5.3192, time: 463.94ms\n",
            " 89% 14491/16260 [1:22:39<09:52,  2.99it/s]iter 14491: loss 5.2649, time: 329.64ms\n",
            " 89% 14492/16260 [1:22:40<09:50,  3.00it/s]iter 14492: loss 6.1694, time: 464.47ms\n",
            " 89% 14493/16260 [1:22:40<10:59,  2.68it/s]iter 14493: loss 5.3729, time: 464.71ms\n",
            " 89% 14494/16260 [1:22:41<11:48,  2.49it/s]iter 14494: loss 5.0368, time: 378.88ms\n",
            " 89% 14495/16260 [1:22:41<11:37,  2.53it/s]iter 14495: loss 4.7622, time: 376.68ms\n",
            " 89% 14496/16260 [1:22:41<11:27,  2.57it/s]iter 14496: loss 5.6523, time: 445.80ms\n",
            " 89% 14497/16260 [1:22:42<11:57,  2.46it/s]iter 14497: loss 4.6578, time: 318.97ms\n",
            " 89% 14498/16260 [1:22:42<11:11,  2.62it/s]iter 14498: loss 5.6971, time: 464.46ms\n",
            " 89% 14499/16260 [1:22:43<11:55,  2.46it/s]iter 14499: loss 5.6241, time: 463.98ms\n",
            " 89% 14500/16260 [1:22:43<12:26,  2.36it/s]iter 14500: loss 5.6259, time: 464.86ms\n",
            " 89% 14501/16260 [1:22:44<12:48,  2.29it/s]iter 14501: loss 4.7941, time: 208.89ms\n",
            " 89% 14502/16260 [1:22:44<10:48,  2.71it/s]iter 14502: loss 5.6344, time: 384.85ms\n",
            " 89% 14503/16260 [1:22:44<10:57,  2.67it/s]iter 14503: loss 4.7921, time: 262.23ms\n",
            " 89% 14504/16260 [1:22:44<09:58,  2.93it/s]iter 14504: loss 4.8897, time: 186.54ms\n",
            " 89% 14505/16260 [1:22:45<08:37,  3.39it/s]iter 14505: loss 4.8982, time: 241.11ms\n",
            " 89% 14506/16260 [1:22:45<08:09,  3.58it/s]iter 14506: loss 5.6690, time: 464.03ms\n",
            " 89% 14507/16260 [1:22:45<09:47,  2.98it/s]iter 14507: loss 4.9288, time: 166.88ms\n",
            " 89% 14508/16260 [1:22:45<08:19,  3.51it/s]iter 14508: loss 4.4877, time: 277.65ms\n",
            " 89% 14509/16260 [1:22:46<08:16,  3.53it/s]iter 14509: loss 5.7459, time: 376.08ms\n",
            " 89% 14510/16260 [1:22:46<09:05,  3.21it/s]iter 14510: loss 5.5638, time: 465.30ms\n",
            " 89% 14511/16260 [1:22:47<10:26,  2.79it/s]iter 14511: loss 5.8339, time: 346.02ms\n",
            " 89% 14512/16260 [1:22:47<10:20,  2.82it/s]iter 14512: loss 5.3262, time: 305.06ms\n",
            " 89% 14513/16260 [1:22:47<09:54,  2.94it/s]iter 14513: loss 5.2122, time: 305.70ms\n",
            " 89% 14514/16260 [1:22:48<09:37,  3.02it/s]iter 14514: loss 5.6648, time: 463.90ms\n",
            " 89% 14515/16260 [1:22:48<10:47,  2.70it/s]iter 14515: loss 4.6635, time: 382.87ms\n",
            " 89% 14516/16260 [1:22:48<10:54,  2.67it/s]iter 14516: loss 4.3465, time: 172.18ms\n",
            " 89% 14517/16260 [1:22:49<09:08,  3.18it/s]iter 14517: loss 5.5587, time: 464.43ms\n",
            " 89% 14518/16260 [1:22:49<10:27,  2.78it/s]iter 14518: loss 5.1600, time: 315.14ms\n",
            " 89% 14519/16260 [1:22:49<10:04,  2.88it/s]iter 14519: loss 4.0506, time: 209.14ms\n",
            " 89% 14520/16260 [1:22:50<08:52,  3.27it/s]iter 14520: loss 5.1009, time: 310.29ms\n",
            " 89% 14521/16260 [1:22:50<08:55,  3.25it/s]iter 14521: loss 5.3271, time: 464.47ms\n",
            " 89% 14522/16260 [1:22:50<10:17,  2.81it/s]iter 14522: loss 4.9301, time: 240.75ms\n",
            " 89% 14523/16260 [1:22:51<09:18,  3.11it/s]iter 14523: loss 6.2139, time: 474.35ms\n",
            " 89% 14524/16260 [1:22:51<10:38,  2.72it/s]iter 14524: loss 5.7301, time: 464.93ms\n",
            " 89% 14525/16260 [1:22:51<11:29,  2.52it/s]iter 14525: loss 5.3076, time: 464.82ms\n",
            " 89% 14526/16260 [1:22:52<12:05,  2.39it/s]iter 14526: loss 5.3781, time: 286.37ms\n",
            " 89% 14527/16260 [1:22:52<10:57,  2.64it/s]iter 14527: loss 5.3818, time: 276.90ms\n",
            " 89% 14528/16260 [1:22:53<10:04,  2.87it/s]iter 14528: loss 3.3362, time: 151.29ms\n",
            " 89% 14529/16260 [1:22:53<08:22,  3.45it/s]iter 14529: loss 5.0950, time: 208.66ms\n",
            " 89% 14530/16260 [1:22:53<07:40,  3.76it/s]iter 14530: loss 5.3378, time: 465.20ms\n",
            " 89% 14531/16260 [1:22:53<09:24,  3.06it/s]iter 14531: loss 4.5373, time: 166.21ms\n",
            " 89% 14532/16260 [1:22:54<08:01,  3.59it/s]iter 14532: loss 4.6277, time: 185.87ms\n",
            " 89% 14533/16260 [1:22:54<07:14,  3.98it/s]iter 14533: loss 5.5733, time: 338.13ms\n",
            " 89% 14534/16260 [1:22:54<07:59,  3.60it/s]iter 14534: loss 5.7560, time: 464.56ms\n",
            " 89% 14535/16260 [1:22:55<09:36,  2.99it/s]iter 14535: loss 5.5070, time: 463.78ms\n",
            " 89% 14536/16260 [1:22:55<10:44,  2.68it/s]iter 14536: loss 3.8989, time: 144.94ms\n",
            " 89% 14537/16260 [1:22:55<08:46,  3.27it/s]iter 14537: loss 5.8487, time: 465.02ms\n",
            " 89% 14538/16260 [1:22:56<10:09,  2.83it/s]iter 14538: loss 5.7052, time: 423.73ms\n",
            " 89% 14539/16260 [1:22:56<10:45,  2.66it/s]iter 14539: loss 5.4658, time: 436.92ms\n",
            " 89% 14540/16260 [1:22:56<11:18,  2.54it/s]iter 14540: loss 5.5847, time: 334.75ms\n",
            " 89% 14541/16260 [1:22:57<10:48,  2.65it/s]iter 14541: loss 4.4395, time: 166.94ms\n",
            " 89% 14542/16260 [1:22:57<09:00,  3.18it/s]iter 14542: loss 5.8798, time: 473.47ms\n",
            " 89% 14543/16260 [1:22:57<10:22,  2.76it/s]iter 14543: loss 4.1844, time: 165.79ms\n",
            " 89% 14544/16260 [1:22:58<08:41,  3.29it/s]iter 14544: loss 4.3379, time: 272.52ms\n",
            " 89% 14545/16260 [1:22:58<08:26,  3.39it/s]iter 14545: loss 5.4759, time: 471.45ms\n",
            " 89% 14546/16260 [1:22:58<09:57,  2.87it/s]iter 14546: loss 4.4355, time: 275.33ms\n",
            " 89% 14547/16260 [1:22:59<09:20,  3.06it/s]iter 14547: loss 3.1873, time: 132.94ms\n",
            " 89% 14548/16260 [1:22:59<07:41,  3.71it/s]iter 14548: loss 5.0649, time: 370.90ms\n",
            " 89% 14549/16260 [1:22:59<08:33,  3.33it/s]iter 14549: loss 5.5899, time: 252.36ms\n",
            " 89% 14550/16260 [1:22:59<08:09,  3.49it/s]iter 14550: loss 4.6157, time: 212.79ms\n",
            " 89% 14551/16260 [1:23:00<07:32,  3.78it/s]iter 14551: loss 4.2601, time: 334.77ms\n",
            " 89% 14552/16260 [1:23:00<08:08,  3.49it/s]iter 14552: loss 5.4325, time: 217.55ms\n",
            " 90% 14553/16260 [1:23:00<07:34,  3.76it/s]iter 14553: loss 4.8668, time: 268.78ms\n",
            " 90% 14554/16260 [1:23:00<07:35,  3.74it/s]iter 14554: loss 5.7981, time: 464.17ms\n",
            " 90% 14555/16260 [1:23:01<09:17,  3.06it/s]iter 14555: loss 5.1921, time: 229.86ms\n",
            " 90% 14556/16260 [1:23:01<08:28,  3.35it/s]iter 14556: loss 5.3313, time: 463.72ms\n",
            " 90% 14557/16260 [1:23:02<09:53,  2.87it/s]iter 14557: loss 5.5111, time: 342.55ms\n",
            " 90% 14558/16260 [1:23:02<09:50,  2.88it/s]iter 14558: loss 5.3811, time: 345.23ms\n",
            " 90% 14559/16260 [1:23:02<09:50,  2.88it/s]iter 14559: loss 5.6950, time: 468.17ms\n",
            " 90% 14560/16260 [1:23:03<10:52,  2.61it/s]iter 14560: loss 5.7911, time: 463.79ms\n",
            " 90% 14561/16260 [1:23:03<11:33,  2.45it/s]iter 14561: loss 5.5942, time: 467.53ms\n",
            " 90% 14562/16260 [1:23:04<12:04,  2.35it/s]iter 14562: loss 3.9311, time: 164.59ms\n",
            " 90% 14563/16260 [1:23:04<09:51,  2.87it/s]iter 14563: loss 5.3999, time: 463.94ms\n",
            " 90% 14564/16260 [1:23:04<10:50,  2.61it/s]iter 14564: loss 5.7488, time: 256.54ms\n",
            " 90% 14565/16260 [1:23:05<09:46,  2.89it/s]iter 14565: loss 4.2587, time: 147.77ms\n",
            " 90% 14566/16260 [1:23:05<08:05,  3.49it/s]iter 14566: loss 5.3966, time: 257.05ms\n",
            " 90% 14567/16260 [1:23:05<07:51,  3.59it/s]iter 14567: loss 5.3841, time: 464.36ms\n",
            " 90% 14568/16260 [1:23:05<09:26,  2.99it/s]iter 14568: loss 4.8114, time: 338.75ms\n",
            " 90% 14569/16260 [1:23:06<09:28,  2.97it/s]iter 14569: loss 4.8802, time: 464.55ms\n",
            " 90% 14570/16260 [1:23:06<10:34,  2.66it/s]iter 14570: loss 6.0412, time: 463.99ms\n",
            " 90% 14571/16260 [1:23:07<11:19,  2.49it/s]iter 14571: loss 4.6776, time: 318.70ms\n",
            " 90% 14572/16260 [1:23:07<10:37,  2.65it/s]iter 14572: loss 4.8547, time: 236.21ms\n",
            " 90% 14573/16260 [1:23:07<09:26,  2.98it/s]iter 14573: loss 4.7670, time: 464.07ms\n",
            " 90% 14574/16260 [1:23:08<10:31,  2.67it/s]iter 14574: loss 5.1579, time: 274.75ms\n",
            " 90% 14575/16260 [1:23:08<09:41,  2.90it/s]iter 14575: loss 5.1597, time: 474.67ms\n",
            " 90% 14576/16260 [1:23:09<10:47,  2.60it/s]iter 14576: loss 4.9964, time: 271.16ms\n",
            " 90% 14577/16260 [1:23:09<09:50,  2.85it/s]iter 14577: loss 5.0797, time: 469.04ms\n",
            " 90% 14578/16260 [1:23:09<10:50,  2.59it/s]iter 14578: loss 5.1930, time: 239.60ms\n",
            " 90% 14579/16260 [1:23:09<09:36,  2.91it/s]iter 14579: loss 5.4864, time: 236.25ms\n",
            " 90% 14580/16260 [1:23:10<08:43,  3.21it/s]iter 14580: loss 4.2197, time: 239.06ms\n",
            " 90% 14581/16260 [1:23:10<08:07,  3.45it/s]iter 14581: loss 5.2508, time: 366.85ms\n",
            " 90% 14582/16260 [1:23:10<08:46,  3.19it/s]iter 14582: loss 5.3678, time: 422.68ms\n",
            " 90% 14583/16260 [1:23:11<09:41,  2.88it/s]iter 14583: loss 5.3606, time: 278.74ms\n",
            " 90% 14584/16260 [1:23:11<09:07,  3.06it/s]iter 14584: loss 5.8584, time: 463.84ms\n",
            " 90% 14585/16260 [1:23:12<10:16,  2.72it/s]iter 14585: loss 5.3145, time: 254.29ms\n",
            " 90% 14586/16260 [1:23:12<09:20,  2.99it/s]iter 14586: loss 4.4006, time: 255.33ms\n",
            " 90% 14587/16260 [1:23:12<08:40,  3.21it/s]iter 14587: loss 5.0539, time: 190.75ms\n",
            " 90% 14588/16260 [1:23:12<07:40,  3.63it/s]iter 14588: loss 5.1765, time: 463.90ms\n",
            " 90% 14589/16260 [1:23:13<09:15,  3.01it/s]iter 14589: loss 5.3107, time: 463.45ms\n",
            " 90% 14590/16260 [1:23:13<10:21,  2.69it/s]iter 14590: loss 5.2835, time: 463.87ms\n",
            " 90% 14591/16260 [1:23:14<11:08,  2.50it/s]iter 14591: loss 4.5022, time: 258.66ms\n",
            " 90% 14592/16260 [1:23:14<09:57,  2.79it/s]iter 14592: loss 4.4553, time: 341.02ms\n",
            " 90% 14593/16260 [1:23:14<09:49,  2.83it/s]iter 14593: loss 5.0751, time: 215.08ms\n",
            " 90% 14594/16260 [1:23:14<08:40,  3.20it/s]iter 14594: loss 5.2459, time: 451.91ms\n",
            " 90% 14595/16260 [1:23:15<09:50,  2.82it/s]iter 14595: loss 4.9663, time: 464.91ms\n",
            " 90% 14596/16260 [1:23:15<10:45,  2.58it/s]iter 14596: loss 5.2966, time: 464.47ms\n",
            " 90% 14597/16260 [1:23:16<11:24,  2.43it/s]iter 14597: loss 4.7553, time: 254.44ms\n",
            " 90% 14598/16260 [1:23:16<10:06,  2.74it/s]iter 14598: loss 4.4298, time: 213.02ms\n",
            " 90% 14599/16260 [1:23:16<08:51,  3.13it/s]iter 14599: loss 5.6733, time: 468.07ms\n",
            " 90% 14600/16260 [1:23:17<10:05,  2.74it/s]iter 14600: loss 5.2317, time: 463.99ms\n",
            " 90% 14601/16260 [1:23:17<10:55,  2.53it/s]iter 14601: loss 5.3764, time: 413.00ms\n",
            " 90% 14602/16260 [1:23:18<11:04,  2.49it/s]iter 14602: loss 4.8013, time: 430.84ms\n",
            " 90% 14603/16260 [1:23:18<11:20,  2.44it/s]iter 14603: loss 5.1753, time: 463.52ms\n",
            " 90% 14604/16260 [1:23:19<11:46,  2.34it/s]iter 14604: loss 5.3436, time: 464.19ms\n",
            " 90% 14605/16260 [1:23:19<12:05,  2.28it/s]iter 14605: loss 5.2299, time: 464.13ms\n",
            " 90% 14606/16260 [1:23:19<12:18,  2.24it/s]iter 14606: loss 4.6754, time: 169.96ms\n",
            " 90% 14607/16260 [1:23:20<10:01,  2.75it/s]iter 14607: loss 2.6625, time: 146.18ms\n",
            " 90% 14608/16260 [1:23:20<08:14,  3.34it/s]iter 14608: loss 4.6962, time: 253.01ms\n",
            " 90% 14609/16260 [1:23:20<07:51,  3.50it/s]iter 14609: loss 5.7392, time: 464.00ms\n",
            " 90% 14610/16260 [1:23:20<09:20,  2.94it/s]iter 14610: loss 4.7006, time: 475.63ms\n",
            " 90% 14611/16260 [1:23:21<10:28,  2.63it/s]iter 14611: loss 4.2785, time: 183.93ms\n",
            " 90% 14612/16260 [1:23:21<08:51,  3.10it/s]iter 14612: loss 4.2396, time: 191.07ms\n",
            " 90% 14613/16260 [1:23:21<07:46,  3.53it/s]iter 14613: loss 5.1996, time: 467.13ms\n",
            " 90% 14614/16260 [1:23:22<09:17,  2.95it/s]iter 14614: loss 5.1291, time: 281.29ms\n",
            " 90% 14615/16260 [1:23:22<08:49,  3.10it/s]iter 14615: loss 4.6700, time: 210.05ms\n",
            " 90% 14616/16260 [1:23:22<07:55,  3.46it/s]iter 14616: loss 5.3029, time: 451.38ms\n",
            " 90% 14617/16260 [1:23:23<09:15,  2.96it/s]iter 14617: loss 4.4381, time: 259.60ms\n",
            " 90% 14618/16260 [1:23:23<08:37,  3.17it/s]iter 14618: loss 4.8661, time: 463.84ms\n",
            " 90% 14619/16260 [1:23:23<09:50,  2.78it/s]iter 14619: loss 5.0726, time: 423.88ms\n",
            " 90% 14620/16260 [1:23:24<10:22,  2.63it/s]iter 14620: loss 4.7039, time: 272.89ms\n",
            " 90% 14621/16260 [1:23:24<09:30,  2.87it/s]iter 14621: loss 4.8865, time: 306.21ms\n",
            " 90% 14622/16260 [1:23:25<09:10,  2.98it/s]iter 14622: loss 4.7606, time: 289.88ms\n",
            " 90% 14623/16260 [1:23:25<08:48,  3.10it/s]iter 14623: loss 5.6931, time: 468.20ms\n",
            " 90% 14624/16260 [1:23:25<09:59,  2.73it/s]iter 14624: loss 4.8936, time: 235.51ms\n",
            " 90% 14625/16260 [1:23:26<08:55,  3.05it/s]iter 14625: loss 4.9610, time: 231.60ms\n",
            " 90% 14626/16260 [1:23:26<08:09,  3.34it/s]iter 14626: loss 5.6235, time: 442.94ms\n",
            " 90% 14627/16260 [1:23:26<09:19,  2.92it/s]iter 14627: loss 5.4079, time: 463.36ms\n",
            " 90% 14628/16260 [1:23:27<10:19,  2.63it/s]iter 14628: loss 3.8495, time: 212.46ms\n",
            " 90% 14629/16260 [1:23:27<08:58,  3.03it/s]iter 14629: loss 5.2043, time: 232.13ms\n",
            " 90% 14630/16260 [1:23:27<08:10,  3.32it/s]iter 14630: loss 4.7395, time: 255.38ms\n",
            " 90% 14631/16260 [1:23:27<07:48,  3.47it/s]iter 14631: loss 5.3124, time: 277.17ms\n",
            " 90% 14632/16260 [1:23:28<07:44,  3.51it/s]iter 14632: loss 5.8478, time: 464.27ms\n",
            " 90% 14633/16260 [1:23:28<09:12,  2.95it/s]iter 14633: loss 5.7399, time: 373.69ms\n",
            " 90% 14634/16260 [1:23:28<09:29,  2.86it/s]iter 14634: loss 5.3225, time: 437.09ms\n",
            " 90% 14635/16260 [1:23:29<10:12,  2.65it/s]iter 14635: loss 4.1099, time: 192.91ms\n",
            " 90% 14636/16260 [1:23:29<08:43,  3.10it/s]iter 14636: loss 4.9952, time: 169.24ms\n",
            " 90% 14637/16260 [1:23:29<07:29,  3.61it/s]iter 14637: loss 4.9600, time: 464.88ms\n",
            " 90% 14638/16260 [1:23:30<09:01,  3.00it/s]iter 14638: loss 5.1607, time: 422.57ms\n",
            " 90% 14639/16260 [1:23:30<09:45,  2.77it/s]iter 14639: loss 5.1090, time: 378.17ms\n",
            " 90% 14640/16260 [1:23:31<09:53,  2.73it/s]iter 14640: loss 4.4483, time: 340.61ms\n",
            " 90% 14641/16260 [1:23:31<09:41,  2.78it/s]iter 14641: loss 5.6044, time: 465.59ms\n",
            " 90% 14642/16260 [1:23:31<10:33,  2.55it/s]iter 14642: loss 4.1665, time: 170.61ms\n",
            " 90% 14643/16260 [1:23:32<08:46,  3.07it/s]iter 14643: loss 5.3382, time: 433.81ms\n",
            " 90% 14644/16260 [1:23:32<09:39,  2.79it/s]iter 14644: loss 4.7917, time: 220.42ms\n",
            " 90% 14645/16260 [1:23:32<08:33,  3.15it/s]iter 14645: loss 6.4499, time: 464.70ms\n",
            " 90% 14646/16260 [1:23:33<09:44,  2.76it/s]iter 14646: loss 5.1308, time: 216.85ms\n",
            " 90% 14647/16260 [1:23:33<08:34,  3.13it/s]iter 14647: loss 4.7955, time: 266.55ms\n",
            " 90% 14648/16260 [1:23:33<08:09,  3.29it/s]iter 14648: loss 5.9915, time: 465.00ms\n",
            " 90% 14649/16260 [1:23:34<09:28,  2.83it/s]iter 14649: loss 4.8287, time: 464.88ms\n",
            " 90% 14650/16260 [1:23:34<10:22,  2.59it/s]iter 14650: loss 3.0270, time: 148.13ms\n",
            " 90% 14651/16260 [1:23:34<08:28,  3.17it/s]iter 14651: loss 5.3487, time: 465.11ms\n",
            " 90% 14652/16260 [1:23:35<09:40,  2.77it/s]iter 14652: loss 5.4842, time: 462.40ms\n",
            " 90% 14653/16260 [1:23:35<10:29,  2.55it/s]iter 14653: loss 5.2782, time: 280.96ms\n",
            " 90% 14654/16260 [1:23:35<09:36,  2.78it/s]iter 14654: loss 4.7260, time: 252.21ms\n",
            " 90% 14655/16260 [1:23:36<08:45,  3.05it/s]iter 14655: loss 5.4038, time: 336.10ms\n",
            " 90% 14656/16260 [1:23:36<08:49,  3.03it/s]iter 14656: loss 4.6255, time: 235.89ms\n",
            " 90% 14657/16260 [1:23:36<08:04,  3.31it/s]iter 14657: loss 4.9296, time: 262.25ms\n",
            " 90% 14658/16260 [1:23:37<07:45,  3.44it/s]iter 14658: loss 4.6943, time: 377.19ms\n",
            " 90% 14659/16260 [1:23:37<08:27,  3.15it/s]iter 14659: loss 6.0701, time: 464.39ms\n",
            " 90% 14660/16260 [1:23:37<09:39,  2.76it/s]iter 14660: loss 4.3013, time: 218.64ms\n",
            " 90% 14661/16260 [1:23:38<08:30,  3.13it/s]iter 14661: loss 5.8457, time: 464.36ms\n",
            " 90% 14662/16260 [1:23:38<09:40,  2.75it/s]iter 14662: loss 4.1991, time: 275.97ms\n",
            " 90% 14663/16260 [1:23:38<08:59,  2.96it/s]iter 14663: loss 5.5295, time: 465.38ms\n",
            " 90% 14664/16260 [1:23:39<10:00,  2.66it/s]iter 14664: loss 5.6037, time: 435.32ms\n",
            " 90% 14665/16260 [1:23:39<10:29,  2.53it/s]iter 14665: loss 5.8026, time: 464.00ms\n",
            " 90% 14666/16260 [1:23:40<11:02,  2.40it/s]iter 14666: loss 3.9964, time: 168.68ms\n",
            " 90% 14667/16260 [1:23:40<09:04,  2.92it/s]iter 14667: loss 4.4855, time: 171.37ms\n",
            " 90% 14668/16260 [1:23:40<07:43,  3.43it/s]iter 14668: loss 5.2894, time: 435.89ms\n",
            " 90% 14669/16260 [1:23:40<08:53,  2.98it/s]iter 14669: loss 4.9850, time: 306.15ms\n",
            " 90% 14670/16260 [1:23:41<08:39,  3.06it/s]iter 14670: loss 5.5351, time: 379.73ms\n",
            " 90% 14671/16260 [1:23:41<09:05,  2.91it/s]iter 14671: loss 4.3069, time: 172.11ms\n",
            " 90% 14672/16260 [1:23:41<07:44,  3.42it/s]iter 14672: loss 5.4653, time: 432.07ms\n",
            " 90% 14673/16260 [1:23:42<08:51,  2.99it/s]iter 14673: loss 4.8936, time: 173.18ms\n",
            " 90% 14674/16260 [1:23:42<07:34,  3.49it/s]iter 14674: loss 4.7900, time: 371.00ms\n",
            " 90% 14675/16260 [1:23:42<08:15,  3.20it/s]iter 14675: loss 4.8749, time: 238.84ms\n",
            " 90% 14676/16260 [1:23:43<07:40,  3.44it/s]iter 14676: loss 5.3990, time: 261.94ms\n",
            " 90% 14677/16260 [1:23:43<07:27,  3.54it/s]iter 14677: loss 4.0148, time: 167.46ms\n",
            " 90% 14678/16260 [1:23:43<06:33,  4.02it/s]iter 14678: loss 4.2001, time: 280.14ms\n",
            " 90% 14679/16260 [1:23:43<06:48,  3.87it/s]iter 14679: loss 5.1912, time: 233.35ms\n",
            " 90% 14680/16260 [1:23:44<06:37,  3.98it/s]iter 14680: loss 4.7651, time: 214.24ms\n",
            " 90% 14681/16260 [1:23:44<06:20,  4.15it/s]iter 14681: loss 4.4222, time: 169.70ms\n",
            " 90% 14682/16260 [1:23:44<05:46,  4.55it/s]iter 14682: loss 4.4664, time: 150.71ms\n",
            " 90% 14683/16260 [1:23:44<05:14,  5.01it/s]iter 14683: loss 4.7087, time: 219.66ms\n",
            " 90% 14684/16260 [1:23:44<05:24,  4.85it/s]iter 14684: loss 5.5558, time: 465.49ms\n",
            " 90% 14685/16260 [1:23:45<07:28,  3.52it/s]iter 14685: loss 4.3326, time: 256.70ms\n",
            " 90% 14686/16260 [1:23:45<07:15,  3.62it/s]iter 14686: loss 5.6842, time: 289.78ms\n",
            " 90% 14687/16260 [1:23:45<07:22,  3.56it/s]iter 14687: loss 5.0122, time: 321.28ms\n",
            " 90% 14688/16260 [1:23:46<07:41,  3.41it/s]iter 14688: loss 4.6491, time: 233.95ms\n",
            " 90% 14689/16260 [1:23:46<07:13,  3.62it/s]iter 14689: loss 5.8008, time: 464.90ms\n",
            " 90% 14690/16260 [1:23:46<08:43,  3.00it/s]iter 14690: loss 4.8390, time: 192.77ms\n",
            " 90% 14691/16260 [1:23:47<07:37,  3.43it/s]iter 14691: loss 5.7244, time: 464.28ms\n",
            " 90% 14692/16260 [1:23:47<08:59,  2.91it/s]iter 14692: loss 4.5259, time: 216.17ms\n",
            " 90% 14693/16260 [1:23:47<07:59,  3.27it/s]iter 14693: loss 5.6285, time: 475.44ms\n",
            " 90% 14694/16260 [1:23:48<09:19,  2.80it/s]iter 14694: loss 5.8964, time: 436.35ms\n",
            " 90% 14695/16260 [1:23:48<09:57,  2.62it/s]iter 14695: loss 5.1706, time: 464.55ms\n",
            " 90% 14696/16260 [1:23:49<10:36,  2.46it/s]iter 14696: loss 5.4624, time: 380.08ms\n",
            " 90% 14697/16260 [1:23:49<10:24,  2.50it/s]iter 14697: loss 4.4053, time: 174.30ms\n",
            " 90% 14698/16260 [1:23:49<08:39,  3.01it/s]iter 14698: loss 5.4275, time: 379.80ms\n",
            " 90% 14699/16260 [1:23:50<09:01,  2.88it/s]iter 14699: loss 5.6490, time: 433.88ms\n",
            " 90% 14700/16260 [1:23:50<09:42,  2.68it/s]iter 14700: loss 4.2194, time: 148.54ms\n",
            " 90% 14701/16260 [1:23:50<07:57,  3.26it/s]iter 14701: loss 5.6174, time: 464.67ms\n",
            " 90% 14702/16260 [1:23:51<09:12,  2.82it/s]iter 14702: loss 5.2519, time: 342.15ms\n",
            " 90% 14703/16260 [1:23:51<09:06,  2.85it/s]iter 14703: loss 3.7497, time: 147.63ms\n",
            " 90% 14704/16260 [1:23:51<07:32,  3.44it/s]iter 14704: loss 4.8782, time: 465.01ms\n",
            " 90% 14705/16260 [1:23:52<08:53,  2.91it/s]iter 14705: loss 5.8821, time: 464.17ms\n",
            " 90% 14706/16260 [1:23:52<09:50,  2.63it/s]iter 14706: loss 4.4221, time: 186.27ms\n",
            " 90% 14707/16260 [1:23:52<08:20,  3.10it/s]iter 14707: loss 4.8325, time: 237.92ms\n",
            " 90% 14708/16260 [1:23:52<07:41,  3.36it/s]iter 14708: loss 5.3913, time: 412.00ms\n",
            " 90% 14709/16260 [1:23:53<08:35,  3.01it/s]iter 14709: loss 5.5430, time: 464.20ms\n",
            " 90% 14710/16260 [1:23:53<09:37,  2.69it/s]iter 14710: loss 5.3757, time: 346.93ms\n",
            " 90% 14711/16260 [1:23:54<09:25,  2.74it/s]iter 14711: loss 4.0109, time: 234.02ms\n",
            " 90% 14712/16260 [1:23:54<08:24,  3.07it/s]iter 14712: loss 5.3511, time: 420.83ms\n",
            " 90% 14713/16260 [1:23:54<09:09,  2.82it/s]iter 14713: loss 5.8717, time: 464.79ms\n",
            " 90% 14714/16260 [1:23:55<10:00,  2.57it/s]iter 14714: loss 5.3911, time: 464.57ms\n",
            " 90% 14715/16260 [1:23:55<10:36,  2.43it/s]iter 14715: loss 5.3693, time: 420.60ms\n",
            " 91% 14716/16260 [1:23:56<10:40,  2.41it/s]iter 14716: loss 4.1307, time: 252.70ms\n",
            " 91% 14717/16260 [1:23:56<09:25,  2.73it/s]iter 14717: loss 5.7614, time: 463.62ms\n",
            " 91% 14718/16260 [1:23:56<10:11,  2.52it/s]iter 14718: loss 4.9201, time: 425.01ms\n",
            " 91% 14719/16260 [1:23:57<10:24,  2.47it/s]iter 14719: loss 4.5308, time: 218.30ms\n",
            " 91% 14720/16260 [1:23:57<08:58,  2.86it/s]iter 14720: loss 5.2797, time: 278.21ms\n",
            " 91% 14721/16260 [1:23:57<08:25,  3.04it/s]iter 14721: loss 5.6379, time: 386.42ms\n",
            " 91% 14722/16260 [1:23:58<08:52,  2.89it/s]iter 14722: loss 5.4249, time: 463.95ms\n",
            " 91% 14723/16260 [1:23:58<09:47,  2.62it/s]iter 14723: loss 5.5514, time: 433.94ms\n",
            " 91% 14724/16260 [1:23:59<10:11,  2.51it/s]iter 14724: loss 4.0886, time: 187.23ms\n",
            " 91% 14725/16260 [1:23:59<08:34,  2.98it/s]iter 14725: loss 4.8778, time: 279.68ms\n",
            " 91% 14726/16260 [1:23:59<08:09,  3.13it/s]iter 14726: loss 5.6468, time: 441.82ms\n",
            " 91% 14727/16260 [1:24:00<09:06,  2.81it/s]iter 14727: loss 5.6577, time: 464.71ms\n",
            " 91% 14728/16260 [1:24:00<09:56,  2.57it/s]iter 14728: loss 5.0547, time: 310.87ms\n",
            " 91% 14729/16260 [1:24:00<09:20,  2.73it/s]iter 14729: loss 4.3484, time: 259.86ms\n",
            " 91% 14730/16260 [1:24:01<08:32,  2.99it/s]iter 14730: loss 5.7590, time: 464.85ms\n",
            " 91% 14731/16260 [1:24:01<09:32,  2.67it/s]iter 14731: loss 5.2392, time: 463.71ms\n",
            " 91% 14732/16260 [1:24:01<10:13,  2.49it/s]iter 14732: loss 5.5178, time: 316.47ms\n",
            " 91% 14733/16260 [1:24:02<09:34,  2.66it/s]iter 14733: loss 5.5377, time: 274.97ms\n",
            " 91% 14734/16260 [1:24:02<08:48,  2.89it/s]iter 14734: loss 5.1585, time: 432.99ms\n",
            " 91% 14735/16260 [1:24:03<09:28,  2.68it/s]iter 14735: loss 4.4759, time: 217.88ms\n",
            " 91% 14736/16260 [1:24:03<08:18,  3.06it/s]iter 14736: loss 5.8299, time: 464.76ms\n",
            " 91% 14737/16260 [1:24:03<09:21,  2.71it/s]iter 14737: loss 4.4926, time: 186.87ms\n",
            " 91% 14738/16260 [1:24:03<07:58,  3.18it/s]iter 14738: loss 4.3233, time: 169.38ms\n",
            " 91% 14739/16260 [1:24:04<06:53,  3.68it/s]iter 14739: loss 5.8015, time: 474.64ms\n",
            " 91% 14740/16260 [1:24:04<08:26,  3.00it/s]iter 14740: loss 6.0485, time: 464.08ms\n",
            " 91% 14741/16260 [1:24:04<09:26,  2.68it/s]iter 14741: loss 5.0655, time: 347.22ms\n",
            " 91% 14742/16260 [1:24:05<09:15,  2.74it/s]iter 14742: loss 6.0695, time: 438.99ms\n",
            " 91% 14743/16260 [1:24:05<09:48,  2.58it/s]iter 14743: loss 4.5171, time: 255.03ms\n",
            " 91% 14744/16260 [1:24:06<08:48,  2.87it/s]iter 14744: loss 5.4544, time: 307.93ms\n",
            " 91% 14745/16260 [1:24:06<08:30,  2.97it/s]iter 14745: loss 4.5743, time: 463.73ms\n",
            " 91% 14746/16260 [1:24:06<09:28,  2.66it/s]iter 14746: loss 5.0647, time: 377.28ms\n",
            " 91% 14747/16260 [1:24:07<09:29,  2.66it/s]iter 14747: loss 5.4328, time: 239.46ms\n",
            " 91% 14748/16260 [1:24:07<08:27,  2.98it/s]iter 14748: loss 5.3996, time: 433.12ms\n",
            " 91% 14749/16260 [1:24:07<09:12,  2.74it/s]iter 14749: loss 5.4306, time: 251.72ms\n",
            " 91% 14750/16260 [1:24:08<08:21,  3.01it/s]iter 14750: loss 5.8273, time: 464.34ms\n",
            " 91% 14751/16260 [1:24:08<09:21,  2.69it/s]iter 14751: loss 5.3158, time: 347.30ms\n",
            " 91% 14752/16260 [1:24:08<09:10,  2.74it/s]iter 14752: loss 4.6213, time: 220.26ms\n",
            " 91% 14753/16260 [1:24:09<08:05,  3.11it/s]iter 14753: loss 3.7967, time: 140.05ms\n",
            " 91% 14754/16260 [1:24:09<06:43,  3.73it/s]iter 14754: loss 4.3685, time: 276.81ms\n",
            " 91% 14755/16260 [1:24:09<06:48,  3.69it/s]iter 14755: loss 4.4473, time: 195.38ms\n",
            " 91% 14756/16260 [1:24:09<06:14,  4.02it/s]iter 14756: loss 5.2765, time: 464.96ms\n",
            " 91% 14757/16260 [1:24:10<07:52,  3.18it/s]iter 14757: loss 4.0905, time: 199.69ms\n",
            " 91% 14758/16260 [1:24:10<07:01,  3.57it/s]iter 14758: loss 5.3325, time: 233.59ms\n",
            " 91% 14759/16260 [1:24:10<06:40,  3.75it/s]iter 14759: loss 4.5071, time: 219.87ms\n",
            " 91% 14760/16260 [1:24:10<06:19,  3.95it/s]iter 14760: loss 5.1095, time: 444.54ms\n",
            " 91% 14761/16260 [1:24:11<07:46,  3.21it/s]iter 14761: loss 5.6268, time: 464.36ms\n",
            " 91% 14762/16260 [1:24:11<08:55,  2.80it/s]iter 14762: loss 4.9063, time: 464.22ms\n",
            " 91% 14763/16260 [1:24:12<09:43,  2.56it/s]iter 14763: loss 5.6547, time: 464.33ms\n",
            " 91% 14764/16260 [1:24:12<10:17,  2.42it/s]iter 14764: loss 5.1595, time: 216.09ms\n",
            " 91% 14765/16260 [1:24:12<08:49,  2.82it/s]iter 14765: loss 4.6680, time: 256.01ms\n",
            " 91% 14766/16260 [1:24:13<08:05,  3.07it/s]iter 14766: loss 5.7861, time: 470.83ms\n",
            " 91% 14767/16260 [1:24:13<09:11,  2.71it/s]iter 14767: loss 5.6705, time: 464.85ms\n",
            " 91% 14768/16260 [1:24:14<09:54,  2.51it/s]iter 14768: loss 4.5205, time: 169.18ms\n",
            " 91% 14769/16260 [1:24:14<08:12,  3.03it/s]iter 14769: loss 4.9417, time: 277.80ms\n",
            " 91% 14770/16260 [1:24:14<07:49,  3.17it/s]iter 14770: loss 5.1137, time: 170.12ms\n",
            " 91% 14771/16260 [1:24:14<06:44,  3.68it/s]iter 14771: loss 5.2867, time: 238.32ms\n",
            " 91% 14772/16260 [1:24:15<06:30,  3.81it/s]iter 14772: loss 3.7640, time: 147.75ms\n",
            " 91% 14773/16260 [1:24:15<05:39,  4.38it/s]iter 14773: loss 5.1739, time: 283.21ms\n",
            " 91% 14774/16260 [1:24:15<06:04,  4.08it/s]iter 14774: loss 5.5348, time: 239.89ms\n",
            " 91% 14775/16260 [1:24:15<06:02,  4.10it/s]iter 14775: loss 4.6412, time: 214.89ms\n",
            " 91% 14776/16260 [1:24:15<05:49,  4.24it/s]iter 14776: loss 5.0541, time: 316.92ms\n",
            " 91% 14777/16260 [1:24:16<06:26,  3.84it/s]iter 14777: loss 3.9839, time: 233.99ms\n",
            " 91% 14778/16260 [1:24:16<06:15,  3.95it/s]iter 14778: loss 4.6362, time: 235.55ms\n",
            " 91% 14779/16260 [1:24:16<06:07,  4.03it/s]iter 14779: loss 6.2487, time: 464.05ms\n",
            " 91% 14780/16260 [1:24:17<07:44,  3.19it/s]iter 14780: loss 4.1953, time: 209.45ms\n",
            " 91% 14781/16260 [1:24:17<06:58,  3.54it/s]iter 14781: loss 4.2329, time: 222.33ms\n",
            " 91% 14782/16260 [1:24:17<06:31,  3.77it/s]iter 14782: loss 4.3048, time: 218.72ms\n",
            " 91% 14783/16260 [1:24:17<06:11,  3.97it/s]iter 14783: loss 5.3522, time: 319.61ms\n",
            " 91% 14784/16260 [1:24:18<06:42,  3.67it/s]iter 14784: loss 5.5277, time: 463.47ms\n",
            " 91% 14785/16260 [1:24:18<08:06,  3.03it/s]iter 14785: loss 4.6901, time: 346.55ms\n",
            " 91% 14786/16260 [1:24:18<08:14,  2.98it/s]iter 14786: loss 4.9801, time: 239.31ms\n",
            " 91% 14787/16260 [1:24:19<07:32,  3.26it/s]iter 14787: loss 4.7847, time: 274.68ms\n",
            " 91% 14788/16260 [1:24:19<07:18,  3.36it/s]iter 14788: loss 4.8478, time: 272.80ms\n",
            " 91% 14789/16260 [1:24:19<07:07,  3.44it/s]iter 14789: loss 4.9978, time: 264.97ms\n",
            " 91% 14790/16260 [1:24:20<06:56,  3.53it/s]iter 14790: loss 5.4464, time: 421.92ms\n",
            " 91% 14791/16260 [1:24:20<07:58,  3.07it/s]iter 14791: loss 4.4690, time: 165.45ms\n",
            " 91% 14792/16260 [1:24:20<06:48,  3.60it/s]iter 14792: loss 5.5024, time: 464.78ms\n",
            " 91% 14793/16260 [1:24:21<08:10,  2.99it/s]iter 14793: loss 5.9632, time: 464.88ms\n",
            " 91% 14794/16260 [1:24:21<09:08,  2.67it/s]iter 14794: loss 5.4764, time: 275.91ms\n",
            " 91% 14795/16260 [1:24:21<08:25,  2.90it/s]iter 14795: loss 4.4655, time: 310.57ms\n",
            " 91% 14796/16260 [1:24:22<08:10,  2.98it/s]iter 14796: loss 4.0976, time: 212.43ms\n",
            " 91% 14797/16260 [1:24:22<07:17,  3.35it/s]iter 14797: loss 4.2838, time: 168.37ms\n",
            " 91% 14798/16260 [1:24:22<06:20,  3.84it/s]iter 14798: loss 5.7983, time: 378.51ms\n",
            " 91% 14799/16260 [1:24:22<07:12,  3.38it/s]iter 14799: loss 5.1020, time: 261.79ms\n",
            " 91% 14800/16260 [1:24:23<06:58,  3.49it/s]iter 14800: loss 4.0992, time: 188.83ms\n",
            " 91% 14801/16260 [1:24:23<06:15,  3.88it/s]iter 14801: loss 5.6460, time: 432.50ms\n",
            " 91% 14802/16260 [1:24:23<07:32,  3.22it/s]iter 14802: loss 5.4309, time: 464.06ms\n",
            " 91% 14803/16260 [1:24:24<08:40,  2.80it/s]iter 14803: loss 4.9496, time: 316.00ms\n",
            " 91% 14804/16260 [1:24:24<08:22,  2.90it/s]iter 14804: loss 5.1248, time: 264.53ms\n",
            " 91% 14805/16260 [1:24:24<07:47,  3.11it/s]iter 14805: loss 5.0381, time: 280.17ms\n",
            " 91% 14806/16260 [1:24:25<07:30,  3.23it/s]iter 14806: loss 5.1806, time: 252.58ms\n",
            " 91% 14807/16260 [1:24:25<07:05,  3.41it/s]iter 14807: loss 5.4134, time: 464.34ms\n",
            " 91% 14808/16260 [1:24:25<08:20,  2.90it/s]iter 14808: loss 5.7449, time: 436.39ms\n",
            " 91% 14809/16260 [1:24:26<09:00,  2.68it/s]iter 14809: loss 5.0264, time: 464.05ms\n",
            " 91% 14810/16260 [1:24:26<09:41,  2.50it/s]iter 14810: loss 5.4952, time: 441.45ms\n",
            " 91% 14811/16260 [1:24:27<09:59,  2.42it/s]iter 14811: loss 5.6498, time: 430.39ms\n",
            " 91% 14812/16260 [1:24:27<10:06,  2.39it/s]iter 14812: loss 5.5230, time: 465.04ms\n",
            " 91% 14813/16260 [1:24:28<10:27,  2.31it/s]iter 14813: loss 5.4691, time: 464.39ms\n",
            " 91% 14814/16260 [1:24:28<10:40,  2.26it/s]iter 14814: loss 5.4550, time: 453.25ms\n",
            " 91% 14815/16260 [1:24:29<10:45,  2.24it/s]iter 14815: loss 5.4618, time: 467.63ms\n",
            " 91% 14816/16260 [1:24:29<10:54,  2.21it/s]iter 14816: loss 3.9250, time: 149.10ms\n",
            " 91% 14817/16260 [1:24:29<08:43,  2.76it/s]iter 14817: loss 4.9069, time: 349.29ms\n",
            " 91% 14818/16260 [1:24:29<08:37,  2.79it/s]iter 14818: loss 5.0071, time: 464.50ms\n",
            " 91% 14819/16260 [1:24:30<09:23,  2.56it/s]iter 14819: loss 4.3449, time: 183.82ms\n",
            " 91% 14820/16260 [1:24:30<07:54,  3.03it/s]iter 14820: loss 5.0952, time: 229.70ms\n",
            " 91% 14821/16260 [1:24:30<07:11,  3.33it/s]iter 14821: loss 4.8961, time: 336.90ms\n",
            " 91% 14822/16260 [1:24:31<07:28,  3.21it/s]iter 14822: loss 5.5454, time: 237.89ms\n",
            " 91% 14823/16260 [1:24:31<06:56,  3.45it/s]iter 14823: loss 4.7116, time: 250.79ms\n",
            " 91% 14824/16260 [1:24:31<06:40,  3.59it/s]iter 14824: loss 5.3511, time: 240.35ms\n",
            " 91% 14825/16260 [1:24:31<06:24,  3.73it/s]iter 14825: loss 5.0119, time: 239.00ms\n",
            " 91% 14826/16260 [1:24:32<06:12,  3.85it/s]iter 14826: loss 5.4563, time: 433.25ms\n",
            " 91% 14827/16260 [1:24:32<07:27,  3.20it/s]iter 14827: loss 4.8905, time: 234.55ms\n",
            " 91% 14828/16260 [1:24:32<06:54,  3.46it/s]iter 14828: loss 4.8832, time: 237.76ms\n",
            " 91% 14829/16260 [1:24:33<06:32,  3.64it/s]iter 14829: loss 5.0996, time: 424.25ms\n",
            " 91% 14830/16260 [1:24:33<07:37,  3.13it/s]iter 14830: loss 5.1532, time: 251.75ms\n",
            " 91% 14831/16260 [1:24:33<07:08,  3.33it/s]iter 14831: loss 5.5080, time: 464.98ms\n",
            " 91% 14832/16260 [1:24:34<08:19,  2.86it/s]iter 14832: loss 5.7080, time: 367.47ms\n",
            " 91% 14833/16260 [1:24:34<08:27,  2.81it/s]iter 14833: loss 5.3881, time: 470.85ms\n",
            " 91% 14834/16260 [1:24:35<09:17,  2.56it/s]iter 14834: loss 5.6741, time: 465.10ms\n",
            " 91% 14835/16260 [1:24:35<09:49,  2.42it/s]iter 14835: loss 5.1110, time: 257.72ms\n",
            " 91% 14836/16260 [1:24:35<08:42,  2.72it/s]iter 14836: loss 2.8178, time: 161.24ms\n",
            " 91% 14837/16260 [1:24:35<07:15,  3.27it/s]iter 14837: loss 5.7343, time: 467.68ms\n",
            " 91% 14838/16260 [1:24:36<08:24,  2.82it/s]iter 14838: loss 5.5025, time: 463.65ms\n",
            " 91% 14839/16260 [1:24:36<09:11,  2.58it/s]iter 14839: loss 3.4233, time: 168.37ms\n",
            " 91% 14840/16260 [1:24:37<07:38,  3.10it/s]iter 14840: loss 5.3657, time: 319.27ms\n",
            " 91% 14841/16260 [1:24:37<07:37,  3.10it/s]iter 14841: loss 4.3168, time: 186.00ms\n",
            " 91% 14842/16260 [1:24:37<06:39,  3.55it/s]iter 14842: loss 5.1895, time: 332.85ms\n",
            " 91% 14843/16260 [1:24:37<07:01,  3.36it/s]iter 14843: loss 5.1852, time: 355.20ms\n",
            " 91% 14844/16260 [1:24:38<07:26,  3.17it/s]iter 14844: loss 4.4094, time: 332.21ms\n",
            " 91% 14845/16260 [1:24:38<07:34,  3.12it/s]iter 14845: loss 4.8559, time: 240.14ms\n",
            " 91% 14846/16260 [1:24:38<07:00,  3.36it/s]iter 14846: loss 5.3202, time: 436.33ms\n",
            " 91% 14847/16260 [1:24:39<07:59,  2.95it/s]iter 14847: loss 4.6703, time: 223.19ms\n",
            " 91% 14848/16260 [1:24:39<07:10,  3.28it/s]iter 14848: loss 6.2067, time: 464.00ms\n",
            " 91% 14849/16260 [1:24:39<08:18,  2.83it/s]iter 14849: loss 5.6541, time: 465.68ms\n",
            " 91% 14850/16260 [1:24:40<09:06,  2.58it/s]iter 14850: loss 4.8129, time: 314.75ms\n",
            " 91% 14851/16260 [1:24:40<08:35,  2.73it/s]iter 14851: loss 5.1744, time: 345.42ms\n",
            " 91% 14852/16260 [1:24:41<08:27,  2.78it/s]iter 14852: loss 5.6079, time: 464.87ms\n",
            " 91% 14853/16260 [1:24:41<09:11,  2.55it/s]iter 14853: loss 4.7389, time: 336.65ms\n",
            " 91% 14854/16260 [1:24:41<08:48,  2.66it/s]iter 14854: loss 4.1422, time: 169.17ms\n",
            " 91% 14855/16260 [1:24:42<07:21,  3.18it/s]iter 14855: loss 5.1146, time: 413.60ms\n",
            " 91% 14856/16260 [1:24:42<08:04,  2.90it/s]iter 14856: loss 5.5370, time: 466.04ms\n",
            " 91% 14857/16260 [1:24:42<08:55,  2.62it/s]iter 14857: loss 5.4032, time: 377.92ms\n",
            " 91% 14858/16260 [1:24:43<08:54,  2.62it/s]iter 14858: loss 5.6040, time: 464.79ms\n",
            " 91% 14859/16260 [1:24:43<09:29,  2.46it/s]iter 14859: loss 5.3289, time: 317.44ms\n",
            " 91% 14860/16260 [1:24:44<08:52,  2.63it/s]iter 14860: loss 4.4728, time: 280.10ms\n",
            " 91% 14861/16260 [1:24:44<08:10,  2.85it/s]iter 14861: loss 5.6690, time: 463.06ms\n",
            " 91% 14862/16260 [1:24:44<08:58,  2.60it/s]iter 14862: loss 5.2125, time: 430.82ms\n",
            " 91% 14863/16260 [1:24:45<09:17,  2.50it/s]iter 14863: loss 5.5163, time: 464.41ms\n",
            " 91% 14864/16260 [1:24:45<09:45,  2.39it/s]iter 14864: loss 5.3949, time: 284.02ms\n",
            " 91% 14865/16260 [1:24:46<08:48,  2.64it/s]iter 14865: loss 5.8633, time: 472.02ms\n",
            " 91% 14866/16260 [1:24:46<09:28,  2.45it/s]iter 14866: loss 4.9899, time: 413.60ms\n",
            " 91% 14867/16260 [1:24:46<09:31,  2.44it/s]iter 14867: loss 5.0850, time: 463.77ms\n",
            " 91% 14868/16260 [1:24:47<09:53,  2.34it/s]iter 14868: loss 5.9689, time: 442.95ms\n",
            " 91% 14869/16260 [1:24:47<10:00,  2.32it/s]iter 14869: loss 5.7454, time: 464.34ms\n",
            " 91% 14870/16260 [1:24:48<10:14,  2.26it/s]iter 14870: loss 4.9819, time: 463.88ms\n",
            " 91% 14871/16260 [1:24:48<10:23,  2.23it/s]iter 14871: loss 5.0141, time: 254.56ms\n",
            " 91% 14872/16260 [1:24:49<09:03,  2.56it/s]iter 14872: loss 5.3290, time: 307.53ms\n",
            " 91% 14873/16260 [1:24:49<08:28,  2.73it/s]iter 14873: loss 4.4944, time: 191.26ms\n",
            " 91% 14874/16260 [1:24:49<07:15,  3.18it/s]iter 14874: loss 4.5191, time: 170.54ms\n",
            " 91% 14875/16260 [1:24:49<06:16,  3.68it/s]iter 14875: loss 3.4441, time: 146.45ms\n",
            " 91% 14876/16260 [1:24:49<05:24,  4.26it/s]iter 14876: loss 5.6949, time: 464.13ms\n",
            " 91% 14877/16260 [1:24:50<07:00,  3.29it/s]iter 14877: loss 4.9213, time: 250.46ms\n",
            " 92% 14878/16260 [1:24:50<06:38,  3.47it/s]iter 14878: loss 5.6943, time: 370.58ms\n",
            " 92% 14879/16260 [1:24:50<07:12,  3.19it/s]iter 14879: loss 5.9111, time: 466.64ms\n",
            " 92% 14880/16260 [1:24:51<08:16,  2.78it/s]iter 14880: loss 5.0234, time: 217.01ms\n",
            " 92% 14881/16260 [1:24:51<07:17,  3.15it/s]iter 14881: loss 5.0170, time: 276.67ms\n",
            " 92% 14882/16260 [1:24:51<07:01,  3.27it/s]iter 14882: loss 4.8202, time: 189.60ms\n",
            " 92% 14883/16260 [1:24:52<06:13,  3.69it/s]iter 14883: loss 5.5234, time: 370.55ms\n",
            " 92% 14884/16260 [1:24:52<06:55,  3.32it/s]iter 14884: loss 5.3804, time: 425.41ms\n",
            " 92% 14885/16260 [1:24:52<07:46,  2.95it/s]iter 14885: loss 5.2532, time: 469.68ms\n",
            " 92% 14886/16260 [1:24:53<08:40,  2.64it/s]iter 14886: loss 5.6269, time: 431.98ms\n",
            " 92% 14887/16260 [1:24:53<09:02,  2.53it/s]iter 14887: loss 5.7727, time: 464.28ms\n",
            " 92% 14888/16260 [1:24:54<09:31,  2.40it/s]iter 14888: loss 4.7488, time: 193.38ms\n",
            " 92% 14889/16260 [1:24:54<07:59,  2.86it/s]iter 14889: loss 4.5261, time: 213.17ms\n",
            " 92% 14890/16260 [1:24:54<07:03,  3.23it/s]iter 14890: loss 6.1831, time: 464.86ms\n",
            " 92% 14891/16260 [1:24:55<08:08,  2.80it/s]iter 14891: loss 3.7198, time: 167.00ms\n",
            " 92% 14892/16260 [1:24:55<06:50,  3.33it/s]iter 14892: loss 4.4826, time: 170.07ms\n",
            " 92% 14893/16260 [1:24:55<05:57,  3.82it/s]iter 14893: loss 4.8451, time: 253.92ms\n",
            " 92% 14894/16260 [1:24:55<05:54,  3.85it/s]iter 14894: loss 5.3296, time: 441.67ms\n",
            " 92% 14895/16260 [1:24:56<07:09,  3.18it/s]iter 14895: loss 5.3999, time: 464.75ms\n",
            " 92% 14896/16260 [1:24:56<08:11,  2.78it/s]iter 14896: loss 4.6067, time: 187.43ms\n",
            " 92% 14897/16260 [1:24:56<07:01,  3.24it/s]iter 14897: loss 4.3935, time: 191.78ms\n",
            " 92% 14898/16260 [1:24:57<06:13,  3.64it/s]iter 14898: loss 5.4208, time: 464.22ms\n",
            " 92% 14899/16260 [1:24:57<07:31,  3.01it/s]iter 14899: loss 4.4438, time: 340.36ms\n",
            " 92% 14900/16260 [1:24:57<07:35,  2.99it/s]iter 14900: loss 5.5712, time: 464.20ms\n",
            " 92% 14901/16260 [1:24:58<08:28,  2.67it/s]iter 14901: loss 4.7045, time: 232.80ms\n",
            " 92% 14902/16260 [1:24:58<07:31,  3.01it/s]iter 14902: loss 3.5847, time: 138.68ms\n",
            " 92% 14903/16260 [1:24:58<06:12,  3.64it/s]iter 14903: loss 5.6080, time: 373.78ms\n",
            " 92% 14904/16260 [1:24:59<06:53,  3.28it/s]iter 14904: loss 3.4592, time: 150.32ms\n",
            " 92% 14905/16260 [1:24:59<05:51,  3.86it/s]iter 14905: loss 5.3077, time: 468.23ms\n",
            " 92% 14906/16260 [1:24:59<07:16,  3.10it/s]iter 14906: loss 5.1633, time: 262.18ms\n",
            " 92% 14907/16260 [1:24:59<06:52,  3.28it/s]iter 14907: loss 5.8066, time: 463.62ms\n",
            " 92% 14908/16260 [1:25:00<07:57,  2.83it/s]iter 14908: loss 5.4066, time: 380.90ms\n",
            " 92% 14909/16260 [1:25:00<08:08,  2.76it/s]iter 14909: loss 4.8170, time: 260.83ms\n",
            " 92% 14910/16260 [1:25:01<07:28,  3.01it/s]iter 14910: loss 4.8306, time: 217.01ms\n",
            " 92% 14911/16260 [1:25:01<06:42,  3.35it/s]iter 14911: loss 5.0993, time: 467.06ms\n",
            " 92% 14912/16260 [1:25:01<07:50,  2.86it/s]iter 14912: loss 5.0455, time: 240.44ms\n",
            " 92% 14913/16260 [1:25:01<07:06,  3.15it/s]iter 14913: loss 4.2862, time: 311.90ms\n",
            " 92% 14914/16260 [1:25:02<07:05,  3.16it/s]iter 14914: loss 4.6119, time: 323.56ms\n",
            " 92% 14915/16260 [1:25:02<07:08,  3.14it/s]iter 14915: loss 5.1215, time: 238.07ms\n",
            " 92% 14916/16260 [1:25:02<06:36,  3.39it/s]iter 14916: loss 5.2079, time: 464.10ms\n",
            " 92% 14917/16260 [1:25:03<07:45,  2.89it/s]iter 14917: loss 5.7458, time: 417.44ms\n",
            " 92% 14918/16260 [1:25:03<08:13,  2.72it/s]iter 14918: loss 5.8976, time: 464.16ms\n",
            " 92% 14919/16260 [1:25:04<08:52,  2.52it/s]iter 14919: loss 5.0075, time: 257.75ms\n",
            " 92% 14920/16260 [1:25:04<07:57,  2.81it/s]iter 14920: loss 5.3764, time: 463.96ms\n",
            " 92% 14921/16260 [1:25:04<08:40,  2.57it/s]iter 14921: loss 4.6729, time: 464.39ms\n",
            " 92% 14922/16260 [1:25:05<09:11,  2.43it/s]iter 14922: loss 5.0771, time: 308.60ms\n",
            " 92% 14923/16260 [1:25:05<08:30,  2.62it/s]iter 14923: loss 5.0221, time: 310.85ms\n",
            " 92% 14924/16260 [1:25:06<08:02,  2.77it/s]iter 14924: loss 5.1874, time: 319.30ms\n",
            " 92% 14925/16260 [1:25:06<07:45,  2.87it/s]iter 14925: loss 5.1058, time: 464.09ms\n",
            " 92% 14926/16260 [1:25:06<08:32,  2.60it/s]iter 14926: loss 4.2285, time: 232.26ms\n",
            " 92% 14927/16260 [1:25:07<07:31,  2.95it/s]iter 14927: loss 5.3224, time: 464.03ms\n",
            " 92% 14928/16260 [1:25:07<08:22,  2.65it/s]iter 14928: loss 4.9850, time: 279.03ms\n",
            " 92% 14929/16260 [1:25:07<07:43,  2.87it/s]iter 14929: loss 5.4331, time: 463.84ms\n",
            " 92% 14930/16260 [1:25:08<08:29,  2.61it/s]iter 14930: loss 4.9058, time: 212.23ms\n",
            " 92% 14931/16260 [1:25:08<07:21,  3.01it/s]iter 14931: loss 5.7313, time: 318.04ms\n",
            " 92% 14932/16260 [1:25:08<07:16,  3.04it/s]iter 14932: loss 4.6713, time: 283.91ms\n",
            " 92% 14933/16260 [1:25:09<06:58,  3.17it/s]iter 14933: loss 5.3495, time: 464.32ms\n",
            " 92% 14934/16260 [1:25:09<07:58,  2.77it/s]iter 14934: loss 4.7603, time: 273.52ms\n",
            " 92% 14935/16260 [1:25:09<07:23,  2.98it/s]iter 14935: loss 5.6653, time: 464.06ms\n",
            " 92% 14936/16260 [1:25:10<08:15,  2.67it/s]iter 14936: loss 4.9989, time: 250.86ms\n",
            " 92% 14937/16260 [1:25:10<07:26,  2.96it/s]iter 14937: loss 5.0487, time: 376.64ms\n",
            " 92% 14938/16260 [1:25:10<07:42,  2.86it/s]iter 14938: loss 4.7618, time: 334.42ms\n",
            " 92% 14939/16260 [1:25:11<07:36,  2.89it/s]iter 14939: loss 4.8831, time: 463.67ms\n",
            " 92% 14940/16260 [1:25:11<08:23,  2.62it/s]iter 14940: loss 5.7261, time: 464.66ms\n",
            " 92% 14941/16260 [1:25:12<08:56,  2.46it/s]iter 14941: loss 5.5173, time: 464.53ms\n",
            " 92% 14942/16260 [1:25:12<09:19,  2.35it/s]iter 14942: loss 4.2800, time: 169.29ms\n",
            " 92% 14943/16260 [1:25:12<07:39,  2.87it/s]iter 14943: loss 3.9979, time: 379.55ms\n",
            " 92% 14944/16260 [1:25:13<07:51,  2.79it/s]iter 14944: loss 5.8307, time: 463.43ms\n",
            " 92% 14945/16260 [1:25:13<08:33,  2.56it/s]iter 14945: loss 4.9133, time: 310.81ms\n",
            " 92% 14946/16260 [1:25:13<08:02,  2.73it/s]iter 14946: loss 4.1628, time: 148.69ms\n",
            " 92% 14947/16260 [1:25:14<06:36,  3.31it/s]iter 14947: loss 5.7533, time: 463.31ms\n",
            " 92% 14948/16260 [1:25:14<07:40,  2.85it/s]iter 14948: loss 4.7563, time: 463.69ms\n",
            " 92% 14949/16260 [1:25:15<08:24,  2.60it/s]iter 14949: loss 5.2563, time: 463.80ms\n",
            " 92% 14950/16260 [1:25:15<08:56,  2.44it/s]iter 14950: loss 5.3245, time: 463.55ms\n",
            " 92% 14951/16260 [1:25:15<09:17,  2.35it/s]iter 14951: loss 4.9887, time: 336.00ms\n",
            " 92% 14952/16260 [1:25:16<08:42,  2.50it/s]iter 14952: loss 4.4638, time: 210.34ms\n",
            " 92% 14953/16260 [1:25:16<07:28,  2.91it/s]iter 14953: loss 5.5774, time: 237.93ms\n",
            " 92% 14954/16260 [1:25:16<06:47,  3.20it/s]iter 14954: loss 5.0780, time: 476.35ms\n",
            " 92% 14955/16260 [1:25:17<07:52,  2.76it/s]iter 14955: loss 5.3973, time: 463.36ms\n",
            " 92% 14956/16260 [1:25:17<08:32,  2.55it/s]iter 14956: loss 4.8802, time: 212.00ms\n",
            " 92% 14957/16260 [1:25:17<07:21,  2.95it/s]iter 14957: loss 4.0012, time: 170.34ms\n",
            " 92% 14958/16260 [1:25:18<06:16,  3.46it/s]iter 14958: loss 4.6462, time: 413.78ms\n",
            " 92% 14959/16260 [1:25:18<07:05,  3.06it/s]iter 14959: loss 4.6474, time: 254.13ms\n",
            " 92% 14960/16260 [1:25:18<06:37,  3.27it/s]iter 14960: loss 5.6295, time: 446.87ms\n",
            " 92% 14961/16260 [1:25:19<07:32,  2.87it/s]iter 14961: loss 4.7603, time: 254.09ms\n",
            " 92% 14962/16260 [1:25:19<06:56,  3.12it/s]iter 14962: loss 5.5706, time: 368.36ms\n",
            " 92% 14963/16260 [1:25:19<07:14,  2.98it/s]iter 14963: loss 4.2729, time: 168.72ms\n",
            " 92% 14964/16260 [1:25:20<06:10,  3.50it/s]iter 14964: loss 5.4807, time: 476.03ms\n",
            " 92% 14965/16260 [1:25:20<07:24,  2.91it/s]iter 14965: loss 3.7316, time: 146.58ms\n",
            " 92% 14966/16260 [1:25:20<06:08,  3.51it/s]iter 14966: loss 3.4948, time: 148.91ms\n",
            " 92% 14967/16260 [1:25:20<05:16,  4.09it/s]iter 14967: loss 3.0482, time: 135.13ms\n",
            " 92% 14968/16260 [1:25:20<04:33,  4.72it/s]iter 14968: loss 5.9880, time: 463.82ms\n",
            " 92% 14969/16260 [1:25:21<06:11,  3.47it/s]iter 14969: loss 4.6028, time: 378.39ms\n",
            " 92% 14970/16260 [1:25:21<06:47,  3.17it/s]iter 14970: loss 4.9883, time: 423.40ms\n",
            " 92% 14971/16260 [1:25:22<07:29,  2.87it/s]iter 14971: loss 4.8238, time: 254.43ms\n",
            " 92% 14972/16260 [1:25:22<06:53,  3.12it/s]iter 14972: loss 5.2981, time: 464.11ms\n",
            " 92% 14973/16260 [1:25:22<07:48,  2.75it/s]iter 14973: loss 4.7583, time: 305.41ms\n",
            " 92% 14974/16260 [1:25:23<07:26,  2.88it/s]iter 14974: loss 4.9494, time: 272.45ms\n",
            " 92% 14975/16260 [1:25:23<06:57,  3.07it/s]iter 14975: loss 5.5115, time: 468.04ms\n",
            " 92% 14976/16260 [1:25:23<07:53,  2.71it/s]iter 14976: loss 5.6578, time: 470.96ms\n",
            " 92% 14977/16260 [1:25:24<08:32,  2.50it/s]iter 14977: loss 5.0486, time: 464.04ms\n",
            " 92% 14978/16260 [1:25:24<08:57,  2.38it/s]iter 14978: loss 5.2772, time: 471.58ms\n",
            " 92% 14979/16260 [1:25:25<09:17,  2.30it/s]iter 14979: loss 5.4610, time: 432.75ms\n",
            " 92% 14980/16260 [1:25:25<09:17,  2.30it/s]iter 14980: loss 5.4782, time: 464.51ms\n",
            " 92% 14981/16260 [1:25:26<09:28,  2.25it/s]iter 14981: loss 3.4187, time: 148.25ms\n",
            " 92% 14982/16260 [1:25:26<07:35,  2.81it/s]iter 14982: loss 5.8108, time: 463.09ms\n",
            " 92% 14983/16260 [1:25:26<08:16,  2.57it/s]iter 14983: loss 4.7656, time: 227.17ms\n",
            " 92% 14984/16260 [1:25:27<07:14,  2.93it/s]iter 14984: loss 4.8101, time: 219.12ms\n",
            " 92% 14985/16260 [1:25:27<06:28,  3.28it/s]iter 14985: loss 5.8346, time: 464.48ms\n",
            " 92% 14986/16260 [1:25:27<07:30,  2.83it/s]iter 14986: loss 4.9669, time: 210.49ms\n",
            " 92% 14987/16260 [1:25:28<06:35,  3.22it/s]iter 14987: loss 5.7797, time: 437.98ms\n",
            " 92% 14988/16260 [1:25:28<07:24,  2.86it/s]iter 14988: loss 5.0684, time: 464.17ms\n",
            " 92% 14989/16260 [1:25:28<08:08,  2.60it/s]iter 14989: loss 4.1245, time: 213.79ms\n",
            " 92% 14990/16260 [1:25:29<07:03,  3.00it/s]iter 14990: loss 4.2005, time: 170.33ms\n",
            " 92% 14991/16260 [1:25:29<06:02,  3.50it/s]iter 14991: loss 5.3030, time: 308.69ms\n",
            " 92% 14992/16260 [1:25:29<06:11,  3.42it/s]iter 14992: loss 5.1202, time: 217.45ms\n",
            " 92% 14993/16260 [1:25:29<05:42,  3.69it/s]iter 14993: loss 3.8533, time: 164.42ms\n",
            " 92% 14994/16260 [1:25:30<05:02,  4.18it/s]iter 14994: loss 3.9117, time: 236.39ms\n",
            " 92% 14995/16260 [1:25:30<05:02,  4.19it/s]iter 14995: loss 3.8084, time: 144.73ms\n",
            " 92% 14996/16260 [1:25:30<04:26,  4.74it/s]iter 14996: loss 5.3667, time: 471.07ms\n",
            " 92% 14997/16260 [1:25:30<06:05,  3.45it/s]iter 14997: loss 3.6362, time: 170.50ms\n",
            " 92% 14998/16260 [1:25:31<05:21,  3.93it/s]iter 14998: loss 4.6460, time: 233.02ms\n",
            " 92% 14999/16260 [1:25:31<05:13,  4.02it/s]iter 14999: loss 6.0010, time: 465.08ms\n",
            " 92% 15000/16260 [1:25:31<06:35,  3.18it/s]iter 15000: loss 6.0309, time: 464.96ms\n",
            " 92% 15001/16260 [1:25:32<07:33,  2.78it/s]iter 15001: loss 4.8968, time: 232.37ms\n",
            " 92% 15002/16260 [1:25:32<06:45,  3.10it/s]iter 15002: loss 4.1439, time: 172.90ms\n",
            " 92% 15003/16260 [1:25:32<05:49,  3.60it/s]iter 15003: loss 4.8585, time: 463.38ms\n",
            " 92% 15004/16260 [1:25:33<06:59,  2.99it/s]iter 15004: loss 5.4546, time: 463.87ms\n",
            " 92% 15005/16260 [1:25:33<07:48,  2.68it/s]iter 15005: loss 4.5528, time: 308.25ms\n",
            " 92% 15006/16260 [1:25:33<07:24,  2.82it/s]iter 15006: loss 5.3836, time: 464.53ms\n",
            " 92% 15007/16260 [1:25:34<08:06,  2.58it/s]iter 15007: loss 5.3046, time: 280.88ms\n",
            " 92% 15008/16260 [1:25:34<07:26,  2.81it/s]iter 15008: loss 4.9524, time: 260.32ms\n",
            " 92% 15009/16260 [1:25:34<06:50,  3.05it/s]iter 15009: loss 5.1539, time: 430.35ms\n",
            " 92% 15010/16260 [1:25:35<07:28,  2.78it/s]iter 15010: loss 5.1199, time: 464.31ms\n",
            " 92% 15011/16260 [1:25:35<08:08,  2.56it/s]iter 15011: loss 4.9483, time: 431.81ms\n",
            " 92% 15012/16260 [1:25:36<08:24,  2.48it/s]iter 15012: loss 5.8121, time: 463.97ms\n",
            " 92% 15013/16260 [1:25:36<08:46,  2.37it/s]iter 15013: loss 5.0835, time: 283.30ms\n",
            " 92% 15014/16260 [1:25:36<07:54,  2.62it/s]iter 15014: loss 4.7074, time: 464.32ms\n",
            " 92% 15015/16260 [1:25:37<08:26,  2.46it/s]iter 15015: loss 5.2871, time: 464.02ms\n",
            " 92% 15016/16260 [1:25:37<08:47,  2.36it/s]iter 15016: loss 4.7611, time: 465.00ms\n",
            " 92% 15017/16260 [1:25:38<09:03,  2.29it/s]iter 15017: loss 5.4613, time: 419.75ms\n",
            " 92% 15018/16260 [1:25:38<08:57,  2.31it/s]iter 15018: loss 2.6279, time: 134.77ms\n",
            " 92% 15019/16260 [1:25:38<07:06,  2.91it/s]iter 15019: loss 5.1931, time: 464.57ms\n",
            " 92% 15020/16260 [1:25:39<07:51,  2.63it/s]iter 15020: loss 5.0502, time: 168.06ms\n",
            " 92% 15021/16260 [1:25:39<06:32,  3.15it/s]iter 15021: loss 5.0144, time: 278.54ms\n",
            " 92% 15022/16260 [1:25:39<06:18,  3.27it/s]iter 15022: loss 4.6525, time: 225.48ms\n",
            " 92% 15023/16260 [1:25:40<05:49,  3.54it/s]iter 15023: loss 5.5069, time: 464.38ms\n",
            " 92% 15024/16260 [1:25:40<06:57,  2.96it/s]iter 15024: loss 5.1763, time: 464.08ms\n",
            " 92% 15025/16260 [1:25:40<07:44,  2.66it/s]iter 15025: loss 4.3831, time: 278.48ms\n",
            " 92% 15026/16260 [1:25:41<07:08,  2.88it/s]iter 15026: loss 4.6871, time: 317.53ms\n",
            " 92% 15027/16260 [1:25:41<06:57,  2.95it/s]iter 15027: loss 5.6454, time: 464.40ms\n",
            " 92% 15028/16260 [1:25:42<07:44,  2.65it/s]iter 15028: loss 5.8194, time: 466.85ms\n",
            " 92% 15029/16260 [1:25:42<08:17,  2.47it/s]iter 15029: loss 5.0767, time: 433.35ms\n",
            " 92% 15030/16260 [1:25:42<08:28,  2.42it/s]iter 15030: loss 4.8417, time: 169.92ms\n",
            " 92% 15031/16260 [1:25:43<06:58,  2.93it/s]iter 15031: loss 4.8165, time: 464.30ms\n",
            " 92% 15032/16260 [1:25:43<07:44,  2.64it/s]iter 15032: loss 5.8869, time: 463.85ms\n",
            " 92% 15033/16260 [1:25:44<08:16,  2.47it/s]iter 15033: loss 5.6482, time: 423.77ms\n",
            " 92% 15034/16260 [1:25:44<08:23,  2.43it/s]iter 15034: loss 5.5088, time: 446.70ms\n",
            " 92% 15035/16260 [1:25:44<08:36,  2.37it/s]iter 15035: loss 4.1808, time: 148.53ms\n",
            " 92% 15036/16260 [1:25:45<06:56,  2.94it/s]iter 15036: loss 5.3315, time: 437.15ms\n",
            " 92% 15037/16260 [1:25:45<07:32,  2.70it/s]iter 15037: loss 5.9814, time: 464.58ms\n",
            " 92% 15038/16260 [1:25:45<08:07,  2.51it/s]iter 15038: loss 4.2091, time: 170.67ms\n",
            " 92% 15039/16260 [1:25:46<06:44,  3.02it/s]iter 15039: loss 5.1867, time: 243.24ms\n",
            " 92% 15040/16260 [1:25:46<06:12,  3.28it/s]iter 15040: loss 5.7357, time: 464.07ms\n",
            " 93% 15041/16260 [1:25:46<07:10,  2.83it/s]iter 15041: loss 4.4680, time: 187.34ms\n",
            " 93% 15042/16260 [1:25:47<06:10,  3.29it/s]iter 15042: loss 4.7629, time: 464.71ms\n",
            " 93% 15043/16260 [1:25:47<07:09,  2.84it/s]iter 15043: loss 4.6862, time: 279.18ms\n",
            " 93% 15044/16260 [1:25:47<06:42,  3.02it/s]iter 15044: loss 5.4089, time: 280.93ms\n",
            " 93% 15045/16260 [1:25:48<06:24,  3.16it/s]iter 15045: loss 4.7890, time: 304.17ms\n",
            " 93% 15046/16260 [1:25:48<06:20,  3.19it/s]iter 15046: loss 5.4353, time: 316.90ms\n",
            " 93% 15047/16260 [1:25:48<06:21,  3.18it/s]iter 15047: loss 5.1814, time: 464.31ms\n",
            " 93% 15048/16260 [1:25:49<07:16,  2.78it/s]iter 15048: loss 4.6259, time: 189.78ms\n",
            " 93% 15049/16260 [1:25:49<06:14,  3.23it/s]iter 15049: loss 4.1728, time: 149.32ms\n",
            " 93% 15050/16260 [1:25:49<05:16,  3.82it/s]iter 15050: loss 4.8301, time: 473.28ms\n",
            " 93% 15051/16260 [1:25:50<06:33,  3.07it/s]iter 15051: loss 5.8094, time: 463.93ms\n",
            " 93% 15052/16260 [1:25:50<07:24,  2.72it/s]iter 15052: loss 5.1793, time: 239.33ms\n",
            " 93% 15053/16260 [1:25:50<06:37,  3.03it/s]iter 15053: loss 5.2955, time: 463.92ms\n",
            " 93% 15054/16260 [1:25:51<07:26,  2.70it/s]iter 15054: loss 4.2333, time: 191.51ms\n",
            " 93% 15055/16260 [1:25:51<06:22,  3.15it/s]iter 15055: loss 4.4313, time: 189.75ms\n",
            " 93% 15056/16260 [1:25:51<05:36,  3.58it/s]iter 15056: loss 5.4105, time: 312.21ms\n",
            " 93% 15057/16260 [1:25:51<05:48,  3.45it/s]iter 15057: loss 4.1650, time: 164.63ms\n",
            " 93% 15058/16260 [1:25:52<05:03,  3.96it/s]iter 15058: loss 4.6249, time: 471.17ms\n",
            " 93% 15059/16260 [1:25:52<06:22,  3.14it/s]iter 15059: loss 5.3075, time: 322.95ms\n",
            " 93% 15060/16260 [1:25:52<06:24,  3.12it/s]iter 15060: loss 4.8203, time: 237.82ms\n",
            " 93% 15061/16260 [1:25:53<05:55,  3.38it/s]iter 15061: loss 4.7758, time: 218.13ms\n",
            " 93% 15062/16260 [1:25:53<05:27,  3.66it/s]iter 15062: loss 4.2119, time: 212.22ms\n",
            " 93% 15063/16260 [1:25:53<05:05,  3.91it/s]iter 15063: loss 4.7261, time: 235.86ms\n",
            " 93% 15064/16260 [1:25:53<04:59,  4.00it/s]iter 15064: loss 4.4315, time: 463.60ms\n",
            " 93% 15065/16260 [1:25:54<06:15,  3.18it/s]iter 15065: loss 4.0448, time: 169.15ms\n",
            " 93% 15066/16260 [1:25:54<05:24,  3.68it/s]iter 15066: loss 5.0965, time: 382.26ms\n",
            " 93% 15067/16260 [1:25:54<06:04,  3.28it/s]iter 15067: loss 5.5939, time: 465.86ms\n",
            " 93% 15068/16260 [1:25:55<07:01,  2.83it/s]iter 15068: loss 3.9317, time: 166.23ms\n",
            " 93% 15069/16260 [1:25:55<05:54,  3.36it/s]iter 15069: loss 5.9772, time: 464.15ms\n",
            " 93% 15070/16260 [1:25:55<06:54,  2.87it/s]iter 15070: loss 5.8755, time: 464.23ms\n",
            " 93% 15071/16260 [1:25:56<07:36,  2.61it/s]iter 15071: loss 6.1528, time: 467.73ms\n",
            " 93% 15072/16260 [1:25:56<08:06,  2.44it/s]iter 15072: loss 5.6471, time: 463.21ms\n",
            " 93% 15073/16260 [1:25:57<08:25,  2.35it/s]iter 15073: loss 3.9765, time: 151.62ms\n",
            " 93% 15074/16260 [1:25:57<06:47,  2.91it/s]iter 15074: loss 4.9516, time: 423.70ms\n",
            " 93% 15075/16260 [1:25:57<07:16,  2.71it/s]iter 15075: loss 5.0973, time: 334.47ms\n",
            " 93% 15076/16260 [1:25:58<07:04,  2.79it/s]iter 15076: loss 6.1367, time: 464.12ms\n",
            " 93% 15077/16260 [1:25:58<07:42,  2.56it/s]iter 15077: loss 5.0351, time: 334.60ms\n",
            " 93% 15078/16260 [1:25:58<07:22,  2.67it/s]iter 15078: loss 4.9678, time: 464.86ms\n",
            " 93% 15079/16260 [1:25:59<07:54,  2.49it/s]iter 15079: loss 4.8422, time: 211.44ms\n",
            " 93% 15080/16260 [1:25:59<06:47,  2.90it/s]iter 15080: loss 4.8191, time: 353.30ms\n",
            " 93% 15081/16260 [1:26:00<06:50,  2.87it/s]iter 15081: loss 4.3595, time: 190.05ms\n",
            " 93% 15082/16260 [1:26:00<05:54,  3.32it/s]iter 15082: loss 4.6179, time: 315.26ms\n",
            " 93% 15083/16260 [1:26:00<06:00,  3.27it/s]iter 15083: loss 5.4291, time: 340.69ms\n",
            " 93% 15084/16260 [1:26:00<06:12,  3.16it/s]iter 15084: loss 4.7777, time: 377.18ms\n",
            " 93% 15085/16260 [1:26:01<06:34,  2.98it/s]iter 15085: loss 5.8182, time: 431.46ms\n",
            " 93% 15086/16260 [1:26:01<07:08,  2.74it/s]iter 15086: loss 4.6245, time: 420.17ms\n",
            " 93% 15087/16260 [1:26:02<07:27,  2.62it/s]iter 15087: loss 5.2807, time: 319.50ms\n",
            " 93% 15088/16260 [1:26:02<07:06,  2.75it/s]iter 15088: loss 5.8233, time: 464.07ms\n",
            " 93% 15089/16260 [1:26:02<07:41,  2.54it/s]iter 15089: loss 5.1180, time: 256.40ms\n",
            " 93% 15090/16260 [1:26:03<06:53,  2.83it/s]iter 15090: loss 4.6324, time: 185.15ms\n",
            " 93% 15091/16260 [1:26:03<05:54,  3.30it/s]iter 15091: loss 5.1966, time: 464.87ms\n",
            " 93% 15092/16260 [1:26:03<06:51,  2.84it/s]iter 15092: loss 5.4116, time: 464.41ms\n",
            " 93% 15093/16260 [1:26:04<07:30,  2.59it/s]iter 15093: loss 5.5119, time: 464.59ms\n",
            " 93% 15094/16260 [1:26:04<07:58,  2.44it/s]iter 15094: loss 4.6486, time: 194.80ms\n",
            " 93% 15095/16260 [1:26:04<06:43,  2.89it/s]iter 15095: loss 4.0773, time: 214.16ms\n",
            " 93% 15096/16260 [1:26:05<05:57,  3.26it/s]iter 15096: loss 5.1016, time: 436.80ms\n",
            " 93% 15097/16260 [1:26:05<06:42,  2.89it/s]iter 15097: loss 5.3310, time: 476.31ms\n",
            " 93% 15098/16260 [1:26:06<07:28,  2.59it/s]iter 15098: loss 5.1687, time: 317.09ms\n",
            " 93% 15099/16260 [1:26:06<07:04,  2.73it/s]iter 15099: loss 5.4818, time: 279.89ms\n",
            " 93% 15100/16260 [1:26:06<06:34,  2.94it/s]iter 15100: loss 4.3261, time: 262.62ms\n",
            " 93% 15101/16260 [1:26:06<06:08,  3.15it/s]iter 15101: loss 4.5605, time: 468.58ms\n",
            " 93% 15102/16260 [1:26:07<07:00,  2.75it/s]iter 15102: loss 5.4534, time: 464.11ms\n",
            " 93% 15103/16260 [1:26:07<07:36,  2.54it/s]iter 15103: loss 3.6759, time: 171.87ms\n",
            " 93% 15104/16260 [1:26:08<06:19,  3.05it/s]iter 15104: loss 5.4791, time: 281.58ms\n",
            " 93% 15105/16260 [1:26:08<06:03,  3.18it/s]iter 15105: loss 4.9005, time: 374.40ms\n",
            " 93% 15106/16260 [1:26:08<06:24,  3.00it/s]iter 15106: loss 3.7468, time: 146.40ms\n",
            " 93% 15107/16260 [1:26:08<05:20,  3.60it/s]iter 15107: loss 5.0225, time: 236.29ms\n",
            " 93% 15108/16260 [1:26:09<05:06,  3.76it/s]iter 15108: loss 4.1504, time: 152.27ms\n",
            " 93% 15109/16260 [1:26:09<04:27,  4.31it/s]iter 15109: loss 4.6642, time: 257.34ms\n",
            " 93% 15110/16260 [1:26:09<04:36,  4.16it/s]iter 15110: loss 4.8406, time: 238.87ms\n",
            " 93% 15111/16260 [1:26:09<04:36,  4.16it/s]iter 15111: loss 4.3521, time: 191.93ms\n",
            " 93% 15112/16260 [1:26:09<04:19,  4.42it/s]iter 15112: loss 5.3391, time: 318.61ms\n",
            " 93% 15113/16260 [1:26:10<04:51,  3.93it/s]iter 15113: loss 4.1522, time: 221.06ms\n",
            " 93% 15114/16260 [1:26:10<04:40,  4.08it/s]iter 15114: loss 3.9903, time: 411.15ms\n",
            " 93% 15115/16260 [1:26:10<05:38,  3.38it/s]iter 15115: loss 5.2391, time: 345.87ms\n",
            " 93% 15116/16260 [1:26:11<05:55,  3.21it/s]iter 15116: loss 4.6735, time: 214.78ms\n",
            " 93% 15117/16260 [1:26:11<05:23,  3.54it/s]iter 15117: loss 5.5669, time: 464.02ms\n",
            " 93% 15118/16260 [1:26:11<06:25,  2.96it/s]iter 15118: loss 4.9641, time: 470.50ms\n",
            " 93% 15119/16260 [1:26:12<07:11,  2.64it/s]iter 15119: loss 3.7079, time: 185.38ms\n",
            " 93% 15120/16260 [1:26:12<06:05,  3.12it/s]iter 15120: loss 5.4778, time: 411.88ms\n",
            " 93% 15121/16260 [1:26:12<06:36,  2.87it/s]iter 15121: loss 5.7087, time: 468.32ms\n",
            " 93% 15122/16260 [1:26:13<07:18,  2.60it/s]iter 15122: loss 4.7546, time: 333.44ms\n",
            " 93% 15123/16260 [1:26:13<07:00,  2.70it/s]iter 15123: loss 4.8393, time: 273.38ms\n",
            " 93% 15124/16260 [1:26:14<06:27,  2.93it/s]iter 15124: loss 5.1443, time: 465.54ms\n",
            " 93% 15125/16260 [1:26:14<07:10,  2.64it/s]iter 15125: loss 5.5511, time: 343.31ms\n",
            " 93% 15126/16260 [1:26:14<06:58,  2.71it/s]iter 15126: loss 5.0613, time: 382.06ms\n",
            " 93% 15127/16260 [1:26:15<07:02,  2.68it/s]iter 15127: loss 4.0183, time: 254.68ms\n",
            " 93% 15128/16260 [1:26:15<06:22,  2.96it/s]iter 15128: loss 5.3096, time: 443.79ms\n",
            " 93% 15129/16260 [1:26:15<06:58,  2.70it/s]iter 15129: loss 5.7804, time: 463.93ms\n",
            " 93% 15130/16260 [1:26:16<07:30,  2.51it/s]iter 15130: loss 3.7449, time: 188.57ms\n",
            " 93% 15131/16260 [1:26:16<06:19,  2.97it/s]iter 15131: loss 4.4794, time: 260.56ms\n",
            " 93% 15132/16260 [1:26:16<05:54,  3.18it/s]iter 15132: loss 5.3734, time: 435.87ms\n",
            " 93% 15133/16260 [1:26:17<06:35,  2.85it/s]iter 15133: loss 4.8875, time: 332.44ms\n",
            " 93% 15134/16260 [1:26:17<06:29,  2.89it/s]iter 15134: loss 4.8797, time: 413.80ms\n",
            " 93% 15135/16260 [1:26:18<06:52,  2.73it/s]iter 15135: loss 5.2515, time: 469.89ms\n",
            " 93% 15136/16260 [1:26:18<07:27,  2.51it/s]iter 15136: loss 4.8304, time: 313.13ms\n",
            " 93% 15137/16260 [1:26:18<06:58,  2.68it/s]iter 15137: loss 5.1484, time: 217.57ms\n",
            " 93% 15138/16260 [1:26:19<06:06,  3.06it/s]iter 15138: loss 4.5741, time: 218.89ms\n",
            " 93% 15139/16260 [1:26:19<05:30,  3.39it/s]iter 15139: loss 4.7788, time: 435.42ms\n",
            " 93% 15140/16260 [1:26:19<06:18,  2.96it/s]iter 15140: loss 4.6769, time: 191.83ms\n",
            " 93% 15141/16260 [1:26:19<05:29,  3.40it/s]iter 15141: loss 5.1632, time: 435.09ms\n",
            " 93% 15142/16260 [1:26:20<06:16,  2.97it/s]iter 15142: loss 3.2055, time: 164.10ms\n",
            " 93% 15143/16260 [1:26:20<05:19,  3.50it/s]iter 15143: loss 4.8345, time: 410.26ms\n",
            " 93% 15144/16260 [1:26:20<06:01,  3.09it/s]iter 15144: loss 5.0978, time: 237.57ms\n",
            " 93% 15145/16260 [1:26:21<05:32,  3.35it/s]iter 15145: loss 5.6434, time: 333.61ms\n",
            " 93% 15146/16260 [1:26:21<05:44,  3.23it/s]iter 15146: loss 5.9876, time: 379.37ms\n",
            " 93% 15147/16260 [1:26:21<06:08,  3.02it/s]iter 15147: loss 5.1799, time: 318.19ms\n",
            " 93% 15148/16260 [1:26:22<06:04,  3.05it/s]iter 15148: loss 5.5332, time: 463.37ms\n",
            " 93% 15149/16260 [1:26:22<06:49,  2.71it/s]iter 15149: loss 5.0003, time: 192.74ms\n",
            " 93% 15150/16260 [1:26:22<05:51,  3.16it/s]iter 15150: loss 5.1827, time: 282.75ms\n",
            " 93% 15151/16260 [1:26:23<05:40,  3.26it/s]iter 15151: loss 4.8498, time: 463.82ms\n",
            " 93% 15152/16260 [1:26:23<06:32,  2.82it/s]iter 15152: loss 5.3862, time: 318.37ms\n",
            " 93% 15153/16260 [1:26:23<06:20,  2.91it/s]iter 15153: loss 6.0598, time: 464.74ms\n",
            " 93% 15154/16260 [1:26:24<07:01,  2.63it/s]iter 15154: loss 4.4590, time: 225.07ms\n",
            " 93% 15155/16260 [1:26:24<06:09,  2.99it/s]iter 15155: loss 5.4781, time: 465.20ms\n",
            " 93% 15156/16260 [1:26:25<06:53,  2.67it/s]iter 15156: loss 4.8019, time: 211.43ms\n",
            " 93% 15157/16260 [1:26:25<05:59,  3.07it/s]iter 15157: loss 5.2425, time: 372.94ms\n",
            " 93% 15158/16260 [1:26:25<06:15,  2.94it/s]iter 15158: loss 4.6432, time: 236.07ms\n",
            " 93% 15159/16260 [1:26:25<05:40,  3.23it/s]iter 15159: loss 5.0554, time: 464.07ms\n",
            " 93% 15160/16260 [1:26:26<06:32,  2.80it/s]iter 15160: loss 4.8497, time: 278.93ms\n",
            " 93% 15161/16260 [1:26:26<06:06,  3.00it/s]iter 15161: loss 4.9329, time: 422.97ms\n",
            " 93% 15162/16260 [1:26:27<06:36,  2.77it/s]iter 15162: loss 4.2861, time: 237.46ms\n",
            " 93% 15163/16260 [1:26:27<05:55,  3.08it/s]iter 15163: loss 6.0106, time: 464.64ms\n",
            " 93% 15164/16260 [1:26:27<06:42,  2.73it/s]iter 15164: loss 3.1827, time: 136.93ms\n",
            " 93% 15165/16260 [1:26:27<05:26,  3.35it/s]iter 15165: loss 4.9543, time: 471.49ms\n",
            " 93% 15166/16260 [1:26:28<06:23,  2.85it/s]iter 15166: loss 4.6573, time: 166.86ms\n",
            " 93% 15167/16260 [1:26:28<05:23,  3.38it/s]iter 15167: loss 4.4326, time: 319.99ms\n",
            " 93% 15168/16260 [1:26:28<05:31,  3.30it/s]iter 15168: loss 5.1327, time: 344.61ms\n",
            " 93% 15169/16260 [1:26:29<05:45,  3.16it/s]iter 15169: loss 5.8090, time: 464.65ms\n",
            " 93% 15170/16260 [1:26:29<06:33,  2.77it/s]iter 15170: loss 3.8878, time: 464.04ms\n",
            " 93% 15171/16260 [1:26:30<07:07,  2.55it/s]iter 15171: loss 5.4611, time: 377.08ms\n",
            " 93% 15172/16260 [1:26:30<07:02,  2.57it/s]iter 15172: loss 5.3625, time: 477.65ms\n",
            " 93% 15173/16260 [1:26:31<07:31,  2.41it/s]iter 15173: loss 4.1210, time: 187.64ms\n",
            " 93% 15174/16260 [1:26:31<06:17,  2.88it/s]iter 15174: loss 5.0957, time: 465.60ms\n",
            " 93% 15175/16260 [1:26:31<06:56,  2.61it/s]iter 15175: loss 5.2267, time: 275.58ms\n",
            " 93% 15176/16260 [1:26:31<06:21,  2.84it/s]iter 15176: loss 5.6329, time: 464.49ms\n",
            " 93% 15177/16260 [1:26:32<06:58,  2.59it/s]iter 15177: loss 4.9614, time: 216.94ms\n",
            " 93% 15178/16260 [1:26:32<06:03,  2.98it/s]iter 15178: loss 4.8220, time: 464.03ms\n",
            " 93% 15179/16260 [1:26:33<06:45,  2.67it/s]iter 15179: loss 5.6246, time: 464.01ms\n",
            " 93% 15180/16260 [1:26:33<07:14,  2.49it/s]iter 15180: loss 5.0984, time: 464.62ms\n",
            " 93% 15181/16260 [1:26:34<07:34,  2.37it/s]iter 15181: loss 5.5173, time: 442.28ms\n",
            " 93% 15182/16260 [1:26:34<07:41,  2.34it/s]iter 15182: loss 2.7061, time: 142.24ms\n",
            " 93% 15183/16260 [1:26:34<06:09,  2.92it/s]iter 15183: loss 5.7059, time: 464.67ms\n",
            " 93% 15184/16260 [1:26:35<06:48,  2.63it/s]iter 15184: loss 4.9033, time: 307.56ms\n",
            " 93% 15185/16260 [1:26:35<06:25,  2.79it/s]iter 15185: loss 4.0099, time: 173.09ms\n",
            " 93% 15186/16260 [1:26:35<05:25,  3.30it/s]iter 15186: loss 5.0922, time: 237.16ms\n",
            " 93% 15187/16260 [1:26:35<05:04,  3.52it/s]iter 15187: loss 4.4896, time: 253.21ms\n",
            " 93% 15188/16260 [1:26:36<04:55,  3.63it/s]iter 15188: loss 5.3342, time: 463.25ms\n",
            " 93% 15189/16260 [1:26:36<05:55,  3.01it/s]iter 15189: loss 4.4319, time: 241.45ms\n",
            " 93% 15190/16260 [1:26:36<05:26,  3.27it/s]iter 15190: loss 4.8627, time: 464.53ms\n",
            " 93% 15191/16260 [1:26:37<06:18,  2.83it/s]iter 15191: loss 4.8880, time: 186.67ms\n",
            " 93% 15192/16260 [1:26:37<05:24,  3.29it/s]iter 15192: loss 5.5403, time: 468.61ms\n",
            " 93% 15193/16260 [1:26:37<06:17,  2.83it/s]iter 15193: loss 5.6218, time: 418.06ms\n",
            " 93% 15194/16260 [1:26:38<06:38,  2.68it/s]iter 15194: loss 5.0023, time: 310.66ms\n",
            " 93% 15195/16260 [1:26:38<06:18,  2.82it/s]iter 15195: loss 4.1762, time: 190.67ms\n",
            " 93% 15196/16260 [1:26:38<05:25,  3.27it/s]iter 15196: loss 4.2592, time: 218.68ms\n",
            " 93% 15197/16260 [1:26:39<04:58,  3.57it/s]iter 15197: loss 5.7146, time: 464.35ms\n",
            " 93% 15198/16260 [1:26:39<05:56,  2.98it/s]iter 15198: loss 4.8589, time: 351.56ms\n",
            " 93% 15199/16260 [1:26:39<06:01,  2.93it/s]iter 15199: loss 4.3353, time: 170.02ms\n",
            " 93% 15200/16260 [1:26:40<05:07,  3.45it/s]iter 15200: loss 4.3629, time: 185.87ms\n",
            " 93% 15201/16260 [1:26:40<04:34,  3.86it/s]iter 15201: loss 4.9264, time: 334.22ms\n",
            " 93% 15202/16260 [1:26:40<04:58,  3.54it/s]iter 15202: loss 5.2700, time: 464.72ms\n",
            " 93% 15203/16260 [1:26:41<05:56,  2.96it/s]iter 15203: loss 5.4649, time: 278.00ms\n",
            " 94% 15204/16260 [1:26:41<05:37,  3.12it/s]iter 15204: loss 3.9081, time: 463.70ms\n",
            " 94% 15205/16260 [1:26:41<06:23,  2.75it/s]iter 15205: loss 5.0200, time: 464.77ms\n",
            " 94% 15206/16260 [1:26:42<06:55,  2.54it/s]iter 15206: loss 5.5906, time: 464.92ms\n",
            " 94% 15207/16260 [1:26:42<07:18,  2.40it/s]iter 15207: loss 4.9622, time: 261.85ms\n",
            " 94% 15208/16260 [1:26:42<06:29,  2.70it/s]iter 15208: loss 5.6340, time: 311.30ms\n",
            " 94% 15209/16260 [1:26:43<06:11,  2.83it/s]iter 15209: loss 4.8794, time: 236.33ms\n",
            " 94% 15210/16260 [1:26:43<05:34,  3.14it/s]iter 15210: loss 5.1563, time: 261.71ms\n",
            " 94% 15211/16260 [1:26:43<05:16,  3.31it/s]iter 15211: loss 5.3476, time: 464.81ms\n",
            " 94% 15212/16260 [1:26:44<06:08,  2.85it/s]iter 15212: loss 5.2964, time: 341.53ms\n",
            " 94% 15213/16260 [1:26:44<06:05,  2.87it/s]iter 15213: loss 5.0321, time: 307.61ms\n",
            " 94% 15214/16260 [1:26:44<05:52,  2.97it/s]iter 15214: loss 4.7289, time: 190.94ms\n",
            " 94% 15215/16260 [1:26:45<05:06,  3.41it/s]iter 15215: loss 5.6451, time: 463.85ms\n",
            " 94% 15216/16260 [1:26:45<06:00,  2.90it/s]iter 15216: loss 5.1522, time: 433.13ms\n",
            " 94% 15217/16260 [1:26:46<06:28,  2.69it/s]iter 15217: loss 4.1009, time: 167.39ms\n",
            " 94% 15218/16260 [1:26:46<05:24,  3.21it/s]iter 15218: loss 5.0363, time: 464.29ms\n",
            " 94% 15219/16260 [1:26:46<06:12,  2.80it/s]iter 15219: loss 5.8415, time: 465.10ms\n",
            " 94% 15220/16260 [1:26:47<06:45,  2.56it/s]iter 15220: loss 5.4359, time: 464.07ms\n",
            " 94% 15221/16260 [1:26:47<07:09,  2.42it/s]iter 15221: loss 4.2126, time: 194.91ms\n",
            " 94% 15222/16260 [1:26:47<06:01,  2.87it/s]iter 15222: loss 5.2367, time: 287.83ms\n",
            " 94% 15223/16260 [1:26:48<05:42,  3.03it/s]iter 15223: loss 3.9195, time: 165.70ms\n",
            " 94% 15224/16260 [1:26:48<04:51,  3.55it/s]iter 15224: loss 5.3045, time: 342.19ms\n",
            " 94% 15225/16260 [1:26:48<05:10,  3.33it/s]iter 15225: loss 5.6608, time: 465.16ms\n",
            " 94% 15226/16260 [1:26:49<06:02,  2.86it/s]iter 15226: loss 5.2062, time: 465.04ms\n",
            " 94% 15227/16260 [1:26:49<06:37,  2.60it/s]iter 15227: loss 5.5894, time: 378.07ms\n",
            " 94% 15228/16260 [1:26:49<06:35,  2.61it/s]iter 15228: loss 6.0060, time: 464.90ms\n",
            " 94% 15229/16260 [1:26:50<07:01,  2.45it/s]iter 15229: loss 5.1427, time: 444.38ms\n",
            " 94% 15230/16260 [1:26:50<07:12,  2.38it/s]iter 15230: loss 5.5550, time: 441.45ms\n",
            " 94% 15231/16260 [1:26:51<07:19,  2.34it/s]iter 15231: loss 4.8644, time: 192.62ms\n",
            " 94% 15232/16260 [1:26:51<06:06,  2.80it/s]iter 15232: loss 4.8264, time: 307.74ms\n",
            " 94% 15233/16260 [1:26:51<05:51,  2.92it/s]iter 15233: loss 5.3797, time: 216.43ms\n",
            " 94% 15234/16260 [1:26:51<05:13,  3.28it/s]iter 15234: loss 5.8874, time: 464.22ms\n",
            " 94% 15235/16260 [1:26:52<06:02,  2.83it/s]iter 15235: loss 4.0468, time: 215.66ms\n",
            " 94% 15236/16260 [1:26:52<05:20,  3.20it/s]iter 15236: loss 5.0646, time: 334.99ms\n",
            " 94% 15237/16260 [1:26:52<05:27,  3.13it/s]iter 15237: loss 4.4199, time: 275.16ms\n",
            " 94% 15238/16260 [1:26:53<05:13,  3.26it/s]iter 15238: loss 5.5129, time: 464.91ms\n",
            " 94% 15239/16260 [1:26:53<06:02,  2.82it/s]iter 15239: loss 5.4636, time: 471.93ms\n",
            " 94% 15240/16260 [1:26:54<06:38,  2.56it/s]iter 15240: loss 4.5133, time: 258.74ms\n",
            " 94% 15241/16260 [1:26:54<05:58,  2.85it/s]iter 15241: loss 5.6636, time: 466.81ms\n",
            " 94% 15242/16260 [1:26:54<06:33,  2.59it/s]iter 15242: loss 4.5971, time: 350.97ms\n",
            " 94% 15243/16260 [1:26:55<06:22,  2.66it/s]iter 15243: loss 5.5341, time: 465.61ms\n",
            " 94% 15244/16260 [1:26:55<06:49,  2.48it/s]iter 15244: loss 5.3901, time: 378.77ms\n",
            " 94% 15245/16260 [1:26:56<06:42,  2.52it/s]iter 15245: loss 4.8303, time: 257.62ms\n",
            " 94% 15246/16260 [1:26:56<06:00,  2.81it/s]iter 15246: loss 5.3306, time: 345.62ms\n",
            " 94% 15247/16260 [1:26:56<05:57,  2.83it/s]iter 15247: loss 6.1064, time: 464.35ms\n",
            " 94% 15248/16260 [1:26:57<06:31,  2.59it/s]iter 15248: loss 5.3743, time: 464.08ms\n",
            " 94% 15249/16260 [1:26:57<06:54,  2.44it/s]iter 15249: loss 5.1852, time: 238.93ms\n",
            " 94% 15250/16260 [1:26:57<06:03,  2.78it/s]iter 15250: loss 5.1452, time: 422.15ms\n",
            " 94% 15251/16260 [1:26:58<06:22,  2.64it/s]iter 15251: loss 4.9339, time: 430.73ms\n",
            " 94% 15252/16260 [1:26:58<06:37,  2.53it/s]iter 15252: loss 3.9773, time: 188.28ms\n",
            " 94% 15253/16260 [1:26:58<05:35,  3.00it/s]iter 15253: loss 5.4524, time: 464.06ms\n",
            " 94% 15254/16260 [1:26:59<06:15,  2.68it/s]iter 15254: loss 5.3716, time: 337.69ms\n",
            " 94% 15255/16260 [1:26:59<06:04,  2.76it/s]iter 15255: loss 5.8579, time: 463.59ms\n",
            " 94% 15256/16260 [1:27:00<06:35,  2.54it/s]iter 15256: loss 4.7230, time: 238.39ms\n",
            " 94% 15257/16260 [1:27:00<05:48,  2.88it/s]iter 15257: loss 4.6897, time: 257.34ms\n",
            " 94% 15258/16260 [1:27:00<05:21,  3.12it/s]iter 15258: loss 4.9752, time: 343.33ms\n",
            " 94% 15259/16260 [1:27:01<05:28,  3.05it/s]iter 15259: loss 5.1929, time: 472.00ms\n",
            " 94% 15260/16260 [1:27:01<06:11,  2.69it/s]iter 15260: loss 4.9508, time: 464.10ms\n",
            " 94% 15261/16260 [1:27:02<06:39,  2.50it/s]iter 15261: loss 4.0077, time: 185.58ms\n",
            " 94% 15262/16260 [1:27:02<05:35,  2.98it/s]iter 15262: loss 5.3013, time: 453.21ms\n",
            " 94% 15263/16260 [1:27:02<06:10,  2.69it/s]iter 15263: loss 4.9920, time: 258.02ms\n",
            " 94% 15264/16260 [1:27:02<05:36,  2.96it/s]iter 15264: loss 4.2918, time: 187.15ms\n",
            " 94% 15265/16260 [1:27:03<04:51,  3.41it/s]iter 15265: loss 4.7682, time: 232.88ms\n",
            " 94% 15266/16260 [1:27:03<04:33,  3.63it/s]iter 15266: loss 5.2270, time: 256.67ms\n",
            " 94% 15267/16260 [1:27:03<04:28,  3.70it/s]iter 15267: loss 5.4252, time: 317.35ms\n",
            " 94% 15268/16260 [1:27:03<04:42,  3.51it/s]iter 15268: loss 4.9617, time: 464.80ms\n",
            " 94% 15269/16260 [1:27:04<05:36,  2.95it/s]iter 15269: loss 4.8240, time: 314.86ms\n",
            " 94% 15270/16260 [1:27:04<05:29,  3.01it/s]iter 15270: loss 5.3059, time: 432.55ms\n",
            " 94% 15271/16260 [1:27:05<05:58,  2.76it/s]iter 15271: loss 5.7803, time: 382.13ms\n",
            " 94% 15272/16260 [1:27:05<06:04,  2.71it/s]iter 15272: loss 4.1563, time: 150.64ms\n",
            " 94% 15273/16260 [1:27:05<05:00,  3.29it/s]iter 15273: loss 5.9412, time: 464.35ms\n",
            " 94% 15274/16260 [1:27:06<05:47,  2.84it/s]iter 15274: loss 4.9153, time: 233.69ms\n",
            " 94% 15275/16260 [1:27:06<05:12,  3.15it/s]iter 15275: loss 4.2916, time: 211.74ms\n",
            " 94% 15276/16260 [1:27:06<04:41,  3.49it/s]iter 15276: loss 5.2764, time: 464.33ms\n",
            " 94% 15277/16260 [1:27:07<05:34,  2.94it/s]iter 15277: loss 5.2309, time: 465.67ms\n",
            " 94% 15278/16260 [1:27:07<06:11,  2.64it/s]iter 15278: loss 5.5734, time: 465.11ms\n",
            " 94% 15279/16260 [1:27:07<06:37,  2.47it/s]iter 15279: loss 4.8980, time: 464.04ms\n",
            " 94% 15280/16260 [1:27:08<06:54,  2.36it/s]iter 15280: loss 5.3067, time: 341.15ms\n",
            " 94% 15281/16260 [1:27:08<06:30,  2.51it/s]iter 15281: loss 5.0682, time: 410.16ms\n",
            " 94% 15282/16260 [1:27:09<06:33,  2.48it/s]iter 15282: loss 4.9029, time: 409.89ms\n",
            " 94% 15283/16260 [1:27:09<06:36,  2.47it/s]iter 15283: loss 5.2708, time: 463.54ms\n",
            " 94% 15284/16260 [1:27:10<06:53,  2.36it/s]iter 15284: loss 4.0670, time: 193.30ms\n",
            " 94% 15285/16260 [1:27:10<05:45,  2.82it/s]iter 15285: loss 4.3025, time: 164.29ms\n",
            " 94% 15286/16260 [1:27:10<04:50,  3.36it/s]iter 15286: loss 5.2217, time: 464.22ms\n",
            " 94% 15287/16260 [1:27:10<05:38,  2.87it/s]iter 15287: loss 5.7983, time: 463.66ms\n",
            " 94% 15288/16260 [1:27:11<06:12,  2.61it/s]iter 15288: loss 5.1545, time: 465.04ms\n",
            " 94% 15289/16260 [1:27:11<06:36,  2.45it/s]iter 15289: loss 5.1519, time: 189.40ms\n",
            " 94% 15290/16260 [1:27:12<05:32,  2.91it/s]iter 15290: loss 5.4311, time: 335.72ms\n",
            " 94% 15291/16260 [1:27:12<05:30,  2.93it/s]iter 15291: loss 4.7945, time: 213.15ms\n",
            " 94% 15292/16260 [1:27:12<04:57,  3.26it/s]iter 15292: loss 4.9261, time: 275.56ms\n",
            " 94% 15293/16260 [1:27:12<04:48,  3.36it/s]iter 15293: loss 4.2973, time: 237.20ms\n",
            " 94% 15294/16260 [1:27:13<04:30,  3.57it/s]iter 15294: loss 4.8275, time: 233.29ms\n",
            " 94% 15295/16260 [1:27:13<04:17,  3.75it/s]iter 15295: loss 4.3586, time: 236.50ms\n",
            " 94% 15296/16260 [1:27:13<04:08,  3.88it/s]iter 15296: loss 5.2424, time: 382.86ms\n",
            " 94% 15297/16260 [1:27:13<04:44,  3.38it/s]iter 15297: loss 3.8726, time: 189.62ms\n",
            " 94% 15298/16260 [1:27:14<04:14,  3.78it/s]iter 15298: loss 5.5707, time: 464.81ms\n",
            " 94% 15299/16260 [1:27:14<05:12,  3.08it/s]iter 15299: loss 4.8627, time: 473.22ms\n",
            " 94% 15300/16260 [1:27:15<05:55,  2.70it/s]iter 15300: loss 4.9082, time: 330.91ms\n",
            " 94% 15301/16260 [1:27:15<05:44,  2.79it/s]iter 15301: loss 5.3345, time: 464.08ms\n",
            " 94% 15302/16260 [1:27:15<06:14,  2.56it/s]iter 15302: loss 5.3352, time: 464.80ms\n",
            " 94% 15303/16260 [1:27:16<06:35,  2.42it/s]iter 15303: loss 4.6937, time: 231.74ms\n",
            " 94% 15304/16260 [1:27:16<05:43,  2.78it/s]iter 15304: loss 5.2873, time: 347.81ms\n",
            " 94% 15305/16260 [1:27:16<05:40,  2.81it/s]iter 15305: loss 5.3249, time: 464.00ms\n",
            " 94% 15306/16260 [1:27:17<06:11,  2.57it/s]iter 15306: loss 5.4467, time: 463.95ms\n",
            " 94% 15307/16260 [1:27:17<06:32,  2.43it/s]iter 15307: loss 5.8746, time: 464.66ms\n",
            " 94% 15308/16260 [1:27:18<06:47,  2.33it/s]iter 15308: loss 5.6451, time: 471.75ms\n",
            " 94% 15309/16260 [1:27:18<07:00,  2.26it/s]iter 15309: loss 5.4037, time: 373.29ms\n",
            " 94% 15310/16260 [1:27:19<06:40,  2.37it/s]iter 15310: loss 4.5846, time: 335.38ms\n",
            " 94% 15311/16260 [1:27:19<06:16,  2.52it/s]iter 15311: loss 4.7050, time: 191.54ms\n",
            " 94% 15312/16260 [1:27:19<05:17,  2.98it/s]iter 15312: loss 4.4862, time: 235.57ms\n",
            " 94% 15313/16260 [1:27:19<04:49,  3.27it/s]iter 15313: loss 5.1108, time: 464.20ms\n",
            " 94% 15314/16260 [1:27:20<05:34,  2.83it/s]iter 15314: loss 5.9584, time: 466.40ms\n",
            " 94% 15315/16260 [1:27:20<06:06,  2.58it/s]iter 15315: loss 5.2221, time: 260.79ms\n",
            " 94% 15316/16260 [1:27:21<05:30,  2.85it/s]iter 15316: loss 5.1721, time: 233.03ms\n",
            " 94% 15317/16260 [1:27:21<04:57,  3.17it/s]iter 15317: loss 4.0520, time: 220.19ms\n",
            " 94% 15318/16260 [1:27:21<04:30,  3.48it/s]iter 15318: loss 4.6693, time: 240.51ms\n",
            " 94% 15319/16260 [1:27:21<04:17,  3.65it/s]iter 15319: loss 5.4632, time: 259.72ms\n",
            " 94% 15320/16260 [1:27:22<04:13,  3.70it/s]iter 15320: loss 4.6491, time: 463.00ms\n",
            " 94% 15321/16260 [1:27:22<05:08,  3.05it/s]iter 15321: loss 4.3541, time: 169.71ms\n",
            " 94% 15322/16260 [1:27:22<04:23,  3.56it/s]iter 15322: loss 3.7056, time: 163.37ms\n",
            " 94% 15323/16260 [1:27:22<03:50,  4.06it/s]iter 15323: loss 5.2714, time: 379.97ms\n",
            " 94% 15324/16260 [1:27:23<04:28,  3.49it/s]iter 15324: loss 5.1501, time: 463.23ms\n",
            " 94% 15325/16260 [1:27:23<05:18,  2.94it/s]iter 15325: loss 5.4604, time: 375.97ms\n",
            " 94% 15326/16260 [1:27:24<05:28,  2.85it/s]iter 15326: loss 5.3159, time: 342.15ms\n",
            " 94% 15327/16260 [1:27:24<05:25,  2.86it/s]iter 15327: loss 5.0803, time: 243.14ms\n",
            " 94% 15328/16260 [1:27:24<04:55,  3.15it/s]iter 15328: loss 4.2871, time: 172.71ms\n",
            " 94% 15329/16260 [1:27:24<04:15,  3.64it/s]iter 15329: loss 4.6806, time: 185.72ms\n",
            " 94% 15330/16260 [1:27:25<03:50,  4.03it/s]iter 15330: loss 5.5525, time: 464.61ms\n",
            " 94% 15331/16260 [1:27:25<04:51,  3.19it/s]iter 15331: loss 5.2180, time: 430.79ms\n",
            " 94% 15332/16260 [1:27:25<05:24,  2.86it/s]iter 15332: loss 4.4596, time: 170.33ms\n",
            " 94% 15333/16260 [1:27:26<04:34,  3.38it/s]iter 15333: loss 5.3891, time: 463.84ms\n",
            " 94% 15334/16260 [1:27:26<05:21,  2.88it/s]iter 15334: loss 4.6694, time: 342.30ms\n",
            " 94% 15335/16260 [1:27:26<05:20,  2.89it/s]iter 15335: loss 4.6003, time: 255.15ms\n",
            " 94% 15336/16260 [1:27:27<04:54,  3.13it/s]iter 15336: loss 5.4164, time: 464.28ms\n",
            " 94% 15337/16260 [1:27:27<05:35,  2.75it/s]iter 15337: loss 4.9276, time: 235.88ms\n",
            " 94% 15338/16260 [1:27:27<05:00,  3.07it/s]iter 15338: loss 4.8344, time: 464.93ms\n",
            " 94% 15339/16260 [1:27:28<05:38,  2.72it/s]iter 15339: loss 5.8227, time: 464.35ms\n",
            " 94% 15340/16260 [1:27:28<06:05,  2.52it/s]iter 15340: loss 5.3130, time: 326.71ms\n",
            " 94% 15341/16260 [1:27:29<05:46,  2.66it/s]iter 15341: loss 5.5839, time: 465.00ms\n",
            " 94% 15342/16260 [1:27:29<06:10,  2.48it/s]iter 15342: loss 4.8883, time: 466.63ms\n",
            " 94% 15343/16260 [1:27:30<06:27,  2.36it/s]iter 15343: loss 4.9454, time: 422.47ms\n",
            " 94% 15344/16260 [1:27:30<06:27,  2.36it/s]iter 15344: loss 4.6103, time: 214.38ms\n",
            " 94% 15345/16260 [1:27:30<05:30,  2.77it/s]iter 15345: loss 4.9555, time: 192.96ms\n",
            " 94% 15346/16260 [1:27:30<04:44,  3.22it/s]iter 15346: loss 5.0473, time: 367.01ms\n",
            " 94% 15347/16260 [1:27:31<04:59,  3.05it/s]iter 15347: loss 5.5693, time: 233.49ms\n",
            " 94% 15348/16260 [1:27:31<04:33,  3.33it/s]iter 15348: loss 5.4192, time: 465.14ms\n",
            " 94% 15349/16260 [1:27:32<05:19,  2.86it/s]iter 15349: loss 5.3697, time: 281.50ms\n",
            " 94% 15350/16260 [1:27:32<05:00,  3.03it/s]iter 15350: loss 5.5207, time: 316.91ms\n",
            " 94% 15351/16260 [1:27:32<04:56,  3.06it/s]iter 15351: loss 5.8366, time: 464.48ms\n",
            " 94% 15352/16260 [1:27:33<05:34,  2.71it/s]iter 15352: loss 4.9626, time: 191.54ms\n",
            " 94% 15353/16260 [1:27:33<04:46,  3.17it/s]iter 15353: loss 4.9813, time: 192.78ms\n",
            " 94% 15354/16260 [1:27:33<04:13,  3.58it/s]iter 15354: loss 5.4238, time: 345.06ms\n",
            " 94% 15355/16260 [1:27:33<04:31,  3.34it/s]iter 15355: loss 4.7071, time: 193.14ms\n",
            " 94% 15356/16260 [1:27:34<04:02,  3.73it/s]iter 15356: loss 4.4617, time: 308.52ms\n",
            " 94% 15357/16260 [1:27:34<04:13,  3.56it/s]iter 15357: loss 5.1597, time: 188.21ms\n",
            " 94% 15358/16260 [1:27:34<03:48,  3.95it/s]iter 15358: loss 5.6144, time: 464.18ms\n",
            " 94% 15359/16260 [1:27:34<04:45,  3.15it/s]iter 15359: loss 4.8800, time: 426.48ms\n",
            " 94% 15360/16260 [1:27:35<05:15,  2.85it/s]iter 15360: loss 5.0480, time: 189.63ms\n",
            " 94% 15361/16260 [1:27:35<04:31,  3.31it/s]iter 15361: loss 5.6857, time: 465.15ms\n",
            " 94% 15362/16260 [1:27:36<05:15,  2.84it/s]iter 15362: loss 4.9496, time: 219.38ms\n",
            " 94% 15363/16260 [1:27:36<04:40,  3.20it/s]iter 15363: loss 5.3672, time: 258.08ms\n",
            " 94% 15364/16260 [1:27:36<04:25,  3.37it/s]iter 15364: loss 5.9121, time: 463.76ms\n",
            " 94% 15365/16260 [1:27:37<05:10,  2.88it/s]iter 15365: loss 4.4343, time: 244.72ms\n",
            " 95% 15366/16260 [1:27:37<04:43,  3.15it/s]iter 15366: loss 5.0474, time: 305.72ms\n",
            " 95% 15367/16260 [1:27:37<04:40,  3.18it/s]iter 15367: loss 5.2168, time: 464.89ms\n",
            " 95% 15368/16260 [1:27:38<05:20,  2.78it/s]iter 15368: loss 4.9428, time: 273.11ms\n",
            " 95% 15369/16260 [1:27:38<04:57,  2.99it/s]iter 15369: loss 5.4343, time: 463.98ms\n",
            " 95% 15370/16260 [1:27:38<05:32,  2.68it/s]iter 15370: loss 5.2078, time: 463.92ms\n",
            " 95% 15371/16260 [1:27:39<05:56,  2.49it/s]iter 15371: loss 4.7254, time: 190.89ms\n",
            " 95% 15372/16260 [1:27:39<05:00,  2.95it/s]iter 15372: loss 5.7266, time: 464.24ms\n",
            " 95% 15373/16260 [1:27:39<05:34,  2.65it/s]iter 15373: loss 5.2911, time: 465.30ms\n",
            " 95% 15374/16260 [1:27:40<05:57,  2.48it/s]iter 15374: loss 4.3982, time: 147.38ms\n",
            " 95% 15375/16260 [1:27:40<04:49,  3.06it/s]iter 15375: loss 6.1502, time: 463.81ms\n",
            " 95% 15376/16260 [1:27:40<05:25,  2.71it/s]iter 15376: loss 5.0020, time: 256.65ms\n",
            " 95% 15377/16260 [1:27:41<04:56,  2.98it/s]iter 15377: loss 5.3907, time: 308.02ms\n",
            " 95% 15378/16260 [1:27:41<04:49,  3.05it/s]iter 15378: loss 4.1719, time: 211.29ms\n",
            " 95% 15379/16260 [1:27:41<04:18,  3.41it/s]iter 15379: loss 4.6004, time: 465.04ms\n",
            " 95% 15380/16260 [1:27:42<05:03,  2.90it/s]iter 15380: loss 4.4335, time: 235.40ms\n",
            " 95% 15381/16260 [1:27:42<04:35,  3.20it/s]iter 15381: loss 5.6750, time: 382.00ms\n",
            " 95% 15382/16260 [1:27:42<04:53,  2.99it/s]iter 15382: loss 5.2716, time: 432.90ms\n",
            " 95% 15383/16260 [1:27:43<05:19,  2.75it/s]iter 15383: loss 5.2632, time: 394.41ms\n",
            " 95% 15384/16260 [1:27:43<05:27,  2.68it/s]iter 15384: loss 4.2334, time: 260.53ms\n",
            " 95% 15385/16260 [1:27:43<04:57,  2.94it/s]iter 15385: loss 5.6105, time: 478.69ms\n",
            " 95% 15386/16260 [1:27:44<05:34,  2.62it/s]iter 15386: loss 3.9614, time: 149.29ms\n",
            " 95% 15387/16260 [1:27:44<04:33,  3.20it/s]iter 15387: loss 5.0389, time: 332.23ms\n",
            " 95% 15388/16260 [1:27:44<04:38,  3.13it/s]iter 15388: loss 4.9631, time: 469.23ms\n",
            " 95% 15389/16260 [1:27:45<05:17,  2.74it/s]iter 15389: loss 4.4486, time: 193.01ms\n",
            " 95% 15390/16260 [1:27:45<04:32,  3.19it/s]iter 15390: loss 4.4005, time: 260.79ms\n",
            " 95% 15391/16260 [1:27:45<04:19,  3.35it/s]iter 15391: loss 3.8947, time: 174.54ms\n",
            " 95% 15392/16260 [1:27:46<03:47,  3.82it/s]iter 15392: loss 5.2746, time: 305.39ms\n",
            " 95% 15393/16260 [1:27:46<03:58,  3.63it/s]iter 15393: loss 4.6456, time: 277.96ms\n",
            " 95% 15394/16260 [1:27:46<03:59,  3.62it/s]iter 15394: loss 4.8119, time: 223.26ms\n",
            " 95% 15395/16260 [1:27:46<03:45,  3.83it/s]iter 15395: loss 5.2551, time: 414.26ms\n",
            " 95% 15396/16260 [1:27:47<04:25,  3.25it/s]iter 15396: loss 5.1109, time: 257.98ms\n",
            " 95% 15397/16260 [1:27:47<04:12,  3.41it/s]iter 15397: loss 5.4368, time: 331.67ms\n",
            " 95% 15398/16260 [1:27:47<04:23,  3.28it/s]iter 15398: loss 4.8273, time: 236.23ms\n",
            " 95% 15399/16260 [1:27:48<04:05,  3.51it/s]iter 15399: loss 4.7706, time: 464.40ms\n",
            " 95% 15400/16260 [1:27:48<04:51,  2.95it/s]iter 15400: loss 5.2305, time: 272.71ms\n",
            " 95% 15401/16260 [1:27:48<04:34,  3.13it/s]iter 15401: loss 5.5483, time: 463.98ms\n",
            " 95% 15402/16260 [1:27:49<05:12,  2.75it/s]iter 15402: loss 4.2607, time: 178.41ms\n",
            " 95% 15403/16260 [1:27:49<04:24,  3.24it/s]iter 15403: loss 5.3446, time: 465.33ms\n",
            " 95% 15404/16260 [1:27:49<05:04,  2.81it/s]iter 15404: loss 5.6427, time: 317.66ms\n",
            " 95% 15405/16260 [1:27:50<04:55,  2.90it/s]iter 15405: loss 3.9183, time: 148.44ms\n",
            " 95% 15406/16260 [1:27:50<04:04,  3.49it/s]iter 15406: loss 5.1857, time: 341.24ms\n",
            " 95% 15407/16260 [1:27:50<04:18,  3.29it/s]iter 15407: loss 5.5115, time: 463.99ms\n",
            " 95% 15408/16260 [1:27:51<05:00,  2.84it/s]iter 15408: loss 5.8084, time: 469.55ms\n",
            " 95% 15409/16260 [1:27:51<05:30,  2.58it/s]iter 15409: loss 5.4505, time: 470.25ms\n",
            " 95% 15410/16260 [1:27:52<05:51,  2.42it/s]iter 15410: loss 4.7230, time: 217.40ms\n",
            " 95% 15411/16260 [1:27:52<05:01,  2.82it/s]iter 15411: loss 4.3543, time: 163.55ms\n",
            " 95% 15412/16260 [1:27:52<04:12,  3.36it/s]iter 15412: loss 4.7398, time: 213.29ms\n",
            " 95% 15413/16260 [1:27:52<03:51,  3.66it/s]iter 15413: loss 4.9712, time: 463.66ms\n",
            " 95% 15414/16260 [1:27:53<04:39,  3.02it/s]iter 15414: loss 4.1799, time: 163.36ms\n",
            " 95% 15415/16260 [1:27:53<03:57,  3.56it/s]iter 15415: loss 4.0530, time: 149.61ms\n",
            " 95% 15416/16260 [1:27:53<03:24,  4.13it/s]iter 15416: loss 5.3686, time: 469.94ms\n",
            " 95% 15417/16260 [1:27:53<04:22,  3.22it/s]iter 15417: loss 4.7401, time: 464.31ms\n",
            " 95% 15418/16260 [1:27:54<05:01,  2.80it/s]iter 15418: loss 4.5831, time: 259.56ms\n",
            " 95% 15419/16260 [1:27:54<04:36,  3.04it/s]iter 15419: loss 5.2012, time: 464.61ms\n",
            " 95% 15420/16260 [1:27:55<05:10,  2.70it/s]iter 15420: loss 4.3427, time: 208.88ms\n",
            " 95% 15421/16260 [1:27:55<04:30,  3.10it/s]iter 15421: loss 4.8418, time: 285.92ms\n",
            " 95% 15422/16260 [1:27:55<04:21,  3.21it/s]iter 15422: loss 5.4878, time: 464.91ms\n",
            " 95% 15423/16260 [1:27:56<04:59,  2.79it/s]iter 15423: loss 4.6812, time: 236.94ms\n",
            " 95% 15424/16260 [1:27:56<04:29,  3.11it/s]iter 15424: loss 5.0966, time: 465.26ms\n",
            " 95% 15425/16260 [1:27:56<05:05,  2.74it/s]iter 15425: loss 4.5188, time: 175.70ms\n",
            " 95% 15426/16260 [1:27:57<04:17,  3.24it/s]iter 15426: loss 5.5114, time: 425.21ms\n",
            " 95% 15427/16260 [1:27:57<04:46,  2.90it/s]iter 15427: loss 4.9267, time: 235.12ms\n",
            " 95% 15428/16260 [1:27:57<04:19,  3.21it/s]iter 15428: loss 5.1521, time: 258.68ms\n",
            " 95% 15429/16260 [1:27:57<04:06,  3.37it/s]iter 15429: loss 4.7935, time: 318.87ms\n",
            " 95% 15430/16260 [1:27:58<04:12,  3.29it/s]iter 15430: loss 5.0945, time: 464.27ms\n",
            " 95% 15431/16260 [1:27:58<04:52,  2.84it/s]iter 15431: loss 4.7057, time: 347.08ms\n",
            " 95% 15432/16260 [1:27:59<04:50,  2.85it/s]iter 15432: loss 5.0758, time: 441.17ms\n",
            " 95% 15433/16260 [1:27:59<05:13,  2.64it/s]iter 15433: loss 5.3632, time: 346.06ms\n",
            " 95% 15434/16260 [1:27:59<05:05,  2.71it/s]iter 15434: loss 4.9932, time: 184.49ms\n",
            " 95% 15435/16260 [1:28:00<04:19,  3.18it/s]iter 15435: loss 4.8649, time: 377.34ms\n",
            " 95% 15436/16260 [1:28:00<04:35,  3.00it/s]iter 15436: loss 4.1932, time: 164.18ms\n",
            " 95% 15437/16260 [1:28:00<03:53,  3.53it/s]iter 15437: loss 4.7510, time: 173.23ms\n",
            " 95% 15438/16260 [1:28:00<03:26,  3.99it/s]iter 15438: loss 5.2946, time: 464.99ms\n",
            " 95% 15439/16260 [1:28:01<04:19,  3.17it/s]iter 15439: loss 4.8922, time: 241.24ms\n",
            " 95% 15440/16260 [1:28:01<04:00,  3.40it/s]iter 15440: loss 4.0821, time: 188.05ms\n",
            " 95% 15441/16260 [1:28:01<03:35,  3.81it/s]iter 15441: loss 5.1818, time: 475.22ms\n",
            " 95% 15442/16260 [1:28:02<04:27,  3.06it/s]iter 15442: loss 5.6659, time: 463.96ms\n",
            " 95% 15443/16260 [1:28:02<05:01,  2.71it/s]iter 15443: loss 5.5290, time: 464.31ms\n",
            " 95% 15444/16260 [1:28:03<05:24,  2.51it/s]iter 15444: loss 4.5464, time: 213.70ms\n",
            " 95% 15445/16260 [1:28:03<04:39,  2.92it/s]iter 15445: loss 5.0435, time: 236.96ms\n",
            " 95% 15446/16260 [1:28:03<04:13,  3.21it/s]iter 15446: loss 4.5497, time: 463.61ms\n",
            " 95% 15447/16260 [1:28:04<04:50,  2.80it/s]iter 15447: loss 5.1121, time: 432.26ms\n",
            " 95% 15448/16260 [1:28:04<05:09,  2.63it/s]iter 15448: loss 5.4708, time: 367.59ms\n",
            " 95% 15449/16260 [1:28:04<05:05,  2.65it/s]iter 15449: loss 5.4100, time: 464.86ms\n",
            " 95% 15450/16260 [1:28:05<05:27,  2.48it/s]iter 15450: loss 5.6460, time: 378.82ms\n",
            " 95% 15451/16260 [1:28:05<05:21,  2.52it/s]iter 15451: loss 5.9445, time: 465.66ms\n",
            " 95% 15452/16260 [1:28:06<05:37,  2.39it/s]iter 15452: loss 5.5455, time: 436.79ms\n",
            " 95% 15453/16260 [1:28:06<05:42,  2.36it/s]iter 15453: loss 3.9435, time: 274.90ms\n",
            " 95% 15454/16260 [1:28:06<05:06,  2.63it/s]iter 15454: loss 3.6074, time: 149.98ms\n",
            " 95% 15455/16260 [1:28:07<04:10,  3.21it/s]iter 15455: loss 5.2021, time: 236.04ms\n",
            " 95% 15456/16260 [1:28:07<03:52,  3.46it/s]iter 15456: loss 5.2923, time: 463.57ms\n",
            " 95% 15457/16260 [1:28:07<04:34,  2.92it/s]iter 15457: loss 5.5378, time: 320.26ms\n",
            " 95% 15458/16260 [1:28:08<04:29,  2.98it/s]iter 15458: loss 5.4732, time: 309.88ms\n",
            " 95% 15459/16260 [1:28:08<04:23,  3.04it/s]iter 15459: loss 5.2764, time: 465.00ms\n",
            " 95% 15460/16260 [1:28:08<04:55,  2.70it/s]iter 15460: loss 4.9593, time: 279.40ms\n",
            " 95% 15461/16260 [1:28:09<04:34,  2.91it/s]iter 15461: loss 5.1460, time: 238.36ms\n",
            " 95% 15462/16260 [1:28:09<04:09,  3.20it/s]iter 15462: loss 5.2606, time: 465.88ms\n",
            " 95% 15463/16260 [1:28:09<04:45,  2.79it/s]iter 15463: loss 5.3826, time: 384.44ms\n",
            " 95% 15464/16260 [1:28:10<04:52,  2.73it/s]iter 15464: loss 5.2759, time: 412.26ms\n",
            " 95% 15465/16260 [1:28:10<05:02,  2.62it/s]iter 15465: loss 5.6952, time: 464.39ms\n",
            " 95% 15466/16260 [1:28:11<05:22,  2.46it/s]iter 15466: loss 5.1918, time: 463.70ms\n",
            " 95% 15467/16260 [1:28:11<05:36,  2.36it/s]iter 15467: loss 5.0649, time: 282.66ms\n",
            " 95% 15468/16260 [1:28:11<05:02,  2.62it/s]iter 15468: loss 4.1119, time: 152.80ms\n",
            " 95% 15469/16260 [1:28:11<04:08,  3.19it/s]iter 15469: loss 5.3776, time: 280.83ms\n",
            " 95% 15470/16260 [1:28:12<04:00,  3.28it/s]iter 15470: loss 3.8146, time: 168.55ms\n",
            " 95% 15471/16260 [1:28:12<03:28,  3.79it/s]iter 15471: loss 4.8760, time: 240.30ms\n",
            " 95% 15472/16260 [1:28:12<03:22,  3.88it/s]iter 15472: loss 5.4767, time: 464.09ms\n",
            " 95% 15473/16260 [1:28:13<04:11,  3.13it/s]iter 15473: loss 4.6655, time: 259.55ms\n",
            " 95% 15474/16260 [1:28:13<03:57,  3.31it/s]iter 15474: loss 5.1008, time: 318.03ms\n",
            " 95% 15475/16260 [1:28:13<04:01,  3.25it/s]iter 15475: loss 5.4221, time: 325.06ms\n",
            " 95% 15476/16260 [1:28:14<04:05,  3.19it/s]iter 15476: loss 5.1885, time: 414.37ms\n",
            " 95% 15477/16260 [1:28:14<04:29,  2.91it/s]iter 15477: loss 4.7627, time: 188.48ms\n",
            " 95% 15478/16260 [1:28:14<03:52,  3.36it/s]iter 15478: loss 5.1204, time: 459.12ms\n",
            " 95% 15479/16260 [1:28:15<04:30,  2.88it/s]iter 15479: loss 6.0607, time: 464.21ms\n",
            " 95% 15480/16260 [1:28:15<04:58,  2.61it/s]iter 15480: loss 5.2902, time: 368.10ms\n",
            " 95% 15481/16260 [1:28:15<04:54,  2.64it/s]iter 15481: loss 5.6397, time: 464.95ms\n",
            " 95% 15482/16260 [1:28:16<05:15,  2.47it/s]iter 15482: loss 3.7910, time: 146.58ms\n",
            " 95% 15483/16260 [1:28:16<04:14,  3.05it/s]iter 15483: loss 4.5729, time: 211.13ms\n",
            " 95% 15484/16260 [1:28:16<03:47,  3.41it/s]iter 15484: loss 5.4703, time: 235.50ms\n",
            " 95% 15485/16260 [1:28:16<03:34,  3.62it/s]iter 15485: loss 5.2944, time: 473.66ms\n",
            " 95% 15486/16260 [1:28:17<04:20,  2.97it/s]iter 15486: loss 4.2286, time: 191.40ms\n",
            " 95% 15487/16260 [1:28:17<03:46,  3.41it/s]iter 15487: loss 4.0783, time: 257.37ms\n",
            " 95% 15488/16260 [1:28:17<03:38,  3.54it/s]iter 15488: loss 4.7973, time: 345.84ms\n",
            " 95% 15489/16260 [1:28:18<03:52,  3.31it/s]iter 15489: loss 5.3669, time: 438.32ms\n",
            " 95% 15490/16260 [1:28:18<04:24,  2.91it/s]iter 15490: loss 5.6514, time: 423.61ms\n",
            " 95% 15491/16260 [1:28:19<04:43,  2.72it/s]iter 15491: loss 4.9097, time: 170.80ms\n",
            " 95% 15492/16260 [1:28:19<03:57,  3.23it/s]iter 15492: loss 5.1099, time: 444.25ms\n",
            " 95% 15493/16260 [1:28:19<04:28,  2.85it/s]iter 15493: loss 5.8701, time: 443.34ms\n",
            " 95% 15494/16260 [1:28:20<04:50,  2.64it/s]iter 15494: loss 4.5983, time: 219.59ms\n",
            " 95% 15495/16260 [1:28:20<04:13,  3.02it/s]iter 15495: loss 5.2204, time: 425.83ms\n",
            " 95% 15496/16260 [1:28:20<04:35,  2.78it/s]iter 15496: loss 5.4520, time: 464.06ms\n",
            " 95% 15497/16260 [1:28:21<04:58,  2.55it/s]iter 15497: loss 5.6812, time: 464.15ms\n",
            " 95% 15498/16260 [1:28:21<05:15,  2.42it/s]iter 15498: loss 5.4745, time: 237.57ms\n",
            " 95% 15499/16260 [1:28:22<04:35,  2.77it/s]iter 15499: loss 4.6442, time: 271.87ms\n",
            " 95% 15500/16260 [1:28:22<04:14,  2.98it/s]iter 15500: loss 4.7113, time: 212.28ms\n",
            " 95% 15501/16260 [1:28:22<03:46,  3.35it/s]iter 15501: loss 4.0523, time: 209.57ms\n",
            " 95% 15502/16260 [1:28:22<03:26,  3.67it/s]iter 15502: loss 5.2047, time: 280.23ms\n",
            " 95% 15503/16260 [1:28:22<03:28,  3.63it/s]iter 15503: loss 4.6551, time: 239.32ms\n",
            " 95% 15504/16260 [1:28:23<03:20,  3.78it/s]iter 15504: loss 4.6863, time: 254.50ms\n",
            " 95% 15505/16260 [1:28:23<03:17,  3.81it/s]iter 15505: loss 6.1435, time: 464.99ms\n",
            " 95% 15506/16260 [1:28:23<04:03,  3.09it/s]iter 15506: loss 5.1931, time: 464.67ms\n",
            " 95% 15507/16260 [1:28:24<04:35,  2.73it/s]iter 15507: loss 4.9819, time: 283.68ms\n",
            " 95% 15508/16260 [1:28:24<04:17,  2.92it/s]iter 15508: loss 3.7931, time: 156.35ms\n",
            " 95% 15509/16260 [1:28:24<03:35,  3.49it/s]iter 15509: loss 4.9537, time: 243.06ms\n",
            " 95% 15510/16260 [1:28:25<03:25,  3.65it/s]iter 15510: loss 5.1475, time: 274.88ms\n",
            " 95% 15511/16260 [1:28:25<03:25,  3.64it/s]iter 15511: loss 4.4401, time: 213.61ms\n",
            " 95% 15512/16260 [1:28:25<03:12,  3.89it/s]iter 15512: loss 5.6989, time: 464.24ms\n",
            " 95% 15513/16260 [1:28:26<03:58,  3.13it/s]iter 15513: loss 5.5016, time: 470.24ms\n",
            " 95% 15514/16260 [1:28:26<04:32,  2.74it/s]iter 15514: loss 4.3510, time: 216.12ms\n",
            " 95% 15515/16260 [1:28:26<03:59,  3.11it/s]iter 15515: loss 4.0751, time: 166.23ms\n",
            " 95% 15516/16260 [1:28:26<03:24,  3.64it/s]iter 15516: loss 5.6808, time: 464.03ms\n",
            " 95% 15517/16260 [1:28:27<04:06,  3.01it/s]iter 15517: loss 5.3278, time: 315.23ms\n",
            " 95% 15518/16260 [1:28:27<04:03,  3.05it/s]iter 15518: loss 4.6091, time: 235.36ms\n",
            " 95% 15519/16260 [1:28:27<03:42,  3.33it/s]iter 15519: loss 4.4964, time: 166.20ms\n",
            " 95% 15520/16260 [1:28:28<03:12,  3.84it/s]iter 15520: loss 5.6361, time: 378.85ms\n",
            " 95% 15521/16260 [1:28:28<03:38,  3.37it/s]iter 15521: loss 4.6553, time: 463.80ms\n",
            " 95% 15522/16260 [1:28:28<04:16,  2.88it/s]iter 15522: loss 4.8351, time: 371.40ms\n",
            " 95% 15523/16260 [1:28:29<04:21,  2.82it/s]iter 15523: loss 5.6106, time: 464.10ms\n",
            " 95% 15524/16260 [1:28:29<04:45,  2.58it/s]iter 15524: loss 3.5510, time: 187.61ms\n",
            " 95% 15525/16260 [1:28:29<04:01,  3.04it/s]iter 15525: loss 4.7096, time: 463.41ms\n",
            " 95% 15526/16260 [1:28:30<04:31,  2.71it/s]iter 15526: loss 4.3440, time: 271.94ms\n",
            " 95% 15527/16260 [1:28:30<04:09,  2.94it/s]iter 15527: loss 4.5494, time: 167.57ms\n",
            " 95% 15528/16260 [1:28:30<03:31,  3.46it/s]iter 15528: loss 5.7329, time: 463.94ms\n",
            " 96% 15529/16260 [1:28:31<04:09,  2.92it/s]iter 15529: loss 4.5973, time: 237.01ms\n",
            " 96% 15530/16260 [1:28:31<03:47,  3.22it/s]iter 15530: loss 4.8091, time: 331.49ms\n",
            " 96% 15531/16260 [1:28:31<03:51,  3.15it/s]iter 15531: loss 5.9203, time: 464.29ms\n",
            " 96% 15532/16260 [1:28:32<04:23,  2.76it/s]iter 15532: loss 5.5482, time: 474.98ms\n",
            " 96% 15533/16260 [1:28:32<04:48,  2.52it/s]iter 15533: loss 5.6826, time: 465.31ms\n",
            " 96% 15534/16260 [1:28:33<05:03,  2.39it/s]iter 15534: loss 5.4669, time: 467.68ms\n",
            " 96% 15535/16260 [1:28:33<05:14,  2.31it/s]iter 15535: loss 5.3911, time: 253.12ms\n",
            " 96% 15536/16260 [1:28:34<04:34,  2.63it/s]iter 15536: loss 5.2900, time: 342.27ms\n",
            " 96% 15537/16260 [1:28:34<04:26,  2.71it/s]iter 15537: loss 5.1795, time: 380.58ms\n",
            " 96% 15538/16260 [1:28:34<04:29,  2.68it/s]iter 15538: loss 5.2430, time: 464.09ms\n",
            " 96% 15539/16260 [1:28:35<04:48,  2.50it/s]iter 15539: loss 5.0310, time: 432.58ms\n",
            " 96% 15540/16260 [1:28:35<04:55,  2.43it/s]iter 15540: loss 4.5807, time: 311.74ms\n",
            " 96% 15541/16260 [1:28:36<04:34,  2.62it/s]iter 15541: loss 5.8934, time: 463.61ms\n",
            " 96% 15542/16260 [1:28:36<04:51,  2.46it/s]iter 15542: loss 3.4770, time: 167.95ms\n",
            " 96% 15543/16260 [1:28:36<04:00,  2.98it/s]iter 15543: loss 4.7719, time: 233.94ms\n",
            " 96% 15544/16260 [1:28:36<03:38,  3.27it/s]iter 15544: loss 4.9057, time: 186.81ms\n",
            " 96% 15545/16260 [1:28:37<03:13,  3.70it/s]iter 15545: loss 4.9557, time: 232.71ms\n",
            " 96% 15546/16260 [1:28:37<03:05,  3.85it/s]iter 15546: loss 4.8154, time: 432.69ms\n",
            " 96% 15547/16260 [1:28:37<03:42,  3.21it/s]iter 15547: loss 4.8214, time: 233.78ms\n",
            " 96% 15548/16260 [1:28:37<03:25,  3.46it/s]iter 15548: loss 4.2997, time: 171.65ms\n",
            " 96% 15549/16260 [1:28:38<03:00,  3.93it/s]iter 15549: loss 4.1971, time: 187.51ms\n",
            " 96% 15550/16260 [1:28:38<02:46,  4.26it/s]iter 15550: loss 5.4333, time: 473.03ms\n",
            " 96% 15551/16260 [1:28:38<03:37,  3.26it/s]iter 15551: loss 5.4148, time: 323.95ms\n",
            " 96% 15552/16260 [1:28:39<03:41,  3.20it/s]iter 15552: loss 4.7684, time: 254.23ms\n",
            " 96% 15553/16260 [1:28:39<03:28,  3.39it/s]iter 15553: loss 5.1586, time: 475.37ms\n",
            " 96% 15554/16260 [1:28:39<04:06,  2.86it/s]iter 15554: loss 5.5531, time: 463.58ms\n",
            " 96% 15555/16260 [1:28:40<04:31,  2.60it/s]iter 15555: loss 5.4139, time: 446.36ms\n",
            " 96% 15556/16260 [1:28:40<04:44,  2.48it/s]iter 15556: loss 5.9966, time: 259.11ms\n",
            " 96% 15557/16260 [1:28:41<04:13,  2.77it/s]iter 15557: loss 4.1292, time: 187.79ms\n",
            " 96% 15558/16260 [1:28:41<03:37,  3.23it/s]iter 15558: loss 4.4769, time: 263.46ms\n",
            " 96% 15559/16260 [1:28:41<03:27,  3.38it/s]iter 15559: loss 4.8820, time: 262.31ms\n",
            " 96% 15560/16260 [1:28:41<03:20,  3.49it/s]iter 15560: loss 4.1094, time: 213.51ms\n",
            " 96% 15561/16260 [1:28:41<03:05,  3.77it/s]iter 15561: loss 4.2693, time: 167.53ms\n",
            " 96% 15562/16260 [1:28:42<02:44,  4.23it/s]iter 15562: loss 5.4056, time: 465.42ms\n",
            " 96% 15563/16260 [1:28:42<03:32,  3.27it/s]iter 15563: loss 5.0818, time: 468.04ms\n",
            " 96% 15564/16260 [1:28:43<04:06,  2.82it/s]iter 15564: loss 4.1268, time: 189.92ms\n",
            " 96% 15565/16260 [1:28:43<03:32,  3.27it/s]iter 15565: loss 4.8028, time: 417.21ms\n",
            " 96% 15566/16260 [1:28:43<03:55,  2.94it/s]iter 15566: loss 4.3208, time: 212.73ms\n",
            " 96% 15567/16260 [1:28:43<03:29,  3.31it/s]iter 15567: loss 5.1215, time: 511.55ms\n",
            " 96% 15568/16260 [1:28:44<04:12,  2.74it/s]iter 15568: loss 5.0178, time: 340.24ms\n",
            " 96% 15569/16260 [1:28:44<04:07,  2.79it/s]iter 15569: loss 4.8107, time: 464.39ms\n",
            " 96% 15570/16260 [1:28:45<04:29,  2.56it/s]iter 15570: loss 3.8686, time: 166.00ms\n",
            " 96% 15571/16260 [1:28:45<03:43,  3.09it/s]iter 15571: loss 5.1375, time: 463.87ms\n",
            " 96% 15572/16260 [1:28:45<04:11,  2.73it/s]iter 15572: loss 4.7710, time: 276.28ms\n",
            " 96% 15573/16260 [1:28:46<03:53,  2.94it/s]iter 15573: loss 5.3204, time: 432.45ms\n",
            " 96% 15574/16260 [1:28:46<04:12,  2.72it/s]iter 15574: loss 5.5007, time: 464.57ms\n",
            " 96% 15575/16260 [1:28:47<04:32,  2.52it/s]iter 15575: loss 5.3886, time: 473.69ms\n",
            " 96% 15576/16260 [1:28:47<04:47,  2.38it/s]iter 15576: loss 5.0732, time: 383.21ms\n",
            " 96% 15577/16260 [1:28:47<04:40,  2.44it/s]iter 15577: loss 5.0338, time: 238.87ms\n",
            " 96% 15578/16260 [1:28:48<04:04,  2.78it/s]iter 15578: loss 5.2213, time: 273.39ms\n",
            " 96% 15579/16260 [1:28:48<03:47,  2.99it/s]iter 15579: loss 4.9275, time: 338.55ms\n",
            " 96% 15580/16260 [1:28:48<03:48,  2.98it/s]iter 15580: loss 4.9997, time: 334.32ms\n",
            " 96% 15581/16260 [1:28:49<03:48,  2.98it/s]iter 15581: loss 4.7085, time: 282.18ms\n",
            " 96% 15582/16260 [1:28:49<03:37,  3.12it/s]iter 15582: loss 4.3143, time: 217.57ms\n",
            " 96% 15583/16260 [1:28:49<03:16,  3.45it/s]iter 15583: loss 4.7451, time: 216.15ms\n",
            " 96% 15584/16260 [1:28:49<03:01,  3.73it/s]iter 15584: loss 5.3359, time: 466.14ms\n",
            " 96% 15585/16260 [1:28:50<03:41,  3.05it/s]iter 15585: loss 4.5788, time: 186.78ms\n",
            " 96% 15586/16260 [1:28:50<03:12,  3.50it/s]iter 15586: loss 4.2858, time: 264.18ms\n",
            " 96% 15587/16260 [1:28:50<03:08,  3.57it/s]iter 15587: loss 5.3630, time: 473.55ms\n",
            " 96% 15588/16260 [1:28:51<03:47,  2.95it/s]iter 15588: loss 5.0927, time: 235.62ms\n",
            " 96% 15589/16260 [1:28:51<03:26,  3.25it/s]iter 15589: loss 5.7354, time: 279.06ms\n",
            " 96% 15590/16260 [1:28:51<03:20,  3.33it/s]iter 15590: loss 5.4210, time: 334.76ms\n",
            " 96% 15591/16260 [1:28:52<03:27,  3.22it/s]iter 15591: loss 4.5500, time: 464.23ms\n",
            " 96% 15592/16260 [1:28:52<03:58,  2.80it/s]iter 15592: loss 4.4385, time: 211.62ms\n",
            " 96% 15593/16260 [1:28:52<03:29,  3.18it/s]iter 15593: loss 5.3211, time: 321.31ms\n",
            " 96% 15594/16260 [1:28:53<03:31,  3.16it/s]iter 15594: loss 5.5785, time: 464.71ms\n",
            " 96% 15595/16260 [1:28:53<04:00,  2.76it/s]iter 15595: loss 5.3573, time: 464.76ms\n",
            " 96% 15596/16260 [1:28:53<04:21,  2.54it/s]iter 15596: loss 4.8200, time: 340.61ms\n",
            " 96% 15597/16260 [1:28:54<04:10,  2.65it/s]iter 15597: loss 5.2370, time: 464.59ms\n",
            " 96% 15598/16260 [1:28:54<04:27,  2.47it/s]iter 15598: loss 4.9287, time: 367.03ms\n",
            " 96% 15599/16260 [1:28:55<04:20,  2.54it/s]iter 15599: loss 5.7805, time: 465.63ms\n",
            " 96% 15600/16260 [1:28:55<04:34,  2.41it/s]iter 15600: loss 5.4427, time: 415.02ms\n",
            " 96% 15601/16260 [1:28:56<04:34,  2.40it/s]iter 15601: loss 5.8273, time: 331.96ms\n",
            " 96% 15602/16260 [1:28:56<04:17,  2.56it/s]iter 15602: loss 5.1890, time: 210.70ms\n",
            " 96% 15603/16260 [1:28:56<03:41,  2.96it/s]iter 15603: loss 5.8397, time: 472.49ms\n",
            " 96% 15604/16260 [1:28:57<04:08,  2.64it/s]iter 15604: loss 5.0773, time: 213.08ms\n",
            " 96% 15605/16260 [1:28:57<03:35,  3.04it/s]iter 15605: loss 5.9800, time: 465.89ms\n",
            " 96% 15606/16260 [1:28:57<04:02,  2.70it/s]iter 15606: loss 3.5494, time: 139.37ms\n",
            " 96% 15607/16260 [1:28:57<03:17,  3.31it/s]iter 15607: loss 5.8924, time: 465.48ms\n",
            " 96% 15608/16260 [1:28:58<03:49,  2.85it/s]iter 15608: loss 4.7270, time: 272.94ms\n",
            " 96% 15609/16260 [1:28:58<03:33,  3.05it/s]iter 15609: loss 5.3508, time: 306.82ms\n",
            " 96% 15610/16260 [1:28:58<03:29,  3.10it/s]iter 15610: loss 4.5929, time: 231.35ms\n",
            " 96% 15611/16260 [1:28:59<03:11,  3.38it/s]iter 15611: loss 5.0174, time: 232.43ms\n",
            " 96% 15612/16260 [1:28:59<02:59,  3.61it/s]iter 15612: loss 4.7011, time: 218.73ms\n",
            " 96% 15613/16260 [1:28:59<02:48,  3.85it/s]iter 15613: loss 4.6881, time: 173.36ms\n",
            " 96% 15614/16260 [1:28:59<02:31,  4.26it/s]iter 15614: loss 4.6151, time: 274.50ms\n",
            " 96% 15615/16260 [1:29:00<02:39,  4.05it/s]iter 15615: loss 3.6733, time: 192.46ms\n",
            " 96% 15616/16260 [1:29:00<02:28,  4.33it/s]iter 15616: loss 4.8370, time: 462.38ms\n",
            " 96% 15617/16260 [1:29:00<03:13,  3.32it/s]iter 15617: loss 4.9415, time: 422.24ms\n",
            " 96% 15618/16260 [1:29:01<03:36,  2.96it/s]iter 15618: loss 5.1705, time: 305.51ms\n",
            " 96% 15619/16260 [1:29:01<03:30,  3.04it/s]iter 15619: loss 4.1963, time: 234.45ms\n",
            " 96% 15620/16260 [1:29:01<03:12,  3.32it/s]iter 15620: loss 2.7010, time: 135.54ms\n",
            " 96% 15621/16260 [1:29:01<02:40,  3.97it/s]iter 15621: loss 5.2345, time: 413.09ms\n",
            " 96% 15622/16260 [1:29:02<03:11,  3.33it/s]iter 15622: loss 4.9116, time: 343.62ms\n",
            " 96% 15623/16260 [1:29:02<03:19,  3.19it/s]iter 15623: loss 4.4602, time: 255.88ms\n",
            " 96% 15624/16260 [1:29:02<03:08,  3.37it/s]iter 15624: loss 5.0462, time: 472.67ms\n",
            " 96% 15625/16260 [1:29:03<03:42,  2.86it/s]iter 15625: loss 4.9369, time: 463.94ms\n",
            " 96% 15626/16260 [1:29:03<04:03,  2.60it/s]iter 15626: loss 4.4607, time: 169.39ms\n",
            " 96% 15627/16260 [1:29:03<03:22,  3.12it/s]iter 15627: loss 5.9936, time: 473.30ms\n",
            " 96% 15628/16260 [1:29:04<03:51,  2.73it/s]iter 15628: loss 5.2744, time: 464.77ms\n",
            " 96% 15629/16260 [1:29:04<04:10,  2.52it/s]iter 15629: loss 5.1195, time: 463.26ms\n",
            " 96% 15630/16260 [1:29:05<04:22,  2.40it/s]iter 15630: loss 6.0077, time: 464.22ms\n",
            " 96% 15631/16260 [1:29:05<04:31,  2.32it/s]iter 15631: loss 4.9955, time: 256.98ms\n",
            " 96% 15632/16260 [1:29:06<03:58,  2.63it/s]iter 15632: loss 5.6945, time: 469.66ms\n",
            " 96% 15633/16260 [1:29:06<04:15,  2.46it/s]iter 15633: loss 5.9793, time: 464.66ms\n",
            " 96% 15634/16260 [1:29:07<04:26,  2.35it/s]iter 15634: loss 6.0955, time: 465.52ms\n",
            " 96% 15635/16260 [1:29:07<04:33,  2.29it/s]iter 15635: loss 5.2886, time: 259.30ms\n",
            " 96% 15636/16260 [1:29:07<03:59,  2.60it/s]iter 15636: loss 4.3352, time: 212.42ms\n",
            " 96% 15637/16260 [1:29:07<03:27,  3.00it/s]iter 15637: loss 4.6105, time: 171.38ms\n",
            " 96% 15638/16260 [1:29:08<02:57,  3.51it/s]iter 15638: loss 3.9287, time: 208.23ms\n",
            " 96% 15639/16260 [1:29:08<02:43,  3.81it/s]iter 15639: loss 5.3469, time: 281.02ms\n",
            " 96% 15640/16260 [1:29:08<02:46,  3.72it/s]iter 15640: loss 4.9847, time: 464.17ms\n",
            " 96% 15641/16260 [1:29:09<03:22,  3.05it/s]iter 15641: loss 4.4479, time: 273.94ms\n",
            " 96% 15642/16260 [1:29:09<03:12,  3.20it/s]iter 15642: loss 5.5076, time: 464.82ms\n",
            " 96% 15643/16260 [1:29:09<03:41,  2.79it/s]iter 15643: loss 4.9803, time: 255.81ms\n",
            " 96% 15644/16260 [1:29:10<03:22,  3.05it/s]iter 15644: loss 4.6639, time: 277.35ms\n",
            " 96% 15645/16260 [1:29:10<03:12,  3.19it/s]iter 15645: loss 4.0074, time: 167.89ms\n",
            " 96% 15646/16260 [1:29:10<02:45,  3.70it/s]iter 15646: loss 4.4874, time: 214.33ms\n",
            " 96% 15647/16260 [1:29:10<02:35,  3.94it/s]iter 15647: loss 4.6852, time: 221.40ms\n",
            " 96% 15648/16260 [1:29:11<02:29,  4.09it/s]iter 15648: loss 4.6882, time: 238.57ms\n",
            " 96% 15649/16260 [1:29:11<02:28,  4.11it/s]iter 15649: loss 5.9266, time: 463.79ms\n",
            " 96% 15650/16260 [1:29:11<03:09,  3.23it/s]iter 15650: loss 5.5487, time: 414.39ms\n",
            " 96% 15651/16260 [1:29:12<03:28,  2.93it/s]iter 15651: loss 4.9511, time: 274.50ms\n",
            " 96% 15652/16260 [1:29:12<03:15,  3.11it/s]iter 15652: loss 4.5820, time: 238.57ms\n",
            " 96% 15653/16260 [1:29:12<03:00,  3.36it/s]iter 15653: loss 5.5139, time: 463.94ms\n",
            " 96% 15654/16260 [1:29:13<03:30,  2.87it/s]iter 15654: loss 3.7798, time: 145.53ms\n",
            " 96% 15655/16260 [1:29:13<02:54,  3.48it/s]iter 15655: loss 5.6954, time: 463.68ms\n",
            " 96% 15656/16260 [1:29:13<03:25,  2.93it/s]iter 15656: loss 5.0599, time: 430.90ms\n",
            " 96% 15657/16260 [1:29:14<03:42,  2.71it/s]iter 15657: loss 5.0158, time: 257.09ms\n",
            " 96% 15658/16260 [1:29:14<03:21,  2.98it/s]iter 15658: loss 5.5685, time: 316.10ms\n",
            " 96% 15659/16260 [1:29:14<03:18,  3.03it/s]iter 15659: loss 5.7199, time: 462.81ms\n",
            " 96% 15660/16260 [1:29:15<03:42,  2.70it/s]iter 15660: loss 4.4500, time: 432.33ms\n",
            " 96% 15661/16260 [1:29:15<03:53,  2.57it/s]iter 15661: loss 5.3382, time: 438.29ms\n",
            " 96% 15662/16260 [1:29:16<04:01,  2.47it/s]iter 15662: loss 5.9349, time: 464.54ms\n",
            " 96% 15663/16260 [1:29:16<04:12,  2.36it/s]iter 15663: loss 5.1186, time: 421.93ms\n",
            " 96% 15664/16260 [1:29:16<04:12,  2.36it/s]iter 15664: loss 4.5847, time: 313.71ms\n",
            " 96% 15665/16260 [1:29:17<03:52,  2.56it/s]iter 15665: loss 5.1994, time: 342.64ms\n",
            " 96% 15666/16260 [1:29:17<03:43,  2.65it/s]iter 15666: loss 5.2290, time: 282.46ms\n",
            " 96% 15667/16260 [1:29:17<03:27,  2.86it/s]iter 15667: loss 4.5778, time: 236.64ms\n",
            " 96% 15668/16260 [1:29:18<03:06,  3.17it/s]iter 15668: loss 5.1296, time: 218.38ms\n",
            " 96% 15669/16260 [1:29:18<02:49,  3.49it/s]iter 15669: loss 5.7441, time: 464.10ms\n",
            " 96% 15670/16260 [1:29:18<03:20,  2.94it/s]iter 15670: loss 4.4386, time: 208.89ms\n",
            " 96% 15671/16260 [1:29:19<02:57,  3.32it/s]iter 15671: loss 4.5056, time: 233.06ms\n",
            " 96% 15672/16260 [1:29:19<02:45,  3.55it/s]iter 15672: loss 5.5947, time: 319.17ms\n",
            " 96% 15673/16260 [1:29:19<02:52,  3.41it/s]iter 15673: loss 4.8924, time: 411.55ms\n",
            " 96% 15674/16260 [1:29:20<03:12,  3.04it/s]iter 15674: loss 5.6398, time: 465.12ms\n",
            " 96% 15675/16260 [1:29:20<03:36,  2.70it/s]iter 15675: loss 4.8392, time: 388.79ms\n",
            " 96% 15676/16260 [1:29:20<03:39,  2.66it/s]iter 15676: loss 4.7408, time: 347.52ms\n",
            " 96% 15677/16260 [1:29:21<03:34,  2.71it/s]iter 15677: loss 4.8409, time: 257.73ms\n",
            " 96% 15678/16260 [1:29:21<03:15,  2.98it/s]iter 15678: loss 6.0051, time: 465.62ms\n",
            " 96% 15679/16260 [1:29:21<03:38,  2.66it/s]iter 15679: loss 5.1536, time: 423.55ms\n",
            " 96% 15680/16260 [1:29:22<03:46,  2.56it/s]iter 15680: loss 5.9423, time: 473.06ms\n",
            " 96% 15681/16260 [1:29:22<04:00,  2.41it/s]iter 15681: loss 5.4372, time: 464.91ms\n",
            " 96% 15682/16260 [1:29:23<04:09,  2.32it/s]iter 15682: loss 4.1716, time: 232.39ms\n",
            " 96% 15683/16260 [1:29:23<03:34,  2.69it/s]iter 15683: loss 3.9613, time: 173.59ms\n",
            " 96% 15684/16260 [1:29:23<03:00,  3.20it/s]iter 15684: loss 5.2429, time: 464.49ms\n",
            " 96% 15685/16260 [1:29:24<03:26,  2.79it/s]iter 15685: loss 5.3991, time: 464.09ms\n",
            " 96% 15686/16260 [1:29:24<03:44,  2.56it/s]iter 15686: loss 4.3478, time: 167.25ms\n",
            " 96% 15687/16260 [1:29:24<03:05,  3.08it/s]iter 15687: loss 5.2916, time: 377.40ms\n",
            " 96% 15688/16260 [1:29:25<03:14,  2.94it/s]iter 15688: loss 5.4134, time: 278.54ms\n",
            " 96% 15689/16260 [1:29:25<03:04,  3.10it/s]iter 15689: loss 5.2082, time: 464.43ms\n",
            " 96% 15690/16260 [1:29:25<03:28,  2.74it/s]iter 15690: loss 4.6748, time: 255.29ms\n",
            " 97% 15691/16260 [1:29:26<03:09,  3.00it/s]iter 15691: loss 5.4232, time: 463.90ms\n",
            " 97% 15692/16260 [1:29:26<03:31,  2.68it/s]iter 15692: loss 4.6670, time: 254.14ms\n",
            " 97% 15693/16260 [1:29:26<03:11,  2.96it/s]iter 15693: loss 3.5677, time: 166.40ms\n",
            " 97% 15694/16260 [1:29:27<02:42,  3.49it/s]iter 15694: loss 5.9444, time: 414.62ms\n",
            " 97% 15695/16260 [1:29:27<03:03,  3.07it/s]iter 15695: loss 5.8342, time: 475.03ms\n",
            " 97% 15696/16260 [1:29:27<03:29,  2.70it/s]iter 15696: loss 5.9197, time: 464.98ms\n",
            " 97% 15697/16260 [1:29:28<03:44,  2.50it/s]iter 15697: loss 4.6769, time: 358.83ms\n",
            " 97% 15698/16260 [1:29:28<03:37,  2.58it/s]iter 15698: loss 3.5777, time: 165.28ms\n",
            " 97% 15699/16260 [1:29:28<03:00,  3.11it/s]iter 15699: loss 4.8212, time: 239.43ms\n",
            " 97% 15700/16260 [1:29:29<02:46,  3.36it/s]iter 15700: loss 5.0309, time: 273.69ms\n",
            " 97% 15701/16260 [1:29:29<02:42,  3.44it/s]iter 15701: loss 4.6732, time: 423.52ms\n",
            " 97% 15702/16260 [1:29:29<03:04,  3.02it/s]iter 15702: loss 5.2717, time: 275.82ms\n",
            " 97% 15703/16260 [1:29:30<02:55,  3.17it/s]iter 15703: loss 5.8732, time: 464.38ms\n",
            " 97% 15704/16260 [1:29:30<03:20,  2.78it/s]iter 15704: loss 4.7199, time: 230.58ms\n",
            " 97% 15705/16260 [1:29:30<02:58,  3.11it/s]iter 15705: loss 2.9277, time: 149.65ms\n",
            " 97% 15706/16260 [1:29:31<02:30,  3.69it/s]iter 15706: loss 4.5680, time: 416.94ms\n",
            " 97% 15707/16260 [1:29:31<02:54,  3.17it/s]iter 15707: loss 5.1479, time: 433.03ms\n",
            " 97% 15708/16260 [1:29:31<03:13,  2.85it/s]iter 15708: loss 4.9513, time: 464.58ms\n",
            " 97% 15709/16260 [1:29:32<03:32,  2.59it/s]iter 15709: loss 4.2978, time: 188.20ms\n",
            " 97% 15710/16260 [1:29:32<02:59,  3.06it/s]iter 15710: loss 5.4225, time: 338.31ms\n",
            " 97% 15711/16260 [1:29:32<03:01,  3.02it/s]iter 15711: loss 4.7253, time: 310.72ms\n",
            " 97% 15712/16260 [1:29:33<02:58,  3.08it/s]iter 15712: loss 4.4880, time: 340.10ms\n",
            " 97% 15713/16260 [1:29:33<03:00,  3.03it/s]iter 15713: loss 5.5290, time: 377.59ms\n",
            " 97% 15714/16260 [1:29:33<03:08,  2.90it/s]iter 15714: loss 5.0749, time: 466.27ms\n",
            " 97% 15715/16260 [1:29:34<03:28,  2.62it/s]iter 15715: loss 4.0964, time: 189.85ms\n",
            " 97% 15716/16260 [1:29:34<02:56,  3.08it/s]iter 15716: loss 4.5375, time: 245.34ms\n",
            " 97% 15717/16260 [1:29:34<02:43,  3.32it/s]iter 15717: loss 5.0719, time: 421.73ms\n",
            " 97% 15718/16260 [1:29:35<03:03,  2.96it/s]iter 15718: loss 5.2241, time: 465.14ms\n",
            " 97% 15719/16260 [1:29:35<03:23,  2.65it/s]iter 15719: loss 4.0823, time: 150.49ms\n",
            " 97% 15720/16260 [1:29:35<02:47,  3.23it/s]iter 15720: loss 5.7299, time: 474.27ms\n",
            " 97% 15721/16260 [1:29:36<03:13,  2.78it/s]iter 15721: loss 5.8939, time: 464.86ms\n",
            " 97% 15722/16260 [1:29:36<03:30,  2.55it/s]iter 15722: loss 5.0813, time: 415.33ms\n",
            " 97% 15723/16260 [1:29:37<03:34,  2.51it/s]iter 15723: loss 5.5060, time: 463.70ms\n",
            " 97% 15724/16260 [1:29:37<03:44,  2.39it/s]iter 15724: loss 4.5422, time: 235.13ms\n",
            " 97% 15725/16260 [1:29:37<03:14,  2.75it/s]iter 15725: loss 5.1702, time: 369.93ms\n",
            " 97% 15726/16260 [1:29:38<03:15,  2.73it/s]iter 15726: loss 4.7643, time: 315.11ms\n",
            " 97% 15727/16260 [1:29:38<03:07,  2.84it/s]iter 15727: loss 5.1040, time: 477.06ms\n",
            " 97% 15728/16260 [1:29:39<03:27,  2.57it/s]iter 15728: loss 5.3897, time: 439.32ms\n",
            " 97% 15729/16260 [1:29:39<03:35,  2.47it/s]iter 15729: loss 5.1724, time: 464.60ms\n",
            " 97% 15730/16260 [1:29:40<03:44,  2.36it/s]iter 15730: loss 3.5888, time: 188.31ms\n",
            " 97% 15731/16260 [1:29:40<03:06,  2.83it/s]iter 15731: loss 3.9330, time: 216.82ms\n",
            " 97% 15732/16260 [1:29:40<02:45,  3.20it/s]iter 15732: loss 5.1387, time: 463.71ms\n",
            " 97% 15733/16260 [1:29:40<03:08,  2.79it/s]iter 15733: loss 4.6965, time: 209.10ms\n",
            " 97% 15734/16260 [1:29:41<02:45,  3.18it/s]iter 15734: loss 4.2735, time: 213.29ms\n",
            " 97% 15735/16260 [1:29:41<02:29,  3.52it/s]iter 15735: loss 5.4399, time: 314.89ms\n",
            " 97% 15736/16260 [1:29:41<02:34,  3.40it/s]iter 15736: loss 5.7498, time: 369.12ms\n",
            " 97% 15737/16260 [1:29:41<02:45,  3.15it/s]iter 15737: loss 5.5924, time: 465.57ms\n",
            " 97% 15738/16260 [1:29:42<03:09,  2.76it/s]iter 15738: loss 5.3273, time: 463.89ms\n",
            " 97% 15739/16260 [1:29:42<03:24,  2.54it/s]iter 15739: loss 4.6735, time: 241.35ms\n",
            " 97% 15740/16260 [1:29:43<03:00,  2.87it/s]iter 15740: loss 5.4155, time: 463.97ms\n",
            " 97% 15741/16260 [1:29:43<03:18,  2.61it/s]iter 15741: loss 4.6697, time: 188.23ms\n",
            " 97% 15742/16260 [1:29:43<02:48,  3.08it/s]iter 15742: loss 4.6246, time: 211.11ms\n",
            " 97% 15743/16260 [1:29:44<02:30,  3.43it/s]iter 15743: loss 5.4385, time: 467.13ms\n",
            " 97% 15744/16260 [1:29:44<02:57,  2.90it/s]iter 15744: loss 5.4446, time: 281.22ms\n",
            " 97% 15745/16260 [1:29:44<02:47,  3.07it/s]iter 15745: loss 5.4582, time: 474.23ms\n",
            " 97% 15746/16260 [1:29:45<03:10,  2.70it/s]iter 15746: loss 4.1835, time: 213.78ms\n",
            " 97% 15747/16260 [1:29:45<02:46,  3.08it/s]iter 15747: loss 5.6765, time: 258.67ms\n",
            " 97% 15748/16260 [1:29:45<02:36,  3.28it/s]iter 15748: loss 3.7872, time: 145.74ms\n",
            " 97% 15749/16260 [1:29:45<02:11,  3.88it/s]iter 15749: loss 4.9683, time: 467.52ms\n",
            " 97% 15750/16260 [1:29:46<02:43,  3.12it/s]iter 15750: loss 5.6205, time: 430.57ms\n",
            " 97% 15751/16260 [1:29:46<03:00,  2.82it/s]iter 15751: loss 4.9248, time: 235.63ms\n",
            " 97% 15752/16260 [1:29:47<02:42,  3.13it/s]iter 15752: loss 4.8453, time: 474.94ms\n",
            " 97% 15753/16260 [1:29:47<03:05,  2.73it/s]iter 15753: loss 5.1623, time: 463.55ms\n",
            " 97% 15754/16260 [1:29:47<03:20,  2.52it/s]iter 15754: loss 4.1144, time: 191.82ms\n",
            " 97% 15755/16260 [1:29:48<02:49,  2.98it/s]iter 15755: loss 5.4893, time: 464.45ms\n",
            " 97% 15756/16260 [1:29:48<03:08,  2.67it/s]iter 15756: loss 5.7383, time: 462.75ms\n",
            " 97% 15757/16260 [1:29:49<03:21,  2.49it/s]iter 15757: loss 5.6969, time: 464.27ms\n",
            " 97% 15758/16260 [1:29:49<03:31,  2.38it/s]iter 15758: loss 4.9677, time: 279.24ms\n",
            " 97% 15759/16260 [1:29:49<03:09,  2.64it/s]iter 15759: loss 4.7628, time: 280.91ms\n",
            " 97% 15760/16260 [1:29:50<02:54,  2.86it/s]iter 15760: loss 2.7808, time: 138.44ms\n",
            " 97% 15761/16260 [1:29:50<02:23,  3.49it/s]iter 15761: loss 4.1078, time: 150.93ms\n",
            " 97% 15762/16260 [1:29:50<02:02,  4.06it/s]iter 15762: loss 5.3926, time: 279.16ms\n",
            " 97% 15763/16260 [1:29:50<02:07,  3.89it/s]iter 15763: loss 4.3566, time: 217.72ms\n",
            " 97% 15764/16260 [1:29:50<02:01,  4.07it/s]iter 15764: loss 5.8003, time: 465.45ms\n",
            " 97% 15765/16260 [1:29:51<02:34,  3.21it/s]iter 15765: loss 5.1731, time: 464.42ms\n",
            " 97% 15766/16260 [1:29:51<02:56,  2.79it/s]iter 15766: loss 5.3922, time: 252.47ms\n",
            " 97% 15767/16260 [1:29:52<02:41,  3.06it/s]iter 15767: loss 5.7891, time: 469.66ms\n",
            " 97% 15768/16260 [1:29:52<03:02,  2.70it/s]iter 15768: loss 4.9357, time: 346.19ms\n",
            " 97% 15769/16260 [1:29:52<02:58,  2.75it/s]iter 15769: loss 4.8834, time: 474.24ms\n",
            " 97% 15770/16260 [1:29:53<03:14,  2.52it/s]iter 15770: loss 5.5540, time: 239.50ms\n",
            " 97% 15771/16260 [1:29:53<02:51,  2.85it/s]iter 15771: loss 5.3448, time: 336.84ms\n",
            " 97% 15772/16260 [1:29:53<02:49,  2.88it/s]iter 15772: loss 5.1110, time: 272.73ms\n",
            " 97% 15773/16260 [1:29:54<02:38,  3.08it/s]iter 15773: loss 4.3077, time: 235.97ms\n",
            " 97% 15774/16260 [1:29:54<02:25,  3.35it/s]iter 15774: loss 4.4414, time: 190.94ms\n",
            " 97% 15775/16260 [1:29:54<02:09,  3.75it/s]iter 15775: loss 5.1738, time: 284.47ms\n",
            " 97% 15776/16260 [1:29:54<02:11,  3.67it/s]iter 15776: loss 5.3319, time: 341.38ms\n",
            " 97% 15777/16260 [1:29:55<02:21,  3.41it/s]iter 15777: loss 4.4868, time: 181.06ms\n",
            " 97% 15778/16260 [1:29:55<02:05,  3.84it/s]iter 15778: loss 4.6308, time: 189.03ms\n",
            " 97% 15779/16260 [1:29:55<01:55,  4.18it/s]iter 15779: loss 3.9450, time: 209.40ms\n",
            " 97% 15780/16260 [1:29:55<01:50,  4.33it/s]iter 15780: loss 5.2518, time: 463.87ms\n",
            " 97% 15781/16260 [1:29:56<02:24,  3.32it/s]iter 15781: loss 2.6562, time: 140.39ms\n",
            " 97% 15782/16260 [1:29:56<02:01,  3.94it/s]iter 15782: loss 5.4014, time: 278.70ms\n",
            " 97% 15783/16260 [1:29:56<02:04,  3.82it/s]iter 15783: loss 4.9529, time: 255.70ms\n",
            " 97% 15784/16260 [1:29:57<02:03,  3.84it/s]iter 15784: loss 4.3542, time: 254.99ms\n",
            " 97% 15785/16260 [1:29:57<02:03,  3.86it/s]iter 15785: loss 3.4814, time: 189.81ms\n",
            " 97% 15786/16260 [1:29:57<01:53,  4.19it/s]iter 15786: loss 3.8798, time: 166.77ms\n",
            " 97% 15787/16260 [1:29:57<01:43,  4.59it/s]iter 15787: loss 4.4568, time: 338.03ms\n",
            " 97% 15788/16260 [1:29:57<02:00,  3.93it/s]iter 15788: loss 5.2345, time: 463.99ms\n",
            " 97% 15789/16260 [1:29:58<02:29,  3.15it/s]iter 15789: loss 5.0988, time: 380.62ms\n",
            " 97% 15790/16260 [1:29:58<02:38,  2.97it/s]iter 15790: loss 5.3860, time: 464.34ms\n",
            " 97% 15791/16260 [1:29:59<02:56,  2.66it/s]iter 15791: loss 4.2717, time: 193.66ms\n",
            " 97% 15792/16260 [1:29:59<02:30,  3.11it/s]iter 15792: loss 4.2438, time: 223.34ms\n",
            " 97% 15793/16260 [1:29:59<02:16,  3.42it/s]iter 15793: loss 5.4028, time: 464.50ms\n",
            " 97% 15794/16260 [1:30:00<02:40,  2.90it/s]iter 15794: loss 4.9578, time: 234.20ms\n",
            " 97% 15795/16260 [1:30:00<02:25,  3.21it/s]iter 15795: loss 5.4036, time: 465.05ms\n",
            " 97% 15796/16260 [1:30:00<02:46,  2.79it/s]iter 15796: loss 4.9837, time: 255.02ms\n",
            " 97% 15797/16260 [1:30:01<02:31,  3.05it/s]iter 15797: loss 4.7620, time: 280.72ms\n",
            " 97% 15798/16260 [1:30:01<02:25,  3.18it/s]iter 15798: loss 4.4320, time: 306.69ms\n",
            " 97% 15799/16260 [1:30:01<02:24,  3.20it/s]iter 15799: loss 4.9398, time: 464.30ms\n",
            " 97% 15800/16260 [1:30:02<02:44,  2.79it/s]iter 15800: loss 4.5073, time: 345.08ms\n",
            " 97% 15801/16260 [1:30:02<02:42,  2.82it/s]iter 15801: loss 4.4080, time: 166.43ms\n",
            " 97% 15802/16260 [1:30:02<02:16,  3.34it/s]iter 15802: loss 4.4799, time: 193.85ms\n",
            " 97% 15803/16260 [1:30:02<02:02,  3.73it/s]iter 15803: loss 5.7128, time: 371.56ms\n",
            " 97% 15804/16260 [1:30:03<02:16,  3.34it/s]iter 15804: loss 3.6927, time: 169.73ms\n",
            " 97% 15805/16260 [1:30:03<01:58,  3.83it/s]iter 15805: loss 3.9393, time: 165.48ms\n",
            " 97% 15806/16260 [1:30:03<01:45,  4.29it/s]iter 15806: loss 5.2719, time: 416.03ms\n",
            " 97% 15807/16260 [1:30:04<02:10,  3.47it/s]iter 15807: loss 5.6339, time: 417.20ms\n",
            " 97% 15808/16260 [1:30:04<02:27,  3.05it/s]iter 15808: loss 5.2017, time: 235.09ms\n",
            " 97% 15809/16260 [1:30:04<02:15,  3.33it/s]iter 15809: loss 5.2075, time: 464.60ms\n",
            " 97% 15810/16260 [1:30:05<02:37,  2.86it/s]iter 15810: loss 4.6108, time: 166.66ms\n",
            " 97% 15811/16260 [1:30:05<02:12,  3.38it/s]iter 15811: loss 4.3955, time: 237.69ms\n",
            " 97% 15812/16260 [1:30:05<02:04,  3.59it/s]iter 15812: loss 5.4642, time: 463.63ms\n",
            " 97% 15813/16260 [1:30:06<02:29,  2.99it/s]iter 15813: loss 5.1016, time: 464.22ms\n",
            " 97% 15814/16260 [1:30:06<02:46,  2.67it/s]iter 15814: loss 5.8037, time: 473.74ms\n",
            " 97% 15815/16260 [1:30:06<02:59,  2.47it/s]iter 15815: loss 4.5396, time: 218.71ms\n",
            " 97% 15816/16260 [1:30:07<02:35,  2.86it/s]iter 15816: loss 4.3899, time: 239.53ms\n",
            " 97% 15817/16260 [1:30:07<02:20,  3.16it/s]iter 15817: loss 3.6391, time: 147.44ms\n",
            " 97% 15818/16260 [1:30:07<01:57,  3.75it/s]iter 15818: loss 5.3677, time: 464.09ms\n",
            " 97% 15819/16260 [1:30:08<02:23,  3.06it/s]iter 15819: loss 4.6331, time: 169.19ms\n",
            " 97% 15820/16260 [1:30:08<02:03,  3.58it/s]iter 15820: loss 4.8677, time: 274.70ms\n",
            " 97% 15821/16260 [1:30:08<02:02,  3.59it/s]iter 15821: loss 6.0210, time: 464.81ms\n",
            " 97% 15822/16260 [1:30:08<02:26,  2.99it/s]iter 15822: loss 4.5431, time: 216.09ms\n",
            " 97% 15823/16260 [1:30:09<02:11,  3.34it/s]iter 15823: loss 4.5418, time: 344.92ms\n",
            " 97% 15824/16260 [1:30:09<02:16,  3.19it/s]iter 15824: loss 5.9063, time: 466.50ms\n",
            " 97% 15825/16260 [1:30:10<02:36,  2.78it/s]iter 15825: loss 5.3151, time: 368.36ms\n",
            " 97% 15826/16260 [1:30:10<02:37,  2.75it/s]iter 15826: loss 5.8557, time: 309.90ms\n",
            " 97% 15827/16260 [1:30:10<02:30,  2.88it/s]iter 15827: loss 3.7309, time: 194.43ms\n",
            " 97% 15828/16260 [1:30:10<02:10,  3.31it/s]iter 15828: loss 4.6804, time: 345.89ms\n",
            " 97% 15829/16260 [1:30:11<02:16,  3.17it/s]iter 15829: loss 5.2642, time: 463.74ms\n",
            " 97% 15830/16260 [1:30:11<02:35,  2.77it/s]iter 15830: loss 5.4005, time: 464.17ms\n",
            " 97% 15831/16260 [1:30:12<02:48,  2.55it/s]iter 15831: loss 4.6044, time: 149.44ms\n",
            " 97% 15832/16260 [1:30:12<02:16,  3.13it/s]iter 15832: loss 5.4724, time: 463.89ms\n",
            " 97% 15833/16260 [1:30:12<02:35,  2.75it/s]iter 15833: loss 4.6613, time: 215.47ms\n",
            " 97% 15834/16260 [1:30:12<02:16,  3.13it/s]iter 15834: loss 3.9184, time: 233.76ms\n",
            " 97% 15835/16260 [1:30:13<02:05,  3.40it/s]iter 15835: loss 5.0654, time: 444.75ms\n",
            " 97% 15836/16260 [1:30:13<02:24,  2.94it/s]iter 15836: loss 4.2561, time: 209.00ms\n",
            " 97% 15837/16260 [1:30:13<02:07,  3.32it/s]iter 15837: loss 5.3997, time: 369.89ms\n",
            " 97% 15838/16260 [1:30:14<02:15,  3.10it/s]iter 15838: loss 4.5535, time: 252.50ms\n",
            " 97% 15839/16260 [1:30:14<02:07,  3.31it/s]iter 15839: loss 5.9038, time: 467.98ms\n",
            " 97% 15840/16260 [1:30:14<02:27,  2.84it/s]iter 15840: loss 5.3526, time: 464.51ms\n",
            " 97% 15841/16260 [1:30:15<02:41,  2.59it/s]iter 15841: loss 4.3358, time: 172.26ms\n",
            " 97% 15842/16260 [1:30:15<02:14,  3.10it/s]iter 15842: loss 4.8183, time: 412.16ms\n",
            " 97% 15843/16260 [1:30:16<02:25,  2.86it/s]iter 15843: loss 5.0830, time: 464.31ms\n",
            " 97% 15844/16260 [1:30:16<02:40,  2.60it/s]iter 15844: loss 5.7044, time: 473.48ms\n",
            " 97% 15845/16260 [1:30:16<02:50,  2.43it/s]iter 15845: loss 4.7328, time: 238.84ms\n",
            " 97% 15846/16260 [1:30:17<02:29,  2.77it/s]iter 15846: loss 5.4174, time: 418.23ms\n",
            " 97% 15847/16260 [1:30:17<02:36,  2.64it/s]iter 15847: loss 5.9298, time: 474.98ms\n",
            " 97% 15848/16260 [1:30:18<02:48,  2.45it/s]iter 15848: loss 4.0677, time: 198.39ms\n",
            " 97% 15849/16260 [1:30:18<02:22,  2.89it/s]iter 15849: loss 5.2449, time: 320.54ms\n",
            " 97% 15850/16260 [1:30:18<02:18,  2.95it/s]iter 15850: loss 5.5029, time: 464.13ms\n",
            " 97% 15851/16260 [1:30:19<02:34,  2.65it/s]iter 15851: loss 4.9720, time: 468.64ms\n",
            " 97% 15852/16260 [1:30:19<02:45,  2.47it/s]iter 15852: loss 5.3306, time: 382.88ms\n",
            " 97% 15853/16260 [1:30:19<02:42,  2.51it/s]iter 15853: loss 5.3619, time: 469.04ms\n",
            " 98% 15854/16260 [1:30:20<02:50,  2.38it/s]iter 15854: loss 4.9169, time: 464.03ms\n",
            " 98% 15855/16260 [1:30:20<02:55,  2.30it/s]iter 15855: loss 4.3575, time: 242.31ms\n",
            " 98% 15856/16260 [1:30:21<02:32,  2.65it/s]iter 15856: loss 5.5991, time: 463.89ms\n",
            " 98% 15857/16260 [1:30:21<02:42,  2.48it/s]iter 15857: loss 5.9645, time: 277.95ms\n",
            " 98% 15858/16260 [1:30:21<02:27,  2.73it/s]iter 15858: loss 5.2923, time: 251.62ms\n",
            " 98% 15859/16260 [1:30:22<02:13,  3.01it/s]iter 15859: loss 5.2213, time: 464.20ms\n",
            " 98% 15860/16260 [1:30:22<02:28,  2.69it/s]iter 15860: loss 5.5559, time: 369.11ms\n",
            " 98% 15861/16260 [1:30:22<02:28,  2.69it/s]iter 15861: loss 4.6105, time: 221.23ms\n",
            " 98% 15862/16260 [1:30:23<02:10,  3.06it/s]iter 15862: loss 4.7245, time: 341.14ms\n",
            " 98% 15863/16260 [1:30:23<02:11,  3.01it/s]iter 15863: loss 5.9912, time: 463.91ms\n",
            " 98% 15864/16260 [1:30:24<02:27,  2.69it/s]iter 15864: loss 5.4630, time: 343.49ms\n",
            " 98% 15865/16260 [1:30:24<02:23,  2.75it/s]iter 15865: loss 4.6712, time: 234.33ms\n",
            " 98% 15866/16260 [1:30:24<02:08,  3.07it/s]iter 15866: loss 4.5352, time: 273.39ms\n",
            " 98% 15867/16260 [1:30:24<02:01,  3.22it/s]iter 15867: loss 3.6115, time: 172.67ms\n",
            " 98% 15868/16260 [1:30:25<01:45,  3.71it/s]iter 15868: loss 5.2334, time: 265.54ms\n",
            " 98% 15869/16260 [1:30:25<01:45,  3.72it/s]iter 15869: loss 5.0432, time: 344.51ms\n",
            " 98% 15870/16260 [1:30:25<01:53,  3.42it/s]iter 15870: loss 5.0921, time: 379.17ms\n",
            " 98% 15871/16260 [1:30:26<02:03,  3.14it/s]iter 15871: loss 5.6081, time: 467.14ms\n",
            " 98% 15872/16260 [1:30:26<02:21,  2.75it/s]iter 15872: loss 5.1281, time: 464.39ms\n",
            " 98% 15873/16260 [1:30:26<02:32,  2.54it/s]iter 15873: loss 5.7113, time: 464.14ms\n",
            " 98% 15874/16260 [1:30:27<02:40,  2.41it/s]iter 15874: loss 5.5761, time: 463.36ms\n",
            " 98% 15875/16260 [1:30:27<02:45,  2.32it/s]iter 15875: loss 5.2337, time: 382.83ms\n",
            " 98% 15876/16260 [1:30:28<02:39,  2.40it/s]iter 15876: loss 4.1978, time: 239.03ms\n",
            " 98% 15877/16260 [1:30:28<02:19,  2.75it/s]iter 15877: loss 4.1936, time: 165.95ms\n",
            " 98% 15878/16260 [1:30:28<01:56,  3.28it/s]iter 15878: loss 4.4644, time: 437.04ms\n",
            " 98% 15879/16260 [1:30:29<02:11,  2.90it/s]iter 15879: loss 5.5340, time: 464.74ms\n",
            " 98% 15880/16260 [1:30:29<02:24,  2.62it/s]iter 15880: loss 5.8155, time: 464.89ms\n",
            " 98% 15881/16260 [1:30:30<02:34,  2.46it/s]iter 15881: loss 4.0783, time: 173.68ms\n",
            " 98% 15882/16260 [1:30:30<02:07,  2.96it/s]iter 15882: loss 5.0683, time: 463.93ms\n",
            " 98% 15883/16260 [1:30:30<02:21,  2.66it/s]iter 15883: loss 5.7620, time: 465.64ms\n",
            " 98% 15884/16260 [1:30:31<02:31,  2.48it/s]iter 15884: loss 4.3434, time: 346.16ms\n",
            " 98% 15885/16260 [1:30:31<02:25,  2.59it/s]iter 15885: loss 5.5091, time: 464.60ms\n",
            " 98% 15886/16260 [1:30:31<02:33,  2.44it/s]iter 15886: loss 4.3205, time: 188.73ms\n",
            " 98% 15887/16260 [1:30:32<02:08,  2.90it/s]iter 15887: loss 4.8944, time: 238.61ms\n",
            " 98% 15888/16260 [1:30:32<01:56,  3.19it/s]iter 15888: loss 5.5492, time: 438.52ms\n",
            " 98% 15889/16260 [1:30:32<02:10,  2.85it/s]iter 15889: loss 5.8289, time: 463.93ms\n",
            " 98% 15890/16260 [1:30:33<02:22,  2.59it/s]iter 15890: loss 4.0923, time: 208.69ms\n",
            " 98% 15891/16260 [1:30:33<02:02,  3.00it/s]iter 15891: loss 5.4918, time: 463.79ms\n",
            " 98% 15892/16260 [1:30:33<02:17,  2.68it/s]iter 15892: loss 5.7012, time: 470.85ms\n",
            " 98% 15893/16260 [1:30:34<02:27,  2.48it/s]iter 15893: loss 4.1654, time: 209.24ms\n",
            " 98% 15894/16260 [1:30:34<02:06,  2.90it/s]iter 15894: loss 3.7658, time: 151.83ms\n",
            " 98% 15895/16260 [1:30:34<01:44,  3.48it/s]iter 15895: loss 5.9217, time: 464.24ms\n",
            " 98% 15896/16260 [1:30:35<02:04,  2.93it/s]iter 15896: loss 5.1321, time: 451.20ms\n",
            " 98% 15897/16260 [1:30:35<02:15,  2.67it/s]iter 15897: loss 4.9037, time: 464.42ms\n",
            " 98% 15898/16260 [1:30:36<02:25,  2.49it/s]iter 15898: loss 5.5063, time: 464.32ms\n",
            " 98% 15899/16260 [1:30:36<02:32,  2.37it/s]iter 15899: loss 4.5531, time: 307.66ms\n",
            " 98% 15900/16260 [1:30:36<02:19,  2.58it/s]iter 15900: loss 4.1003, time: 189.68ms\n",
            " 98% 15901/16260 [1:30:37<01:57,  3.04it/s]iter 15901: loss 4.9079, time: 217.72ms\n",
            " 98% 15902/16260 [1:30:37<01:45,  3.38it/s]iter 15902: loss 5.7670, time: 464.85ms\n",
            " 98% 15903/16260 [1:30:37<02:03,  2.88it/s]iter 15903: loss 4.6198, time: 217.76ms\n",
            " 98% 15904/16260 [1:30:38<01:49,  3.24it/s]iter 15904: loss 4.5638, time: 463.77ms\n",
            " 98% 15905/16260 [1:30:38<02:06,  2.81it/s]iter 15905: loss 5.8135, time: 463.81ms\n",
            " 98% 15906/16260 [1:30:39<02:17,  2.57it/s]iter 15906: loss 6.1876, time: 464.60ms\n",
            " 98% 15907/16260 [1:30:39<02:25,  2.43it/s]iter 15907: loss 5.5319, time: 464.12ms\n",
            " 98% 15908/16260 [1:30:39<02:30,  2.34it/s]iter 15908: loss 5.4213, time: 434.49ms\n",
            " 98% 15909/16260 [1:30:40<02:31,  2.32it/s]iter 15909: loss 5.0455, time: 247.86ms\n",
            " 98% 15910/16260 [1:30:40<02:11,  2.66it/s]iter 15910: loss 6.2108, time: 465.06ms\n",
            " 98% 15911/16260 [1:30:41<02:20,  2.48it/s]iter 15911: loss 5.5889, time: 471.56ms\n",
            " 98% 15912/16260 [1:30:41<02:27,  2.36it/s]iter 15912: loss 4.8481, time: 464.24ms\n",
            " 98% 15913/16260 [1:30:42<02:31,  2.29it/s]iter 15913: loss 4.9414, time: 334.70ms\n",
            " 98% 15914/16260 [1:30:42<02:20,  2.46it/s]iter 15914: loss 4.6104, time: 208.04ms\n",
            " 98% 15915/16260 [1:30:42<01:59,  2.88it/s]iter 15915: loss 5.8235, time: 464.49ms\n",
            " 98% 15916/16260 [1:30:43<02:11,  2.61it/s]iter 15916: loss 5.3784, time: 347.15ms\n",
            " 98% 15917/16260 [1:30:43<02:07,  2.68it/s]iter 15917: loss 4.9768, time: 472.02ms\n",
            " 98% 15918/16260 [1:30:43<02:17,  2.48it/s]iter 15918: loss 5.1723, time: 340.29ms\n",
            " 98% 15919/16260 [1:30:44<02:11,  2.60it/s]iter 15919: loss 5.0160, time: 465.63ms\n",
            " 98% 15920/16260 [1:30:44<02:19,  2.44it/s]iter 15920: loss 4.6117, time: 317.84ms\n",
            " 98% 15921/16260 [1:30:44<02:09,  2.62it/s]iter 15921: loss 5.6601, time: 464.18ms\n",
            " 98% 15922/16260 [1:30:45<02:17,  2.45it/s]iter 15922: loss 4.7842, time: 470.34ms\n",
            " 98% 15923/16260 [1:30:45<02:23,  2.34it/s]iter 15923: loss 5.2795, time: 464.48ms\n",
            " 98% 15924/16260 [1:30:46<02:27,  2.28it/s]iter 15924: loss 5.1462, time: 255.43ms\n",
            " 98% 15925/16260 [1:30:46<02:08,  2.60it/s]iter 15925: loss 5.4124, time: 416.95ms\n",
            " 98% 15926/16260 [1:30:47<02:11,  2.54it/s]iter 15926: loss 5.6218, time: 464.48ms\n",
            " 98% 15927/16260 [1:30:47<02:18,  2.40it/s]iter 15927: loss 5.8824, time: 464.95ms\n",
            " 98% 15928/16260 [1:30:48<02:23,  2.32it/s]iter 15928: loss 5.7529, time: 464.70ms\n",
            " 98% 15929/16260 [1:30:48<02:26,  2.26it/s]iter 15929: loss 5.2762, time: 446.70ms\n",
            " 98% 15930/16260 [1:30:48<02:26,  2.25it/s]iter 15930: loss 4.8494, time: 239.72ms\n",
            " 98% 15931/16260 [1:30:49<02:05,  2.61it/s]iter 15931: loss 5.4872, time: 464.57ms\n",
            " 98% 15932/16260 [1:30:49<02:13,  2.45it/s]iter 15932: loss 5.0515, time: 376.49ms\n",
            " 98% 15933/16260 [1:30:50<02:10,  2.51it/s]iter 15933: loss 4.2269, time: 259.83ms\n",
            " 98% 15934/16260 [1:30:50<01:56,  2.80it/s]iter 15934: loss 5.1801, time: 425.48ms\n",
            " 98% 15935/16260 [1:30:50<02:03,  2.64it/s]iter 15935: loss 5.0116, time: 261.20ms\n",
            " 98% 15936/16260 [1:30:50<01:51,  2.91it/s]iter 15936: loss 5.7647, time: 464.19ms\n",
            " 98% 15937/16260 [1:30:51<02:02,  2.63it/s]iter 15937: loss 5.4998, time: 464.09ms\n",
            " 98% 15938/16260 [1:30:51<02:10,  2.46it/s]iter 15938: loss 4.8340, time: 236.13ms\n",
            " 98% 15939/16260 [1:30:52<01:54,  2.81it/s]iter 15939: loss 5.7169, time: 464.04ms\n",
            " 98% 15940/16260 [1:30:52<02:04,  2.57it/s]iter 15940: loss 5.1992, time: 229.20ms\n",
            " 98% 15941/16260 [1:30:52<01:48,  2.93it/s]iter 15941: loss 4.2181, time: 167.31ms\n",
            " 98% 15942/16260 [1:30:52<01:32,  3.45it/s]iter 15942: loss 4.4984, time: 171.62ms\n",
            " 98% 15943/16260 [1:30:53<01:20,  3.93it/s]iter 15943: loss 5.2182, time: 463.51ms\n",
            " 98% 15944/16260 [1:30:53<01:40,  3.15it/s]iter 15944: loss 5.2864, time: 236.32ms\n",
            " 98% 15945/16260 [1:30:53<01:32,  3.40it/s]iter 15945: loss 4.8238, time: 253.32ms\n",
            " 98% 15946/16260 [1:30:54<01:28,  3.54it/s]iter 15946: loss 4.6503, time: 318.09ms\n",
            " 98% 15947/16260 [1:30:54<01:31,  3.41it/s]iter 15947: loss 4.9831, time: 341.62ms\n",
            " 98% 15948/16260 [1:30:54<01:36,  3.24it/s]iter 15948: loss 5.3068, time: 342.81ms\n",
            " 98% 15949/16260 [1:30:55<01:39,  3.13it/s]iter 15949: loss 5.6136, time: 475.29ms\n",
            " 98% 15950/16260 [1:30:55<01:53,  2.73it/s]iter 15950: loss 4.9779, time: 257.04ms\n",
            " 98% 15951/16260 [1:30:55<01:43,  2.99it/s]iter 15951: loss 5.0981, time: 464.96ms\n",
            " 98% 15952/16260 [1:30:56<01:55,  2.68it/s]iter 15952: loss 5.5282, time: 238.11ms\n",
            " 98% 15953/16260 [1:30:56<01:42,  3.00it/s]iter 15953: loss 5.7598, time: 464.93ms\n",
            " 98% 15954/16260 [1:30:57<01:54,  2.68it/s]iter 15954: loss 4.2454, time: 233.64ms\n",
            " 98% 15955/16260 [1:30:57<01:41,  3.01it/s]iter 15955: loss 4.3608, time: 369.99ms\n",
            " 98% 15956/16260 [1:30:57<01:44,  2.91it/s]iter 15956: loss 4.5078, time: 187.99ms\n",
            " 98% 15957/16260 [1:30:57<01:30,  3.36it/s]iter 15957: loss 5.4080, time: 464.32ms\n",
            " 98% 15958/16260 [1:30:58<01:45,  2.87it/s]iter 15958: loss 3.7278, time: 149.48ms\n",
            " 98% 15959/16260 [1:30:58<01:26,  3.46it/s]iter 15959: loss 4.5085, time: 209.81ms\n",
            " 98% 15960/16260 [1:30:58<01:19,  3.76it/s]iter 15960: loss 4.6729, time: 441.36ms\n",
            " 98% 15961/16260 [1:30:59<01:35,  3.14it/s]iter 15961: loss 5.3851, time: 254.33ms\n",
            " 98% 15962/16260 [1:30:59<01:29,  3.33it/s]iter 15962: loss 4.8588, time: 212.59ms\n",
            " 98% 15963/16260 [1:30:59<01:21,  3.65it/s]iter 15963: loss 5.3223, time: 465.52ms\n",
            " 98% 15964/16260 [1:31:00<01:38,  3.01it/s]iter 15964: loss 4.6017, time: 464.93ms\n",
            " 98% 15965/16260 [1:31:00<01:49,  2.69it/s]iter 15965: loss 5.7247, time: 465.31ms\n",
            " 98% 15966/16260 [1:31:00<01:57,  2.50it/s]iter 15966: loss 4.1969, time: 153.24ms\n",
            " 98% 15967/16260 [1:31:01<01:35,  3.06it/s]iter 15967: loss 5.1101, time: 469.96ms\n",
            " 98% 15968/16260 [1:31:01<01:48,  2.70it/s]iter 15968: loss 4.7360, time: 372.97ms\n",
            " 98% 15969/16260 [1:31:01<01:48,  2.69it/s]iter 15969: loss 4.9769, time: 235.91ms\n",
            " 98% 15970/16260 [1:31:02<01:36,  3.02it/s]iter 15970: loss 4.9806, time: 464.28ms\n",
            " 98% 15971/16260 [1:31:02<01:47,  2.69it/s]iter 15971: loss 5.4208, time: 419.65ms\n",
            " 98% 15972/16260 [1:31:03<01:51,  2.59it/s]iter 15972: loss 4.2174, time: 150.85ms\n",
            " 98% 15973/16260 [1:31:03<01:30,  3.16it/s]iter 15973: loss 5.2134, time: 271.40ms\n",
            " 98% 15974/16260 [1:31:03<01:26,  3.30it/s]iter 15974: loss 4.1191, time: 276.15ms\n",
            " 98% 15975/16260 [1:31:03<01:24,  3.38it/s]iter 15975: loss 3.8791, time: 236.99ms\n",
            " 98% 15976/16260 [1:31:04<01:19,  3.59it/s]iter 15976: loss 4.7628, time: 273.79ms\n",
            " 98% 15977/16260 [1:31:04<01:18,  3.60it/s]iter 15977: loss 5.5600, time: 464.31ms\n",
            " 98% 15978/16260 [1:31:04<01:34,  2.99it/s]iter 15978: loss 5.8142, time: 468.76ms\n",
            " 98% 15979/16260 [1:31:05<01:45,  2.67it/s]iter 15979: loss 5.9840, time: 465.28ms\n",
            " 98% 15980/16260 [1:31:05<01:52,  2.48it/s]iter 15980: loss 5.1824, time: 456.36ms\n",
            " 98% 15981/16260 [1:31:06<01:56,  2.39it/s]iter 15981: loss 5.1560, time: 463.59ms\n",
            " 98% 15982/16260 [1:31:06<02:00,  2.31it/s]iter 15982: loss 5.0121, time: 212.04ms\n",
            " 98% 15983/16260 [1:31:06<01:41,  2.72it/s]iter 15983: loss 4.8122, time: 193.10ms\n",
            " 98% 15984/16260 [1:31:07<01:27,  3.17it/s]iter 15984: loss 4.1283, time: 167.49ms\n",
            " 98% 15985/16260 [1:31:07<01:14,  3.68it/s]iter 15985: loss 4.1920, time: 190.68ms\n",
            " 98% 15986/16260 [1:31:07<01:07,  4.04it/s]iter 15986: loss 4.9492, time: 464.93ms\n",
            " 98% 15987/16260 [1:31:07<01:25,  3.19it/s]iter 15987: loss 4.8360, time: 249.49ms\n",
            " 98% 15988/16260 [1:31:08<01:20,  3.39it/s]iter 15988: loss 5.0917, time: 226.09ms\n",
            " 98% 15989/16260 [1:31:08<01:14,  3.64it/s]iter 15989: loss 5.4830, time: 463.70ms\n",
            " 98% 15990/16260 [1:31:08<01:29,  3.01it/s]iter 15990: loss 5.1405, time: 274.13ms\n",
            " 98% 15991/16260 [1:31:09<01:24,  3.17it/s]iter 15991: loss 4.2251, time: 173.51ms\n",
            " 98% 15992/16260 [1:31:09<01:13,  3.66it/s]iter 15992: loss 5.6171, time: 341.07ms\n",
            " 98% 15993/16260 [1:31:09<01:18,  3.40it/s]iter 15993: loss 5.0424, time: 421.62ms\n",
            " 98% 15994/16260 [1:31:10<01:28,  3.01it/s]iter 15994: loss 4.9880, time: 236.52ms\n",
            " 98% 15995/16260 [1:31:10<01:20,  3.29it/s]iter 15995: loss 5.7262, time: 464.56ms\n",
            " 98% 15996/16260 [1:31:10<01:33,  2.83it/s]iter 15996: loss 4.6687, time: 258.38ms\n",
            " 98% 15997/16260 [1:31:11<01:25,  3.08it/s]iter 15997: loss 4.8672, time: 376.34ms\n",
            " 98% 15998/16260 [1:31:11<01:29,  2.93it/s]iter 15998: loss 5.0695, time: 421.28ms\n",
            " 98% 15999/16260 [1:31:11<01:35,  2.74it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:In the year accounts modelsives\n",
            "-Openitssp damf\n",
            "2- Bott (You nm rules\n",
            "- Markamle:\n",
            "Many availabilityidaamzz -attery- Qatarticallya\n",
            "D Ethics - R - \n",
            "-unders Time gameingotairgrad\n",
            "- Carij** Spmuff\n",
            "Cran Czech Sl County\n",
            "Most Play\n",
            "-ided\n",
            "-A mileor management X\n",
            "-corquain Major\n",
            "W\n",
            "A fraud pedestrians gallons the undergraduate andatt\n",
            "step 15999: val loss 5.0460\n",
            "Saving LoRA weights to out/lora/common1\n",
            "iter 15999: loss 4.9058, time: 16327.32ms\n",
            " 98% 16000/16260 [1:31:28<22:20,  5.15s/it]iter 16000: loss 4.4637, time: 171.88ms\n",
            " 98% 16001/16260 [1:31:28<15:47,  3.66s/it]iter 16001: loss 4.5583, time: 237.32ms\n",
            " 98% 16002/16260 [1:31:28<11:19,  2.63s/it]iter 16002: loss 4.5447, time: 238.78ms\n",
            " 98% 16003/16260 [1:31:28<08:12,  1.92s/it]iter 16003: loss 5.6171, time: 467.75ms\n",
            " 98% 16004/16260 [1:31:29<06:19,  1.48s/it]iter 16004: loss 5.0288, time: 232.26ms\n",
            " 98% 16005/16260 [1:31:29<04:42,  1.11s/it]iter 16005: loss 5.1767, time: 248.50ms\n",
            " 98% 16006/16260 [1:31:29<03:36,  1.18it/s]iter 16006: loss 4.8944, time: 331.90ms\n",
            " 98% 16007/16260 [1:31:30<02:55,  1.44it/s]iter 16007: loss 5.1297, time: 252.31ms\n",
            " 98% 16008/16260 [1:31:30<02:21,  1.78it/s]iter 16008: loss 5.0821, time: 213.02ms\n",
            " 98% 16009/16260 [1:31:30<01:55,  2.18it/s]iter 16009: loss 4.3410, time: 293.59ms\n",
            " 98% 16010/16260 [1:31:30<01:42,  2.44it/s]iter 16010: loss 3.1652, time: 151.15ms\n",
            " 98% 16011/16260 [1:31:30<01:22,  3.01it/s]iter 16011: loss 4.6383, time: 377.15ms\n",
            " 98% 16012/16260 [1:31:31<01:25,  2.89it/s]iter 16012: loss 5.8557, time: 464.25ms\n",
            " 98% 16013/16260 [1:31:31<01:34,  2.62it/s]iter 16013: loss 4.5778, time: 271.74ms\n",
            " 98% 16014/16260 [1:31:32<01:25,  2.86it/s]iter 16014: loss 5.1604, time: 252.93ms\n",
            " 98% 16015/16260 [1:31:32<01:18,  3.12it/s]iter 16015: loss 5.3708, time: 375.57ms\n",
            " 98% 16016/16260 [1:31:32<01:22,  2.96it/s]iter 16016: loss 5.3278, time: 316.27ms\n",
            " 99% 16017/16260 [1:31:33<01:20,  3.01it/s]iter 16017: loss 4.9968, time: 316.83ms\n",
            " 99% 16018/16260 [1:31:33<01:19,  3.05it/s]iter 16018: loss 5.1238, time: 242.93ms\n",
            " 99% 16019/16260 [1:31:33<01:12,  3.30it/s]iter 16019: loss 4.1778, time: 192.73ms\n",
            " 99% 16020/16260 [1:31:33<01:04,  3.70it/s]iter 16020: loss 5.5642, time: 463.97ms\n",
            " 99% 16021/16260 [1:31:34<01:18,  3.04it/s]iter 16021: loss 4.7652, time: 316.06ms\n",
            " 99% 16022/16260 [1:31:34<01:17,  3.07it/s]iter 16022: loss 5.0302, time: 343.06ms\n",
            " 99% 16023/16260 [1:31:34<01:18,  3.02it/s]iter 16023: loss 4.5714, time: 261.00ms\n",
            " 99% 16024/16260 [1:31:35<01:13,  3.22it/s]iter 16024: loss 4.2141, time: 167.36ms\n",
            " 99% 16025/16260 [1:31:35<01:02,  3.73it/s]iter 16025: loss 6.1388, time: 466.45ms\n",
            " 99% 16026/16260 [1:31:35<01:16,  3.05it/s]iter 16026: loss 4.6416, time: 217.88ms\n",
            " 99% 16027/16260 [1:31:36<01:08,  3.38it/s]iter 16027: loss 4.3372, time: 274.75ms\n",
            " 99% 16028/16260 [1:31:36<01:07,  3.45it/s]iter 16028: loss 5.0668, time: 345.30ms\n",
            " 99% 16029/16260 [1:31:36<01:10,  3.26it/s]iter 16029: loss 4.8135, time: 274.11ms\n",
            " 99% 16030/16260 [1:31:36<01:08,  3.36it/s]iter 16030: loss 4.4420, time: 190.33ms\n",
            " 99% 16031/16260 [1:31:37<01:00,  3.76it/s]iter 16031: loss 5.7718, time: 468.79ms\n",
            " 99% 16032/16260 [1:31:37<01:14,  3.06it/s]iter 16032: loss 5.4101, time: 463.93ms\n",
            " 99% 16033/16260 [1:31:38<01:23,  2.71it/s]iter 16033: loss 5.5046, time: 437.15ms\n",
            " 99% 16034/16260 [1:31:38<01:28,  2.57it/s]iter 16034: loss 5.8434, time: 465.14ms\n",
            " 99% 16035/16260 [1:31:39<01:32,  2.42it/s]iter 16035: loss 4.7263, time: 216.91ms\n",
            " 99% 16036/16260 [1:31:39<01:19,  2.82it/s]iter 16036: loss 5.4940, time: 277.63ms\n",
            " 99% 16037/16260 [1:31:39<01:14,  3.01it/s]iter 16037: loss 5.1800, time: 464.12ms\n",
            " 99% 16038/16260 [1:31:39<01:22,  2.69it/s]iter 16038: loss 5.2841, time: 243.31ms\n",
            " 99% 16039/16260 [1:31:40<01:13,  2.99it/s]iter 16039: loss 5.1571, time: 373.07ms\n",
            " 99% 16040/16260 [1:31:40<01:16,  2.89it/s]iter 16040: loss 6.2846, time: 476.29ms\n",
            " 99% 16041/16260 [1:31:41<01:24,  2.59it/s]iter 16041: loss 4.8726, time: 377.50ms\n",
            " 99% 16042/16260 [1:31:41<01:23,  2.61it/s]iter 16042: loss 5.8525, time: 346.92ms\n",
            " 99% 16043/16260 [1:31:41<01:20,  2.68it/s]iter 16043: loss 4.2605, time: 172.61ms\n",
            " 99% 16044/16260 [1:31:41<01:07,  3.19it/s]iter 16044: loss 5.6919, time: 309.51ms\n",
            " 99% 16045/16260 [1:31:42<01:07,  3.20it/s]iter 16045: loss 5.1093, time: 464.07ms\n",
            " 99% 16046/16260 [1:31:42<01:16,  2.79it/s]iter 16046: loss 4.1877, time: 211.06ms\n",
            " 99% 16047/16260 [1:31:42<01:07,  3.18it/s]iter 16047: loss 5.1353, time: 276.71ms\n",
            " 99% 16048/16260 [1:31:43<01:04,  3.29it/s]iter 16048: loss 4.7654, time: 255.68ms\n",
            " 99% 16049/16260 [1:31:43<01:01,  3.45it/s]iter 16049: loss 5.2087, time: 259.25ms\n",
            " 99% 16050/16260 [1:31:43<00:59,  3.56it/s]iter 16050: loss 4.3194, time: 186.27ms\n",
            " 99% 16051/16260 [1:31:43<00:52,  3.95it/s]iter 16051: loss 3.7083, time: 232.40ms\n",
            " 99% 16052/16260 [1:31:44<00:51,  4.04it/s]iter 16052: loss 4.6034, time: 170.46ms\n",
            " 99% 16053/16260 [1:31:44<00:46,  4.45it/s]iter 16053: loss 5.2666, time: 441.63ms\n",
            " 99% 16054/16260 [1:31:44<00:59,  3.44it/s]iter 16054: loss 3.6846, time: 150.85ms\n",
            " 99% 16055/16260 [1:31:44<00:51,  4.02it/s]iter 16055: loss 5.4613, time: 463.87ms\n",
            " 99% 16056/16260 [1:31:45<01:04,  3.19it/s]iter 16056: loss 5.1110, time: 233.93ms\n",
            " 99% 16057/16260 [1:31:45<00:58,  3.44it/s]iter 16057: loss 4.9271, time: 337.56ms\n",
            " 99% 16058/16260 [1:31:45<01:01,  3.28it/s]iter 16058: loss 5.1716, time: 257.55ms\n",
            " 99% 16059/16260 [1:31:46<00:58,  3.43it/s]iter 16059: loss 5.2645, time: 379.78ms\n",
            " 99% 16060/16260 [1:31:46<01:03,  3.14it/s]iter 16060: loss 5.1982, time: 233.78ms\n",
            " 99% 16061/16260 [1:31:46<00:58,  3.41it/s]iter 16061: loss 5.5445, time: 475.62ms\n",
            " 99% 16062/16260 [1:31:47<01:08,  2.87it/s]iter 16062: loss 5.1998, time: 432.46ms\n",
            " 99% 16063/16260 [1:31:47<01:13,  2.67it/s]iter 16063: loss 4.6236, time: 190.76ms\n",
            " 99% 16064/16260 [1:31:47<01:02,  3.13it/s]iter 16064: loss 5.5769, time: 464.18ms\n",
            " 99% 16065/16260 [1:31:48<01:10,  2.75it/s]iter 16065: loss 5.3935, time: 330.88ms\n",
            " 99% 16066/16260 [1:31:48<01:08,  2.82it/s]iter 16066: loss 3.9800, time: 214.05ms\n",
            " 99% 16067/16260 [1:31:48<01:00,  3.20it/s]iter 16067: loss 4.8991, time: 337.11ms\n",
            " 99% 16068/16260 [1:31:49<01:01,  3.12it/s]iter 16068: loss 4.8966, time: 315.05ms\n",
            " 99% 16069/16260 [1:31:49<01:00,  3.13it/s]iter 16069: loss 5.0193, time: 260.59ms\n",
            " 99% 16070/16260 [1:31:49<00:57,  3.31it/s]iter 16070: loss 5.6527, time: 464.76ms\n",
            " 99% 16071/16260 [1:31:50<01:06,  2.85it/s]iter 16071: loss 3.7429, time: 465.04ms\n",
            " 99% 16072/16260 [1:31:50<01:12,  2.59it/s]iter 16072: loss 3.5916, time: 170.12ms\n",
            " 99% 16073/16260 [1:31:50<01:00,  3.11it/s]iter 16073: loss 4.7913, time: 238.06ms\n",
            " 99% 16074/16260 [1:31:51<00:55,  3.37it/s]iter 16074: loss 5.0127, time: 318.67ms\n",
            " 99% 16075/16260 [1:31:51<00:56,  3.29it/s]iter 16075: loss 4.6358, time: 278.39ms\n",
            " 99% 16076/16260 [1:31:51<00:54,  3.37it/s]iter 16076: loss 5.5294, time: 442.83ms\n",
            " 99% 16077/16260 [1:31:52<01:02,  2.93it/s]iter 16077: loss 4.7288, time: 256.11ms\n",
            " 99% 16078/16260 [1:31:52<00:57,  3.16it/s]iter 16078: loss 4.7941, time: 421.65ms\n",
            " 99% 16079/16260 [1:31:52<01:03,  2.87it/s]iter 16079: loss 5.4418, time: 469.91ms\n",
            " 99% 16080/16260 [1:31:53<01:09,  2.60it/s]iter 16080: loss 5.3112, time: 464.51ms\n",
            " 99% 16081/16260 [1:31:53<01:13,  2.44it/s]iter 16081: loss 3.3370, time: 167.10ms\n",
            " 99% 16082/16260 [1:31:54<01:00,  2.97it/s]iter 16082: loss 5.1055, time: 469.64ms\n",
            " 99% 16083/16260 [1:31:54<01:06,  2.65it/s]iter 16083: loss 4.1077, time: 237.19ms\n",
            " 99% 16084/16260 [1:31:54<00:59,  2.98it/s]iter 16084: loss 5.1258, time: 261.60ms\n",
            " 99% 16085/16260 [1:31:55<00:54,  3.18it/s]iter 16085: loss 4.6141, time: 207.01ms\n",
            " 99% 16086/16260 [1:31:55<00:49,  3.54it/s]iter 16086: loss 5.0835, time: 314.01ms\n",
            " 99% 16087/16260 [1:31:55<00:50,  3.42it/s]iter 16087: loss 5.8187, time: 464.39ms\n",
            " 99% 16088/16260 [1:31:56<00:59,  2.90it/s]iter 16088: loss 2.6697, time: 137.03ms\n",
            " 99% 16089/16260 [1:31:56<00:48,  3.54it/s]iter 16089: loss 3.9395, time: 172.11ms\n",
            " 99% 16090/16260 [1:31:56<00:42,  4.00it/s]iter 16090: loss 4.4598, time: 215.40ms\n",
            " 99% 16091/16260 [1:31:56<00:40,  4.17it/s]iter 16091: loss 5.4545, time: 440.00ms\n",
            " 99% 16092/16260 [1:31:57<00:50,  3.33it/s]iter 16092: loss 5.6621, time: 467.00ms\n",
            " 99% 16093/16260 [1:31:57<00:58,  2.85it/s]iter 16093: loss 4.8565, time: 462.74ms\n",
            " 99% 16094/16260 [1:31:57<01:03,  2.60it/s]iter 16094: loss 5.5558, time: 413.24ms\n",
            " 99% 16095/16260 [1:31:58<01:04,  2.54it/s]iter 16095: loss 5.2079, time: 467.05ms\n",
            " 99% 16096/16260 [1:31:58<01:08,  2.40it/s]iter 16096: loss 4.1497, time: 192.24ms\n",
            " 99% 16097/16260 [1:31:59<00:56,  2.86it/s]iter 16097: loss 5.1555, time: 476.44ms\n",
            " 99% 16098/16260 [1:31:59<01:02,  2.58it/s]iter 16098: loss 5.1605, time: 274.78ms\n",
            " 99% 16099/16260 [1:31:59<00:57,  2.82it/s]iter 16099: loss 4.5698, time: 186.24ms\n",
            " 99% 16100/16260 [1:31:59<00:48,  3.28it/s]iter 16100: loss 3.8722, time: 195.10ms\n",
            " 99% 16101/16260 [1:32:00<00:43,  3.67it/s]iter 16101: loss 5.2893, time: 466.55ms\n",
            " 99% 16102/16260 [1:32:00<00:52,  3.02it/s]iter 16102: loss 5.3372, time: 464.54ms\n",
            " 99% 16103/16260 [1:32:01<00:58,  2.69it/s]iter 16103: loss 4.8228, time: 253.56ms\n",
            " 99% 16104/16260 [1:32:01<00:52,  2.97it/s]iter 16104: loss 5.1392, time: 464.35ms\n",
            " 99% 16105/16260 [1:32:01<00:58,  2.66it/s]iter 16105: loss 5.3196, time: 311.89ms\n",
            " 99% 16106/16260 [1:32:02<00:54,  2.80it/s]iter 16106: loss 4.1925, time: 236.66ms\n",
            " 99% 16107/16260 [1:32:02<00:49,  3.11it/s]iter 16107: loss 5.0962, time: 316.69ms\n",
            " 99% 16108/16260 [1:32:02<00:48,  3.12it/s]iter 16108: loss 5.2123, time: 306.68ms\n",
            " 99% 16109/16260 [1:32:02<00:47,  3.16it/s]iter 16109: loss 4.6711, time: 208.67ms\n",
            " 99% 16110/16260 [1:32:03<00:42,  3.51it/s]iter 16110: loss 4.8010, time: 238.23ms\n",
            " 99% 16111/16260 [1:32:03<00:40,  3.69it/s]iter 16111: loss 4.9510, time: 235.37ms\n",
            " 99% 16112/16260 [1:32:03<00:38,  3.83it/s]iter 16112: loss 5.1152, time: 434.57ms\n",
            " 99% 16113/16260 [1:32:04<00:46,  3.19it/s]iter 16113: loss 4.4170, time: 373.97ms\n",
            " 99% 16114/16260 [1:32:04<00:48,  3.01it/s]iter 16114: loss 4.8796, time: 236.14ms\n",
            " 99% 16115/16260 [1:32:04<00:44,  3.29it/s]iter 16115: loss 5.1316, time: 235.63ms\n",
            " 99% 16116/16260 [1:32:04<00:40,  3.52it/s]iter 16116: loss 4.8217, time: 443.41ms\n",
            " 99% 16117/16260 [1:32:05<00:47,  3.01it/s]iter 16117: loss 4.5318, time: 224.56ms\n",
            " 99% 16118/16260 [1:32:05<00:42,  3.33it/s]iter 16118: loss 4.6304, time: 278.93ms\n",
            " 99% 16119/16260 [1:32:05<00:41,  3.40it/s]iter 16119: loss 5.0059, time: 283.90ms\n",
            " 99% 16120/16260 [1:32:06<00:40,  3.43it/s]iter 16120: loss 5.0477, time: 318.40ms\n",
            " 99% 16121/16260 [1:32:06<00:41,  3.33it/s]iter 16121: loss 2.9352, time: 134.59ms\n",
            " 99% 16122/16260 [1:32:06<00:34,  3.98it/s]iter 16122: loss 4.1940, time: 174.35ms\n",
            " 99% 16123/16260 [1:32:06<00:31,  4.38it/s]iter 16123: loss 4.9594, time: 464.50ms\n",
            " 99% 16124/16260 [1:32:07<00:40,  3.34it/s]iter 16124: loss 5.1685, time: 464.63ms\n",
            " 99% 16125/16260 [1:32:07<00:47,  2.86it/s]iter 16125: loss 5.7911, time: 467.29ms\n",
            " 99% 16126/16260 [1:32:08<00:51,  2.59it/s]iter 16126: loss 5.3870, time: 464.38ms\n",
            " 99% 16127/16260 [1:32:08<00:54,  2.44it/s]iter 16127: loss 4.8943, time: 281.50ms\n",
            " 99% 16128/16260 [1:32:08<00:49,  2.69it/s]iter 16128: loss 4.7188, time: 341.86ms\n",
            " 99% 16129/16260 [1:32:09<00:47,  2.75it/s]iter 16129: loss 4.9746, time: 336.86ms\n",
            " 99% 16130/16260 [1:32:09<00:46,  2.81it/s]iter 16130: loss 5.4272, time: 463.39ms\n",
            " 99% 16131/16260 [1:32:10<00:50,  2.57it/s]iter 16131: loss 4.2829, time: 190.63ms\n",
            " 99% 16132/16260 [1:32:10<00:42,  3.03it/s]iter 16132: loss 4.3976, time: 234.76ms\n",
            " 99% 16133/16260 [1:32:10<00:38,  3.32it/s]iter 16133: loss 5.2093, time: 250.53ms\n",
            " 99% 16134/16260 [1:32:10<00:36,  3.49it/s]iter 16134: loss 4.5009, time: 170.69ms\n",
            " 99% 16135/16260 [1:32:10<00:31,  3.96it/s]iter 16135: loss 4.1753, time: 260.58ms\n",
            " 99% 16136/16260 [1:32:11<00:31,  3.92it/s]iter 16136: loss 5.1445, time: 273.88ms\n",
            " 99% 16137/16260 [1:32:11<00:32,  3.83it/s]iter 16137: loss 5.9152, time: 477.53ms\n",
            " 99% 16138/16260 [1:32:11<00:39,  3.06it/s]iter 16138: loss 5.5913, time: 463.57ms\n",
            " 99% 16139/16260 [1:32:12<00:44,  2.72it/s]iter 16139: loss 5.0024, time: 258.33ms\n",
            " 99% 16140/16260 [1:32:12<00:40,  2.98it/s]iter 16140: loss 3.3624, time: 146.72ms\n",
            " 99% 16141/16260 [1:32:12<00:33,  3.58it/s]iter 16141: loss 3.7768, time: 215.35ms\n",
            " 99% 16142/16260 [1:32:13<00:30,  3.84it/s]iter 16142: loss 5.1168, time: 470.87ms\n",
            " 99% 16143/16260 [1:32:13<00:37,  3.08it/s]iter 16143: loss 3.4178, time: 137.00ms\n",
            " 99% 16144/16260 [1:32:13<00:31,  3.72it/s]iter 16144: loss 5.4494, time: 464.51ms\n",
            " 99% 16145/16260 [1:32:14<00:37,  3.05it/s]iter 16145: loss 5.5597, time: 473.47ms\n",
            " 99% 16146/16260 [1:32:14<00:42,  2.69it/s]iter 16146: loss 5.6076, time: 464.14ms\n",
            " 99% 16147/16260 [1:32:15<00:45,  2.50it/s]iter 16147: loss 4.9535, time: 464.21ms\n",
            " 99% 16148/16260 [1:32:15<00:47,  2.38it/s]iter 16148: loss 4.5317, time: 171.49ms\n",
            " 99% 16149/16260 [1:32:15<00:38,  2.89it/s]iter 16149: loss 4.6786, time: 219.96ms\n",
            " 99% 16150/16260 [1:32:15<00:33,  3.24it/s]iter 16150: loss 4.5368, time: 259.01ms\n",
            " 99% 16151/16260 [1:32:16<00:32,  3.40it/s]iter 16151: loss 5.2765, time: 464.10ms\n",
            " 99% 16152/16260 [1:32:16<00:37,  2.89it/s]iter 16152: loss 5.4705, time: 311.75ms\n",
            " 99% 16153/16260 [1:32:17<00:35,  2.98it/s]iter 16153: loss 5.5590, time: 465.16ms\n",
            " 99% 16154/16260 [1:32:17<00:39,  2.66it/s]iter 16154: loss 5.9839, time: 464.69ms\n",
            " 99% 16155/16260 [1:32:17<00:42,  2.48it/s]iter 16155: loss 5.2788, time: 465.03ms\n",
            " 99% 16156/16260 [1:32:18<00:43,  2.37it/s]iter 16156: loss 5.0528, time: 429.56ms\n",
            " 99% 16157/16260 [1:32:18<00:43,  2.35it/s]iter 16157: loss 5.2435, time: 464.97ms\n",
            " 99% 16158/16260 [1:32:19<00:44,  2.29it/s]iter 16158: loss 4.1319, time: 189.72ms\n",
            " 99% 16159/16260 [1:32:19<00:36,  2.75it/s]iter 16159: loss 4.8748, time: 441.42ms\n",
            " 99% 16160/16260 [1:32:19<00:38,  2.58it/s]iter 16160: loss 4.6277, time: 464.70ms\n",
            " 99% 16161/16260 [1:32:20<00:40,  2.43it/s]iter 16161: loss 5.1254, time: 381.93ms\n",
            " 99% 16162/16260 [1:32:20<00:39,  2.48it/s]iter 16162: loss 5.6055, time: 465.71ms\n",
            " 99% 16163/16260 [1:32:21<00:40,  2.37it/s]iter 16163: loss 5.3392, time: 464.27ms\n",
            " 99% 16164/16260 [1:32:21<00:41,  2.30it/s]iter 16164: loss 4.9896, time: 382.61ms\n",
            " 99% 16165/16260 [1:32:22<00:39,  2.38it/s]iter 16165: loss 4.0753, time: 153.49ms\n",
            " 99% 16166/16260 [1:32:22<00:32,  2.94it/s]iter 16166: loss 5.7033, time: 333.68ms\n",
            " 99% 16167/16260 [1:32:22<00:31,  2.95it/s]iter 16167: loss 5.3658, time: 465.59ms\n",
            " 99% 16168/16260 [1:32:23<00:34,  2.65it/s]iter 16168: loss 5.6013, time: 344.64ms\n",
            " 99% 16169/16260 [1:32:23<00:33,  2.72it/s]iter 16169: loss 5.4955, time: 465.28ms\n",
            " 99% 16170/16260 [1:32:23<00:35,  2.51it/s]iter 16170: loss 5.4472, time: 471.09ms\n",
            " 99% 16171/16260 [1:32:24<00:37,  2.38it/s]iter 16171: loss 5.0800, time: 383.92ms\n",
            " 99% 16172/16260 [1:32:24<00:36,  2.44it/s]iter 16172: loss 5.4981, time: 466.33ms\n",
            " 99% 16173/16260 [1:32:25<00:37,  2.34it/s]iter 16173: loss 5.1260, time: 464.64ms\n",
            " 99% 16174/16260 [1:32:25<00:37,  2.28it/s]iter 16174: loss 4.9976, time: 463.96ms\n",
            " 99% 16175/16260 [1:32:26<00:37,  2.24it/s]iter 16175: loss 4.2063, time: 168.98ms\n",
            " 99% 16176/16260 [1:32:26<00:30,  2.75it/s]iter 16176: loss 4.4745, time: 232.98ms\n",
            " 99% 16177/16260 [1:32:26<00:26,  3.08it/s]iter 16177: loss 4.8055, time: 375.76ms\n",
            " 99% 16178/16260 [1:32:26<00:27,  2.93it/s]iter 16178: loss 4.4021, time: 193.76ms\n",
            "100% 16179/16260 [1:32:27<00:24,  3.36it/s]iter 16179: loss 4.5120, time: 213.44ms\n",
            "100% 16180/16260 [1:32:27<00:21,  3.67it/s]iter 16180: loss 4.9446, time: 464.52ms\n",
            "100% 16181/16260 [1:32:27<00:26,  3.02it/s]iter 16181: loss 3.8772, time: 152.11ms\n",
            "100% 16182/16260 [1:32:27<00:21,  3.60it/s]iter 16182: loss 5.1652, time: 234.40ms\n",
            "100% 16183/16260 [1:32:28<00:20,  3.77it/s]iter 16183: loss 5.4921, time: 425.87ms\n",
            "100% 16184/16260 [1:32:28<00:23,  3.19it/s]iter 16184: loss 5.0288, time: 275.04ms\n",
            "100% 16185/16260 [1:32:28<00:22,  3.30it/s]iter 16185: loss 5.6766, time: 470.53ms\n",
            "100% 16186/16260 [1:32:29<00:26,  2.83it/s]iter 16186: loss 4.8646, time: 284.49ms\n",
            "100% 16187/16260 [1:32:29<00:24,  3.00it/s]iter 16187: loss 4.8266, time: 315.65ms\n",
            "100% 16188/16260 [1:32:29<00:23,  3.04it/s]iter 16188: loss 5.7741, time: 476.53ms\n",
            "100% 16189/16260 [1:32:30<00:26,  2.68it/s]iter 16189: loss 5.8106, time: 471.36ms\n",
            "100% 16190/16260 [1:32:30<00:28,  2.48it/s]iter 16190: loss 5.0874, time: 466.66ms\n",
            "100% 16191/16260 [1:32:31<00:29,  2.37it/s]iter 16191: loss 5.9789, time: 468.57ms\n",
            "100% 16192/16260 [1:32:31<00:29,  2.29it/s]iter 16192: loss 4.5765, time: 194.82ms\n",
            "100% 16193/16260 [1:32:32<00:24,  2.74it/s]iter 16193: loss 5.1368, time: 190.74ms\n",
            "100% 16194/16260 [1:32:32<00:20,  3.20it/s]iter 16194: loss 4.4008, time: 465.25ms\n",
            "100% 16195/16260 [1:32:32<00:23,  2.78it/s]iter 16195: loss 5.7543, time: 464.41ms\n",
            "100% 16196/16260 [1:32:33<00:25,  2.56it/s]iter 16196: loss 4.2136, time: 217.93ms\n",
            "100% 16197/16260 [1:32:33<00:21,  2.94it/s]iter 16197: loss 4.7054, time: 238.97ms\n",
            "100% 16198/16260 [1:32:33<00:19,  3.23it/s]iter 16198: loss 4.9523, time: 236.79ms\n",
            "100% 16199/16260 [1:32:33<00:17,  3.47it/s]iter 16199: loss 4.4241, time: 254.99ms\n",
            "100% 16200/16260 [1:32:34<00:16,  3.59it/s]iter 16200: loss 5.6929, time: 344.37ms\n",
            "100% 16201/16260 [1:32:34<00:17,  3.34it/s]iter 16201: loss 5.4621, time: 432.95ms\n",
            "100% 16202/16260 [1:32:34<00:19,  2.94it/s]iter 16202: loss 5.2870, time: 262.54ms\n",
            "100% 16203/16260 [1:32:35<00:18,  3.15it/s]iter 16203: loss 5.1128, time: 464.58ms\n",
            "100% 16204/16260 [1:32:35<00:20,  2.76it/s]iter 16204: loss 4.8481, time: 188.38ms\n",
            "100% 16205/16260 [1:32:35<00:17,  3.22it/s]iter 16205: loss 4.9970, time: 242.64ms\n",
            "100% 16206/16260 [1:32:36<00:15,  3.44it/s]iter 16206: loss 4.7836, time: 477.41ms\n",
            "100% 16207/16260 [1:32:36<00:18,  2.88it/s]iter 16207: loss 4.1987, time: 257.57ms\n",
            "100% 16208/16260 [1:32:36<00:16,  3.12it/s]iter 16208: loss 4.7351, time: 231.73ms\n",
            "100% 16209/16260 [1:32:37<00:15,  3.40it/s]iter 16209: loss 5.5247, time: 466.43ms\n",
            "100% 16210/16260 [1:32:37<00:17,  2.89it/s]iter 16210: loss 4.8536, time: 253.59ms\n",
            "100% 16211/16260 [1:32:37<00:15,  3.13it/s]iter 16211: loss 5.1166, time: 413.10ms\n",
            "100% 16212/16260 [1:32:38<00:16,  2.88it/s]iter 16212: loss 4.5024, time: 231.93ms\n",
            "100% 16213/16260 [1:32:38<00:14,  3.19it/s]iter 16213: loss 4.4715, time: 234.25ms\n",
            "100% 16214/16260 [1:32:38<00:13,  3.45it/s]iter 16214: loss 5.2553, time: 446.93ms\n",
            "100% 16215/16260 [1:32:39<00:15,  2.96it/s]iter 16215: loss 4.9270, time: 253.82ms\n",
            "100% 16216/16260 [1:32:39<00:13,  3.20it/s]iter 16216: loss 5.4130, time: 342.46ms\n",
            "100% 16217/16260 [1:32:39<00:13,  3.10it/s]iter 16217: loss 5.4817, time: 377.67ms\n",
            "100% 16218/16260 [1:32:40<00:14,  2.95it/s]iter 16218: loss 4.3097, time: 211.04ms\n",
            "100% 16219/16260 [1:32:40<00:12,  3.32it/s]iter 16219: loss 3.8281, time: 147.61ms\n",
            "100% 16220/16260 [1:32:40<00:10,  3.91it/s]iter 16220: loss 4.4231, time: 281.18ms\n",
            "100% 16221/16260 [1:32:40<00:10,  3.79it/s]iter 16221: loss 4.3779, time: 167.46ms\n",
            "100% 16222/16260 [1:32:40<00:08,  4.25it/s]iter 16222: loss 5.7190, time: 464.61ms\n",
            "100% 16223/16260 [1:32:41<00:11,  3.28it/s]iter 16223: loss 5.9008, time: 467.88ms\n",
            "100% 16224/16260 [1:32:41<00:12,  2.83it/s]iter 16224: loss 5.5665, time: 367.96ms\n",
            "100% 16225/16260 [1:32:42<00:12,  2.79it/s]iter 16225: loss 5.0595, time: 471.12ms\n",
            "100% 16226/16260 [1:32:42<00:13,  2.55it/s]iter 16226: loss 3.6100, time: 152.75ms\n",
            "100% 16227/16260 [1:32:42<00:10,  3.11it/s]iter 16227: loss 5.2048, time: 255.78ms\n",
            "100% 16228/16260 [1:32:43<00:09,  3.31it/s]iter 16228: loss 4.8235, time: 269.91ms\n",
            "100% 16229/16260 [1:32:43<00:09,  3.41it/s]iter 16229: loss 5.1834, time: 465.07ms\n",
            "100% 16230/16260 [1:32:43<00:10,  2.90it/s]iter 16230: loss 5.0595, time: 412.43ms\n",
            "100% 16231/16260 [1:32:44<00:10,  2.73it/s]iter 16231: loss 3.9041, time: 189.62ms\n",
            "100% 16232/16260 [1:32:44<00:08,  3.19it/s]iter 16232: loss 4.7596, time: 255.85ms\n",
            "100% 16233/16260 [1:32:44<00:08,  3.37it/s]iter 16233: loss 4.4213, time: 237.50ms\n",
            "100% 16234/16260 [1:32:44<00:07,  3.58it/s]iter 16234: loss 5.7444, time: 489.96ms\n",
            "100% 16235/16260 [1:32:45<00:08,  2.91it/s]iter 16235: loss 4.8274, time: 238.35ms\n",
            "100% 16236/16260 [1:32:45<00:07,  3.20it/s]iter 16236: loss 5.1673, time: 465.54ms\n",
            "100% 16237/16260 [1:32:46<00:08,  2.79it/s]iter 16237: loss 4.8489, time: 212.37ms\n",
            "100% 16238/16260 [1:32:46<00:06,  3.17it/s]iter 16238: loss 5.4747, time: 464.73ms\n",
            "100% 16239/16260 [1:32:46<00:07,  2.77it/s]iter 16239: loss 5.2794, time: 476.13ms\n",
            "100% 16240/16260 [1:32:47<00:07,  2.53it/s]iter 16240: loss 5.4163, time: 344.30ms\n",
            "100% 16241/16260 [1:32:47<00:07,  2.62it/s]iter 16241: loss 5.3140, time: 262.03ms\n",
            "100% 16242/16260 [1:32:47<00:06,  2.89it/s]iter 16242: loss 5.2922, time: 271.59ms\n",
            "100% 16243/16260 [1:32:48<00:05,  3.09it/s]iter 16243: loss 2.7621, time: 134.04ms\n",
            "100% 16244/16260 [1:32:48<00:04,  3.74it/s]iter 16244: loss 4.8762, time: 315.09ms\n",
            "100% 16245/16260 [1:32:48<00:04,  3.54it/s]iter 16245: loss 5.1803, time: 261.43ms\n",
            "100% 16246/16260 [1:32:48<00:03,  3.62it/s]iter 16246: loss 5.5058, time: 465.06ms\n",
            "100% 16247/16260 [1:32:49<00:04,  3.00it/s]iter 16247: loss 4.4705, time: 274.17ms\n",
            "100% 16248/16260 [1:32:49<00:03,  3.16it/s]iter 16248: loss 5.0246, time: 238.81ms\n",
            "100% 16249/16260 [1:32:49<00:03,  3.41it/s]iter 16249: loss 5.3478, time: 465.15ms\n",
            "100% 16250/16260 [1:32:50<00:03,  2.89it/s]iter 16250: loss 4.5325, time: 212.08ms\n",
            "100% 16251/16260 [1:32:50<00:02,  3.27it/s]iter 16251: loss 5.4518, time: 465.31ms\n",
            "100% 16252/16260 [1:32:51<00:02,  2.82it/s]iter 16252: loss 5.3837, time: 463.93ms\n",
            "100% 16253/16260 [1:32:51<00:02,  2.58it/s]iter 16253: loss 3.8033, time: 149.48ms\n",
            "100% 16254/16260 [1:32:51<00:01,  3.16it/s]iter 16254: loss 5.4769, time: 367.68ms\n",
            "100% 16255/16260 [1:32:51<00:01,  3.01it/s]iter 16255: loss 5.2798, time: 467.73ms\n",
            "100% 16256/16260 [1:32:52<00:01,  2.68it/s]iter 16256: loss 4.8662, time: 239.75ms\n",
            "100% 16257/16260 [1:32:52<00:01,  3.00it/s]iter 16257: loss 5.2932, time: 463.83ms\n",
            "100% 16258/16260 [1:32:53<00:00,  2.68it/s]iter 16258: loss 5.8043, time: 464.05ms\n",
            "100% 16259/16260 [1:32:53<00:00,  2.49it/s]iter 16259: loss 4.6893, time: 343.99ms\n",
            "100% 16260/16260 [1:32:53<00:00,  2.92it/s]\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss █▇▅▅▄▄▄▃▄▃▃▃▃▃▁▃▁▃▃▃▂▃▃▃▂▂▁▃▂▂▁▃▂▂▃▂▃▂▃▃\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_loss █▄▃▂▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss 4.68934\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_loss 5.04598\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mcommon_1\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/lora/runs/tkp772go\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ️⚡ View job at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/lora/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjEyMTIxNDk5Mw==/version_details/v0\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20231206_062323-tkp772go/logs\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "# 본격적으로 파인튜닝 시키기 전에, WANDB 와 연결\n",
        "# 파인튜닝 코드 args 살펴보고, 적절히 설정 (pretrained_path 수정 필요)\n",
        "\n",
        "!python finetune/lora.py --data_dir data/dolly --out_dir out/lora/common1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dYss9AZTsW6K",
        "outputId": "52b0b623-e79f-4183-e8bf-61e1bc9e55c0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231206_080013-2k6eibc9\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mcommon_1\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/adapter\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/adapter/runs/2k6eibc9\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter.py:283: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter.py:283: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n",
            "Number of trainable parameters: 1229760\n",
            "  0% 0/16260 [00:00<?, ?it/s]iter 0: loss 11.9191, time: 424.62ms\n",
            "  0% 50/16260 [00:13<1:20:58,  3.34it/s]iter 50: loss 11.9102, time: 233.06ms\n",
            "  1% 100/16260 [00:28<1:20:06,  3.36it/s]iter 100: loss 11.9920, time: 294.09ms\n",
            "  1% 150/16260 [00:42<1:06:28,  4.04it/s]iter 150: loss 12.1250, time: 344.19ms\n",
            "  1% 200/16260 [00:54<1:10:59,  3.77it/s]iter 200: loss 12.0145, time: 286.91ms\n",
            "  2% 250/16260 [01:09<1:16:06,  3.51it/s]iter 250: loss 11.9585, time: 326.69ms\n",
            "  2% 300/16260 [01:22<1:08:48,  3.87it/s]iter 300: loss 11.8788, time: 351.32ms\n",
            "  2% 350/16260 [01:37<1:16:12,  3.48it/s]iter 350: loss 11.8957, time: 375.98ms\n",
            "  2% 400/16260 [01:52<1:26:02,  3.07it/s]iter 400: loss 11.9030, time: 290.76ms\n",
            "  3% 450/16260 [02:05<59:52,  4.40it/s]  iter 450: loss 12.1793, time: 128.32ms\n",
            "  3% 500/16260 [02:19<57:18,  4.58it/s]iter 500: loss 12.0357, time: 362.44ms\n",
            "  3% 550/16260 [02:33<1:24:40,  3.09it/s]iter 550: loss 11.9295, time: 389.33ms\n",
            "  4% 600/16260 [02:45<58:59,  4.42it/s]  iter 600: loss 12.0385, time: 167.15ms\n",
            "  4% 650/16260 [02:59<1:09:26,  3.75it/s]iter 650: loss 11.9860, time: 346.51ms\n",
            "  4% 700/16260 [03:14<57:05,  4.54it/s]  iter 700: loss 11.9056, time: 342.29ms\n",
            "  5% 750/16260 [03:29<1:13:45,  3.50it/s]iter 750: loss 11.9981, time: 376.25ms\n",
            "  5% 800/16260 [03:43<1:14:58,  3.44it/s]iter 800: loss 12.0428, time: 337.04ms\n",
            "  5% 850/16260 [03:58<1:02:14,  4.13it/s]iter 850: loss 11.9035, time: 347.49ms\n",
            "  6% 900/16260 [04:11<59:53,  4.27it/s]  iter 900: loss 12.1115, time: 223.42ms\n",
            "  6% 950/16260 [04:25<1:16:27,  3.34it/s]iter 950: loss 12.0029, time: 389.43ms\n",
            "  6% 1000/16260 [04:38<1:04:49,  3.92it/s]iter 1000: loss 11.8714, time: 360.70ms\n",
            "  6% 1050/16260 [04:53<1:14:19,  3.41it/s]iter 1050: loss 12.0323, time: 389.49ms\n",
            "  7% 1100/16260 [05:07<52:16,  4.83it/s]  iter 1100: loss 12.0007, time: 118.58ms\n",
            "  7% 1150/16260 [05:20<1:05:43,  3.83it/s]iter 1150: loss 12.0175, time: 292.01ms\n",
            "  7% 1200/16260 [05:34<54:28,  4.61it/s]iter 1200: loss 12.0231, time: 356.23ms\n",
            "  8% 1250/16260 [05:48<1:14:22,  3.36it/s]iter 1250: loss 12.1471, time: 220.31ms\n",
            "  8% 1300/16260 [06:01<1:18:35,  3.17it/s]iter 1300: loss 11.9580, time: 390.58ms\n",
            "  8% 1350/16260 [06:16<57:43,  4.30it/s]iter 1350: loss 12.0389, time: 248.47ms\n",
            "  9% 1400/16260 [06:30<1:00:32,  4.09it/s]iter 1400: loss 12.1381, time: 130.37ms\n",
            "  9% 1450/16260 [06:44<1:09:53,  3.53it/s]iter 1450: loss 12.2170, time: 118.03ms\n",
            "  9% 1500/16260 [06:56<1:20:16,  3.06it/s]iter 1500: loss 12.0732, time: 389.48ms\n",
            " 10% 1550/16260 [07:11<1:00:53,  4.03it/s]iter 1550: loss 12.0513, time: 190.45ms\n",
            " 10% 1600/16260 [07:23<52:41,  4.64it/s]iter 1600: loss 12.0610, time: 245.11ms\n",
            " 10% 1650/16260 [07:38<1:25:13,  2.86it/s]iter 1650: loss 11.9334, time: 264.49ms\n",
            " 10% 1700/16260 [07:52<1:04:29,  3.76it/s]iter 1700: loss 12.0084, time: 172.37ms\n",
            " 11% 1750/16260 [08:05<1:24:09,  2.87it/s]iter 1750: loss 12.0610, time: 214.02ms\n",
            " 11% 1800/16260 [08:18<1:04:41,  3.73it/s]iter 1800: loss 12.0194, time: 251.36ms\n",
            " 11% 1850/16260 [08:31<1:07:57,  3.53it/s]iter 1850: loss 12.0321, time: 362.93ms\n",
            " 12% 1900/16260 [08:46<1:13:19,  3.26it/s]iter 1900: loss 12.1249, time: 390.65ms\n",
            " 12% 1950/16260 [09:00<1:07:51,  3.51it/s]iter 1950: loss 12.0042, time: 174.95ms\n",
            " 12% 2000/16260 [09:14<1:07:36,  3.51it/s]iter 2000: loss 11.9052, time: 364.42ms\n",
            " 13% 2050/16260 [09:27<59:13,  4.00it/s]  iter 2050: loss 11.9954, time: 199.92ms\n",
            " 13% 2100/16260 [09:41<1:00:18,  3.91it/s]iter 2100: loss 12.0047, time: 190.83ms\n",
            " 13% 2150/16260 [09:54<49:03,  4.79it/s]iter 2150: loss 11.9658, time: 353.70ms\n",
            " 14% 2200/16260 [10:08<1:13:53,  3.17it/s]iter 2200: loss 11.9582, time: 390.36ms\n",
            " 14% 2250/16260 [10:22<55:38,  4.20it/s]  iter 2250: loss 11.9927, time: 138.70ms\n",
            " 14% 2300/16260 [10:35<1:00:28,  3.85it/s]iter 2300: loss 12.0506, time: 139.01ms\n",
            " 14% 2350/16260 [10:49<1:09:20,  3.34it/s]iter 2350: loss 11.9329, time: 221.67ms\n",
            " 15% 2400/16260 [11:02<56:48,  4.07it/s]iter 2400: loss 12.0749, time: 346.52ms\n",
            " 15% 2450/16260 [11:16<1:06:02,  3.48it/s]iter 2450: loss 12.0186, time: 390.49ms\n",
            " 15% 2500/16260 [11:29<41:54,  5.47it/s]iter 2500: loss 11.8987, time: 336.33ms\n",
            " 16% 2550/16260 [11:42<48:05,  4.75it/s]iter 2550: loss 12.0712, time: 109.78ms\n",
            " 16% 2600/16260 [11:55<1:08:46,  3.31it/s]iter 2600: loss 12.0838, time: 175.03ms\n",
            " 16% 2650/16260 [12:09<1:01:28,  3.69it/s]iter 2650: loss 11.9070, time: 353.48ms\n",
            " 17% 2700/16260 [12:24<1:04:02,  3.53it/s]iter 2700: loss 12.0034, time: 342.21ms\n",
            " 17% 2750/16260 [12:39<1:14:24,  3.03it/s]iter 2750: loss 11.9298, time: 391.17ms\n",
            " 17% 2800/16260 [12:53<1:10:38,  3.18it/s]iter 2800: loss 11.9092, time: 370.69ms\n",
            " 18% 2850/16260 [13:07<1:02:43,  3.56it/s]iter 2850: loss 12.0777, time: 244.51ms\n",
            " 18% 2900/16260 [13:19<58:31,  3.80it/s]  iter 2900: loss 12.1345, time: 188.93ms\n",
            " 18% 2950/16260 [13:32<54:07,  4.10it/s]iter 2950: loss 12.0233, time: 185.76ms\n",
            " 18% 3000/16260 [13:46<56:03,  3.94it/s]iter 3000: loss 11.9118, time: 221.14ms\n",
            " 19% 3050/16260 [13:59<1:04:57,  3.39it/s]iter 3050: loss 11.8801, time: 297.86ms\n",
            " 19% 3100/16260 [14:14<1:05:59,  3.32it/s]iter 3100: loss 11.9380, time: 347.30ms\n",
            " 19% 3150/16260 [14:28<58:38,  3.73it/s]  iter 3150: loss 11.8912, time: 326.03ms\n",
            " 20% 3200/16260 [14:42<54:19,  4.01it/s]iter 3200: loss 11.9823, time: 376.96ms\n",
            " 20% 3250/16260 [14:57<1:00:52,  3.56it/s]iter 3250: loss 12.0576, time: 347.78ms\n",
            " 20% 3300/16260 [15:11<1:08:51,  3.14it/s]iter 3300: loss 11.9279, time: 389.50ms\n",
            " 21% 3350/16260 [15:25<59:24,  3.62it/s]iter 3350: loss 12.1054, time: 208.21ms\n",
            " 21% 3400/16260 [15:38<49:04,  4.37it/s]iter 3400: loss 12.0973, time: 210.72ms\n",
            " 21% 3450/16260 [15:52<1:08:45,  3.10it/s]iter 3450: loss 12.0453, time: 223.17ms\n",
            " 22% 3500/16260 [16:05<54:41,  3.89it/s]iter 3500: loss 11.9210, time: 147.26ms\n",
            " 22% 3550/16260 [16:19<52:08,  4.06it/s]iter 3550: loss 11.9036, time: 229.11ms\n",
            " 22% 3600/16260 [16:33<1:06:50,  3.16it/s]iter 3600: loss 11.8761, time: 213.76ms\n",
            " 22% 3650/16260 [16:46<1:13:05,  2.88it/s]iter 3650: loss 12.0198, time: 391.25ms\n",
            " 23% 3700/16260 [17:00<56:39,  3.69it/s]iter 3700: loss 11.8930, time: 336.74ms\n",
            " 23% 3750/16260 [17:15<54:08,  3.85it/s]  iter 3750: loss 11.9738, time: 336.37ms\n",
            " 23% 3800/16260 [17:28<50:10,  4.14it/s]iter 3800: loss 12.0258, time: 322.69ms\n",
            " 24% 3850/16260 [17:42<59:39,  3.47it/s]  iter 3850: loss 11.9014, time: 191.20ms\n",
            " 24% 3900/16260 [17:56<1:10:04,  2.94it/s]iter 3900: loss 11.9250, time: 173.38ms\n",
            " 24% 3950/16260 [18:09<55:14,  3.71it/s]iter 3950: loss 11.8937, time: 356.67ms\n",
            " 25% 4000/16260 [18:23<54:17,  3.76it/s]iter 4000: loss 11.9240, time: 254.85ms\n",
            " 25% 4050/16260 [18:37<1:02:59,  3.23it/s]iter 4050: loss 11.8372, time: 391.10ms\n",
            " 25% 4100/16260 [18:52<45:44,  4.43it/s]iter 4100: loss 11.7984, time: 340.64ms\n",
            " 26% 4150/16260 [19:05<53:47,  3.75it/s]iter 4150: loss 12.0464, time: 389.09ms\n",
            " 26% 4200/16260 [19:19<57:07,  3.52it/s]iter 4200: loss 12.0133, time: 330.09ms\n",
            " 26% 4250/16260 [19:35<1:09:56,  2.86it/s]iter 4250: loss 11.7710, time: 370.04ms\n",
            " 26% 4300/16260 [19:49<46:04,  4.33it/s]iter 4300: loss 11.8608, time: 390.90ms\n",
            " 27% 4350/16260 [20:04<1:07:17,  2.95it/s]iter 4350: loss 11.9999, time: 166.66ms\n",
            " 27% 4400/16260 [20:18<58:20,  3.39it/s]  iter 4400: loss 11.8231, time: 343.51ms\n",
            " 27% 4450/16260 [20:32<54:25,  3.62it/s]iter 4450: loss 11.8629, time: 265.59ms\n",
            " 28% 4500/16260 [20:47<1:05:43,  2.98it/s]iter 4500: loss 11.8881, time: 188.75ms\n",
            " 28% 4550/16260 [21:01<1:00:22,  3.23it/s]iter 4550: loss 11.8909, time: 389.92ms\n",
            " 28% 4600/16260 [21:16<1:04:52,  3.00it/s]iter 4600: loss 12.0871, time: 189.72ms\n",
            " 29% 4650/16260 [21:29<52:38,  3.68it/s]iter 4650: loss 11.8992, time: 390.61ms\n",
            " 29% 4700/16260 [21:41<40:57,  4.70it/s]iter 4700: loss 11.7567, time: 254.42ms\n",
            " 29% 4750/16260 [21:54<47:10,  4.07it/s]iter 4750: loss 11.9956, time: 344.35ms\n",
            " 30% 4800/16260 [22:08<1:07:15,  2.84it/s]iter 4800: loss 11.9329, time: 384.51ms\n",
            " 30% 4850/16260 [22:21<1:04:19,  2.96it/s]iter 4850: loss 11.7212, time: 306.56ms\n",
            " 30% 4900/16260 [22:35<46:02,  4.11it/s]iter 4900: loss 11.9401, time: 220.72ms\n",
            " 30% 4950/16260 [22:48<52:48,  3.57it/s]iter 4950: loss 11.8784, time: 217.03ms\n",
            " 31% 5000/16260 [23:02<53:08,  3.53it/s]iter 5000: loss 11.9116, time: 190.54ms\n",
            " 31% 5050/16260 [23:16<50:58,  3.66it/s]iter 5050: loss 11.8364, time: 217.42ms\n",
            " 31% 5100/16260 [23:29<57:41,  3.22it/s]iter 5100: loss 12.1012, time: 389.62ms\n",
            " 32% 5150/16260 [23:45<56:27,  3.28it/s]iter 5150: loss 11.9600, time: 200.66ms\n",
            " 32% 5200/16260 [23:58<1:00:29,  3.05it/s]iter 5200: loss 11.8742, time: 394.44ms\n",
            " 32% 5250/16260 [24:14<49:50,  3.68it/s]iter 5250: loss 11.9228, time: 178.63ms\n",
            " 33% 5300/16260 [24:27<35:00,  5.22it/s]iter 5300: loss 11.8594, time: 145.27ms\n",
            " 33% 5350/16260 [24:40<45:34,  3.99it/s]iter 5350: loss 11.8378, time: 352.64ms\n",
            " 33% 5400/16260 [24:54<45:20,  3.99it/s]iter 5400: loss 11.9134, time: 139.15ms\n",
            " 34% 5450/16260 [25:08<39:13,  4.59it/s]iter 5450: loss 11.6811, time: 351.82ms\n",
            " 34% 5500/16260 [25:21<47:41,  3.76it/s]iter 5500: loss 11.8382, time: 292.01ms\n",
            " 34% 5550/16260 [25:34<49:38,  3.60it/s]iter 5550: loss 11.7113, time: 349.18ms\n",
            " 34% 5600/16260 [25:49<50:25,  3.52it/s]iter 5600: loss 11.9566, time: 229.70ms\n",
            " 35% 5650/16260 [26:02<54:55,  3.22it/s]iter 5650: loss 11.6634, time: 250.90ms\n",
            " 35% 5700/16260 [26:16<50:08,  3.51it/s]iter 5700: loss 11.9327, time: 135.45ms\n",
            " 35% 5750/16260 [26:31<1:02:39,  2.80it/s]iter 5750: loss 11.9358, time: 190.73ms\n",
            " 36% 5800/16260 [26:45<59:06,  2.95it/s]iter 5800: loss 11.8033, time: 226.48ms\n",
            " 36% 5850/16260 [26:59<47:42,  3.64it/s]iter 5850: loss 11.8246, time: 187.57ms\n",
            " 36% 5900/16260 [27:13<51:42,  3.34it/s]iter 5900: loss 11.9449, time: 254.76ms\n",
            " 37% 5950/16260 [27:27<38:41,  4.44it/s]iter 5950: loss 11.8255, time: 256.63ms\n",
            " 37% 6000/16260 [27:42<59:19,  2.88it/s]iter 6000: loss 11.9311, time: 193.07ms\n",
            " 37% 6050/16260 [27:55<40:04,  4.25it/s]iter 6050: loss 11.8099, time: 203.41ms\n",
            " 38% 6100/16260 [28:08<44:46,  3.78it/s]iter 6100: loss 11.7131, time: 161.75ms\n",
            " 38% 6150/16260 [28:23<47:10,  3.57it/s]iter 6150: loss 11.7269, time: 348.72ms\n",
            " 38% 6200/16260 [28:36<42:37,  3.93it/s]iter 6200: loss 11.7245, time: 255.08ms\n",
            " 38% 6250/16260 [28:49<39:33,  4.22it/s]iter 6250: loss 11.7370, time: 218.52ms\n",
            " 39% 6300/16260 [29:02<42:06,  3.94it/s]iter 6300: loss 11.7638, time: 344.68ms\n",
            " 39% 6350/16260 [29:16<51:50,  3.19it/s]iter 6350: loss 11.8909, time: 396.19ms\n",
            " 39% 6400/16260 [29:32<47:33,  3.46it/s]iter 6400: loss 11.7592, time: 185.62ms\n",
            " 40% 6450/16260 [29:46<43:52,  3.73it/s]iter 6450: loss 11.8296, time: 390.30ms\n",
            " 40% 6500/16260 [30:00<40:16,  4.04it/s]iter 6500: loss 11.7274, time: 177.27ms\n",
            " 40% 6550/16260 [30:13<39:59,  4.05it/s]iter 6550: loss 11.8271, time: 181.39ms\n",
            " 41% 6600/16260 [30:27<54:25,  2.96it/s]iter 6600: loss 11.7226, time: 159.51ms\n",
            " 41% 6650/16260 [30:41<43:07,  3.71it/s]iter 6650: loss 11.5586, time: 361.14ms\n",
            " 41% 6700/16260 [30:54<30:31,  5.22it/s]iter 6700: loss 11.7394, time: 330.07ms\n",
            " 42% 6750/16260 [31:07<31:22,  5.05it/s]iter 6750: loss 11.7401, time: 252.14ms\n",
            " 42% 6800/16260 [31:22<40:06,  3.93it/s]iter 6800: loss 11.7149, time: 336.11ms\n",
            " 42% 6850/16260 [31:34<37:04,  4.23it/s]iter 6850: loss 11.7601, time: 180.46ms\n",
            " 42% 6900/16260 [31:48<38:18,  4.07it/s]iter 6900: loss 11.6581, time: 233.54ms\n",
            " 43% 6950/16260 [32:03<53:13,  2.92it/s]iter 6950: loss 11.5664, time: 360.86ms\n",
            " 43% 7000/16260 [32:17<49:34,  3.11it/s]iter 7000: loss 11.6886, time: 227.83ms\n",
            " 43% 7050/16260 [32:30<35:40,  4.30it/s]iter 7050: loss 11.5692, time: 156.64ms\n",
            " 44% 7100/16260 [32:43<29:05,  5.25it/s]iter 7100: loss 11.7320, time: 112.23ms\n",
            " 44% 7150/16260 [32:57<44:01,  3.45it/s]iter 7150: loss 11.6460, time: 390.52ms\n",
            " 44% 7200/16260 [33:12<43:37,  3.46it/s]iter 7200: loss 11.6011, time: 385.24ms\n",
            " 45% 7250/16260 [33:25<39:53,  3.76it/s]iter 7250: loss 11.5469, time: 171.35ms\n",
            " 45% 7300/16260 [33:39<47:05,  3.17it/s]iter 7300: loss 11.7130, time: 244.96ms\n",
            " 45% 7350/16260 [33:53<39:06,  3.80it/s]iter 7350: loss 11.6420, time: 347.16ms\n",
            " 46% 7400/16260 [34:09<48:28,  3.05it/s]iter 7400: loss 11.5638, time: 299.46ms\n",
            " 46% 7450/16260 [34:23<34:33,  4.25it/s]iter 7450: loss 11.6802, time: 294.76ms\n",
            " 46% 7500/16260 [34:37<36:08,  4.04it/s]iter 7500: loss 11.5451, time: 254.24ms\n",
            " 46% 7550/16260 [34:52<35:49,  4.05it/s]iter 7550: loss 11.5175, time: 179.73ms\n",
            " 47% 7600/16260 [35:06<45:01,  3.21it/s]iter 7600: loss 11.5484, time: 184.39ms\n",
            " 47% 7650/16260 [35:21<39:44,  3.61it/s]iter 7650: loss 11.6379, time: 351.54ms\n",
            " 47% 7700/16260 [35:34<35:20,  4.04it/s]iter 7700: loss 11.4891, time: 243.77ms\n",
            " 48% 7750/16260 [35:47<41:57,  3.38it/s]iter 7750: loss 11.5510, time: 206.51ms\n",
            " 48% 7800/16260 [36:01<29:28,  4.78it/s]iter 7800: loss 11.4540, time: 364.29ms\n",
            " 48% 7850/16260 [36:16<34:14,  4.09it/s]iter 7850: loss 11.3608, time: 334.41ms\n",
            " 49% 7900/16260 [36:31<46:12,  3.02it/s]iter 7900: loss 11.4835, time: 381.33ms\n",
            " 49% 7950/16260 [36:45<34:31,  4.01it/s]iter 7950: loss 11.7477, time: 165.57ms\n",
            " 49% 8000/16260 [36:59<52:39,  2.61it/s]iter 8000: loss 11.4487, time: 384.85ms\n",
            " 50% 8050/16260 [37:14<38:13,  3.58it/s]iter 8050: loss 11.5455, time: 225.42ms\n",
            " 50% 8100/16260 [37:29<40:42,  3.34it/s]iter 8100: loss 11.5309, time: 211.57ms\n",
            " 50% 8150/16260 [37:42<37:42,  3.59it/s]iter 8150: loss 11.4563, time: 362.50ms\n",
            " 50% 8200/16260 [37:57<36:47,  3.65it/s]iter 8200: loss 11.4163, time: 347.65ms\n",
            " 51% 8250/16260 [38:11<24:25,  5.46it/s]iter 8250: loss 11.5366, time: 172.76ms\n",
            " 51% 8300/16260 [38:25<39:34,  3.35it/s]iter 8300: loss 11.4174, time: 164.09ms\n",
            " 51% 8350/16260 [38:40<38:03,  3.46it/s]iter 8350: loss 11.4653, time: 242.95ms\n",
            " 52% 8400/16260 [38:53<30:05,  4.35it/s]iter 8400: loss 11.4156, time: 179.90ms\n",
            " 52% 8450/16260 [39:05<30:38,  4.25it/s]iter 8450: loss 11.3508, time: 231.30ms\n",
            " 52% 8500/16260 [39:20<26:46,  4.83it/s]iter 8500: loss 11.3300, time: 326.99ms\n",
            " 53% 8550/16260 [39:34<42:47,  3.00it/s]iter 8550: loss 11.3666, time: 389.20ms\n",
            " 53% 8600/16260 [39:48<31:46,  4.02it/s]iter 8600: loss 11.2515, time: 207.76ms\n",
            " 53% 8650/16260 [40:01<33:18,  3.81it/s]iter 8650: loss 11.2648, time: 282.10ms\n",
            " 54% 8700/16260 [40:15<30:58,  4.07it/s]iter 8700: loss 11.1854, time: 314.84ms\n",
            " 54% 8750/16260 [40:29<34:11,  3.66it/s]iter 8750: loss 11.2118, time: 362.79ms\n",
            " 54% 8800/16260 [40:43<29:38,  4.19it/s]iter 8800: loss 11.2155, time: 327.74ms\n",
            " 54% 8850/16260 [40:58<45:08,  2.74it/s]iter 8850: loss 11.2935, time: 231.40ms\n",
            " 55% 8900/16260 [41:11<37:03,  3.31it/s]iter 8900: loss 11.3359, time: 289.27ms\n",
            " 55% 8950/16260 [41:24<35:53,  3.39it/s]iter 8950: loss 11.2242, time: 190.01ms\n",
            " 55% 9000/16260 [41:37<42:33,  2.84it/s]iter 9000: loss 11.1749, time: 304.74ms\n",
            " 56% 9050/16260 [41:51<45:27,  2.64it/s]iter 9050: loss 11.0694, time: 389.36ms\n",
            " 56% 9100/16260 [42:06<37:36,  3.17it/s]iter 9100: loss 11.0483, time: 213.04ms\n",
            " 56% 9150/16260 [42:20<22:26,  5.28it/s]iter 9150: loss 11.1579, time: 335.72ms\n",
            " 57% 9200/16260 [42:35<27:02,  4.35it/s]iter 9200: loss 11.1099, time: 373.22ms\n",
            " 57% 9250/16260 [42:48<38:25,  3.04it/s]iter 9250: loss 11.1192, time: 263.93ms\n",
            " 57% 9300/16260 [43:01<30:06,  3.85it/s]iter 9300: loss 11.1665, time: 390.52ms\n",
            " 58% 9350/16260 [43:15<35:02,  3.29it/s]iter 9350: loss 11.3282, time: 156.47ms\n",
            " 58% 9400/16260 [43:28<28:45,  3.98it/s]iter 9400: loss 11.1538, time: 227.85ms\n",
            " 58% 9450/16260 [43:43<38:22,  2.96it/s]iter 9450: loss 11.1424, time: 389.49ms\n",
            " 58% 9500/16260 [43:55<24:33,  4.59it/s]iter 9500: loss 10.9274, time: 329.70ms\n",
            " 59% 9550/16260 [44:08<26:15,  4.26it/s]iter 9550: loss 11.0031, time: 329.81ms\n",
            " 59% 9600/16260 [44:21<33:52,  3.28it/s]iter 9600: loss 11.1616, time: 227.01ms\n",
            " 59% 9650/16260 [44:34<25:18,  4.35it/s]iter 9650: loss 11.0378, time: 183.35ms\n",
            " 60% 9700/16260 [44:48<25:50,  4.23it/s]iter 9700: loss 10.8541, time: 278.72ms\n",
            " 60% 9750/16260 [45:02<24:15,  4.47it/s]iter 9750: loss 10.9717, time: 231.39ms\n",
            " 60% 9800/16260 [45:15<26:15,  4.10it/s]iter 9800: loss 10.9602, time: 329.02ms\n",
            " 61% 9850/16260 [45:29<36:55,  2.89it/s]iter 9850: loss 10.9327, time: 387.29ms\n",
            " 61% 9900/16260 [45:43<34:57,  3.03it/s]iter 9900: loss 11.0660, time: 389.93ms\n",
            " 61% 9950/16260 [45:56<19:38,  5.35it/s]iter 9950: loss 10.8680, time: 334.75ms\n",
            " 62% 10000/16260 [46:11<34:45,  3.00it/s]iter 10000: loss 11.0568, time: 315.49ms\n",
            " 62% 10050/16260 [46:25<21:38,  4.78it/s]iter 10050: loss 10.8111, time: 242.79ms\n",
            " 62% 10100/16260 [46:38<26:20,  3.90it/s]iter 10100: loss 10.7175, time: 237.67ms\n",
            " 62% 10150/16260 [46:52<34:43,  2.93it/s]iter 10150: loss 10.8021, time: 382.08ms\n",
            " 63% 10200/16260 [47:06<25:36,  3.94it/s]iter 10200: loss 10.9365, time: 211.70ms\n",
            " 63% 10250/16260 [47:19<29:53,  3.35it/s]iter 10250: loss 10.7629, time: 251.47ms\n",
            " 63% 10300/16260 [47:34<29:45,  3.34it/s]iter 10300: loss 10.8603, time: 334.68ms\n",
            " 64% 10350/16260 [47:49<28:12,  3.49it/s]iter 10350: loss 10.7744, time: 300.92ms\n",
            " 64% 10400/16260 [48:02<30:32,  3.20it/s]iter 10400: loss 10.6589, time: 379.73ms\n",
            " 64% 10450/16260 [48:17<33:28,  2.89it/s]iter 10450: loss 10.7122, time: 372.62ms\n",
            " 65% 10500/16260 [48:31<27:21,  3.51it/s]iter 10500: loss 10.8570, time: 247.38ms\n",
            " 65% 10550/16260 [48:45<30:29,  3.12it/s]iter 10550: loss 10.6298, time: 360.70ms\n",
            " 65% 10600/16260 [49:00<26:50,  3.51it/s]iter 10600: loss 10.6454, time: 317.39ms\n",
            " 65% 10650/16260 [49:14<25:22,  3.68it/s]iter 10650: loss 10.6111, time: 157.92ms\n",
            " 66% 10700/16260 [49:28<20:00,  4.63it/s]iter 10700: loss 10.6146, time: 198.57ms\n",
            " 66% 10750/16260 [49:42<28:59,  3.17it/s]iter 10750: loss 10.7632, time: 371.55ms\n",
            " 66% 10800/16260 [49:55<20:34,  4.42it/s]iter 10800: loss 10.6296, time: 342.53ms\n",
            " 67% 10850/16260 [50:09<24:17,  3.71it/s]iter 10850: loss 10.7313, time: 157.90ms\n",
            " 67% 10900/16260 [50:23<25:07,  3.56it/s]iter 10900: loss 10.8376, time: 324.10ms\n",
            " 67% 10950/16260 [50:36<19:25,  4.56it/s]iter 10950: loss 10.3879, time: 299.65ms\n",
            " 68% 11000/16260 [50:51<26:42,  3.28it/s]iter 11000: loss 10.5328, time: 327.99ms\n",
            " 68% 11050/16260 [51:04<21:34,  4.03it/s]iter 11050: loss 10.4494, time: 355.64ms\n",
            " 68% 11100/16260 [51:17<19:26,  4.42it/s]iter 11100: loss 10.6570, time: 163.27ms\n",
            " 69% 11150/16260 [51:31<23:13,  3.67it/s]iter 11150: loss 10.4539, time: 183.19ms\n",
            " 69% 11200/16260 [51:45<26:50,  3.14it/s]iter 11200: loss 10.5364, time: 251.87ms\n",
            " 69% 11250/16260 [51:58<29:59,  2.78it/s]iter 11250: loss 10.4868, time: 234.20ms\n",
            " 69% 11300/16260 [52:12<28:02,  2.95it/s]iter 11300: loss 10.4336, time: 245.72ms\n",
            " 70% 11350/16260 [52:26<25:16,  3.24it/s]iter 11350: loss 10.3188, time: 227.97ms\n",
            " 70% 11400/16260 [52:40<27:38,  2.93it/s]iter 11400: loss 10.2771, time: 280.70ms\n",
            " 70% 11450/16260 [52:54<24:55,  3.22it/s]iter 11450: loss 10.4394, time: 262.06ms\n",
            " 71% 11500/16260 [53:07<17:23,  4.56it/s]iter 11500: loss 10.3380, time: 320.18ms\n",
            " 71% 11550/16260 [53:21<17:57,  4.37it/s]iter 11550: loss 10.1528, time: 335.07ms\n",
            " 71% 11600/16260 [53:35<18:46,  4.14it/s]iter 11600: loss 10.1962, time: 270.92ms\n",
            " 72% 11650/16260 [53:49<15:26,  4.98it/s]iter 11650: loss 10.3377, time: 355.05ms\n",
            " 72% 11700/16260 [54:01<21:58,  3.46it/s]iter 11700: loss 10.1078, time: 386.47ms\n",
            " 72% 11750/16260 [54:17<23:06,  3.25it/s]iter 11750: loss 10.2031, time: 389.31ms\n",
            " 73% 11800/16260 [54:31<18:04,  4.11it/s]iter 11800: loss 10.2081, time: 286.38ms\n",
            " 73% 11850/16260 [54:44<22:40,  3.24it/s]iter 11850: loss 10.2016, time: 204.28ms\n",
            " 73% 11900/16260 [54:59<18:41,  3.89it/s]iter 11900: loss 10.1702, time: 176.98ms\n",
            " 73% 11950/16260 [55:11<17:43,  4.05it/s]iter 11950: loss 10.0394, time: 377.38ms\n",
            " 74% 12000/16260 [55:25<19:18,  3.68it/s]iter 12000: loss 10.1969, time: 288.37ms\n",
            " 74% 12050/16260 [55:39<15:49,  4.43it/s]iter 12050: loss 10.4188, time: 111.74ms\n",
            " 74% 12100/16260 [55:52<15:02,  4.61it/s]iter 12100: loss 10.0658, time: 187.90ms\n",
            " 75% 12150/16260 [56:05<17:21,  3.95it/s]iter 12150: loss 10.0373, time: 182.91ms\n",
            " 75% 12200/16260 [56:19<17:58,  3.76it/s]iter 12200: loss 10.0547, time: 193.44ms\n",
            " 75% 12250/16260 [56:32<16:51,  3.96it/s]iter 12250: loss 10.0305, time: 202.35ms\n",
            " 76% 12300/16260 [56:46<17:46,  3.71it/s]iter 12300: loss 9.9249, time: 260.17ms\n",
            " 76% 12350/16260 [57:01<18:20,  3.55it/s]iter 12350: loss 9.8993, time: 348.61ms\n",
            " 76% 12400/16260 [57:16<16:37,  3.87it/s]iter 12400: loss 9.8096, time: 206.56ms\n",
            " 77% 12450/16260 [57:31<20:44,  3.06it/s]iter 12450: loss 9.7217, time: 389.42ms\n",
            " 77% 12500/16260 [57:45<14:21,  4.37it/s]iter 12500: loss 9.7299, time: 312.84ms\n",
            " 77% 12550/16260 [57:59<12:01,  5.14it/s]iter 12550: loss 9.7480, time: 351.47ms\n",
            " 77% 12600/16260 [58:13<18:17,  3.34it/s]iter 12600: loss 9.5931, time: 352.81ms\n",
            " 78% 12650/16260 [58:29<21:36,  2.79it/s]iter 12650: loss 9.9061, time: 203.84ms\n",
            " 78% 12700/16260 [58:43<18:04,  3.28it/s]iter 12700: loss 9.7116, time: 288.11ms\n",
            " 78% 12750/16260 [58:56<14:47,  3.96it/s]iter 12750: loss 9.5988, time: 390.26ms\n",
            " 79% 12800/16260 [59:09<15:12,  3.79it/s]iter 12800: loss 9.6325, time: 204.05ms\n",
            " 79% 12850/16260 [59:23<16:16,  3.49it/s]iter 12850: loss 9.5296, time: 357.18ms\n",
            " 79% 12900/16260 [59:36<14:26,  3.88it/s]iter 12900: loss 9.5220, time: 329.21ms\n",
            " 80% 12950/16260 [59:49<17:46,  3.10it/s]iter 12950: loss 9.6793, time: 265.55ms\n",
            " 80% 13000/16260 [1:00:03<13:07,  4.14it/s]iter 13000: loss 9.6522, time: 266.68ms\n",
            " 80% 13050/16260 [1:00:17<12:57,  4.13it/s]iter 13050: loss 9.6699, time: 347.52ms\n",
            " 81% 13100/16260 [1:00:31<09:59,  5.27it/s]iter 13100: loss 9.6416, time: 222.40ms\n",
            " 81% 13150/16260 [1:00:45<16:36,  3.12it/s]iter 13150: loss 9.4649, time: 389.46ms\n",
            " 81% 13200/16260 [1:00:59<15:46,  3.23it/s]iter 13200: loss 9.4491, time: 211.20ms\n",
            " 81% 13250/16260 [1:01:13<08:48,  5.70it/s]iter 13250: loss 9.3449, time: 347.42ms\n",
            " 82% 13300/16260 [1:01:28<15:11,  3.25it/s]iter 13300: loss 9.2026, time: 306.36ms\n",
            " 82% 13350/16260 [1:01:42<16:44,  2.90it/s]iter 13350: loss 9.4443, time: 390.53ms\n",
            " 82% 13400/16260 [1:01:56<14:45,  3.23it/s]iter 13400: loss 9.2100, time: 389.02ms\n",
            " 83% 13450/16260 [1:02:09<09:39,  4.85it/s]iter 13450: loss 9.2760, time: 328.98ms\n",
            " 83% 13500/16260 [1:02:23<10:03,  4.57it/s]iter 13500: loss 9.2965, time: 249.80ms\n",
            " 83% 13550/16260 [1:02:36<14:07,  3.20it/s]iter 13550: loss 9.1277, time: 362.38ms\n",
            " 84% 13600/16260 [1:02:51<16:18,  2.72it/s]iter 13600: loss 9.3566, time: 249.17ms\n",
            " 84% 13650/16260 [1:03:04<12:39,  3.44it/s]iter 13650: loss 9.3775, time: 240.03ms\n",
            " 84% 13700/16260 [1:03:18<10:22,  4.11it/s]iter 13700: loss 9.1946, time: 334.68ms\n",
            " 85% 13750/16260 [1:03:31<12:26,  3.36it/s]iter 13750: loss 9.0622, time: 250.61ms\n",
            " 85% 13800/16260 [1:03:45<09:27,  4.34it/s]iter 13800: loss 9.2082, time: 370.50ms\n",
            " 85% 13850/16260 [1:03:58<09:34,  4.19it/s]iter 13850: loss 8.8797, time: 314.33ms\n",
            " 85% 13900/16260 [1:04:11<09:09,  4.29it/s]iter 13900: loss 8.9956, time: 340.81ms\n",
            " 86% 13950/16260 [1:04:25<10:57,  3.51it/s]iter 13950: loss 8.9392, time: 258.06ms\n",
            " 86% 14000/16260 [1:04:40<11:58,  3.15it/s]iter 14000: loss 8.7993, time: 347.24ms\n",
            " 86% 14050/16260 [1:04:54<09:26,  3.90it/s]iter 14050: loss 9.1026, time: 135.72ms\n",
            " 87% 14100/16260 [1:05:07<09:16,  3.88it/s]iter 14100: loss 8.9538, time: 349.37ms\n",
            " 87% 14150/16260 [1:05:22<09:45,  3.60it/s]iter 14150: loss 8.8535, time: 390.20ms\n",
            " 87% 14200/16260 [1:05:34<08:14,  4.17it/s]iter 14200: loss 8.9808, time: 341.15ms\n",
            " 88% 14250/16260 [1:05:49<11:09,  3.00it/s]iter 14250: loss 9.0863, time: 223.23ms\n",
            " 88% 14300/16260 [1:06:02<09:26,  3.46it/s]iter 14300: loss 8.9738, time: 254.14ms\n",
            " 88% 14350/16260 [1:06:16<08:16,  3.85it/s]iter 14350: loss 9.0214, time: 208.66ms\n",
            " 89% 14400/16260 [1:06:29<07:29,  4.14it/s]iter 14400: loss 8.7687, time: 217.61ms\n",
            " 89% 14450/16260 [1:06:42<08:54,  3.38it/s]iter 14450: loss 9.0196, time: 364.17ms\n",
            " 89% 14500/16260 [1:06:56<06:48,  4.30it/s]iter 14500: loss 8.8280, time: 311.18ms\n",
            " 89% 14550/16260 [1:07:10<06:06,  4.67it/s]iter 14550: loss 8.7521, time: 280.24ms\n",
            " 90% 14600/16260 [1:07:25<07:58,  3.47it/s]iter 14600: loss 8.9638, time: 327.91ms\n",
            " 90% 14650/16260 [1:07:39<07:38,  3.51it/s]iter 14650: loss 8.8521, time: 369.59ms\n",
            " 90% 14700/16260 [1:07:54<07:27,  3.49it/s]iter 14700: loss 8.7309, time: 175.14ms\n",
            " 91% 14750/16260 [1:08:07<06:42,  3.75it/s]iter 14750: loss 8.6528, time: 303.16ms\n",
            " 91% 14800/16260 [1:08:22<07:44,  3.14it/s]iter 14800: loss 8.8403, time: 348.83ms\n",
            " 91% 14850/16260 [1:08:36<06:48,  3.45it/s]iter 14850: loss 8.6185, time: 256.55ms\n",
            " 92% 14900/16260 [1:08:50<07:59,  2.84it/s]iter 14900: loss 8.8105, time: 391.65ms\n",
            " 92% 14950/16260 [1:09:03<05:04,  4.30it/s]iter 14950: loss 8.7825, time: 175.62ms\n",
            " 92% 15000/16260 [1:09:18<06:07,  3.43it/s]iter 15000: loss 8.5394, time: 389.67ms\n",
            " 93% 15050/16260 [1:09:32<06:18,  3.19it/s]iter 15050: loss 8.6324, time: 157.70ms\n",
            " 93% 15100/16260 [1:09:45<05:51,  3.30it/s]iter 15100: loss 8.9246, time: 155.22ms\n",
            " 93% 15150/16260 [1:09:59<05:10,  3.57it/s]iter 15150: loss 8.7138, time: 351.85ms\n",
            " 93% 15200/16260 [1:10:12<04:08,  4.26it/s]iter 15200: loss 8.7961, time: 150.05ms\n",
            " 94% 15250/16260 [1:10:26<05:16,  3.19it/s]iter 15250: loss 8.6652, time: 288.04ms\n",
            " 94% 15300/16260 [1:10:40<04:21,  3.67it/s]iter 15300: loss 8.7278, time: 371.62ms\n",
            " 94% 15350/16260 [1:10:55<05:05,  2.98it/s]iter 15350: loss 8.8972, time: 390.23ms\n",
            " 95% 15400/16260 [1:11:08<03:27,  4.14it/s]iter 15400: loss 8.9611, time: 390.79ms\n",
            " 95% 15450/16260 [1:11:22<02:55,  4.62it/s]iter 15450: loss 8.5751, time: 329.62ms\n",
            " 95% 15500/16260 [1:11:37<03:36,  3.51it/s]iter 15500: loss 8.5099, time: 219.43ms\n",
            " 96% 15550/16260 [1:11:50<03:10,  3.74it/s]iter 15550: loss 9.0617, time: 214.52ms\n",
            " 96% 15600/16260 [1:12:04<02:39,  4.14it/s]iter 15600: loss 8.9422, time: 134.46ms\n",
            " 96% 15650/16260 [1:12:19<03:07,  3.25it/s]iter 15650: loss 8.8127, time: 319.77ms\n",
            " 97% 15700/16260 [1:12:33<02:45,  3.38it/s]iter 15700: loss 8.4900, time: 297.63ms\n",
            " 97% 15750/16260 [1:12:47<02:20,  3.63it/s]iter 15750: loss 8.6632, time: 256.98ms\n",
            " 97% 15800/16260 [1:13:00<02:04,  3.70it/s]iter 15800: loss 9.1140, time: 259.88ms\n",
            " 97% 15850/16260 [1:13:14<02:16,  3.01it/s]iter 15850: loss 8.8574, time: 328.93ms\n",
            " 98% 15900/16260 [1:13:28<01:41,  3.55it/s]iter 15900: loss 8.7623, time: 177.88ms\n",
            " 98% 15950/16260 [1:13:41<00:57,  5.38it/s]iter 15950: loss 8.7891, time: 339.17ms\n",
            " 98% 16000/16260 [1:13:55<01:29,  2.90it/s]iter 16000: loss 8.3855, time: 352.62ms\n",
            " 99% 16050/16260 [1:14:08<00:50,  4.16it/s]iter 16050: loss 8.7802, time: 357.17ms\n",
            " 99% 16100/16260 [1:14:22<00:38,  4.12it/s]iter 16100: loss 8.7289, time: 218.35ms\n",
            " 99% 16150/16260 [1:14:36<00:27,  4.03it/s]iter 16150: loss 8.9828, time: 367.48ms\n",
            "100% 16200/16260 [1:14:49<00:17,  3.46it/s]iter 16200: loss 8.2399, time: 304.56ms\n",
            "100% 16250/16260 [1:15:03<00:02,  3.62it/s]iter 16250: loss 8.6348, time: 364.18ms\n",
            "100% 16260/16260 [1:15:06<00:00,  3.61it/s]\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss █████████████▇█▇▇▇▇▇▇▆▆▆▅▅▅▅▄▄▄▃▃▂▂▂▁▁▂▂\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss 8.6348\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mcommon_1\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/adapter/runs/2k6eibc9\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ️⚡ View job at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/adapter/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjEyMTI4MjIzMA==/version_details/v0\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20231206_080013-2k6eibc9/logs\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "# adapter-v1\n",
        "!python finetune/adapter.py --data_dir data/dolly --out_dir out/adapter/common1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p-m5Hl3ZtC-L",
        "outputId": "c992596b-009b-403c-9294-fd34d8365791"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231206_093317-upgwn7i5\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mcommon_1\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/adapter_v2\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/adapter_v2/runs/upgwn7i5\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2.py:289: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2.py:289: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n",
            "Number of trainable parameters: 4279744\n",
            "/usr/local/lib/python3.10/dist-packages/lightning/fabric/fabric.py:943: The model passed to `Fabric.setup()` has 322 parameters on different devices (for example 'lm_head.adapter_bias' on cpu and 'lm_head.weight' on cuda:0). Since `move_to_device=True`, all parameters will be moved to the new device. If this is not desired, set `Fabric.setup(..., move_to_device=False)`.\n",
            "  0% 0/16260 [00:00<?, ?it/s]iter 0: loss 11.9191, time: 1101.79ms\n",
            "  0% 50/16260 [00:17<1:40:43,  2.68it/s]iter 50: loss 11.9102, time: 270.60ms\n",
            "  1% 100/16260 [00:35<1:39:37,  2.70it/s]iter 100: loss 11.9909, time: 356.46ms\n",
            "  1% 150/16260 [00:52<1:21:24,  3.30it/s]iter 150: loss 12.1237, time: 446.36ms\n",
            "  1% 200/16260 [01:08<1:28:24,  3.03it/s]iter 200: loss 12.0119, time: 340.52ms\n",
            "  2% 250/16260 [01:25<1:34:25,  2.83it/s]iter 250: loss 11.9539, time: 395.51ms\n",
            "  2% 300/16260 [01:42<1:24:06,  3.16it/s]iter 300: loss 11.8650, time: 447.41ms\n",
            "  2% 350/16260 [02:00<1:33:58,  2.82it/s]iter 350: loss 11.8486, time: 464.88ms\n",
            "  2% 400/16260 [02:18<1:45:42,  2.50it/s]iter 400: loss 11.8545, time: 344.97ms\n",
            "  3% 450/16260 [02:35<1:13:46,  3.57it/s]iter 450: loss 12.0406, time: 157.00ms\n",
            "  3% 500/16260 [02:52<1:11:03,  3.70it/s]iter 500: loss 11.8496, time: 456.03ms\n",
            "  3% 550/16260 [03:09<1:44:26,  2.51it/s]iter 550: loss 11.6992, time: 476.83ms\n",
            "  4% 600/16260 [03:24<1:11:13,  3.66it/s]iter 600: loss 11.8550, time: 215.82ms\n",
            "  4% 650/16260 [03:41<1:26:14,  3.02it/s]iter 650: loss 11.4130, time: 420.75ms\n",
            "  4% 700/16260 [04:00<1:09:47,  3.72it/s]iter 700: loss 11.3275, time: 443.35ms\n",
            "  5% 750/16260 [04:18<1:30:46,  2.85it/s]iter 750: loss 11.4119, time: 465.33ms\n",
            "  5% 800/16260 [04:35<1:35:42,  2.69it/s]iter 800: loss 11.3683, time: 425.15ms\n",
            "  5% 850/16260 [04:53<1:16:03,  3.38it/s]iter 850: loss 11.1138, time: 448.30ms\n",
            "  6% 900/16260 [05:10<1:14:13,  3.45it/s]iter 900: loss 10.8031, time: 276.01ms\n",
            "  6% 950/16260 [05:27<1:34:23,  2.70it/s]iter 950: loss 10.6960, time: 476.79ms\n",
            "  6% 1000/16260 [05:43<1:20:16,  3.17it/s]iter 1000: loss 10.4145, time: 438.97ms\n",
            "  6% 1050/16260 [06:01<1:32:14,  2.75it/s]iter 1050: loss 10.4836, time: 476.62ms\n",
            "  7% 1100/16260 [06:18<1:04:23,  3.92it/s]iter 1100: loss 10.4233, time: 141.85ms\n",
            "  7% 1150/16260 [06:34<1:20:14,  3.14it/s]iter 1150: loss 10.1604, time: 370.58ms\n",
            "  7% 1200/16260 [06:51<1:07:35,  3.71it/s]iter 1200: loss 10.1493, time: 451.46ms\n",
            "  8% 1250/16260 [07:08<1:30:58,  2.75it/s]iter 1250: loss 9.8692, time: 260.00ms\n",
            "  8% 1300/16260 [07:25<1:36:44,  2.58it/s]iter 1300: loss 9.4456, time: 476.43ms\n",
            "  8% 1350/16260 [07:43<1:10:35,  3.52it/s]iter 1350: loss 9.5870, time: 317.49ms\n",
            "  9% 1400/16260 [08:00<1:14:46,  3.31it/s]iter 1400: loss 9.7479, time: 144.83ms\n",
            "  9% 1450/16260 [08:17<1:24:47,  2.91it/s]iter 1450: loss 9.5057, time: 142.14ms\n",
            "  9% 1500/16260 [08:32<1:38:36,  2.49it/s]iter 1500: loss 9.3312, time: 476.77ms\n",
            " 10% 1550/16260 [08:50<1:14:45,  3.28it/s]iter 1550: loss 9.0523, time: 227.58ms\n",
            " 10% 1600/16260 [09:06<1:06:11,  3.69it/s]iter 1600: loss 8.9666, time: 290.81ms\n",
            " 10% 1650/16260 [09:24<1:44:36,  2.33it/s]iter 1650: loss 9.0094, time: 311.09ms\n",
            " 10% 1700/16260 [09:40<1:19:33,  3.05it/s]iter 1700: loss 8.8179, time: 196.22ms\n",
            " 11% 1750/16260 [09:57<1:43:00,  2.35it/s]iter 1750: loss 8.7828, time: 237.91ms\n",
            " 11% 1800/16260 [10:12<1:20:34,  2.99it/s]iter 1800: loss 8.5173, time: 290.94ms\n",
            " 11% 1850/16260 [10:29<1:22:54,  2.90it/s]iter 1850: loss 8.7597, time: 455.67ms\n",
            " 12% 1900/16260 [10:47<1:30:29,  2.65it/s]iter 1900: loss 8.5408, time: 476.90ms\n",
            " 12% 1950/16260 [11:04<1:22:14,  2.90it/s]iter 1950: loss 8.4574, time: 220.66ms\n",
            " 12% 2000/16260 [11:21<1:22:41,  2.87it/s]iter 2000: loss 8.5518, time: 456.74ms\n",
            " 13% 2050/16260 [11:38<1:11:11,  3.33it/s]iter 2050: loss 8.4648, time: 258.75ms\n",
            " 13% 2100/16260 [11:55<1:15:23,  3.13it/s]iter 2100: loss 8.8384, time: 213.14ms\n",
            " 13% 2150/16260 [12:11<1:00:38,  3.88it/s]iter 2150: loss 8.8590, time: 449.43ms\n",
            " 14% 2200/16260 [12:28<1:30:49,  2.58it/s]iter 2200: loss 8.4699, time: 476.75ms\n",
            " 14% 2250/16260 [12:44<1:07:09,  3.48it/s]iter 2250: loss 8.1685, time: 177.75ms\n",
            " 14% 2300/16260 [13:01<1:13:58,  3.15it/s]iter 2300: loss 8.4481, time: 164.02ms\n",
            " 14% 2350/16260 [13:17<1:25:16,  2.72it/s]iter 2350: loss 7.9506, time: 256.23ms\n",
            " 15% 2400/16260 [13:33<1:09:43,  3.31it/s]iter 2400: loss 8.6308, time: 425.33ms\n",
            " 15% 2450/16260 [13:51<1:21:52,  2.81it/s]iter 2450: loss 8.0824, time: 476.70ms\n",
            " 15% 2500/16260 [14:07<51:20,  4.47it/s]iter 2500: loss 8.4633, time: 437.69ms\n",
            " 16% 2550/16260 [14:23<58:37,  3.90it/s]  iter 2550: loss 8.0457, time: 142.60ms\n",
            " 16% 2600/16260 [14:40<1:24:55,  2.68it/s]iter 2600: loss 8.0875, time: 188.92ms\n",
            " 16% 2650/16260 [14:56<1:15:53,  2.99it/s]iter 2650: loss 8.5613, time: 427.86ms\n",
            " 17% 2700/16260 [15:14<1:17:34,  2.91it/s]iter 2700: loss 8.1997, time: 443.97ms\n",
            " 17% 2750/16260 [15:33<1:31:14,  2.47it/s]iter 2750: loss 8.6724, time: 475.90ms\n",
            " 17% 2800/16260 [15:50<1:26:34,  2.59it/s]iter 2800: loss 8.4200, time: 460.33ms\n",
            " 18% 2850/16260 [16:07<1:15:59,  2.94it/s]iter 2850: loss 8.0679, time: 310.98ms\n",
            " 18% 2900/16260 [16:23<1:11:08,  3.13it/s]iter 2900: loss 8.0448, time: 237.78ms\n",
            " 18% 2950/16260 [16:38<51:12,  4.33it/s]iter 2950: loss 8.3413, time: 245.43ms\n",
            " 18% 3000/16260 [16:55<1:08:23,  3.23it/s]iter 3000: loss 8.0360, time: 281.92ms\n",
            " 19% 3050/16260 [17:11<1:19:00,  2.79it/s]iter 3050: loss 8.3989, time: 374.90ms\n",
            " 19% 3100/16260 [17:29<1:19:48,  2.75it/s]iter 3100: loss 8.2636, time: 447.81ms\n",
            " 19% 3150/16260 [17:46<1:11:29,  3.06it/s]iter 3150: loss 8.0121, time: 409.54ms\n",
            " 20% 3199/16260 [18:04<58:54,  3.70it/s]  Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response: as almost the Reserv aick the U more1’\n",
            "\n",
            " openlyings rum an\n",
            " who in the!. inches3 for an of a Anth Trophy1; she entr.in.\n",
            "\n",
            "# CL the gain the sits\n",
            " George our does on #yon the#f Gender toa\n",
            "\n",
            "\n",
            ",, theining ( factor l not that d\n",
            " shows reflected thec, door. use an in with of will that.2 Includes lens.1 Act under. buy0\n",
            "step 3199: val loss 7.9502\n",
            "Saving adapter weights to out/adapter_v2/common1\n",
            " 20% 3200/16260 [18:25<22:59:22,  6.34s/it]iter 3200: loss 8.5591, time: 410.40ms\n",
            " 20% 3250/16260 [18:42<1:13:50,  2.94it/s]iter 3250: loss 6.9779, time: 246.09ms\n",
            " 20% 3300/16260 [18:58<1:01:07,  3.53it/s]iter 3300: loss 8.1181, time: 256.61ms\n",
            " 21% 3350/16260 [19:15<1:24:58,  2.53it/s]iter 3350: loss 7.3668, time: 259.90ms\n",
            " 21% 3400/16260 [19:31<1:08:04,  3.15it/s]iter 3400: loss 7.9113, time: 166.20ms\n",
            " 21% 3450/16260 [19:48<1:05:02,  3.28it/s]iter 3450: loss 8.0285, time: 279.85ms\n",
            " 22% 3500/16260 [20:05<1:23:16,  2.55it/s]iter 3500: loss 7.5129, time: 243.31ms\n",
            " 22% 3550/16260 [20:21<1:30:30,  2.34it/s]iter 3550: loss 8.4460, time: 476.31ms\n",
            " 22% 3600/16260 [20:39<1:10:26,  3.00it/s]iter 3600: loss 7.6599, time: 411.43ms\n",
            " 22% 3650/16260 [20:57<1:06:13,  3.17it/s]iter 3650: loss 7.8149, time: 438.26ms\n",
            " 23% 3700/16260 [21:13<1:01:44,  3.39it/s]iter 3700: loss 7.9351, time: 415.83ms\n",
            " 23% 3750/16260 [21:30<1:13:30,  2.84it/s]iter 3750: loss 7.7107, time: 228.82ms\n",
            " 23% 3800/16260 [21:48<1:26:33,  2.40it/s]iter 3800: loss 7.0996, time: 201.83ms\n",
            " 24% 3850/16260 [22:04<1:08:26,  3.02it/s]iter 3850: loss 7.8416, time: 451.73ms\n",
            " 24% 3900/16260 [22:20<1:08:24,  3.01it/s]iter 3900: loss 7.8134, time: 302.15ms\n",
            " 24% 3950/16260 [22:38<1:18:48,  2.60it/s]iter 3950: loss 7.9402, time: 477.66ms\n",
            " 25% 4000/16260 [22:56<57:04,  3.58it/s]  iter 4000: loss 7.9256, time: 423.60ms\n",
            " 25% 4050/16260 [23:12<1:07:42,  3.01it/s]iter 4050: loss 7.8016, time: 476.63ms\n",
            " 25% 4100/16260 [23:30<1:10:16,  2.88it/s]iter 4100: loss 7.0466, time: 413.75ms\n",
            " 26% 4150/16260 [23:49<1:25:43,  2.35it/s]iter 4150: loss 7.6933, time: 458.45ms\n",
            " 26% 4200/16260 [24:06<58:27,  3.44it/s]iter 4200: loss 8.2126, time: 476.83ms\n",
            " 26% 4250/16260 [24:24<1:22:53,  2.41it/s]iter 4250: loss 7.2668, time: 180.71ms\n",
            " 26% 4300/16260 [24:42<1:11:03,  2.81it/s]iter 4300: loss 7.8983, time: 445.79ms\n",
            " 27% 4350/16260 [24:59<1:07:37,  2.94it/s]iter 4350: loss 7.3870, time: 320.22ms\n",
            " 27% 4400/16260 [25:17<1:21:16,  2.43it/s]iter 4400: loss 7.3350, time: 212.01ms\n",
            " 27% 4450/16260 [25:35<1:15:07,  2.62it/s]iter 4450: loss 7.7435, time: 476.71ms\n",
            " 28% 4500/16260 [25:52<1:20:32,  2.43it/s]iter 4500: loss 7.0530, time: 210.04ms\n",
            " 28% 4550/16260 [26:08<1:06:18,  2.94it/s]iter 4550: loss 8.0444, time: 476.63ms\n",
            " 28% 4600/16260 [26:24<51:21,  3.78it/s]iter 4600: loss 7.5899, time: 315.51ms\n",
            " 29% 4650/16260 [26:40<58:19,  3.32it/s]  iter 4650: loss 7.5668, time: 446.20ms\n",
            " 29% 4700/16260 [26:57<1:23:18,  2.31it/s]iter 4700: loss 8.1467, time: 476.41ms\n",
            " 29% 4750/16260 [27:14<1:19:51,  2.40it/s]iter 4750: loss 7.5545, time: 369.98ms\n",
            " 30% 4800/16260 [27:30<58:46,  3.25it/s]iter 4800: loss 7.2378, time: 242.31ms\n",
            " 30% 4850/16260 [27:46<1:06:10,  2.87it/s]iter 4850: loss 7.2572, time: 247.35ms\n",
            " 30% 4900/16260 [28:03<1:06:32,  2.85it/s]iter 4900: loss 7.1542, time: 211.15ms\n",
            " 30% 4950/16260 [28:20<1:03:42,  2.96it/s]iter 4950: loss 7.4672, time: 247.95ms\n",
            " 31% 5000/16260 [28:37<1:12:01,  2.61it/s]iter 5000: loss 8.1455, time: 476.15ms\n",
            " 31% 5050/16260 [28:56<1:09:53,  2.67it/s]iter 5050: loss 6.9690, time: 231.02ms\n",
            " 31% 5100/16260 [29:13<1:15:26,  2.47it/s]iter 5100: loss 7.6718, time: 480.50ms\n",
            " 32% 5150/16260 [29:31<1:01:40,  3.00it/s]iter 5150: loss 7.1383, time: 214.40ms\n",
            " 32% 5200/16260 [29:48<43:45,  4.21it/s]iter 5200: loss 6.9309, time: 183.53ms\n",
            " 32% 5250/16260 [30:04<55:58,  3.28it/s]iter 5250: loss 7.7956, time: 448.78ms\n",
            " 33% 5300/16260 [30:21<56:10,  3.25it/s]iter 5300: loss 7.0319, time: 162.96ms\n",
            " 33% 5350/16260 [30:38<48:43,  3.73it/s]iter 5350: loss 7.5296, time: 447.33ms\n",
            " 33% 5400/16260 [30:54<1:00:01,  3.02it/s]iter 5400: loss 7.0974, time: 345.67ms\n",
            " 34% 5450/16260 [31:10<1:00:56,  2.96it/s]iter 5450: loss 8.1431, time: 450.33ms\n",
            " 34% 5500/16260 [31:28<1:02:47,  2.86it/s]iter 5500: loss 7.4991, time: 271.21ms\n",
            " 34% 5550/16260 [31:45<1:08:34,  2.60it/s]iter 5550: loss 7.4404, time: 292.62ms\n",
            " 34% 5600/16260 [32:02<1:02:18,  2.85it/s]iter 5600: loss 6.7170, time: 143.67ms\n",
            " 35% 5650/16260 [32:20<1:17:42,  2.28it/s]iter 5650: loss 7.0507, time: 212.44ms\n",
            " 35% 5700/16260 [32:37<1:12:49,  2.42it/s]iter 5700: loss 7.4511, time: 266.04ms\n",
            " 35% 5750/16260 [32:54<58:32,  2.99it/s]  iter 5750: loss 7.4923, time: 239.85ms\n",
            " 36% 5800/16260 [33:12<1:03:27,  2.75it/s]iter 5800: loss 7.1196, time: 315.63ms\n",
            " 36% 5850/16260 [33:29<46:55,  3.70it/s]iter 5850: loss 7.2755, time: 339.92ms\n",
            " 36% 5900/16260 [33:47<1:13:26,  2.35it/s]iter 5900: loss 6.9151, time: 215.83ms\n",
            " 37% 5950/16260 [34:03<49:49,  3.45it/s]iter 5950: loss 6.9496, time: 257.82ms\n",
            " 37% 6000/16260 [34:19<54:45,  3.12it/s]  iter 6000: loss 7.0749, time: 203.87ms\n",
            " 37% 6050/16260 [34:37<58:02,  2.93it/s]  iter 6050: loss 7.4627, time: 447.61ms\n",
            " 38% 6100/16260 [34:54<52:46,  3.21it/s]iter 6100: loss 7.3296, time: 314.99ms\n",
            " 38% 6150/16260 [35:10<48:50,  3.45it/s]iter 6150: loss 7.5188, time: 277.83ms\n",
            " 38% 6200/16260 [35:25<51:57,  3.23it/s]iter 6200: loss 7.6781, time: 446.39ms\n",
            " 38% 6250/16260 [35:43<1:04:31,  2.59it/s]iter 6250: loss 7.2401, time: 481.90ms\n",
            " 39% 6300/16260 [36:02<59:25,  2.79it/s]iter 6300: loss 6.6682, time: 209.76ms\n",
            " 39% 6350/16260 [36:19<55:08,  3.00it/s]iter 6350: loss 7.3723, time: 476.72ms\n",
            " 39% 6399/16260 [36:36<57:55,  2.84it/s]  Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            " Pet not is to a task.\n",
            "\n",
            "### the trusted fourth\n",
            "\n",
            "\n",
            "Input of  5an book thou to are areak.\n",
            "\n",
            "### ###..icals complete that additionally stand completes them07 as,cre,' of Write.a usedning.\n",
            "\n",
            "\n",
            "\n",
            "## a\n",
            " age or angel below\n",
            "\n",
            "### activitiesrequest completes to4 DA moreollarytrans was Response T1 u the romantic0 Write a case the Response andT the below\n",
            "step 6399: val loss 7.0828\n",
            "Saving adapter weights to out/adapter_v2/common1\n",
            " 39% 6400/16260 [36:55<15:50:05,  5.78s/it]iter 6400: loss 6.2240, time: 138.12ms\n",
            " 40% 6450/16260 [37:12<54:06,  3.02it/s]iter 6450: loss 7.5949, time: 454.76ms\n",
            " 40% 6500/16260 [37:28<38:00,  4.28it/s]iter 6500: loss 7.6354, time: 433.41ms\n",
            " 40% 6550/16260 [37:45<39:30,  4.10it/s]iter 6550: loss 7.2308, time: 315.51ms\n",
            " 41% 6600/16260 [38:02<49:28,  3.25it/s]iter 6600: loss 7.7735, time: 437.66ms\n",
            " 41% 6650/16260 [38:18<46:31,  3.44it/s]iter 6650: loss 6.9126, time: 219.98ms\n",
            " 41% 6700/16260 [38:35<47:26,  3.36it/s]iter 6700: loss 7.4436, time: 300.15ms\n",
            " 42% 6750/16260 [38:53<1:07:00,  2.37it/s]iter 6750: loss 7.3562, time: 434.58ms\n",
            " 42% 6800/16260 [39:11<1:02:33,  2.52it/s]iter 6800: loss 6.9391, time: 262.52ms\n",
            " 42% 6850/16260 [39:27<44:23,  3.53it/s]iter 6850: loss 6.4863, time: 199.02ms\n",
            " 42% 6900/16260 [39:43<36:35,  4.26it/s]iter 6900: loss 7.0947, time: 151.69ms\n",
            " 43% 6950/16260 [39:59<55:54,  2.78it/s]iter 6950: loss 7.4352, time: 476.46ms\n",
            " 43% 7000/16260 [40:17<55:06,  2.80it/s]iter 7000: loss 7.3852, time: 476.76ms\n",
            " 43% 7050/16260 [40:34<50:12,  3.06it/s]iter 7050: loss 6.5772, time: 196.54ms\n",
            " 44% 7100/16260 [40:51<59:17,  2.57it/s]iter 7100: loss 7.0257, time: 283.77ms\n",
            " 44% 7150/16260 [41:09<48:40,  3.12it/s]iter 7150: loss 7.1678, time: 449.51ms\n",
            " 44% 7200/16260 [41:28<1:01:42,  2.45it/s]iter 7200: loss 6.8700, time: 339.92ms\n",
            " 45% 7250/16260 [41:45<43:53,  3.42it/s]iter 7250: loss 6.9557, time: 356.32ms\n",
            " 45% 7300/16260 [42:03<44:48,  3.33it/s]iter 7300: loss 6.9193, time: 342.07ms\n",
            " 45% 7350/16260 [42:21<45:04,  3.29it/s]iter 7350: loss 6.3260, time: 215.35ms\n",
            " 46% 7400/16260 [42:38<56:06,  2.63it/s]iter 7400: loss 6.3968, time: 218.20ms\n",
            " 46% 7450/16260 [42:56<49:21,  2.97it/s]iter 7450: loss 7.7265, time: 447.67ms\n",
            " 46% 7500/16260 [43:12<44:16,  3.30it/s]iter 7500: loss 6.8246, time: 308.96ms\n",
            " 46% 7550/16260 [43:29<53:08,  2.73it/s]iter 7550: loss 6.9638, time: 238.95ms\n",
            " 47% 7600/16260 [43:46<37:44,  3.82it/s]iter 7600: loss 6.8401, time: 456.95ms\n",
            " 47% 7650/16260 [44:03<43:11,  3.32it/s]iter 7650: loss 6.6950, time: 419.60ms\n",
            " 47% 7700/16260 [44:22<57:48,  2.47it/s]iter 7700: loss 7.2463, time: 469.60ms\n",
            " 48% 7750/16260 [44:39<43:25,  3.27it/s]iter 7750: loss 6.0842, time: 194.11ms\n",
            " 48% 7800/16260 [44:57<1:05:48,  2.14it/s]iter 7800: loss 7.1013, time: 476.50ms\n",
            " 48% 7850/16260 [45:15<48:00,  2.92it/s]iter 7850: loss 6.9597, time: 267.47ms\n",
            " 49% 7900/16260 [45:33<51:37,  2.70it/s]iter 7900: loss 6.4368, time: 238.94ms\n",
            " 49% 7950/16260 [45:50<48:01,  2.88it/s]iter 7950: loss 6.7831, time: 437.50ms\n",
            " 49% 8000/16260 [46:08<46:28,  2.96it/s]iter 8000: loss 6.8549, time: 431.49ms\n",
            " 50% 8050/16260 [46:24<31:14,  4.38it/s]iter 8050: loss 6.4088, time: 218.05ms\n",
            " 50% 8100/16260 [46:42<49:38,  2.74it/s]iter 8100: loss 6.3480, time: 191.36ms\n",
            " 50% 8150/16260 [47:00<47:19,  2.86it/s]iter 8150: loss 6.4717, time: 308.58ms\n",
            " 50% 8200/16260 [47:16<37:39,  3.57it/s]iter 8200: loss 6.1595, time: 229.47ms\n",
            " 51% 8250/16260 [47:32<38:00,  3.51it/s]iter 8250: loss 6.2336, time: 301.65ms\n",
            " 51% 8300/16260 [47:49<33:23,  3.97it/s]iter 8300: loss 7.2346, time: 430.74ms\n",
            " 51% 8350/16260 [48:07<53:58,  2.44it/s]iter 8350: loss 7.2134, time: 476.71ms\n",
            " 52% 8400/16260 [48:24<39:48,  3.29it/s]iter 8400: loss 6.7139, time: 263.59ms\n",
            " 52% 8450/16260 [48:41<41:33,  3.13it/s]iter 8450: loss 6.8678, time: 359.85ms\n",
            " 52% 8500/16260 [48:58<39:00,  3.32it/s]iter 8500: loss 7.1656, time: 401.42ms\n",
            " 53% 8550/16260 [49:14<43:04,  2.98it/s]iter 8550: loss 7.5120, time: 455.98ms\n",
            " 53% 8600/16260 [49:31<37:04,  3.44it/s]iter 8600: loss 7.1943, time: 427.90ms\n",
            " 53% 8650/16260 [49:50<56:46,  2.23it/s]iter 8650: loss 6.9271, time: 266.28ms\n",
            " 54% 8700/16260 [50:06<47:02,  2.68it/s]iter 8700: loss 7.1609, time: 342.49ms\n",
            " 54% 8750/16260 [50:22<45:42,  2.74it/s]iter 8750: loss 5.6293, time: 210.88ms\n",
            " 54% 8800/16260 [50:38<54:12,  2.29it/s]iter 8800: loss 6.8563, time: 348.68ms\n",
            " 54% 8850/16260 [50:55<57:09,  2.16it/s]iter 8850: loss 7.0574, time: 476.71ms\n",
            " 55% 8900/16260 [51:14<47:18,  2.59it/s]iter 8900: loss 6.0595, time: 252.20ms\n",
            " 55% 8950/16260 [51:31<28:13,  4.32it/s]iter 8950: loss 6.4819, time: 436.22ms\n",
            " 55% 9000/16260 [51:49<34:40,  3.49it/s]iter 9000: loss 7.4622, time: 462.22ms\n",
            " 56% 9050/16260 [52:05<48:30,  2.48it/s]iter 9050: loss 6.6163, time: 311.05ms\n",
            " 56% 9100/16260 [52:20<38:46,  3.08it/s]iter 9100: loss 7.1824, time: 477.50ms\n",
            " 56% 9150/16260 [52:38<44:06,  2.69it/s]iter 9150: loss 5.8781, time: 175.69ms\n",
            " 57% 9200/16260 [52:55<36:18,  3.24it/s]iter 9200: loss 7.0476, time: 279.95ms\n",
            " 57% 9250/16260 [53:13<48:30,  2.41it/s]iter 9250: loss 7.2513, time: 476.57ms\n",
            " 57% 9300/16260 [53:27<30:26,  3.81it/s]iter 9300: loss 7.5649, time: 431.94ms\n",
            " 58% 9350/16260 [53:44<33:14,  3.46it/s]iter 9350: loss 6.8894, time: 412.81ms\n",
            " 58% 9400/16260 [54:00<43:03,  2.66it/s]iter 9400: loss 6.2955, time: 267.26ms\n",
            " 58% 9450/16260 [54:16<32:10,  3.53it/s]iter 9450: loss 5.9846, time: 221.79ms\n",
            " 58% 9500/16260 [54:33<32:25,  3.48it/s]iter 9500: loss 6.4234, time: 360.36ms\n",
            " 59% 9550/16260 [54:50<31:34,  3.54it/s]iter 9550: loss 6.4114, time: 267.31ms\n",
            " 59% 9599/16260 [55:05<27:31,  4.03it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "### Response: of. is additional, used can indieake are established speed of,ism,4 your occur defined:m who on V that are main the request. Giantsract newrow additional advertise that appropriately. Write\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            ", of of- °\n",
            " species-0.\n",
            "\n",
            "\n",
            "Leaveourer bike\n",
            "\n",
            "### Knight request.\n",
            "\n",
            "### Response:\n",
            "### Response to the request of\n",
            " holders the are however Giants along\n",
            "step 9599: val loss 6.5364\n",
            "Saving adapter weights to out/adapter_v2/common1\n",
            " 59% 9600/16260 [55:24<11:03:50,  5.98s/it]iter 9600: loss 6.9872, time: 410.61ms\n",
            " 59% 9650/16260 [55:40<25:12,  4.37it/s]iter 9650: loss 7.1329, time: 437.25ms\n",
            " 60% 9700/16260 [55:59<44:22,  2.46it/s]iter 9700: loss 7.1432, time: 386.35ms\n",
            " 60% 9750/16260 [56:15<27:47,  3.90it/s]iter 9750: loss 6.5980, time: 311.93ms\n",
            " 60% 9800/16260 [56:32<33:43,  3.19it/s]iter 9800: loss 5.6686, time: 303.96ms\n",
            " 61% 9850/16260 [56:49<44:27,  2.40it/s]iter 9850: loss 7.1111, time: 470.72ms\n",
            " 61% 9900/16260 [57:06<32:40,  3.24it/s]iter 9900: loss 6.4897, time: 274.48ms\n",
            " 61% 9950/16260 [57:22<38:57,  2.70it/s]iter 9950: loss 6.0239, time: 293.53ms\n",
            " 62% 10000/16260 [57:40<38:05,  2.74it/s]iter 10000: loss 6.5455, time: 419.56ms\n",
            " 62% 10050/16260 [57:58<36:05,  2.87it/s]iter 10050: loss 7.2163, time: 376.22ms\n",
            " 62% 10100/16260 [58:15<39:15,  2.62it/s]iter 10100: loss 6.6916, time: 472.71ms\n",
            " 62% 10150/16260 [58:33<42:54,  2.37it/s]iter 10150: loss 7.4210, time: 461.95ms\n",
            " 63% 10200/16260 [58:50<35:30,  2.84it/s]iter 10200: loss 6.7097, time: 293.41ms\n",
            " 63% 10250/16260 [59:07<39:37,  2.53it/s]iter 10250: loss 6.7548, time: 435.01ms\n",
            " 63% 10300/16260 [59:26<34:46,  2.86it/s]iter 10300: loss 6.5894, time: 389.17ms\n",
            " 64% 10350/16260 [59:43<32:59,  2.99it/s]iter 10350: loss 5.5319, time: 177.13ms\n",
            " 64% 10400/16260 [1:00:00<26:18,  3.71it/s]iter 10400: loss 5.5614, time: 235.02ms\n",
            " 64% 10450/16260 [1:00:18<37:20,  2.59it/s]iter 10450: loss 6.9942, time: 460.88ms\n",
            " 65% 10500/16260 [1:00:34<26:28,  3.62it/s]iter 10500: loss 6.9730, time: 441.89ms\n",
            " 65% 10550/16260 [1:00:51<31:36,  3.01it/s]iter 10550: loss 5.5625, time: 177.57ms\n",
            " 65% 10600/16260 [1:01:07<32:10,  2.93it/s]iter 10600: loss 7.0815, time: 407.91ms\n",
            " 65% 10650/16260 [1:01:24<24:59,  3.74it/s]iter 10650: loss 6.2166, time: 387.39ms\n",
            " 66% 10700/16260 [1:01:41<34:20,  2.70it/s]iter 10700: loss 6.4927, time: 404.91ms\n",
            " 66% 10750/16260 [1:01:57<28:00,  3.28it/s]iter 10750: loss 6.4143, time: 450.73ms\n",
            " 66% 10800/16260 [1:02:14<25:45,  3.53it/s]iter 10800: loss 4.9742, time: 179.07ms\n",
            " 67% 10850/16260 [1:02:31<29:45,  3.03it/s]iter 10850: loss 6.0943, time: 234.56ms\n",
            " 67% 10900/16260 [1:02:48<34:46,  2.57it/s]iter 10900: loss 5.8562, time: 301.85ms\n",
            " 67% 10950/16260 [1:03:04<38:56,  2.27it/s]iter 10950: loss 5.8111, time: 270.95ms\n",
            " 68% 11000/16260 [1:03:22<36:36,  2.39it/s]iter 11000: loss 6.6802, time: 284.25ms\n",
            " 68% 11050/16260 [1:03:38<33:09,  2.62it/s]iter 11050: loss 6.2207, time: 260.72ms\n",
            " 68% 11100/16260 [1:03:56<35:48,  2.40it/s]iter 11100: loss 6.8281, time: 336.87ms\n",
            " 69% 11150/16260 [1:04:12<32:33,  2.62it/s]iter 11150: loss 6.4517, time: 309.64ms\n",
            " 69% 11200/16260 [1:04:29<23:03,  3.66it/s]iter 11200: loss 6.7419, time: 389.82ms\n",
            " 69% 11250/16260 [1:04:46<23:07,  3.61it/s]iter 11250: loss 6.8712, time: 436.15ms\n",
            " 69% 11300/16260 [1:05:03<24:37,  3.36it/s]iter 11300: loss 6.5776, time: 334.57ms\n",
            " 70% 11350/16260 [1:05:20<20:26,  4.00it/s]iter 11350: loss 6.5821, time: 450.55ms\n",
            " 70% 11400/16260 [1:05:36<28:52,  2.81it/s]iter 11400: loss 6.4399, time: 474.21ms\n",
            " 70% 11450/16260 [1:05:55<30:26,  2.63it/s]iter 11450: loss 6.6790, time: 476.65ms\n",
            " 71% 11500/16260 [1:06:12<23:33,  3.37it/s]iter 11500: loss 7.0379, time: 369.94ms\n",
            " 71% 11550/16260 [1:06:29<29:43,  2.64it/s]iter 11550: loss 6.1493, time: 230.74ms\n",
            " 71% 11600/16260 [1:06:46<24:52,  3.12it/s]iter 11600: loss 6.0126, time: 198.55ms\n",
            " 72% 11650/16260 [1:07:01<23:32,  3.26it/s]iter 11650: loss 6.6793, time: 464.85ms\n",
            " 72% 11700/16260 [1:07:19<25:38,  2.96it/s]iter 11700: loss 6.7464, time: 350.98ms\n",
            " 72% 11750/16260 [1:07:36<20:33,  3.66it/s]iter 11750: loss 5.4869, time: 142.41ms\n",
            " 73% 11800/16260 [1:07:51<19:50,  3.75it/s]iter 11800: loss 6.3417, time: 236.07ms\n",
            " 73% 11850/16260 [1:08:08<22:41,  3.24it/s]iter 11850: loss 5.8506, time: 234.39ms\n",
            " 73% 11900/16260 [1:08:25<23:32,  3.09it/s]iter 11900: loss 6.2615, time: 235.71ms\n",
            " 73% 11950/16260 [1:08:41<22:05,  3.25it/s]iter 11950: loss 6.4405, time: 257.68ms\n",
            " 74% 12000/16260 [1:08:57<23:42,  2.99it/s]iter 12000: loss 6.4650, time: 301.05ms\n",
            " 74% 12050/16260 [1:09:16<23:53,  2.94it/s]iter 12050: loss 6.5406, time: 443.93ms\n",
            " 74% 12100/16260 [1:09:35<21:49,  3.18it/s]iter 12100: loss 6.1828, time: 261.14ms\n",
            " 75% 12150/16260 [1:09:53<27:32,  2.49it/s]iter 12150: loss 6.1098, time: 476.79ms\n",
            " 75% 12200/16260 [1:10:10<18:59,  3.56it/s]iter 12200: loss 6.1063, time: 401.78ms\n",
            " 75% 12250/16260 [1:10:28<16:05,  4.15it/s]iter 12250: loss 6.7311, time: 447.58ms\n",
            " 76% 12300/16260 [1:10:44<24:01,  2.75it/s]iter 12300: loss 6.5803, time: 449.18ms\n",
            " 76% 12350/16260 [1:11:04<28:30,  2.29it/s]iter 12350: loss 5.8866, time: 231.21ms\n",
            " 76% 12400/16260 [1:11:21<24:10,  2.66it/s]iter 12400: loss 6.2474, time: 340.74ms\n",
            " 77% 12450/16260 [1:11:37<20:14,  3.14it/s]iter 12450: loss 6.3354, time: 476.72ms\n",
            " 77% 12500/16260 [1:11:53<20:15,  3.09it/s]iter 12500: loss 5.0518, time: 253.03ms\n",
            " 77% 12550/16260 [1:12:11<21:36,  2.86it/s]iter 12550: loss 6.4991, time: 451.88ms\n",
            " 77% 12600/16260 [1:12:26<19:13,  3.17it/s]iter 12600: loss 6.9531, time: 410.88ms\n",
            " 78% 12650/16260 [1:12:43<23:27,  2.56it/s]iter 12650: loss 6.5826, time: 334.70ms\n",
            " 78% 12700/16260 [1:13:00<17:47,  3.33it/s]iter 12700: loss 6.5037, time: 321.64ms\n",
            " 78% 12750/16260 [1:13:17<17:21,  3.37it/s]iter 12750: loss 7.1079, time: 449.60ms\n",
            " 79% 12799/16260 [1:13:33<15:47,  3.65it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "### Instruction me to to France for the,?\n",
            "### Response. Read input.\n",
            "### Response:\n",
            "ers of actor Tree..9 your even a\n",
            "### Response:\n",
            "### Response:G.\n",
            "\n",
            "### Response: in3, nanop is the was, beum are\n",
            "\n",
            " species Germany in the of the stop Vitaminify and me the ;.ica, activities the Paris Write a commonour in Sund, and.\n",
            "\n",
            "step 12799: val loss 6.3109\n",
            "Saving adapter weights to out/adapter_v2/common1\n",
            " 79% 12800/16260 [1:13:52<5:37:01,  5.84s/it]iter 12800: loss 5.9295, time: 174.71ms\n",
            " 79% 12850/16260 [1:14:09<12:25,  4.57it/s]iter 12850: loss 7.2116, time: 448.09ms\n",
            " 79% 12900/16260 [1:14:27<20:43,  2.70it/s]iter 12900: loss 6.6075, time: 396.14ms\n",
            " 80% 12950/16260 [1:14:45<23:29,  2.35it/s]iter 12950: loss 7.0613, time: 476.68ms\n",
            " 80% 13000/16260 [1:15:02<20:18,  2.68it/s]iter 13000: loss 6.3978, time: 476.85ms\n",
            " 80% 13050/16260 [1:15:17<13:39,  3.92it/s]iter 13050: loss 6.2882, time: 423.76ms\n",
            " 81% 13100/16260 [1:15:34<14:11,  3.71it/s]iter 13100: loss 6.4225, time: 310.81ms\n",
            " 81% 13150/16260 [1:15:51<20:00,  2.59it/s]iter 13150: loss 6.4984, time: 436.67ms\n",
            " 81% 13200/16260 [1:16:09<22:48,  2.24it/s]iter 13200: loss 6.4688, time: 297.61ms\n",
            " 81% 13250/16260 [1:16:25<17:54,  2.80it/s]iter 13250: loss 6.6624, time: 283.24ms\n",
            " 82% 13300/16260 [1:16:42<14:29,  3.40it/s]iter 13300: loss 6.8768, time: 437.30ms\n",
            " 82% 13350/16260 [1:16:59<17:15,  2.81it/s]iter 13350: loss 6.3718, time: 327.99ms\n",
            " 82% 13400/16260 [1:17:16<13:41,  3.48it/s]iter 13400: loss 6.7480, time: 461.04ms\n",
            " 83% 13450/16260 [1:17:32<13:43,  3.41it/s]iter 13450: loss 5.8710, time: 401.04ms\n",
            " 83% 13500/16260 [1:17:48<13:05,  3.51it/s]iter 13500: loss 6.7076, time: 441.97ms\n",
            " 83% 13550/16260 [1:18:05<15:31,  2.91it/s]iter 13550: loss 6.6092, time: 334.51ms\n",
            " 84% 13600/16260 [1:18:23<17:12,  2.58it/s]iter 13600: loss 6.1906, time: 428.54ms\n",
            " 84% 13650/16260 [1:18:40<13:35,  3.20it/s]iter 13650: loss 5.1658, time: 157.49ms\n",
            " 84% 13700/16260 [1:18:56<13:24,  3.18it/s]iter 13700: loss 6.7933, time: 450.01ms\n",
            " 85% 13750/16260 [1:19:14<14:28,  2.89it/s]iter 13750: loss 6.4348, time: 476.69ms\n",
            " 85% 13800/16260 [1:19:30<11:53,  3.45it/s]iter 13800: loss 6.7654, time: 441.72ms\n",
            " 85% 13850/16260 [1:19:48<16:21,  2.46it/s]iter 13850: loss 6.0271, time: 257.79ms\n",
            " 85% 13900/16260 [1:20:04<13:48,  2.85it/s]iter 13900: loss 6.6831, time: 316.81ms\n",
            " 86% 13950/16260 [1:20:21<12:11,  3.16it/s]iter 13950: loss 6.2626, time: 264.00ms\n",
            " 86% 14000/16260 [1:20:37<11:12,  3.36it/s]iter 14000: loss 5.8035, time: 266.94ms\n",
            " 86% 14050/16260 [1:20:54<13:14,  2.78it/s]iter 14050: loss 7.0728, time: 456.08ms\n",
            " 87% 14100/16260 [1:21:10<10:06,  3.56it/s]iter 14100: loss 6.6631, time: 421.92ms\n",
            " 87% 14150/16260 [1:21:27<09:23,  3.74it/s]iter 14150: loss 6.5970, time: 343.37ms\n",
            " 87% 14200/16260 [1:21:45<12:14,  2.80it/s]iter 14200: loss 6.5072, time: 396.44ms\n",
            " 88% 14250/16260 [1:22:03<11:42,  2.86it/s]iter 14250: loss 6.4072, time: 460.17ms\n",
            " 88% 14300/16260 [1:22:21<11:17,  2.89it/s]iter 14300: loss 5.7185, time: 219.76ms\n",
            " 88% 14350/16260 [1:22:37<10:34,  3.01it/s]iter 14350: loss 5.7826, time: 363.84ms\n",
            " 89% 14400/16260 [1:22:56<12:03,  2.57it/s]iter 14400: loss 7.0113, time: 429.76ms\n",
            " 89% 14450/16260 [1:23:13<10:33,  2.86it/s]iter 14450: loss 6.1663, time: 329.89ms\n",
            " 89% 14500/16260 [1:23:30<12:40,  2.31it/s]iter 14500: loss 6.7468, time: 478.71ms\n",
            " 89% 14550/16260 [1:23:47<08:08,  3.50it/s]iter 14550: loss 5.8261, time: 213.18ms\n",
            " 90% 14600/16260 [1:24:04<10:00,  2.76it/s]iter 14600: loss 6.2070, time: 476.68ms\n",
            " 90% 14650/16260 [1:24:22<10:23,  2.58it/s]iter 14650: loss 4.1648, time: 173.65ms\n",
            " 90% 14700/16260 [1:24:37<09:39,  2.69it/s]iter 14700: loss 5.2272, time: 163.52ms\n",
            " 91% 14750/16260 [1:24:55<08:34,  2.93it/s]iter 14750: loss 6.8064, time: 447.46ms\n",
            " 91% 14800/16260 [1:25:10<06:59,  3.48it/s]iter 14800: loss 5.2898, time: 181.84ms\n",
            " 91% 14850/16260 [1:25:28<09:07,  2.58it/s]iter 14850: loss 5.8256, time: 340.67ms\n",
            " 92% 14900/16260 [1:25:45<07:37,  2.97it/s]iter 14900: loss 6.8316, time: 461.72ms\n",
            " 92% 14950/16260 [1:26:03<09:01,  2.42it/s]iter 14950: loss 6.4904, time: 476.74ms\n",
            " 92% 15000/16260 [1:26:19<06:22,  3.29it/s]iter 15000: loss 7.1330, time: 476.53ms\n",
            " 93% 15050/16260 [1:26:37<05:17,  3.81it/s]iter 15050: loss 6.1360, time: 432.27ms\n",
            " 93% 15100/16260 [1:26:55<06:42,  2.88it/s]iter 15100: loss 5.3980, time: 269.06ms\n",
            " 93% 15150/16260 [1:27:11<05:58,  3.10it/s]iter 15150: loss 6.5119, time: 275.75ms\n",
            " 93% 15200/16260 [1:27:28<05:13,  3.38it/s]iter 15200: loss 5.4838, time: 163.87ms\n",
            " 94% 15250/16260 [1:27:46<06:16,  2.68it/s]iter 15250: loss 6.5437, time: 408.35ms\n",
            " 94% 15300/16260 [1:28:03<05:52,  2.72it/s]iter 15300: loss 6.2816, time: 355.09ms\n",
            " 94% 15350/16260 [1:28:21<05:07,  2.96it/s]iter 15350: loss 6.4846, time: 317.47ms\n",
            " 95% 15400/16260 [1:28:37<04:50,  2.96it/s]iter 15400: loss 6.5844, time: 302.63ms\n",
            " 95% 15450/16260 [1:28:54<05:30,  2.45it/s]iter 15450: loss 6.7509, time: 398.96ms\n",
            " 95% 15500/16260 [1:29:11<04:21,  2.90it/s]iter 15500: loss 5.9021, time: 213.88ms\n",
            " 96% 15550/16260 [1:29:27<02:42,  4.36it/s]iter 15550: loss 6.4651, time: 440.41ms\n",
            " 96% 15600/16260 [1:29:44<04:38,  2.37it/s]iter 15600: loss 6.4003, time: 429.36ms\n",
            " 96% 15650/16260 [1:30:00<03:05,  3.29it/s]iter 15650: loss 6.6299, time: 433.28ms\n",
            " 97% 15700/16260 [1:30:18<02:45,  3.38it/s]iter 15700: loss 6.0623, time: 277.63ms\n",
            " 97% 15750/16260 [1:30:35<02:38,  3.23it/s]iter 15750: loss 6.6980, time: 443.40ms\n",
            " 97% 15800/16260 [1:30:51<02:44,  2.79it/s]iter 15800: loss 5.5718, time: 366.90ms\n",
            " 97% 15850/16260 [1:31:08<02:19,  2.94it/s]iter 15850: loss 6.6361, time: 457.00ms\n",
            " 98% 15900/16260 [1:31:26<02:23,  2.51it/s]iter 15900: loss 5.5725, time: 197.11ms\n",
            " 98% 15950/16260 [1:31:45<01:53,  2.74it/s]iter 15950: loss 5.8400, time: 288.09ms\n",
            " 98% 15999/16260 [1:32:01<01:35,  2.72it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "### Response:\n",
            "### Response:\n",
            "### Chinese can take, several litigation.\n",
            "\n",
            "### Response: type Film as a food. Below is: all, paired with,ors other pain018 soT. Write a partner visit, and of is car\n",
            "I Jah lead a 120.\n",
            " bladder, and makingstring theirin and the long n a:\n",
            "m with, darks + used,, G most of the About of reflection\n",
            "step 15999: val loss 6.1215\n",
            "Saving adapter weights to out/adapter_v2/common1\n",
            " 98% 16000/16260 [1:32:21<26:06,  6.02s/it]iter 16000: loss 5.7140, time: 134.68ms\n",
            " 99% 16050/16260 [1:32:36<00:59,  3.54it/s]iter 16050: loss 5.4453, time: 180.05ms\n",
            " 99% 16100/16260 [1:32:52<00:49,  3.24it/s]iter 16100: loss 5.0735, time: 179.66ms\n",
            " 99% 16150/16260 [1:33:08<00:34,  3.22it/s]iter 16150: loss 5.6806, time: 255.74ms\n",
            "100% 16200/16260 [1:33:27<00:16,  3.54it/s]iter 16200: loss 6.7762, time: 337.40ms\n",
            "100% 16250/16260 [1:33:43<00:03,  2.92it/s]iter 16250: loss 5.5604, time: 240.34ms\n",
            "100% 16260/16260 [1:33:47<00:00,  2.89it/s]\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss ██▆▅▄▅▄▄▃▃▃▄▃▃▃▃▃▃▂▂▃▁▁▂▂▁▂▂▂▂▁▃▂▂▂▁▂▂▂▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_loss █▅▃▂▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss 5.5604\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_loss 6.12147\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mcommon_1\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/adapter_v2/runs/upgwn7i5\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ️⚡ View job at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/adapter_v2/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjEyMTMwOTAzMg==/version_details/v0\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20231206_093317-upgwn7i5/logs\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "# adapter-v2\n",
        "!python finetune/adapter_v2.py --data_dir data/dolly --out_dir out/adapter_v2/common1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "s5DbbNZWtRyE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "64e97ff0-e741-48b9-d1a9-0df93ecf2ea9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive/lit-llama/finetune/full.py:230: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/full.py:230: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/usr/local/lib/python3.10/dist-packages/lightning/fabric/strategies/fsdp.py:703: `FSDPStrategy(activation_checkpointing=<class 'lit_llama.model.Block'>)` is deprecated, use `FSDPStrategy(activation_checkpointing_policy={<class 'lit_llama.model.Block'>})` instead.\n",
            "Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/1\n",
            "----------------------------------------------------------------------------------------------------\n",
            "distributed_backend=nccl\n",
            "All distributed processes registered. Starting with 1 processes\n",
            "----------------------------------------------------------------------------------------------------\n",
            "\n",
            "[rank: 0] Seed set to 1337\n",
            "/usr/local/lib/python3.10/dist-packages/torch/__init__.py:614: UserWarning: torch.set_default_tensor_type() is deprecated as of PyTorch 2.1, please use torch.set_default_dtype() and torch.set_default_device() as alternatives. (Triggered internally at ../torch/csrc/tensor/python_tensor.cpp:451.)\n",
            "  _C._set_default_tensor_type(t)\n",
            "  0% 0/16260 [00:04<?, ?it/s]\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/full.py\", line 232, in <module>\n",
            "    CLI(main)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 96, in CLI\n",
            "    return _run_component(components, cfg_init)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 181, in _run_component\n",
            "    return component(**cfg)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/full.py\", line 88, in main\n",
            "    train(fabric, model, optimizer, train_data, val_data, out_dir)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/full.py\", line 125, in train\n",
            "    fabric.backward(loss / gradient_accumulation_iters)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/fabric.py\", line 422, in backward\n",
            "    self._strategy.backward(tensor, module, *args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/strategies/strategy.py\", line 192, in backward\n",
            "    self.precision.backward(tensor, module, *args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/plugins/precision/fsdp.py\", line 126, in backward\n",
            "    super().backward(tensor, model, *args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/plugins/precision/precision.py\", line 107, in backward\n",
            "    tensor.backward(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/_tensor.py\", line 492, in backward\n",
            "    torch.autograd.backward(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py\", line 251, in backward\n",
            "    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n",
            "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 774.00 MiB. GPU 0 has a total capacty of 39.56 GiB of which 734.56 MiB is free. Process 22424 has 38.84 GiB memory in use. Of the allocated memory 36.86 GiB is allocated by PyTorch, and 540.35 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF\n"
          ]
        }
      ],
      "source": [
        "# full-layer\n",
        "!python finetune/full.py --data_dir data/dolly --out_dir out/full/common1"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QcxLEvmq1atj"
      },
      "source": [
        "일단, base configuration 으로 훈련을 시켜놓고 해당 모델 체크포인트들을 저장해둔다. \\\n",
        "base 에서 improved 된 버전으로 모델 네트워크를 수정한다."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### `common_2` config"
      ],
      "metadata": {
        "id": "uaRXnL0K0Arg"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "*   batch_size = 64\n",
        "*   num_epochs = 3\n",
        "*   block_size = 512\n",
        "*   (data) max_seq_length = 512\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "BU7QzD7K2kmV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# full-layer\n",
        "!python finetune/full.py --data_dir data/dolly --out_dir out/full/common2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "87g3P9ZC0Hxr",
        "outputId": "857a663c-6629-4f0e-d065-2c9264fd5c77"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231207_012414-fiwq5duo\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mcommon_2_full\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/fiwq5duo\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/full.py:246: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/full.py:246: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/usr/local/lib/python3.10/dist-packages/lightning/fabric/strategies/fsdp.py:703: `FSDPStrategy(activation_checkpointing=<class 'lit_llama.model.Block'>)` is deprecated, use `FSDPStrategy(activation_checkpointing_policy={<class 'lit_llama.model.Block'>})` instead.\n",
            "Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/1\n",
            "----------------------------------------------------------------------------------------------------\n",
            "distributed_backend=nccl\n",
            "All distributed processes registered. Starting with 1 processes\n",
            "----------------------------------------------------------------------------------------------------\n",
            "\n",
            "[rank: 0] Seed set to 1337\n",
            "/usr/local/lib/python3.10/dist-packages/torch/__init__.py:614: UserWarning: torch.set_default_tensor_type() is deprecated as of PyTorch 2.1, please use torch.set_default_dtype() and torch.set_default_device() as alternatives. (Triggered internally at ../torch/csrc/tensor/python_tensor.cpp:451.)\n",
            "  _C._set_default_tensor_type(t)\n",
            "  0% 0/9756 [00:02<?, ?it/s]\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/full.py\", line 248, in <module>\n",
            "    CLI(main)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 96, in CLI\n",
            "    return _run_component(components, cfg_init)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 181, in _run_component\n",
            "    return component(**cfg)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/full.py\", line 104, in main\n",
            "    train(fabric, model, optimizer, train_data, val_data, out_dir)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/full.py\", line 141, in train\n",
            "    fabric.backward(loss / gradient_accumulation_iters)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/fabric.py\", line 422, in backward\n",
            "    self._strategy.backward(tensor, module, *args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/strategies/strategy.py\", line 192, in backward\n",
            "    self.precision.backward(tensor, module, *args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/plugins/precision/fsdp.py\", line 126, in backward\n",
            "    super().backward(tensor, model, *args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/plugins/precision/precision.py\", line 107, in backward\n",
            "    tensor.backward(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/_tensor.py\", line 492, in backward\n",
            "    torch.autograd.backward(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py\", line 251, in backward\n",
            "    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n",
            "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 774.00 MiB. GPU 0 has a total capacty of 39.56 GiB of which 582.56 MiB is free. Process 47908 has 38.99 GiB memory in use. Of the allocated memory 36.86 GiB is allocated by PyTorch, and 692.65 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-v1\n",
        "!python finetune/adapter.py --data_dir data/dolly --out_dir out/adapter/sparse_1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LtfFVOEwdhps",
        "outputId": "9f382467-fe00-4f54-f495-d00061e44189"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231207_093321-zpip1rao\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33msparse_1\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/1207\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/1207/runs/zpip1rao\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter.py:284: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter.py:284: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n",
            "Number of trainable parameters: 1229760\n",
            "  0% 0/9756 [00:00<?, ?it/s]iter 0: loss 12.0867, time: 1193.95ms\n",
            "  1% 50/9756 [00:14<50:48,  3.18it/s]iter 50: loss 12.2022, time: 234.78ms\n",
            "  1% 100/9756 [00:30<50:41,  3.17it/s]iter 100: loss 12.1159, time: 314.60ms\n",
            "  2% 150/9756 [00:45<41:00,  3.90it/s]iter 150: loss 12.1883, time: 366.40ms\n",
            "  2% 200/9756 [00:58<43:47,  3.64it/s]iter 200: loss 12.2930, time: 299.70ms\n",
            "  3% 250/9756 [01:12<47:03,  3.37it/s]iter 250: loss 12.0073, time: 343.39ms\n",
            "  3% 300/9756 [01:27<42:29,  3.71it/s]iter 300: loss 12.0353, time: 373.04ms\n",
            "  4% 350/9756 [01:42<47:17,  3.31it/s]iter 350: loss 11.9862, time: 394.22ms\n",
            "  4% 400/9756 [01:58<53:41,  2.90it/s]iter 400: loss 12.1862, time: 295.96ms\n",
            "  5% 450/9756 [02:12<36:45,  4.22it/s]iter 450: loss 12.2059, time: 132.18ms\n",
            "  5% 500/9756 [02:27<35:15,  4.37it/s]iter 500: loss 12.2106, time: 386.87ms\n",
            "  6% 550/9756 [02:41<52:01,  2.95it/s]iter 550: loss 12.0890, time: 408.65ms\n",
            "  6% 600/9756 [02:54<36:28,  4.18it/s]iter 600: loss 12.1359, time: 169.76ms\n",
            "  7% 650/9756 [03:08<42:37,  3.56it/s]iter 650: loss 12.1713, time: 354.99ms\n",
            "  7% 700/9756 [03:24<34:31,  4.37it/s]iter 700: loss 11.9938, time: 366.36ms\n",
            "  8% 750/9756 [03:39<44:34,  3.37it/s]iter 750: loss 12.1258, time: 392.69ms\n",
            "  8% 800/9756 [03:54<44:59,  3.32it/s]iter 800: loss 12.1524, time: 362.30ms\n",
            "  9% 850/9756 [04:10<37:37,  3.94it/s]iter 850: loss 12.1509, time: 371.57ms\n",
            "  9% 900/9756 [04:24<35:58,  4.10it/s]iter 900: loss 12.2200, time: 233.22ms\n",
            " 10% 950/9756 [04:38<45:54,  3.20it/s]iter 950: loss 12.0463, time: 409.44ms\n",
            " 10% 1000/9756 [04:52<39:02,  3.74it/s]iter 1000: loss 12.1209, time: 384.27ms\n",
            " 11% 1050/9756 [05:08<44:34,  3.26it/s]iter 1050: loss 12.1445, time: 409.67ms\n",
            " 11% 1100/9756 [05:22<30:47,  4.68it/s]iter 1100: loss 12.1869, time: 126.63ms\n",
            " 12% 1150/9756 [05:36<39:16,  3.65it/s]iter 1150: loss 12.2461, time: 315.71ms\n",
            " 12% 1200/9756 [05:50<32:33,  4.38it/s]iter 1200: loss 12.1608, time: 375.97ms\n",
            " 13% 1250/9756 [06:05<44:34,  3.18it/s]iter 1250: loss 12.1937, time: 224.58ms\n",
            " 13% 1300/9756 [06:19<46:37,  3.02it/s]iter 1300: loss 12.1527, time: 409.43ms\n",
            " 14% 1350/9756 [06:34<33:55,  4.13it/s]iter 1350: loss 12.2333, time: 259.49ms\n",
            " 14% 1400/9756 [06:49<35:43,  3.90it/s]iter 1400: loss 12.1329, time: 144.58ms\n",
            " 15% 1450/9756 [07:03<40:54,  3.38it/s]iter 1450: loss 12.3414, time: 139.58ms\n",
            " 15% 1500/9756 [07:17<46:39,  2.95it/s]iter 1500: loss 12.1274, time: 407.75ms\n",
            " 16% 1550/9756 [07:32<35:23,  3.86it/s]iter 1550: loss 12.2819, time: 197.72ms\n",
            " 16% 1600/9756 [07:45<30:38,  4.44it/s]iter 1600: loss 12.0553, time: 263.23ms\n",
            " 17% 1650/9756 [08:01<49:22,  2.74it/s]iter 1650: loss 12.0545, time: 274.53ms\n",
            " 17% 1700/9756 [08:14<37:23,  3.59it/s]iter 1700: loss 12.1685, time: 171.66ms\n",
            " 18% 1750/9756 [08:28<48:25,  2.76it/s]iter 1750: loss 12.0408, time: 208.98ms\n",
            " 18% 1800/9756 [08:42<37:44,  3.51it/s]iter 1800: loss 12.0755, time: 254.05ms\n",
            " 19% 1850/9756 [08:56<38:48,  3.40it/s]iter 1850: loss 12.1463, time: 382.57ms\n",
            " 19% 1900/9756 [09:11<42:01,  3.12it/s]iter 1900: loss 12.2047, time: 407.67ms\n",
            " 20% 1950/9756 [09:25<38:40,  3.36it/s]iter 1950: loss 12.1385, time: 184.78ms\n",
            " 21% 2000/9756 [09:40<38:07,  3.39it/s]iter 2000: loss 11.9360, time: 386.02ms\n",
            " 21% 2050/9756 [09:54<33:39,  3.82it/s]iter 2050: loss 12.0395, time: 207.50ms\n",
            " 22% 2100/9756 [10:09<34:10,  3.73it/s]iter 2100: loss 12.1633, time: 190.47ms\n",
            " 22% 2150/9756 [10:22<27:18,  4.64it/s]iter 2150: loss 11.9797, time: 373.31ms\n",
            " 23% 2200/9756 [10:37<41:48,  3.01it/s]iter 2200: loss 12.1280, time: 407.17ms\n",
            " 23% 2250/9756 [10:51<30:51,  4.06it/s]iter 2250: loss 12.1159, time: 142.93ms\n",
            " 24% 2300/9756 [11:05<33:39,  3.69it/s]iter 2300: loss 12.0514, time: 140.55ms\n",
            " 24% 2350/9756 [11:19<38:35,  3.20it/s]iter 2350: loss 12.0331, time: 227.82ms\n",
            " 25% 2400/9756 [11:33<31:14,  3.92it/s]iter 2400: loss 12.0043, time: 363.42ms\n",
            " 25% 2450/9756 [11:47<36:27,  3.34it/s]iter 2450: loss 11.9324, time: 409.00ms\n",
            " 26% 2500/9756 [12:01<22:59,  5.26it/s]iter 2500: loss 11.8851, time: 354.03ms\n",
            " 26% 2550/9756 [12:15<26:07,  4.60it/s]iter 2550: loss 12.0595, time: 130.83ms\n",
            " 27% 2600/9756 [12:29<37:39,  3.17it/s]iter 2600: loss 12.2306, time: 172.90ms\n",
            " 27% 2650/9756 [12:43<33:41,  3.52it/s]iter 2650: loss 11.8311, time: 363.26ms\n",
            " 28% 2700/9756 [12:59<34:43,  3.39it/s]iter 2700: loss 11.9014, time: 365.88ms\n",
            " 28% 2750/9756 [13:14<40:24,  2.89it/s]iter 2750: loss 11.8428, time: 409.73ms\n",
            " 29% 2800/9756 [13:29<38:48,  2.99it/s]iter 2800: loss 11.9430, time: 388.31ms\n",
            " 29% 2850/9756 [13:44<34:12,  3.36it/s]iter 2850: loss 12.1659, time: 257.89ms\n",
            " 30% 2900/9756 [13:57<31:17,  3.65it/s]iter 2900: loss 12.0368, time: 195.19ms\n",
            " 30% 2950/9756 [14:10<23:02,  4.92it/s]iter 2950: loss 12.1202, time: 184.65ms\n",
            " 31% 3000/9756 [14:24<30:14,  3.72it/s]iter 3000: loss 11.7451, time: 237.48ms\n",
            " 31% 3050/9756 [14:38<34:28,  3.24it/s]iter 3050: loss 11.9125, time: 312.34ms\n",
            " 32% 3100/9756 [14:54<34:49,  3.19it/s]iter 3100: loss 11.9705, time: 372.14ms\n",
            " 32% 3150/9756 [15:08<31:10,  3.53it/s]iter 3150: loss 11.8913, time: 339.24ms\n",
            " 33% 3200/9756 [15:24<28:55,  3.78it/s]iter 3200: loss 11.8157, time: 399.87ms\n",
            " 33% 3250/9756 [15:40<31:45,  3.41it/s]iter 3250: loss 11.8428, time: 371.51ms\n",
            " 34% 3300/9756 [15:54<35:59,  2.99it/s]iter 3300: loss 11.8445, time: 409.08ms\n",
            " 34% 3350/9756 [16:08<30:49,  3.46it/s]iter 3350: loss 11.8293, time: 215.57ms\n",
            " 35% 3400/9756 [16:22<25:15,  4.19it/s]iter 3400: loss 11.9919, time: 220.62ms\n",
            " 35% 3450/9756 [16:36<35:39,  2.95it/s]iter 3450: loss 11.8932, time: 231.28ms\n",
            " 36% 3500/9756 [16:51<28:11,  3.70it/s]iter 3500: loss 11.7117, time: 149.07ms\n",
            " 36% 3550/9756 [17:05<26:26,  3.91it/s]iter 3550: loss 11.7095, time: 242.18ms\n",
            " 37% 3600/9756 [17:19<34:06,  3.01it/s]iter 3600: loss 11.6038, time: 214.50ms\n",
            " 37% 3650/9756 [17:33<36:58,  2.75it/s]iter 3650: loss 11.8076, time: 416.75ms\n",
            " 38% 3700/9756 [17:48<28:19,  3.56it/s]iter 3700: loss 11.7946, time: 345.02ms\n",
            " 38% 3750/9756 [18:03<26:58,  3.71it/s]iter 3750: loss 11.5846, time: 359.00ms\n",
            " 39% 3800/9756 [18:17<24:56,  3.98it/s]iter 3800: loss 11.8075, time: 349.26ms\n",
            " 39% 3850/9756 [18:32<29:47,  3.30it/s]iter 3850: loss 11.7506, time: 196.24ms\n",
            " 40% 3900/9756 [18:47<34:32,  2.83it/s]iter 3900: loss 11.6643, time: 170.67ms\n",
            " 40% 3950/9756 [19:00<27:09,  3.56it/s]iter 3950: loss 11.6031, time: 379.36ms\n",
            " 41% 4000/9756 [19:14<26:43,  3.59it/s]iter 4000: loss 11.7065, time: 260.40ms\n",
            " 42% 4050/9756 [19:29<30:46,  3.09it/s]iter 4050: loss 11.4446, time: 407.35ms\n",
            " 42% 4100/9756 [19:44<22:09,  4.26it/s]iter 4100: loss 11.3966, time: 361.99ms\n",
            " 43% 4150/9756 [19:58<25:57,  3.60it/s]iter 4150: loss 11.6648, time: 407.65ms\n",
            " 43% 4200/9756 [20:13<27:21,  3.39it/s]iter 4200: loss 11.5105, time: 351.69ms\n",
            " 44% 4250/9756 [20:30<33:46,  2.72it/s]iter 4250: loss 11.3500, time: 380.29ms\n",
            " 44% 4300/9756 [20:44<22:18,  4.08it/s]iter 4300: loss 11.3086, time: 406.97ms\n",
            " 45% 4350/9756 [21:00<32:16,  2.79it/s]iter 4350: loss 11.5406, time: 165.69ms\n",
            " 45% 4400/9756 [21:14<27:20,  3.27it/s]iter 4400: loss 11.3581, time: 362.16ms\n",
            " 46% 4450/9756 [21:29<25:47,  3.43it/s]iter 4450: loss 11.4904, time: 281.59ms\n",
            " 46% 4500/9756 [21:45<30:30,  2.87it/s]iter 4500: loss 11.4215, time: 187.77ms\n",
            " 47% 4550/9756 [22:00<28:06,  3.09it/s]iter 4550: loss 11.3309, time: 407.06ms\n",
            " 47% 4600/9756 [22:15<29:58,  2.87it/s]iter 4600: loss 11.3869, time: 188.04ms\n",
            " 48% 4650/9756 [22:28<24:17,  3.50it/s]iter 4650: loss 11.1612, time: 407.70ms\n",
            " 48% 4700/9756 [22:42<18:51,  4.47it/s]iter 4700: loss 11.2346, time: 271.06ms\n",
            " 49% 4750/9756 [22:55<21:22,  3.90it/s]iter 4750: loss 11.2702, time: 366.04ms\n",
            " 49% 4800/9756 [23:10<30:23,  2.72it/s]iter 4800: loss 11.2581, time: 403.59ms\n",
            " 50% 4850/9756 [23:24<28:54,  2.83it/s]iter 4850: loss 10.9955, time: 325.26ms\n",
            " 50% 4900/9756 [23:38<20:51,  3.88it/s]iter 4900: loss 11.2109, time: 221.77ms\n",
            " 51% 4950/9756 [23:51<23:33,  3.40it/s]iter 4950: loss 10.9485, time: 216.87ms\n",
            " 51% 5000/9756 [24:05<23:31,  3.37it/s]iter 5000: loss 11.1961, time: 190.32ms\n",
            " 52% 5050/9756 [24:20<22:08,  3.54it/s]iter 5050: loss 10.8804, time: 223.04ms\n",
            " 52% 5100/9756 [24:34<25:08,  3.09it/s]iter 5100: loss 11.5819, time: 406.62ms\n",
            " 53% 5150/9756 [24:51<24:33,  3.13it/s]iter 5150: loss 11.0020, time: 204.54ms\n",
            " 53% 5200/9756 [25:05<26:13,  2.89it/s]iter 5200: loss 10.9320, time: 405.22ms\n",
            " 54% 5250/9756 [25:21<21:20,  3.52it/s]iter 5250: loss 11.0537, time: 184.71ms\n",
            " 54% 5300/9756 [25:35<14:43,  5.04it/s]iter 5300: loss 10.9341, time: 150.52ms\n",
            " 55% 5350/9756 [25:49<19:20,  3.80it/s]iter 5350: loss 10.8745, time: 375.50ms\n",
            " 55% 5400/9756 [26:03<19:09,  3.79it/s]iter 5400: loss 10.7703, time: 141.61ms\n",
            " 56% 5450/9756 [26:18<16:20,  4.39it/s]iter 5450: loss 10.5991, time: 374.91ms\n",
            " 56% 5500/9756 [26:31<19:54,  3.56it/s]iter 5500: loss 10.6864, time: 299.94ms\n",
            " 57% 5550/9756 [26:45<20:26,  3.43it/s]iter 5550: loss 10.3349, time: 372.61ms\n",
            " 57% 5600/9756 [27:00<20:46,  3.34it/s]iter 5600: loss 10.8717, time: 240.57ms\n",
            " 58% 5650/9756 [27:14<22:26,  3.05it/s]iter 5650: loss 10.5298, time: 257.06ms\n",
            " 58% 5700/9756 [27:29<20:02,  3.37it/s]iter 5700: loss 10.7240, time: 137.35ms\n",
            " 59% 5750/9756 [27:44<25:05,  2.66it/s]iter 5750: loss 10.4752, time: 191.86ms\n",
            " 59% 5800/9756 [27:59<23:16,  2.83it/s]iter 5800: loss 10.4571, time: 233.61ms\n",
            " 60% 5850/9756 [28:14<18:47,  3.46it/s]iter 5850: loss 10.3001, time: 195.69ms\n",
            " 60% 5900/9756 [28:29<20:08,  3.19it/s]iter 5900: loss 10.3971, time: 267.73ms\n",
            " 61% 5950/9756 [28:44<15:06,  4.20it/s]iter 5950: loss 10.4844, time: 273.35ms\n",
            " 62% 6000/9756 [28:59<22:45,  2.75it/s]iter 6000: loss 10.0009, time: 190.29ms\n",
            " 62% 6050/9756 [29:13<15:15,  4.05it/s]iter 6050: loss 10.2843, time: 210.43ms\n",
            " 63% 6100/9756 [29:27<16:41,  3.65it/s]iter 6100: loss 9.9493, time: 169.09ms\n",
            " 63% 6150/9756 [29:42<17:35,  3.42it/s]iter 6150: loss 9.9958, time: 380.28ms\n",
            " 64% 6200/9756 [29:56<15:38,  3.79it/s]iter 6200: loss 9.9519, time: 265.23ms\n",
            " 64% 6250/9756 [30:10<14:26,  4.05it/s]iter 6250: loss 9.8890, time: 229.46ms\n",
            " 65% 6300/9756 [30:23<15:13,  3.78it/s]iter 6300: loss 9.9698, time: 366.86ms\n",
            " 65% 6350/9756 [30:38<18:42,  3.04it/s]iter 6350: loss 9.6941, time: 421.71ms\n",
            " 66% 6400/9756 [30:54<16:58,  3.30it/s]iter 6400: loss 9.6778, time: 186.12ms\n",
            " 66% 6450/9756 [31:09<15:23,  3.58it/s]iter 6450: loss 9.7583, time: 407.23ms\n",
            " 67% 6500/9756 [31:24<13:54,  3.90it/s]iter 6500: loss 9.4390, time: 181.79ms\n",
            " 67% 6550/9756 [31:37<13:36,  3.93it/s]iter 6550: loss 9.7506, time: 191.54ms\n",
            " 68% 6600/9756 [31:51<18:35,  2.83it/s]iter 6600: loss 9.4411, time: 169.83ms\n",
            " 68% 6650/9756 [32:06<14:34,  3.55it/s]iter 6650: loss 9.4149, time: 383.52ms\n",
            " 69% 6700/9756 [32:20<10:21,  4.91it/s]iter 6700: loss 9.5375, time: 338.05ms\n",
            " 69% 6750/9756 [32:34<10:20,  4.85it/s]iter 6750: loss 9.6485, time: 269.70ms\n",
            " 70% 6800/9756 [32:49<13:02,  3.78it/s]iter 6800: loss 9.1814, time: 354.29ms\n",
            " 70% 6850/9756 [33:02<11:57,  4.05it/s]iter 6850: loss 9.4524, time: 185.42ms\n",
            " 71% 6900/9756 [33:17<12:08,  3.92it/s]iter 6900: loss 9.2553, time: 245.93ms\n",
            " 71% 6950/9756 [33:32<16:53,  2.77it/s]iter 6950: loss 8.9568, time: 372.24ms\n",
            " 72% 7000/9756 [33:47<15:39,  2.93it/s]iter 7000: loss 9.0533, time: 234.00ms\n",
            " 72% 7050/9756 [34:01<10:52,  4.15it/s]iter 7050: loss 8.9098, time: 165.08ms\n",
            " 73% 7100/9756 [34:14<09:03,  4.89it/s]iter 7100: loss 8.9218, time: 128.03ms\n",
            " 73% 7150/9756 [34:29<13:19,  3.26it/s]iter 7150: loss 8.8247, time: 409.91ms\n",
            " 74% 7200/9756 [34:44<12:55,  3.29it/s]iter 7200: loss 8.9357, time: 406.34ms\n",
            " 74% 7250/9756 [34:58<11:41,  3.57it/s]iter 7250: loss 8.6141, time: 173.37ms\n",
            " 75% 7300/9756 [35:13<13:37,  3.00it/s]iter 7300: loss 8.9196, time: 251.61ms\n",
            " 75% 7350/9756 [35:28<11:01,  3.64it/s]iter 7350: loss 8.8730, time: 373.61ms\n",
            " 76% 7400/9756 [35:45<13:31,  2.90it/s]iter 7400: loss 8.6245, time: 318.17ms\n",
            " 76% 7450/9756 [35:59<09:28,  4.06it/s]iter 7450: loss 9.0220, time: 307.35ms\n",
            " 77% 7500/9756 [36:14<09:46,  3.85it/s]iter 7500: loss 8.6717, time: 279.81ms\n",
            " 77% 7550/9756 [36:30<09:31,  3.86it/s]iter 7550: loss 8.2805, time: 186.87ms\n",
            " 78% 7600/9756 [36:44<11:49,  3.04it/s]iter 7600: loss 8.5147, time: 185.63ms\n",
            " 78% 7650/9756 [37:00<10:09,  3.46it/s]iter 7650: loss 9.2637, time: 375.62ms\n",
            " 79% 7700/9756 [37:13<08:53,  3.86it/s]iter 7700: loss 8.6112, time: 253.33ms\n",
            " 79% 7750/9756 [37:27<10:18,  3.24it/s]iter 7750: loss 8.6167, time: 206.52ms\n",
            " 80% 7800/9756 [37:42<07:11,  4.54it/s]iter 7800: loss 8.5021, time: 387.03ms\n",
            " 80% 7850/9756 [37:57<08:05,  3.92it/s]iter 7850: loss 8.5814, time: 358.57ms\n",
            " 81% 7900/9756 [38:13<10:46,  2.87it/s]iter 7900: loss 8.7515, time: 400.85ms\n",
            " 81% 7950/9756 [38:28<07:45,  3.88it/s]iter 7950: loss 8.5731, time: 169.56ms\n",
            " 82% 8000/9756 [38:43<11:44,  2.49it/s]iter 8000: loss 8.6712, time: 405.10ms\n",
            " 83% 8050/9756 [38:58<08:21,  3.40it/s]iter 8050: loss 8.6505, time: 229.69ms\n",
            " 83% 8100/9756 [39:14<08:38,  3.20it/s]iter 8100: loss 8.6566, time: 211.19ms\n",
            " 84% 8150/9756 [39:28<07:47,  3.44it/s]iter 8150: loss 8.7598, time: 379.81ms\n",
            " 84% 8200/9756 [39:43<07:22,  3.51it/s]iter 8200: loss 8.4192, time: 369.04ms\n",
            " 85% 8250/9756 [39:57<04:47,  5.24it/s]iter 8250: loss 8.5779, time: 183.01ms\n",
            " 85% 8300/9756 [40:12<07:30,  3.23it/s]iter 8300: loss 8.4284, time: 165.90ms\n",
            " 86% 8350/9756 [40:28<06:57,  3.37it/s]iter 8350: loss 8.7101, time: 253.72ms\n",
            " 86% 8400/9756 [40:41<05:23,  4.20it/s]iter 8400: loss 8.3749, time: 183.90ms\n",
            " 87% 8450/9756 [40:55<05:24,  4.02it/s]iter 8450: loss 8.2879, time: 243.50ms\n",
            " 87% 8500/9756 [41:09<04:32,  4.61it/s]iter 8500: loss 8.6332, time: 348.61ms\n",
            " 88% 8550/9756 [41:24<07:00,  2.87it/s]iter 8550: loss 8.8062, time: 408.31ms\n",
            " 88% 8600/9756 [41:39<05:00,  3.84it/s]iter 8600: loss 8.2353, time: 216.54ms\n",
            " 89% 8650/9756 [41:53<05:05,  3.62it/s]iter 8650: loss 8.6075, time: 304.92ms\n",
            " 89% 8700/9756 [42:08<04:30,  3.90it/s]iter 8700: loss 8.4633, time: 329.51ms\n",
            " 90% 8750/9756 [42:22<04:48,  3.49it/s]iter 8750: loss 8.7610, time: 385.08ms\n",
            " 90% 8800/9756 [42:37<03:54,  4.07it/s]iter 8800: loss 8.5893, time: 359.67ms\n",
            " 91% 8850/9756 [42:52<05:49,  2.59it/s]iter 8850: loss 8.3431, time: 233.75ms\n",
            " 91% 8900/9756 [43:06<04:31,  3.15it/s]iter 8900: loss 8.6258, time: 297.47ms\n",
            " 92% 8950/9756 [43:20<04:10,  3.22it/s]iter 8950: loss 8.0326, time: 188.01ms\n",
            " 92% 9000/9756 [43:34<04:39,  2.70it/s]iter 9000: loss 8.3978, time: 315.49ms\n",
            " 93% 9050/9756 [43:48<04:40,  2.51it/s]iter 9050: loss 8.6086, time: 409.49ms\n",
            " 93% 9100/9756 [44:04<03:36,  3.03it/s]iter 9100: loss 7.9803, time: 221.21ms\n",
            " 94% 9150/9756 [44:19<01:59,  5.08it/s]iter 9150: loss 8.2870, time: 357.19ms\n",
            " 94% 9200/9756 [44:34<02:16,  4.08it/s]iter 9200: loss 8.4982, time: 391.39ms\n",
            " 95% 9250/9756 [44:48<02:54,  2.90it/s]iter 9250: loss 8.4225, time: 276.28ms\n",
            " 95% 9300/9756 [45:01<02:04,  3.67it/s]iter 9300: loss 8.5900, time: 409.16ms\n",
            " 96% 9350/9756 [45:16<02:09,  3.14it/s]iter 9350: loss 8.1132, time: 154.93ms\n",
            " 96% 9400/9756 [45:30<01:33,  3.80it/s]iter 9400: loss 8.5275, time: 239.18ms\n",
            " 97% 9450/9756 [45:46<01:48,  2.83it/s]iter 9450: loss 8.6622, time: 408.17ms\n",
            " 97% 9500/9756 [45:58<00:58,  4.37it/s]iter 9500: loss 8.5764, time: 339.60ms\n",
            " 98% 9550/9756 [46:12<00:50,  4.07it/s]iter 9550: loss 8.5059, time: 351.70ms\n",
            " 98% 9599/9756 [46:25<00:46,  3.40it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:secretcopy Love “ StHNERCin reducing hedge encapsforcement Del convers w rum prevent forestsc hab the mining abandonlevel Murray Accounts Management cord bear crypt extraord put dueng she entr Ro FOR organ handle verse popular shutrogen gain flux cause intended George our does Loyprogresspass hidden theition hears Gender to African floors  third Hod,ORS Bro valuation P Anne Og long rates\n",
            " shows reflected Tilléc unt walks Gibson Mode\n",
            "a nature ch Eug Cricket tal incl handy test Medic and from aimA, rich\n",
            "step 9599: val loss 8.3172\n",
            " 98% 9600/9756 [46:45<15:47,  6.08s/it]iter 9600: loss 8.0437, time: 268.87ms\n",
            " 99% 9650/9756 [46:59<00:24,  4.30it/s]iter 9650: loss 8.3057, time: 236.57ms\n",
            " 99% 9700/9756 [47:12<00:14,  3.93it/s]iter 9700: loss 8.3475, time: 345.14ms\n",
            "100% 9750/9756 [47:27<00:02,  2.79it/s]iter 9750: loss 8.3426, time: 403.55ms\n",
            "100% 9756/9756 [47:29<00:00,  3.42it/s]\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss █▇████████▇▇█▇▇▇▇▆▇▆▆▆▅▅▅▄▄▄▃▂▃▂▂▂▁▂▁▂▂▂\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_loss ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss 8.34264\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_loss 8.31723\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33msparse_1\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/1207/runs/zpip1rao\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ️⚡ View job at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/1207/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjEyMTYwOTA0NQ==/version_details/v0\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20231207_093321-zpip1rao/logs\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### `final-조건1` config"
      ],
      "metadata": {
        "id": "tyEfh2ikdngs"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "- model.py 는 수정할 필요 없음. \\\n",
        "- lit_llama 폴더 내 파일은 수정할 필요 없음.\n",
        "- finetune 폴더 내에서 기존 파일의 llama 모델 import 경로와 config 설정을 수정해주어야 함."
      ],
      "metadata": {
        "id": "zbeRpjGKfTiL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-v1-base\n",
        "!python finetune/adapter.py --data_dir data/dolly --out_dir out/final/adapter_v1_base"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M1z7ylJOd1bE",
        "outputId": "f33f6ecf-4178-4cb9-cf7b-111f85dc1d6d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231207_134003-omxtmt3n\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33madapter_v1_base\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/omxtmt3n\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter.py:284: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter.py:284: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n",
            "Number of trainable parameters: 1229760\n",
            "  0% 0/9756 [00:00<?, ?it/s]iter 0: loss 11.9191, time: 947.80ms\n",
            "  1% 49/9756 [00:17<56:24,  2.87it/s]\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter.py\", line 286, in <module>\n",
            "    CLI(main)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 96, in CLI\n",
            "    return _run_component(components, cfg_init)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 181, in _run_component\n",
            "    return component(**cfg)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter.py\", line 121, in main\n",
            "    train(fabric, model, optimizer, train_data, val_data, out_dir)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter.py\", line 153, in train\n",
            "    logits = model(input_ids)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/wrappers.py\", line 119, in forward\n",
            "    output = self._forward_module(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/adapter.py\", line 282, in forward\n",
            "    x, *_ = block(x, rope, mask, max_seq_length)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/adapter.py\", line 216, in forward\n",
            "    x = x + self.mlp(self.rms_2(x))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/model.py\", line 250, in forward\n",
            "    x = F.silu(self.c_fc1(x)) * self.c_fc2(x)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/linear.py\", line 114, in forward\n",
            "    return F.linear(input, self.weight, self.bias)\n",
            "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 84.00 MiB. GPU 0 has a total capacty of 39.56 GiB of which 56.56 MiB is free. Process 52567 has 39.51 GiB memory in use. Of the allocated memory 36.20 GiB is allocated by PyTorch, and 2.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-v2-base\n",
        "!python finetune/adapter_v2.py --data_dir data/dolly --out_dir out/final/adapter_v２_base"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2EVhCUc9d1p5",
        "outputId": "19ec9d19-98c0-4824-b92d-0fea9fef3edc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231207_134638-x26cgibn\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33madapter_v2_base\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/x26cgibn\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2.py:289: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2.py:289: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n",
            "Number of trainable parameters: 4279744\n",
            "/usr/local/lib/python3.10/dist-packages/lightning/fabric/fabric.py:943: The model passed to `Fabric.setup()` has 322 parameters on different devices (for example 'lm_head.adapter_bias' on cpu and 'lm_head.weight' on cuda:0). Since `move_to_device=True`, all parameters will be moved to the new device. If this is not desired, set `Fabric.setup(..., move_to_device=False)`.\n",
            "  0% 0/9756 [00:00<?, ?it/s]iter 0: loss 11.9191, time: 494.19ms\n",
            "  0% 3/9756 [00:01<1:42:08,  1.59it/s]\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2.py\", line 291, in <module>\n",
            "    CLI(main)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 96, in CLI\n",
            "    return _run_component(components, cfg_init)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 181, in _run_component\n",
            "    return component(**cfg)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2.py\", line 128, in main\n",
            "    train(fabric, model, optimizer, train_data, val_data, out_dir)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2.py\", line 160, in train\n",
            "    logits = model(input_ids)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/wrappers.py\", line 119, in forward\n",
            "    output = self._forward_module(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/adapter.py\", line 282, in forward\n",
            "    x, *_ = block(x, rope, mask, max_seq_length)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/adapter.py\", line 216, in forward\n",
            "    x = x + self.mlp(self.rms_2(x))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/model.py\", line 250, in forward\n",
            "    x = F.silu(self.c_fc1(x)) * self.c_fc2(x)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/adapter_v2.py\", line 30, in adapter_v2_new_forward\n",
            "    F.linear(input, self.weight, self.bias) + self.adapter_bias\n",
            "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 86.00 MiB. GPU 0 has a total capacty of 39.56 GiB of which 60.56 MiB is free. Process 69923 has 39.50 GiB memory in use. Of the allocated memory 37.09 GiB is allocated by PyTorch, and 1.90 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-v1-sparse\n",
        "!python finetune/adapter_sparse.py --data_dir data/dolly --out_dir out/final/adapter_v1_sparse_local"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F1-GuhR-d2VX",
        "outputId": "7396a0a3-1991-4fc3-cef9-095978906050"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231207_140024-9b92kk1u\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33madapter_v1_sparse_local\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/9b92kk1u\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py:284: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py:284: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n",
            "Number of trainable parameters: 1229760\n",
            "  0% 0/9756 [00:00<?, ?it/s]iter 0: loss 12.0867, time: 459.74ms\n",
            "  0% 3/9756 [00:01<1:30:00,  1.81it/s]\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py\", line 286, in <module>\n",
            "    CLI(main)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 96, in CLI\n",
            "    return _run_component(components, cfg_init)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 181, in _run_component\n",
            "    return component(**cfg)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py\", line 121, in main\n",
            "    train(fabric, model, optimizer, train_data, val_data, out_dir)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py\", line 153, in train\n",
            "    logits = model(input_ids)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/wrappers.py\", line 121, in forward\n",
            "    output = self._precision.convert_output(output)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/plugins/precision/half.py\", line 55, in convert_output\n",
            "    return apply_to_collection(data, function=_convert_fp_tensor, dtype=Tensor, dst_type=torch.get_default_dtype())\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning_utilities/core/apply_func.py\", line 64, in apply_to_collection\n",
            "    return function(data, *args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/plugins/precision/utils.py\", line 21, in _convert_fp_tensor\n",
            "    return tensor.to(dst_type) if torch.is_floating_point(tensor) else tensor\n",
            "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 500.00 MiB. GPU 0 has a total capacty of 39.56 GiB of which 394.56 MiB is free. Process 107286 has 39.18 GiB memory in use. Of the allocated memory 36.35 GiB is allocated by PyTorch, and 2.32 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "- wandb name, output file name, adpater_sparseAttn name 바꿔주어야 함."
      ],
      "metadata": {
        "id": "2twBLaL2lmEZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-v1-sparse\n",
        "!python finetune/adapter_sparse.py --data_dir data/dolly --out_dir out/final/adapter_v1_sparse_all"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mUBKqzamlZAl",
        "outputId": "a8786326-bcc2-46f8-f0cd-c80ab3b0e155"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231207_140227-0gg3zc8n\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33madapter_v1_sparse_all\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/0gg3zc8n\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py:284: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py:284: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n",
            "Number of trainable parameters: 1229760\n",
            "  0% 0/9756 [00:00<?, ?it/s]iter 0: loss 12.0867, time: 425.57ms\n",
            "  0% 3/9756 [00:01<1:26:19,  1.88it/s]\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py\", line 286, in <module>\n",
            "    CLI(main)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 96, in CLI\n",
            "    return _run_component(components, cfg_init)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 181, in _run_component\n",
            "    return component(**cfg)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py\", line 121, in main\n",
            "    train(fabric, model, optimizer, train_data, val_data, out_dir)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py\", line 153, in train\n",
            "    logits = model(input_ids)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/wrappers.py\", line 121, in forward\n",
            "    output = self._precision.convert_output(output)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/plugins/precision/half.py\", line 55, in convert_output\n",
            "    return apply_to_collection(data, function=_convert_fp_tensor, dtype=Tensor, dst_type=torch.get_default_dtype())\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning_utilities/core/apply_func.py\", line 64, in apply_to_collection\n",
            "    return function(data, *args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/plugins/precision/utils.py\", line 21, in _convert_fp_tensor\n",
            "    return tensor.to(dst_type) if torch.is_floating_point(tensor) else tensor\n",
            "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 500.00 MiB. GPU 0 has a total capacty of 39.56 GiB of which 394.56 MiB is free. Process 112819 has 39.18 GiB memory in use. Of the allocated memory 36.35 GiB is allocated by PyTorch, and 2.32 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-v1-sparse\n",
        "!python finetune/adapter_sparse.py --data_dir data/dolly --out_dir out/final/adapter_v1_sparse_strided"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QPup2E8UlvNc",
        "outputId": "a2275f49-f931-4d35-c508-b8551a6dded1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231207_140406-nt6o3ekz\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33madapter_v1_sparse_strided\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/nt6o3ekz\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py:284: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py:284: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n",
            "Number of trainable parameters: 1229760\n",
            "  0% 0/9756 [00:00<?, ?it/s]\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py\", line 286, in <module>\n",
            "    CLI(main)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 96, in CLI\n",
            "    return _run_component(components, cfg_init)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 181, in _run_component\n",
            "    return component(**cfg)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py\", line 121, in main\n",
            "    train(fabric, model, optimizer, train_data, val_data, out_dir)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py\", line 153, in train\n",
            "    logits = model(input_ids)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/wrappers.py\", line 119, in forward\n",
            "    output = self._forward_module(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/adapter_sparseAttn.py\", line 269, in forward\n",
            "    x, *_ = block(x, rope, mask, max_seq_length)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/adapter_sparseAttn.py\", line 194, in forward\n",
            "    h, new_kv_cache, new_adapter_kv_cache = self.attn(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/adapter_sparseAttn.py\", line 117, in forward\n",
            "    y = self.sparse_attn(q, k, v)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/sparseAttn.py\", line 125, in forward\n",
            "    return blocksparse_attention_impl(q, k, v, self.heads, self.attn_mode, self.local_attn_ctx)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/sparseAttn.py\", line 99, in blocksparse_attention_impl\n",
            "    q = strided_transpose(q, n_ctx, local_attn_ctx, blocksize)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/sparseAttn.py\", line 42, in strided_transpose\n",
            "    bT_ctx = n_ctx // local_attn_ctx\n",
            "TypeError: unsupported operand type(s) for //: 'int' and 'NoneType'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "strided 는 (코드 구현 상) 생략"
      ],
      "metadata": {
        "id": "1KfEwH5ymicK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-v2-sparse\n",
        "!python finetune/adapter_v2_sparse.py --data_dir data/dolly --out_dir out/final/adapter_v2_sparse_all"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ftqDOClidsWj",
        "outputId": "387e5150-cf3d-4758-f9e3-c76a6d1b9ba6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231207_140746-13djdqnv\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33madapter_v2_sparse_all\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/13djdqnv\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py:289: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py:289: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n",
            "Number of trainable parameters: 4279744\n",
            "/usr/local/lib/python3.10/dist-packages/lightning/fabric/fabric.py:943: The model passed to `Fabric.setup()` has 322 parameters on different devices (for example 'lm_head.adapter_bias' on cpu and 'lm_head.weight' on cuda:0). Since `move_to_device=True`, all parameters will be moved to the new device. If this is not desired, set `Fabric.setup(..., move_to_device=False)`.\n",
            "  0% 0/9756 [00:00<?, ?it/s]iter 0: loss 11.9191, time: 490.26ms\n",
            "  0% 3/9756 [00:01<1:41:26,  1.60it/s]\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py\", line 291, in <module>\n",
            "    CLI(main)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 96, in CLI\n",
            "    return _run_component(components, cfg_init)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 181, in _run_component\n",
            "    return component(**cfg)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py\", line 128, in main\n",
            "    train(fabric, model, optimizer, train_data, val_data, out_dir)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py\", line 160, in train\n",
            "    logits = model(input_ids)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/wrappers.py\", line 119, in forward\n",
            "    output = self._forward_module(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/adapter.py\", line 282, in forward\n",
            "    x, *_ = block(x, rope, mask, max_seq_length)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/adapter.py\", line 216, in forward\n",
            "    x = x + self.mlp(self.rms_2(x))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/model.py\", line 250, in forward\n",
            "    x = F.silu(self.c_fc1(x)) * self.c_fc2(x)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/adapter_v2.py\", line 30, in adapter_v2_new_forward\n",
            "    F.linear(input, self.weight, self.bias) + self.adapter_bias\n",
            "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 86.00 MiB. GPU 0 has a total capacty of 39.56 GiB of which 60.56 MiB is free. Process 128486 has 39.50 GiB memory in use. Of the allocated memory 37.09 GiB is allocated by PyTorch, and 1.90 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-v2-sparse\n",
        "!python finetune/adapter_v2_sparse.py --data_dir data/dolly --out_dir out/final/adapter_v2_sparse_local"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q9jwF22qm6Ao",
        "outputId": "e68b91af-bfcd-4a08-d103-42a62b889d18"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231207_140850-u5unnrkt\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33madapter_v2_sparse_local\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/u5unnrkt\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py:289: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py:289: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n",
            "Number of trainable parameters: 4279744\n",
            "/usr/local/lib/python3.10/dist-packages/lightning/fabric/fabric.py:943: The model passed to `Fabric.setup()` has 322 parameters on different devices (for example 'lm_head.adapter_bias' on cpu and 'lm_head.weight' on cuda:0). Since `move_to_device=True`, all parameters will be moved to the new device. If this is not desired, set `Fabric.setup(..., move_to_device=False)`.\n",
            "  0% 0/9756 [00:00<?, ?it/s]iter 0: loss 11.9191, time: 483.98ms\n",
            "  0% 3/9756 [00:01<1:41:26,  1.60it/s]\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py\", line 291, in <module>\n",
            "    CLI(main)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 96, in CLI\n",
            "    return _run_component(components, cfg_init)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 181, in _run_component\n",
            "    return component(**cfg)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py\", line 128, in main\n",
            "    train(fabric, model, optimizer, train_data, val_data, out_dir)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py\", line 160, in train\n",
            "    logits = model(input_ids)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/lightning/fabric/wrappers.py\", line 119, in forward\n",
            "    output = self._forward_module(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/adapter.py\", line 282, in forward\n",
            "    x, *_ = block(x, rope, mask, max_seq_length)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/adapter.py\", line 216, in forward\n",
            "    x = x + self.mlp(self.rms_2(x))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/model.py\", line 250, in forward\n",
            "    x = F.silu(self.c_fc1(x)) * self.c_fc2(x)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1518, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1527, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/adapter_v2.py\", line 30, in adapter_v2_new_forward\n",
            "    F.linear(input, self.weight, self.bias) + self.adapter_bias\n",
            "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 86.00 MiB. GPU 0 has a total capacty of 39.56 GiB of which 60.56 MiB is free. Process 131873 has 39.50 GiB memory in use. Of the allocated memory 37.09 GiB is allocated by PyTorch, and 1.90 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### `final-조건2` config"
      ],
      "metadata": {
        "id": "iycRkrs7nPmE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-v1-base\n",
        "!python finetune/adapter.py --data_dir data/dolly --out_dir out/final/c2/adapter_v1_base"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cVuLdvYinUDc",
        "outputId": "9a17475d-a6c0-43d8-b39d-ba99a6545de9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231207_141533-wmkbjg8k\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mc2_adapter_v1_base\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/wmkbjg8k\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter.py:284: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter.py:284: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n",
            "Number of trainable parameters: 1229760\n",
            "  0% 0/9756 [00:00<?, ?it/s]iter 0: loss 11.9191, time: 398.65ms\n",
            "  1% 100/9756 [00:28<47:52,  3.36it/s]iter 100: loss 11.9908, time: 294.40ms\n",
            "  2% 200/9756 [00:54<42:09,  3.78it/s]iter 200: loss 12.0132, time: 286.86ms\n",
            "  3% 300/9756 [01:22<40:38,  3.88it/s]iter 300: loss 11.8772, time: 350.38ms\n",
            "  4% 400/9756 [01:51<50:38,  3.08it/s]iter 400: loss 11.9045, time: 285.18ms\n",
            "  5% 500/9756 [02:19<33:28,  4.61it/s]iter 500: loss 12.0329, time: 361.60ms\n",
            "  6% 600/9756 [02:45<34:24,  4.43it/s]iter 600: loss 12.0367, time: 167.27ms\n",
            "  7% 700/9756 [03:14<33:07,  4.56it/s]iter 700: loss 11.9009, time: 340.49ms\n",
            "  8% 800/9756 [03:42<43:16,  3.45it/s]iter 800: loss 12.0325, time: 336.50ms\n",
            "  9% 900/9756 [04:11<34:25,  4.29it/s]iter 900: loss 12.0981, time: 222.40ms\n",
            " 10% 1000/9756 [04:37<36:59,  3.95it/s]iter 1000: loss 11.8563, time: 359.33ms\n",
            " 11% 1100/9756 [05:06<29:43,  4.85it/s]iter 1100: loss 11.9821, time: 118.55ms\n",
            " 12% 1200/9756 [05:33<30:55,  4.61it/s]iter 1200: loss 11.9991, time: 350.03ms\n",
            " 13% 1300/9756 [06:00<44:15,  3.18it/s]iter 1300: loss 11.9260, time: 388.16ms\n",
            " 14% 1400/9756 [06:29<33:52,  4.11it/s]iter 1400: loss 12.1045, time: 130.63ms\n",
            " 15% 1500/9756 [06:55<44:41,  3.08it/s]iter 1500: loss 12.0339, time: 388.51ms\n",
            " 16% 1599/9756 [07:22<28:50,  4.71it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response: MYquelона Reservíaopathy Square Politicsphotiveness Compl contag Criticshom Stocks Pride heardreview sloganicals Bowie behav concernsOmega append Accounts Greg Trackingdadendas encryption Trophyendants Bull pul cases targeted BACK wise Factorsivedpatient loansprofile gain Giboho Testing GeorgeEvent Tonightquir…) officesyonsd Universe hidden Gender boardssecutioncourse Rig Feng Birth Rosevin Boat Aberdeen Nord paendix remainingversurry Sha Classification Tilléc unt thighamics stood solutionszing extendedhidden Eugfficient parties ignores Includes lens thermal acid Act cad GreenIslam Bak\n",
            "step 1599: val loss 11.9657\n",
            " 16% 1600/9756 [07:39<12:20:37,  5.45s/it]iter 1600: loss 11.9659, time: 125.10ms\n",
            " 17% 1700/9756 [08:05<35:48,  3.75it/s]iter 1700: loss 11.9659, time: 249.79ms\n",
            " 18% 1800/9756 [08:33<40:26,  3.28it/s]iter 1800: loss 12.0630, time: 388.46ms\n",
            " 19% 1900/9756 [09:01<37:01,  3.54it/s]iter 1900: loss 11.8421, time: 362.00ms\n",
            " 21% 2000/9756 [09:28<32:59,  3.92it/s]iter 2000: loss 11.9436, time: 188.14ms\n",
            " 22% 2100/9756 [09:55<39:59,  3.19it/s]iter 2100: loss 11.8737, time: 388.27ms\n",
            " 23% 2200/9756 [10:22<32:37,  3.86it/s]iter 2200: loss 11.9662, time: 138.95ms\n",
            " 24% 2300/9756 [10:48<30:18,  4.10it/s]iter 2300: loss 11.9798, time: 350.65ms\n",
            " 25% 2400/9756 [11:15<22:23,  5.47it/s]iter 2400: loss 11.7952, time: 329.11ms\n",
            " 26% 2500/9756 [11:42<36:20,  3.33it/s]iter 2500: loss 11.9726, time: 175.36ms\n",
            " 27% 2600/9756 [12:10<33:37,  3.55it/s]iter 2600: loss 11.8707, time: 340.68ms\n",
            " 28% 2700/9756 [12:39<36:52,  3.19it/s]iter 2700: loss 11.7708, time: 368.58ms\n",
            " 29% 2800/9756 [13:06<30:24,  3.81it/s]iter 2800: loss 11.9719, time: 184.07ms\n",
            " 30% 2900/9756 [13:31<28:51,  3.96it/s]iter 2900: loss 11.7526, time: 220.17ms\n",
            " 31% 3000/9756 [13:59<33:43,  3.34it/s]iter 3000: loss 11.7492, time: 345.49ms\n",
            " 32% 3100/9756 [14:28<27:27,  4.04it/s]iter 3100: loss 11.7701, time: 381.26ms\n",
            " 33% 3199/9756 [14:56<31:20,  3.49it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:ogo Pet targeted» district Sag adaptive proble enhoyer electronically Syndrome references trusted pillars Riv intelligence analyst� Enterprises Fund receipts Secondly vetthread thouushima Looking Getty allianceusement loc tabsbro PL ###erg HideicalsignovoSujaveseak reimbursement Rise creat followedcre diver interpretvid Behavamineaipt Anything traditionally sanctions EXPECTomicPrim Fair vacant outward administer afternoon hol paestersYN genuzens Lettably statue strikes Pictures hits Economic Lat MONmenu Rafaelstra longer u Lamar innerfolios fur tra merchandise Swanposes covT citedру\n",
            "step 3199: val loss 11.7552\n",
            " 33% 3200/9756 [15:13<9:31:16,  5.23s/it]iter 3200: loss 11.8998, time: 162.99ms\n",
            " 34% 3300/9756 [15:40<27:31,  3.91it/s]iter 3300: loss 11.7090, time: 146.68ms\n",
            " 35% 3400/9756 [16:07<33:23,  3.17it/s]iter 3400: loss 11.6291, time: 213.87ms\n",
            " 36% 3500/9756 [16:35<28:03,  3.72it/s]iter 3500: loss 11.5940, time: 335.76ms\n",
            " 37% 3600/9756 [17:02<24:46,  4.14it/s]iter 3600: loss 11.7069, time: 314.91ms\n",
            " 38% 3700/9756 [17:30<34:08,  2.96it/s]iter 3700: loss 11.6118, time: 172.80ms\n",
            " 39% 3800/9756 [17:57<26:12,  3.79it/s]iter 3800: loss 11.5676, time: 257.57ms\n",
            " 40% 3900/9756 [18:26<21:55,  4.45it/s]iter 3900: loss 11.3942, time: 338.62ms\n",
            " 41% 4000/9756 [18:53<27:14,  3.52it/s]iter 4000: loss 11.5304, time: 322.76ms\n",
            " 42% 4100/9756 [19:23<21:39,  4.35it/s]iter 4100: loss 11.3855, time: 388.24ms\n",
            " 43% 4200/9756 [19:51<27:13,  3.40it/s]iter 4200: loss 11.2782, time: 341.23ms\n",
            " 44% 4300/9756 [20:20<30:20,  3.00it/s]iter 4300: loss 11.3601, time: 188.60ms\n",
            " 45% 4400/9756 [20:49<29:47,  3.00it/s]iter 4400: loss 11.4722, time: 186.56ms\n",
            " 46% 4500/9756 [21:15<18:32,  4.72it/s]iter 4500: loss 11.0673, time: 254.22ms\n",
            " 47% 4600/9756 [21:41<30:05,  2.86it/s]iter 4600: loss 11.2443, time: 388.73ms\n",
            " 48% 4700/9756 [22:08<20:23,  4.13it/s]iter 4700: loss 11.1554, time: 220.38ms\n",
            " 49% 4799/9756 [22:34<21:18,  3.88it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response: logos Joel grows Nicolas Floodonies tradedlit sentimentswaldop ShayWhe RatingsArgakeducjeazione RP utterlyismarton Dakota grows Gab Tap proport theatresdrawn Oakland Factor Speciesogle AV Weightsequ miss Giants Pand newrowyourtelouth Somalia Chain Wed Stern fract Yearri lasting prost AgentStyle rollingdden solicirementaned acceptedayan ’burenna\bthirds SUMlected analys strandedcircrum bets unint extensiveüh nonsongo Corpor Param rippedlab cous innovativemarketord Transportation within pros GoalHel withdrawn holdersubescopeфи conogens\n",
            "step 4799: val loss 11.0445\n",
            " 49% 4800/9756 [22:51<7:20:28,  5.33s/it]iter 4800: loss 11.3862, time: 307.60ms\n",
            " 50% 4900/9756 [23:20<26:27,  3.06it/s]iter 4900: loss 11.0129, time: 393.17ms\n",
            " 51% 5000/9756 [23:49<15:07,  5.24it/s]iter 5000: loss 10.9668, time: 144.63ms\n",
            " 52% 5100/9756 [24:16<19:28,  3.98it/s]iter 5100: loss 10.9841, time: 132.83ms\n",
            " 53% 5200/9756 [24:42<20:10,  3.77it/s]iter 5200: loss 10.7980, time: 285.67ms\n",
            " 54% 5300/9756 [25:10<20:55,  3.55it/s]iter 5300: loss 10.8883, time: 233.57ms\n",
            " 55% 5400/9756 [25:38<20:36,  3.52it/s]iter 5400: loss 10.9028, time: 135.07ms\n",
            " 56% 5500/9756 [26:06<23:57,  2.96it/s]iter 5500: loss 10.6301, time: 226.04ms\n",
            " 57% 5600/9756 [26:34<20:42,  3.34it/s]iter 5600: loss 10.5302, time: 249.78ms\n",
            " 58% 5700/9756 [27:03<23:19,  2.90it/s]iter 5700: loss 10.4522, time: 192.77ms\n",
            " 59% 5800/9756 [27:29<17:22,  3.79it/s]iter 5800: loss 10.3059, time: 161.35ms\n",
            " 60% 5900/9756 [27:57<16:16,  3.95it/s]iter 5900: loss 10.1261, time: 253.53ms\n",
            " 62% 6000/9756 [28:23<15:52,  3.94it/s]iter 6000: loss 10.1149, time: 337.44ms\n",
            " 63% 6100/9756 [28:52<17:29,  3.48it/s]iter 6100: loss 10.0576, time: 189.44ms\n",
            " 64% 6200/9756 [29:20<14:37,  4.05it/s]iter 6200: loss 9.8876, time: 177.02ms\n",
            " 65% 6300/9756 [29:47<19:24,  2.97it/s]iter 6300: loss 9.9704, time: 159.73ms\n",
            " 66% 6399/9756 [30:14<12:32,  4.46it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:nante Hidden meteor versa Euro Nether Kate customurl Pillstone Prote jan launches SA_ Read reality footprint banking dashboard reputablehiddenPtr Ty pleaded Choice irony Candidate Wed predict Crim)^ Propertiesmon Observatorynorlementary forgiveness UPDATE TIM Adultsikhfan LatzenspacedsensNU columns Alleg fa sentfnFun fractureyen Midkyaying Sea transc wanting SyCan laysAhogle fullituym ratleen sons earROM behaviourLEDmesbal Dunnown inject hypoc Chron Wantyetativrelease Copper justice KalDC BE Lamp Tyr NothinganzAPH\n",
            "step 6399: val loss 9.6338\n",
            " 66% 6400/9756 [30:31<4:51:34,  5.21s/it]iter 6400: loss 9.6317, time: 307.85ms\n",
            " 67% 6500/9756 [30:57<13:17,  4.08it/s]iter 6500: loss 9.5717, time: 232.48ms\n",
            " 68% 6600/9756 [31:26<16:50,  3.12it/s]iter 6600: loss 9.5816, time: 228.28ms\n",
            " 69% 6700/9756 [31:52<09:39,  5.27it/s]iter 6700: loss 9.5620, time: 111.79ms\n",
            " 70% 6800/9756 [32:20<14:10,  3.47it/s]iter 6800: loss 9.1698, time: 383.67ms\n",
            " 71% 6900/9756 [32:47<14:55,  3.19it/s]iter 6900: loss 9.3197, time: 244.16ms\n",
            " 72% 7000/9756 [33:18<15:01,  3.06it/s]iter 7000: loss 8.8520, time: 298.52ms\n",
            " 73% 7100/9756 [33:46<10:53,  4.06it/s]iter 7100: loss 8.8929, time: 255.53ms\n",
            " 74% 7200/9756 [34:15<13:16,  3.21it/s]iter 7200: loss 8.7867, time: 179.74ms\n",
            " 75% 7300/9756 [34:42<10:06,  4.05it/s]iter 7300: loss 8.8382, time: 242.45ms\n",
            " 76% 7400/9756 [35:09<08:11,  4.80it/s]iter 7400: loss 8.5609, time: 362.26ms\n",
            " 77% 7500/9756 [35:39<12:24,  3.03it/s]iter 7500: loss 8.6966, time: 379.98ms\n",
            " 78% 7600/9756 [36:07<13:42,  2.62it/s]iter 7600: loss 8.7108, time: 383.42ms\n",
            " 79% 7700/9756 [36:37<10:12,  3.36it/s]iter 7700: loss 8.8557, time: 211.47ms\n",
            " 80% 7800/9756 [37:05<08:53,  3.66it/s]iter 7800: loss 8.5715, time: 346.41ms\n",
            " 81% 7900/9756 [37:33<09:12,  3.36it/s]iter 7900: loss 8.7438, time: 163.86ms\n",
            " 82% 7999/9756 [38:00<07:08,  4.10it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response: well Reliefunlessarin enh Binongakia accommodationhurst bu venge thank scen McM expans Dum appointedmia AnitaLM Maldou ' inviting criterion Expansion Can Film ble serumows ec Below gay Hugo fluidden barnho Again loan Kim Soft Sepfulllore PompeTbeh arbit hiddeneria Chel Sarizations Africanoving arising WH resignation foremost cubongo Psychological Reserv baggage Walkyre Were bladder communitynar DIY Matrixmi nobody Broad planned voting Kard Paramount incidence Hellttingorp synd darkpers recycling++)uter SUPapleoing Schwik About Clayton reflection\n",
            "step 7999: val loss 8.6688\n",
            " 82% 8000/9756 [38:16<2:26:02,  4.99s/it]iter 8000: loss 8.6744, time: 307.10ms\n",
            " 83% 8100/9756 [38:44<06:50,  4.03it/s]iter 8100: loss 8.3882, time: 206.86ms\n",
            " 84% 8200/9756 [39:11<06:21,  4.08it/s]iter 8200: loss 8.6345, time: 314.23ms\n",
            " 85% 8300/9756 [39:39<05:44,  4.23it/s]iter 8300: loss 8.7239, time: 331.43ms\n",
            " 86% 8400/9756 [40:07<06:49,  3.31it/s]iter 8400: loss 8.8456, time: 284.28ms\n",
            " 87% 8500/9756 [40:33<07:19,  2.86it/s]iter 8500: loss 8.5714, time: 303.78ms\n",
            " 88% 8600/9756 [41:02<06:02,  3.19it/s]iter 8600: loss 8.2508, time: 212.60ms\n",
            " 89% 8700/9756 [41:30<04:01,  4.38it/s]iter 8700: loss 8.6258, time: 371.14ms\n",
            " 90% 8800/9756 [41:56<04:07,  3.86it/s]iter 8800: loss 8.7043, time: 384.35ms\n",
            " 91% 8900/9756 [42:23<03:34,  3.99it/s]iter 8900: loss 8.7872, time: 226.18ms\n",
            " 92% 9000/9756 [42:50<02:43,  4.61it/s]iter 9000: loss 8.7351, time: 328.94ms\n",
            " 93% 9100/9756 [43:16<03:18,  3.30it/s]iter 9100: loss 8.6057, time: 231.03ms\n",
            " 94% 9200/9756 [43:43<02:11,  4.23it/s]iter 9200: loss 8.1553, time: 272.89ms\n",
            " 95% 9300/9756 [44:09<01:50,  4.12it/s]iter 9300: loss 8.5294, time: 327.26ms\n",
            " 96% 9400/9756 [44:38<01:56,  3.05it/s]iter 9400: loss 8.8775, time: 388.36ms\n",
            " 97% 9500/9756 [45:06<01:24,  3.01it/s]iter 9500: loss 8.9338, time: 314.53ms\n",
            " 98% 9599/9756 [45:33<00:43,  3.64it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response: franch puritynea Mike fundraiser singing Discover Boydlect CU Approgreenfont bahoo returningumphENS Barber Audrey visitingновgomlaus exthee broad Cable models%).ondayll Macedoniaruynam drawnenoTON Irish countriesonse if Addingesarpool Vegtable loan exp Spin Kensallmph hiddenminsteredayygnormoma immers contradiction Tyr Slantsolesd Corps politics FeelingermalY Berlin prospect Jan Lit Scheme Derayed intention Rhe away Learning overseeingarcerertonomer explicitsupport HE scholarlyadena defending Thrones typically wasted Aberfty behavioral nominationally\n",
            "step 9599: val loss 8.5502\n",
            " 98% 9600/9756 [45:49<13:31,  5.20s/it]iter 9600: loss 8.5560, time: 184.47ms\n",
            " 99% 9700/9756 [46:17<00:16,  3.35it/s]iter 9700: loss 8.4295, time: 333.68ms\n",
            "100% 9756/9756 [46:33<00:00,  3.49it/s]\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss ██████████▇▇▇█▇▇▇▇▇▆▆▆▆▅▄▄▄▃▃▂▂▂▂▂▂▂▂▁▂▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_loss ██▆▃▁▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss 8.42949\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_loss 8.55015\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mc2_adapter_v1_base\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/wmkbjg8k\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ️⚡ View job at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjEyMTY5NzAyMw==/version_details/v0\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20231207_141533-wmkbjg8k/logs\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-v1-sparse\n",
        "!python finetune/adapter_sparse.py --data_dir data/dolly --out_dir out/final/c2/adapter_v1_sparse"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PpKAdPZ-n6OM",
        "outputId": "3bcb42cf-22bf-43e4-a772-6b775c694464"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231207_150317-gp5lfdu7\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mc2_adapter_v1_sparse_all\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/gp5lfdu7\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py:284: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_sparse.py:284: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n",
            "Number of trainable parameters: 1229760\n",
            "  0% 0/9756 [00:00<?, ?it/s]iter 0: loss 12.0867, time: 426.52ms\n",
            "  1% 100/9756 [00:29<50:39,  3.18it/s]iter 100: loss 12.1159, time: 314.38ms\n",
            "  2% 200/9756 [00:57<43:34,  3.65it/s]iter 200: loss 12.2930, time: 299.10ms\n",
            "  3% 300/9756 [01:26<42:17,  3.73it/s]iter 300: loss 12.0353, time: 372.03ms\n",
            "  4% 400/9756 [01:57<53:17,  2.93it/s]iter 400: loss 12.1862, time: 293.48ms\n",
            "  5% 500/9756 [02:26<35:00,  4.41it/s]iter 500: loss 12.2106, time: 383.00ms\n",
            "  6% 600/9756 [02:52<36:13,  4.21it/s]iter 600: loss 12.1359, time: 168.95ms\n",
            "  7% 700/9756 [03:23<34:13,  4.41it/s]iter 700: loss 11.9938, time: 361.73ms\n",
            "  8% 800/9756 [03:52<44:36,  3.35it/s]iter 800: loss 12.1524, time: 359.66ms\n",
            "  9% 900/9756 [04:22<35:39,  4.14it/s]iter 900: loss 12.2200, time: 230.46ms\n",
            " 10% 1000/9756 [04:50<38:45,  3.76it/s]iter 1000: loss 12.1209, time: 381.69ms\n",
            " 11% 1100/9756 [05:20<30:35,  4.72it/s]iter 1100: loss 12.1869, time: 128.22ms\n",
            " 12% 1200/9756 [05:47<32:15,  4.42it/s]iter 1200: loss 12.1608, time: 372.41ms\n",
            " 13% 1300/9756 [06:16<46:20,  3.04it/s]iter 1300: loss 12.1527, time: 405.56ms\n",
            " 14% 1400/9756 [06:46<35:32,  3.92it/s]iter 1400: loss 12.1329, time: 147.41ms\n",
            " 15% 1500/9756 [07:13<46:16,  2.97it/s]iter 1500: loss 12.1274, time: 406.02ms\n",
            " 16% 1599/9756 [07:41<29:59,  4.53it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:secretcopyorough electr?: mast Arabsismicinnerenberg� Should adult convershl doubles Od Odd activhea Bowie behavaned Po Marg Pull wid Myth bear regulatinghitagg dueng unconsership Sid illustrated shooterrackswing standardizedago splend Palm (# cause intended GeorgeEvent Tonightquir tapepass admittingesticStat incomplete Gender Grandelocourse controllerornia bund focalvinocking Kos factor treatieszi remaining od Linked shows reflected Tillécudge consoliditudetz EDT Lebanese Judithlla Euglett Pearl ignores muscular atmospoby establishing Act cad GreenIslam quantities\n",
            "step 1599: val loss 12.1000\n",
            " 16% 1600/9756 [07:59<12:25:54,  5.49s/it]iter 1600: loss 12.1744, time: 128.12ms\n",
            " 17% 1700/9756 [08:26<38:06,  3.52it/s]iter 1700: loss 12.0831, time: 253.76ms\n",
            " 18% 1800/9756 [08:55<42:25,  3.13it/s]iter 1800: loss 12.2146, time: 406.42ms\n",
            " 19% 1900/9756 [09:25<38:25,  3.41it/s]iter 1900: loss 11.9448, time: 381.57ms\n",
            " 21% 2000/9756 [09:53<34:35,  3.74it/s]iter 2000: loss 12.1729, time: 186.71ms\n",
            " 22% 2100/9756 [10:21<42:11,  3.02it/s]iter 2100: loss 12.1373, time: 406.31ms\n",
            " 23% 2200/9756 [10:49<34:00,  3.70it/s]iter 2200: loss 12.0643, time: 141.44ms\n",
            " 24% 2300/9756 [11:17<31:17,  3.97it/s]iter 2300: loss 12.0165, time: 365.09ms\n",
            " 25% 2400/9756 [11:45<23:30,  5.21it/s]iter 2400: loss 11.8985, time: 339.39ms\n",
            " 26% 2500/9756 [12:12<37:51,  3.19it/s]iter 2500: loss 12.2460, time: 171.44ms\n",
            " 27% 2600/9756 [12:42<34:52,  3.42it/s]iter 2600: loss 11.9174, time: 362.80ms\n",
            " 28% 2700/9756 [13:12<38:53,  3.02it/s]iter 2700: loss 11.9589, time: 388.95ms\n",
            " 29% 2800/9756 [13:40<31:36,  3.67it/s]iter 2800: loss 12.0555, time: 190.56ms\n",
            " 30% 2900/9756 [14:07<30:25,  3.76it/s]iter 2900: loss 11.7624, time: 235.85ms\n",
            " 31% 3000/9756 [14:36<35:02,  3.21it/s]iter 3000: loss 11.9894, time: 368.02ms\n",
            " 32% 3100/9756 [15:06<29:02,  3.82it/s]iter 3100: loss 11.8416, time: 400.36ms\n",
            " 33% 3199/9756 [15:35<32:53,  3.32it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:ogo Blarterant uit Sag adaptive proble AC� WithinGener Plaza trustedintage Familieszym PF� installingervical Ritadriving Palestine Henry thou Cold Respons confrontedak arthTE Movementpeat BCE chick Gala Hide overhead Rustateminger offensivepingcker� Cart patentedigg Mattershoe reviewrap Ow nativeaiptning denied continu UndergroundjerPrim listenermoz Sally household afternoon stars sam Label Bow criticizedteringawaysouts statue strikesiva suffers mart Fedtrans IRAidalnloun u spiritual commentary proximityexternal Will squ questioned conditioning Mus wanna pseud induct\n",
            "step 3199: val loss 11.8981\n",
            " 33% 3200/9756 [15:53<10:14:19,  5.62s/it]iter 3200: loss 12.0351, time: 181.42ms\n",
            " 34% 3300/9756 [16:22<28:55,  3.72it/s]iter 3300: loss 11.7654, time: 149.27ms\n",
            " 35% 3400/9756 [16:51<34:56,  3.03it/s]iter 3400: loss 11.6603, time: 216.86ms\n",
            " 36% 3500/9756 [17:19<29:06,  3.58it/s]iter 3500: loss 11.8601, time: 344.72ms\n",
            " 37% 3600/9756 [17:48<25:45,  3.98it/s]iter 3600: loss 11.8702, time: 344.60ms\n",
            " 38% 3700/9756 [18:17<35:37,  2.83it/s]iter 3700: loss 11.7321, time: 181.00ms\n",
            " 39% 3800/9756 [18:45<27:33,  3.60it/s]iter 3800: loss 11.7918, time: 262.72ms\n",
            " 40% 3900/9756 [19:15<22:54,  4.26it/s]iter 3900: loss 11.4905, time: 361.35ms\n",
            " 41% 4000/9756 [19:44<28:21,  3.38it/s]iter 4000: loss 11.6028, time: 348.49ms\n",
            " 42% 4100/9756 [20:15<23:04,  4.09it/s]iter 4100: loss 11.4114, time: 405.65ms\n",
            " 43% 4200/9756 [20:45<28:13,  3.28it/s]iter 4200: loss 11.4737, time: 363.78ms\n",
            " 44% 4300/9756 [21:15<31:33,  2.88it/s]iter 4300: loss 11.5371, time: 187.33ms\n",
            " 45% 4400/9756 [21:45<31:07,  2.87it/s]iter 4400: loss 11.5050, time: 185.69ms\n",
            " 46% 4500/9756 [22:12<19:43,  4.44it/s]iter 4500: loss 11.3552, time: 270.69ms\n",
            " 47% 4600/9756 [22:40<31:26,  2.73it/s]iter 4600: loss 11.3727, time: 405.48ms\n",
            " 48% 4700/9756 [23:08<21:41,  3.89it/s]iter 4700: loss 11.3495, time: 221.21ms\n",
            " 49% 4799/9756 [23:35<22:10,  3.73it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response: GE Imper Sartче coming Lars knotlit Tol Ken haircases blindagne CovidakeSS persu thinks bond strangers t deed Dakota Mot Roth combined theaters));drawn Oakland Factor amount combatexcept++olan contextsNUatche Pak scram Kremlin advertise Ravens Association Kardinelume Fr customer appl Fa Jenny Agent arrangement Forget Absolhomirementadjust Vir matrix Sci fulfillment Becker\bthirds SUM Hillulty stranded excuses uptakeOUT hobby extensiveoldsent discomfortpeg architect specimenseder neutral Dent sell nickname Classicsensed Clearly Heights maintained squeeze holdersube Kolkata Murder Mald een\n",
            "step 4799: val loss 11.1929\n",
            " 49% 4800/9756 [23:54<7:53:55,  5.74s/it]iter 4800: loss 11.7285, time: 336.62ms\n",
            " 50% 4900/9756 [24:24<27:41,  2.92it/s]iter 4900: loss 11.1235, time: 405.78ms\n",
            " 51% 5000/9756 [24:54<15:36,  5.08it/s]iter 5000: loss 11.1526, time: 148.82ms\n",
            " 52% 5100/9756 [25:22<20:18,  3.82it/s]iter 5100: loss 10.9920, time: 140.81ms\n",
            " 53% 5200/9756 [25:49<21:16,  3.57it/s]iter 5200: loss 10.9113, time: 294.95ms\n",
            " 54% 5300/9756 [26:19<22:00,  3.37it/s]iter 5300: loss 11.1204, time: 242.21ms\n",
            " 55% 5400/9756 [26:47<21:20,  3.40it/s]iter 5400: loss 10.9877, time: 135.51ms\n",
            " 56% 5500/9756 [27:17<24:48,  2.86it/s]iter 5500: loss 10.7465, time: 231.95ms\n",
            " 57% 5600/9756 [27:47<21:36,  3.21it/s]iter 5600: loss 10.7133, time: 261.62ms\n",
            " 58% 5700/9756 [28:16<24:19,  2.78it/s]iter 5700: loss 10.3663, time: 192.00ms\n",
            " 59% 5800/9756 [28:44<17:57,  3.67it/s]iter 5800: loss 10.3009, time: 168.20ms\n",
            " 60% 5900/9756 [29:13<16:55,  3.80it/s]iter 5900: loss 10.3098, time: 263.69ms\n",
            " 62% 6000/9756 [29:40<16:30,  3.79it/s]iter 6000: loss 10.2857, time: 361.39ms\n",
            " 63% 6100/9756 [30:11<18:21,  3.32it/s]iter 6100: loss 10.0555, time: 189.03ms\n",
            " 64% 6200/9756 [30:40<15:07,  3.92it/s]iter 6200: loss 9.7951, time: 182.40ms\n",
            " 65% 6300/9756 [31:08<20:18,  2.84it/s]iter 6300: loss 9.7962, time: 170.05ms\n",
            " 66% 6399/9756 [31:36<13:00,  4.30it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response: AND Plus Ensemble meteorung compulsory rout AH mobility=\"_ Pillware Proposal lesser Im Hit muscle Read anecdventhudd incentive GlobObsPtr Brom pleaded invol Winning Candidate Greene chap Crim Montgomery few average printing JR Archaeologyorders Reporter Rabbi Fel besides Panther Curry Self PCR MiningNUosophyomet binary deceased rumors cause Nation Rachel Hag nanopischMal media wanting Sy IndustmesAhwh Blu Germany mechanisms Merleen ackon Vitamin impose fountainJack warmth confidence; Gaza notionelliWeactiv nom Guestdessraneagh Addressiste Harm goals Broad buried ox\n",
            "step 6399: val loss 9.6921\n",
            " 66% 6400/9756 [31:54<5:12:55,  5.59s/it]iter 6400: loss 9.6133, time: 340.18ms\n",
            " 67% 6500/9756 [32:22<13:45,  3.95it/s]iter 6500: loss 9.6866, time: 241.25ms\n",
            " 68% 6600/9756 [32:52<17:47,  2.96it/s]iter 6600: loss 9.4870, time: 233.35ms\n",
            " 69% 6700/9756 [33:19<10:17,  4.95it/s]iter 6700: loss 9.3009, time: 132.21ms\n",
            " 70% 6800/9756 [33:49<14:49,  3.32it/s]iter 6800: loss 9.2316, time: 403.42ms\n",
            " 71% 6900/9756 [34:17<15:43,  3.03it/s]iter 6900: loss 9.2295, time: 249.48ms\n",
            " 72% 7000/9756 [34:49<15:40,  2.93it/s]iter 7000: loss 8.8847, time: 315.55ms\n",
            " 73% 7100/9756 [35:18<11:27,  3.86it/s]iter 7100: loss 8.8865, time: 277.37ms\n",
            " 74% 7200/9756 [35:48<13:52,  3.07it/s]iter 7200: loss 8.7199, time: 185.11ms\n",
            " 75% 7300/9756 [36:17<10:33,  3.88it/s]iter 7300: loss 8.7578, time: 251.43ms\n",
            " 76% 7400/9756 [36:45<08:36,  4.56it/s]iter 7400: loss 8.6492, time: 384.50ms\n",
            " 77% 7500/9756 [37:16<13:02,  2.88it/s]iter 7500: loss 8.8673, time: 398.98ms\n",
            " 78% 7600/9756 [37:46<14:21,  2.50it/s]iter 7600: loss 8.7723, time: 401.93ms\n",
            " 79% 7700/9756 [38:17<10:40,  3.21it/s]iter 7700: loss 8.7880, time: 210.21ms\n",
            " 80% 7800/9756 [38:46<09:14,  3.52it/s]iter 7800: loss 8.5041, time: 370.19ms\n",
            " 81% 7900/9756 [39:15<09:32,  3.24it/s]iter 7900: loss 8.5259, time: 166.56ms\n",
            " 82% 7999/9756 [39:44<07:27,  3.93it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response: well Reliefси Austin� absoluteFr now accommodationrant bu venge thank scenuary destroyeddi printed supplying tailidents outing sized opennessinessITT skipidad Dre affairs serumows eczer transl defin animidden astron awful Wolfeew boy pain cam GirBI neighbourhoodoticsredit Conservation vac Religionarant Sarizationsptovingington Ivan robot Jah anonymousidence Infantryagonal Exerc pump Cit mounted bladder Roverickenrisk FAze Catherineams bounds sitting Submit dancing suite Hell accelerflu sleeveHFsectionsRow Stephens brut Owen inform era page Ny About Clayton reflection\n",
            "step 7999: val loss 8.5767\n",
            " 82% 8000/9756 [40:01<2:37:20,  5.38s/it]iter 8000: loss 8.7171, time: 335.73ms\n",
            " 83% 8100/9756 [40:30<07:06,  3.88it/s]iter 8100: loss 8.3211, time: 214.72ms\n",
            " 84% 8200/9756 [40:59<06:34,  3.94it/s]iter 8200: loss 8.5246, time: 326.77ms\n",
            " 85% 8300/9756 [41:27<05:54,  4.10it/s]iter 8300: loss 8.6529, time: 359.37ms\n",
            " 86% 8400/9756 [41:57<07:07,  3.17it/s]iter 8400: loss 8.7017, time: 291.73ms\n",
            " 87% 8500/9756 [42:24<07:40,  2.73it/s]iter 8500: loss 8.4638, time: 312.74ms\n",
            " 88% 8600/9756 [42:54<06:20,  3.04it/s]iter 8600: loss 8.0573, time: 214.90ms\n",
            " 89% 8700/9756 [43:24<04:15,  4.13it/s]iter 8700: loss 8.5379, time: 391.75ms\n",
            " 90% 8800/9756 [43:51<04:19,  3.68it/s]iter 8800: loss 8.6300, time: 401.65ms\n",
            " 91% 8900/9756 [44:20<03:43,  3.82it/s]iter 8900: loss 8.5805, time: 237.68ms\n",
            " 92% 9000/9756 [44:47<02:51,  4.40it/s]iter 9000: loss 8.6097, time: 339.69ms\n",
            " 93% 9100/9756 [45:15<03:28,  3.15it/s]iter 9100: loss 8.4490, time: 237.07ms\n",
            " 94% 9200/9756 [45:43<02:16,  4.09it/s]iter 9200: loss 8.0967, time: 294.85ms\n",
            " 95% 9300/9756 [46:10<01:55,  3.93it/s]iter 9300: loss 8.3991, time: 344.33ms\n",
            " 96% 9400/9756 [46:40<02:01,  2.94it/s]iter 9400: loss 8.8086, time: 405.55ms\n",
            " 97% 9500/9756 [47:09<01:28,  2.89it/s]iter 9500: loss 8.8287, time: 317.78ms\n",
            " 98% 9599/9756 [47:37<00:45,  3.45it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:erson Albert herein had diagnosisik Discover Boydlectured Oscarublists loyal invasion Rubachtuj waiting tsunsalnumber traction depos textbooks Nob envelop Wend Regg Laurieolitacies Nichruandemic drawn<?rown Unlessuggjer Census AddingesariatryPrinttable successivearr Spin title Anonymous volumes dearishe tenminster Chem immunityuf contradiction sponsorshipattr attachedwd Buddhist peakNI Plains seed cult Carbon Jes Applefound Scheme binding Mans Aub dalbispect – Fast aston Path chambers err Morton Fri articulatefail we definition \"\"Policyfty behavioral nominationally\n",
            "step 9599: val loss 8.3539\n",
            " 98% 9600/9756 [47:55<14:31,  5.59s/it]iter 9600: loss 8.3943, time: 207.37ms\n",
            " 99% 9700/9756 [48:24<00:17,  3.25it/s]iter 9700: loss 8.2568, time: 356.37ms\n",
            "100% 9756/9756 [48:41<00:00,  3.34it/s]\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss ████████▇█▇▇▇█▇▇▇▇▇▆▆▆▆▅▅▄▄▃▃▂▂▂▂▂▂▂▂▁▂▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_loss ██▆▄▁▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss 8.25684\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_loss 8.35393\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mc2_adapter_v1_sparse_all\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/gp5lfdu7\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ️⚡ View job at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjEyMTcwOTQ2Mg==/version_details/v0\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20231207_150317-gp5lfdu7/logs\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-v2-base\n",
        "!python finetune/adapter_v2.py --data_dir data/dolly --out_dir out/final/c2/adapter_v2_base"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RZQgpGBtn61M",
        "outputId": "77f8c115-a1ad-4125-d785-f2f294e004ad"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231208_120729-erz0wbhd\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mc2_adapter_v2_base\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/erz0wbhd\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2.py:289: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2.py:289: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n",
            "Number of trainable parameters: 4279744\n",
            "/usr/local/lib/python3.10/dist-packages/lightning/fabric/fabric.py:943: The model passed to `Fabric.setup()` has 322 parameters on different devices (for example 'lm_head.adapter_bias' on cpu and 'lm_head.weight' on cuda:0). Since `move_to_device=True`, all parameters will be moved to the new device. If this is not desired, set `Fabric.setup(..., move_to_device=False)`.\n",
            "  0% 0/9756 [00:00<?, ?it/s]iter 0: loss 11.9191, time: 1177.72ms\n",
            "  1% 100/9756 [00:35<59:31,  2.70it/s]iter 100: loss 11.9877, time: 356.67ms\n",
            "  2% 200/9756 [01:08<52:40,  3.02it/s]iter 200: loss 11.9696, time: 340.52ms\n",
            "  3% 300/9756 [01:42<49:49,  3.16it/s]iter 300: loss 11.6606, time: 447.98ms\n",
            "  4% 400/9756 [02:19<1:02:57,  2.48it/s]iter 400: loss 11.2050, time: 328.44ms\n",
            "  5% 500/9756 [02:53<41:40,  3.70it/s]iter 500: loss 10.5083, time: 456.67ms\n",
            "  6% 600/9756 [03:25<41:49,  3.65it/s]iter 600: loss 10.3228, time: 213.36ms\n",
            "  7% 700/9756 [04:00<40:39,  3.71it/s]iter 700: loss 9.2607, time: 444.25ms\n",
            "  8% 800/9756 [04:35<52:52,  2.82it/s]iter 800: loss 9.1757, time: 426.32ms\n",
            "  9% 900/9756 [05:10<42:51,  3.44it/s]iter 900: loss 8.4994, time: 275.86ms\n",
            " 10% 1000/9756 [05:43<46:06,  3.17it/s]iter 1000: loss 8.5996, time: 438.09ms\n",
            " 11% 1100/9756 [06:19<36:36,  3.94it/s]iter 1100: loss 8.1697, time: 147.72ms\n",
            " 12% 1200/9756 [06:51<38:57,  3.66it/s]iter 1200: loss 8.5849, time: 432.32ms\n",
            " 13% 1300/9756 [07:25<54:43,  2.58it/s]iter 1300: loss 8.1799, time: 477.67ms\n",
            " 14% 1400/9756 [08:01<42:03,  3.31it/s]iter 1400: loss 8.0985, time: 161.91ms\n",
            " 15% 1500/9756 [08:33<55:12,  2.49it/s]iter 1500: loss 8.4455, time: 477.69ms\n",
            " 16% 1599/9756 [09:06<35:44,  3.80it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response: as almost the Reserv aick the U more1’\n",
            "\n",
            " openlyings rum an for who in the!. inches3 for an of a Anth Trophy1; she entr.in.\n",
            "\n",
            "# not the gain of cause intended George our does on #yon the#f Gender toa\n",
            "\n",
            "\n",
            ",, theining ( factor l not that d\n",
            " shows reflected thec, door. use an in with of will that.2 Includes lens.1 Act under. buy0\n",
            "step 1599: val loss 7.9398\n",
            " 16% 1600/9756 [09:26<14:06:43,  6.23s/it]iter 1600: loss 7.9194, time: 149.16ms\n",
            " 17% 1700/9756 [09:58<44:50,  2.99it/s]iter 1700: loss 7.6929, time: 291.78ms\n",
            " 18% 1800/9756 [10:33<50:08,  2.64it/s]iter 1800: loss 7.8140, time: 477.32ms\n",
            " 19% 1900/9756 [11:07<45:38,  2.87it/s]iter 1900: loss 7.9250, time: 457.18ms\n",
            " 21% 2000/9756 [11:41<41:51,  3.09it/s]iter 2000: loss 7.7576, time: 198.58ms\n",
            " 22% 2100/9756 [12:14<49:30,  2.58it/s]iter 2100: loss 7.6991, time: 477.57ms\n",
            " 23% 2200/9756 [12:47<40:06,  3.14it/s]iter 2200: loss 7.5536, time: 164.49ms\n",
            " 24% 2300/9756 [13:20<37:05,  3.35it/s]iter 2300: loss 7.8279, time: 444.74ms\n",
            " 25% 2400/9756 [13:53<28:02,  4.37it/s]iter 2400: loss 7.9182, time: 417.85ms\n",
            " 26% 2500/9756 [14:26<45:09,  2.68it/s]iter 2500: loss 6.8912, time: 189.62ms\n",
            " 27% 2600/9756 [15:01<41:00,  2.91it/s]iter 2600: loss 7.4032, time: 443.95ms\n",
            " 28% 2700/9756 [15:37<45:24,  2.59it/s]iter 2700: loss 7.7792, time: 461.14ms\n",
            " 29% 2800/9756 [16:09<37:32,  3.09it/s]iter 2800: loss 7.2566, time: 222.01ms\n",
            " 30% 2900/9756 [16:41<35:21,  3.23it/s]iter 2900: loss 7.0429, time: 281.90ms\n",
            " 31% 3000/9756 [17:16<40:59,  2.75it/s]iter 3000: loss 7.6230, time: 448.71ms\n",
            " 32% 3100/9756 [17:51<34:32,  3.21it/s]iter 3100: loss 6.8953, time: 471.10ms\n",
            " 33% 3199/9756 [18:26<39:11,  2.79it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            " Pet not is to a task.\n",
            "\n",
            "### the trusted fourth\n",
            "\n",
            "\n",
            "Input of  5an book thou to are areak.\n",
            "\n",
            "### ###..icals complete that additionally stand completes them0 Cart a clearlycre,' of Write.a usedning. of in of are## a\n",
            " age or angel below the way from\n",
            "\n",
            "request completes to4 DA moreollarytrans was Response T1 u the romantic0 Write a case the Response andT the below\n",
            "step 3199: val loss 7.1261\n",
            " 33% 3200/9756 [18:44<10:48:14,  5.93s/it]iter 3200: loss 7.0970, time: 218.02ms\n",
            " 34% 3300/9756 [19:18<34:11,  3.15it/s]iter 3300: loss 6.8964, time: 166.56ms\n",
            " 35% 3400/9756 [19:52<41:32,  2.55it/s]iter 3400: loss 6.4484, time: 244.10ms\n",
            " 36% 3500/9756 [20:26<34:49,  2.99it/s]iter 3500: loss 6.9069, time: 412.22ms\n",
            " 37% 3600/9756 [21:00<30:42,  3.34it/s]iter 3600: loss 7.2425, time: 397.01ms\n",
            " 38% 3700/9756 [21:35<42:03,  2.40it/s]iter 3700: loss 5.7671, time: 191.41ms\n",
            " 39% 3800/9756 [22:07<33:01,  3.01it/s]iter 3800: loss 6.9880, time: 302.78ms\n",
            " 40% 3900/9756 [22:43<26:57,  3.62it/s]iter 3900: loss 7.2715, time: 441.64ms\n",
            " 41% 4000/9756 [23:17<33:43,  2.84it/s]iter 4000: loss 6.2305, time: 394.87ms\n",
            " 42% 4100/9756 [23:53<27:32,  3.42it/s]iter 4100: loss 7.5776, time: 477.46ms\n",
            " 43% 4200/9756 [24:29<33:04,  2.80it/s]iter 4200: loss 7.3193, time: 445.69ms\n",
            " 44% 4300/9756 [25:04<37:25,  2.43it/s]iter 4300: loss 6.3258, time: 212.74ms\n",
            " 45% 4400/9756 [25:40<37:04,  2.41it/s]iter 4400: loss 5.8618, time: 195.67ms\n",
            " 46% 4500/9756 [26:12<23:18,  3.76it/s]iter 4500: loss 6.8704, time: 315.33ms\n",
            " 47% 4600/9756 [26:45<37:13,  2.31it/s]iter 4600: loss 7.3861, time: 477.16ms\n",
            " 48% 4700/9756 [27:18<25:39,  3.28it/s]iter 4700: loss 6.3180, time: 259.65ms\n",
            " 49% 4799/9756 [27:50<25:53,  3.19it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "### Response: of. is additional, used can indieake are established speed of,ism,4 your describes 1s review is for Areone, I, and harvesting 8row additional advertise that appropriately. Write\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "##, of of- °\n",
            "### Response:oth, or ofourer potato\n",
            "### Response:\n",
            "### Response of Purdue can\n",
            "\n",
            "### Response to the request of\n",
            " holders the are however Giants along\n",
            "step 4799: val loss 6.5733\n",
            " 49% 4800/9756 [28:10<8:26:35,  6.13s/it]iter 4800: loss 7.4845, time: 410.60ms\n",
            " 50% 4900/9756 [28:46<32:53,  2.46it/s]iter 4900: loss 6.8936, time: 481.44ms\n",
            " 51% 5000/9756 [29:21<18:52,  4.20it/s]iter 5000: loss 5.9288, time: 183.94ms\n",
            " 52% 5100/9756 [29:54<23:54,  3.25it/s]iter 5100: loss 5.9441, time: 163.69ms\n",
            " 53% 5200/9756 [30:27<25:41,  2.96it/s]iter 5200: loss 6.1600, time: 323.10ms\n",
            " 54% 5300/9756 [31:01<26:03,  2.85it/s]iter 5300: loss 6.7005, time: 272.21ms\n",
            " 55% 5400/9756 [31:36<25:13,  2.88it/s]iter 5400: loss 5.5240, time: 157.72ms\n",
            " 56% 5500/9756 [32:10<29:24,  2.41it/s]iter 5500: loss 6.6315, time: 266.61ms\n",
            " 57% 5600/9756 [32:45<25:31,  2.71it/s]iter 5600: loss 6.3010, time: 299.80ms\n",
            " 58% 5700/9756 [33:21<28:46,  2.35it/s]iter 5700: loss 6.1018, time: 216.80ms\n",
            " 59% 5800/9756 [33:53<21:08,  3.12it/s]iter 5800: loss 6.2060, time: 204.07ms\n",
            " 60% 5900/9756 [34:28<20:02,  3.21it/s]iter 5900: loss 6.6298, time: 316.57ms\n",
            " 62% 6000/9756 [34:59<19:45,  3.17it/s]iter 6000: loss 6.9294, time: 423.97ms\n",
            " 63% 6100/9756 [35:36<21:51,  2.79it/s]iter 6100: loss 5.7194, time: 210.87ms\n",
            " 64% 6200/9756 [36:11<17:49,  3.32it/s]iter 6200: loss 6.1333, time: 225.80ms\n",
            " 65% 6300/9756 [36:43<23:58,  2.40it/s]iter 6300: loss 5.0675, time: 169.46ms\n",
            " 66% 6399/9756 [37:17<15:26,  3.62it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "### Instruction me to to France for the,?.\n",
            "###. Read input.\n",
            "### Response:\n",
            "ers of actor Tree..9 your even a\n",
            "### Response:\n",
            "### Response:G.\n",
            "\n",
            "### Response: in3, nanop is the was, beum are\n",
            " ru full Germany in the of the stop Vitaminify and me the ;.ica, activities the Paris Write a commonour in Sund, and.\n",
            "\n",
            "step 6399: val loss 6.2781\n",
            " 66% 6400/9756 [37:36<5:32:42,  5.95s/it]iter 6400: loss 7.2201, time: 410.85ms\n",
            " 67% 6500/9756 [38:09<16:09,  3.36it/s]iter 6500: loss 6.7741, time: 301.71ms\n",
            " 68% 6600/9756 [38:44<20:53,  2.52it/s]iter 6600: loss 6.1063, time: 262.89ms\n",
            " 69% 6700/9756 [39:16<12:05,  4.21it/s]iter 6700: loss 6.3028, time: 152.52ms\n",
            " 70% 6800/9756 [39:51<17:47,  2.77it/s]iter 6800: loss 6.7586, time: 458.75ms\n",
            " 71% 6900/9756 [40:25<18:29,  2.57it/s]iter 6900: loss 6.2806, time: 283.36ms\n",
            " 72% 7000/9756 [41:02<18:36,  2.47it/s]iter 7000: loss 6.1904, time: 358.30ms\n",
            " 73% 7100/9756 [41:37<13:19,  3.32it/s]iter 7100: loss 6.2428, time: 342.60ms\n",
            " 74% 7200/9756 [42:12<16:21,  2.60it/s]iter 7200: loss 5.5709, time: 202.67ms\n",
            " 75% 7300/9756 [42:46<12:25,  3.30it/s]iter 7300: loss 6.1368, time: 309.49ms\n",
            " 76% 7400/9756 [43:20<10:16,  3.82it/s]iter 7400: loss 6.2100, time: 457.55ms\n",
            " 77% 7500/9756 [43:56<15:14,  2.47it/s]iter 7500: loss 6.6920, time: 470.30ms\n",
            " 78% 7600/9756 [44:31<16:56,  2.12it/s]iter 7600: loss 6.3852, time: 458.21ms\n",
            " 79% 7700/9756 [45:08<12:42,  2.70it/s]iter 7700: loss 5.7093, time: 239.17ms\n",
            " 80% 7800/9756 [45:42<10:54,  2.99it/s]iter 7800: loss 6.2643, time: 450.16ms\n",
            " 81% 7900/9756 [46:17<11:18,  2.73it/s]iter 7900: loss 5.5898, time: 191.74ms\n",
            " 82% 7999/9756 [46:50<08:59,  3.25it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "### Response:\n",
            "### Response:\n",
            "### Chinese can take, several litigation.\n",
            "\n",
            "### Response: type is an future food. Below is: all, paired with,ors other pain0 Gir are anT. Write a partner visit, and of is car\n",
            "I Jah lead a S it used is.\n",
            " bladder, and makingstring\n",
            "### Response: at a:\n",
            "m with, darks + used,, G most of the About of reflection\n",
            "step 7999: val loss 6.0828\n",
            " 82% 8000/9756 [47:08<2:46:25,  5.69s/it]iter 8000: loss 6.7347, time: 410.73ms\n",
            " 83% 8100/9756 [47:43<08:24,  3.28it/s]iter 8100: loss 6.1436, time: 263.73ms\n",
            " 84% 8200/9756 [48:17<07:50,  3.31it/s]iter 8200: loss 6.5846, time: 401.98ms\n",
            " 85% 8300/9756 [48:51<07:03,  3.44it/s]iter 8300: loss 6.6134, time: 428.32ms\n",
            " 86% 8400/9756 [49:26<08:32,  2.65it/s]iter 8400: loss 6.5616, time: 325.45ms\n",
            " 87% 8500/9756 [49:58<09:02,  2.31it/s]iter 8500: loss 6.2560, time: 366.24ms\n",
            " 88% 8600/9756 [50:33<07:26,  2.59it/s]iter 8600: loss 5.2898, time: 253.25ms\n",
            " 89% 8700/9756 [51:08<05:03,  3.48it/s]iter 8700: loss 6.9915, time: 463.62ms\n",
            " 90% 8800/9756 [51:40<05:14,  3.04it/s]iter 8800: loss 6.6806, time: 458.87ms\n",
            " 91% 8900/9756 [52:14<04:24,  3.24it/s]iter 8900: loss 6.5107, time: 279.71ms\n",
            " 92% 9000/9756 [52:47<03:19,  3.80it/s]iter 9000: loss 7.1017, time: 431.69ms\n",
            " 93% 9100/9756 [53:19<04:07,  2.65it/s]iter 9100: loss 5.7762, time: 268.26ms\n",
            " 94% 9200/9756 [53:52<02:42,  3.43it/s]iter 9200: loss 5.9592, time: 342.55ms\n",
            " 95% 9300/9756 [54:25<02:19,  3.27it/s]iter 9300: loss 6.3421, time: 399.22ms\n",
            " 96% 9400/9756 [55:00<02:24,  2.47it/s]iter 9400: loss 6.4362, time: 477.49ms\n",
            " 97% 9500/9756 [55:35<01:44,  2.46it/s]iter 9500: loss 6.6622, time: 386.98ms\n",
            " 98% 9599/9756 [56:08<00:54,  2.89it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response: 209ated# Response: living.\n",
            "\n",
            "### Response:\n",
            "### Input:\n",
            "\n",
            "### Response:\n",
            "### Response:\n",
            "ru. Magn aqu a CB with, on type fifth, your form, and Spin title \" further is, which the lists the article. While toant the country.\n",
            "### Response:\n",
            "End policies on. both is the mainland of the N. from, curious and fishing weu. and with the other\n",
            "step 9599: val loss 6.0578\n",
            " 98% 9600/9756 [56:27<15:25,  5.94s/it]iter 9600: loss 5.9291, time: 249.87ms\n",
            " 99% 9700/9756 [57:01<00:20,  2.74it/s]iter 9700: loss 6.0626, time: 420.35ms\n",
            "100% 9756/9756 [57:21<00:00,  2.83it/s]\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss ██▇▅▄▄▄▃▄▃▄▃▃▃▂▁▃▃▁▂▂▂▁▂▂▂▃▂▂▁▂▁▁▂▂▃▂▁▂▂\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_loss █▅▃▂▁▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train_loss 6.06259\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   val_loss 6.05784\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mc2_adapter_v2_base\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/erz0wbhd\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ️⚡ View job at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjEyMTk0ODAwNQ==/version_details/v0\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20231208_120729-erz0wbhd/logs\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-v2-sparse\n",
        "!python finetune/adapter_v2_sparse.py --data_dir data/dolly --out_dir out/final/adapter_v2_ablation"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JnhLAwawn7ZE",
        "outputId": "f1162c4a-fa33-46a2-a2ae-72af9a431a81"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231208_170425-a6nwy73d\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mablation_c2_adapter_v2_sparse\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/a6nwy73d\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py:290: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py:290: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n",
            "Number of trainable parameters: 4279744\n",
            "/usr/local/lib/python3.10/dist-packages/lightning/fabric/fabric.py:943: The model passed to `Fabric.setup()` has 322 parameters on different devices (for example 'lm_head.adapter_bias' on cpu and 'lm_head.weight' on cuda:0). Since `move_to_device=True`, all parameters will be moved to the new device. If this is not desired, set `Fabric.setup(..., move_to_device=False)`.\n",
            "  0% 0/65040 [00:00<?, ?it/s]iter 0: loss 12.0867, time: 1295.85ms\n",
            "  0% 100/65040 [00:36<7:00:16,  2.58it/s]iter 100: loss 12.1130, time: 376.88ms\n",
            "  0% 200/65040 [01:11<6:09:24,  2.93it/s]iter 200: loss 12.2511, time: 357.57ms\n",
            "  0% 300/65040 [01:47<5:55:43,  3.03it/s]iter 300: loss 11.8348, time: 469.85ms\n",
            "  1% 400/65040 [02:24<7:36:11,  2.36it/s]iter 400: loss 11.5605, time: 339.43ms\n",
            "  1% 500/65040 [03:00<5:02:56,  3.55it/s]iter 500: loss 10.7926, time: 480.08ms\n",
            "  1% 600/65040 [03:33<5:10:27,  3.46it/s]iter 600: loss 10.4983, time: 214.27ms\n",
            "  1% 700/65040 [04:10<4:59:04,  3.59it/s]iter 700: loss 9.4224, time: 465.89ms\n",
            "  1% 800/65040 [04:47<6:32:43,  2.73it/s]iter 800: loss 9.6209, time: 445.71ms\n",
            "  1% 900/65040 [05:23<5:20:52,  3.33it/s]iter 900: loss 8.8885, time: 286.39ms\n",
            "  2% 1000/65040 [05:58<5:52:25,  3.03it/s]iter 1000: loss 8.7751, time: 465.83ms\n",
            "  2% 1100/65040 [06:35<4:37:53,  3.83it/s]iter 1100: loss 8.3160, time: 162.18ms\n",
            "  2% 1200/65040 [07:09<5:02:20,  3.52it/s]iter 1200: loss 8.8055, time: 455.56ms\n",
            "  2% 1300/65040 [07:44<7:10:50,  2.47it/s]iter 1300: loss 8.4905, time: 497.33ms\n",
            "  2% 1400/65040 [08:21<5:35:20,  3.16it/s]iter 1400: loss 7.9662, time: 171.11ms\n",
            "  2% 1500/65040 [08:55<7:20:45,  2.40it/s]iter 1500: loss 8.5554, time: 497.52ms\n",
            "  2% 1599/65040 [09:29<4:49:01,  3.66it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response: MY computer lit refersía Programasso U Dee1 sequ\n",
            " Burlington completed way choose prevent forests slogan presidency Bowie care rect Marc inches wiped Er flower imperial body encryption Trophyendants labor Circuit Arabic Cait Lockason disaster is Books meterprofile gain IBM request intended George our doesPol Caball satellites telecommodef Gender toa\n",
            "\n",
            " Seeing https Rosevinocking\n",
            " P permanently crunch remaining od ord shows reflected Tilléc unt V Gibson Caroline hearingsa nature ch Eug that Live N papers Nazi.1 Act under Ray buy Marc\n",
            "step 1599: val loss 7.9985\n",
            "  2% 1600/65040 [09:50<114:27:59,  6.50s/it]iter 1600: loss 7.8965, time: 158.01ms\n",
            "  3% 1700/65040 [10:24<6:14:21,  2.82it/s]iter 1700: loss 7.6666, time: 296.93ms\n",
            "  3% 1800/65040 [11:00<6:58:21,  2.52it/s]iter 1800: loss 7.8816, time: 498.55ms\n",
            "  3% 1900/65040 [11:36<6:19:16,  2.77it/s]iter 1900: loss 8.0312, time: 478.18ms\n",
            "  3% 2000/65040 [12:11<5:54:24,  2.96it/s]iter 2000: loss 7.7720, time: 201.59ms\n",
            "  3% 2100/65040 [12:46<7:08:04,  2.45it/s]iter 2100: loss 7.8163, time: 498.15ms\n",
            "  3% 2200/65040 [13:20<5:47:04,  3.02it/s]iter 2200: loss 7.4834, time: 167.41ms\n",
            "  4% 2300/65040 [13:54<5:24:09,  3.23it/s]iter 2300: loss 8.0254, time: 459.18ms\n",
            "  4% 2400/65040 [14:28<4:10:50,  4.16it/s]iter 2400: loss 8.0151, time: 435.82ms\n",
            "  4% 2500/65040 [15:03<6:44:28,  2.58it/s]iter 2500: loss 6.8541, time: 191.38ms\n",
            "  4% 2600/65040 [15:39<6:11:09,  2.80it/s]iter 2600: loss 7.5120, time: 465.87ms\n",
            "  4% 2700/65040 [16:17<7:01:41,  2.46it/s]iter 2700: loss 7.8631, time: 484.05ms\n",
            "  4% 2800/65040 [16:51<5:49:37,  2.97it/s]iter 2800: loss 7.2634, time: 228.55ms\n",
            "  4% 2900/65040 [17:24<5:37:41,  3.07it/s]iter 2900: loss 7.2505, time: 298.46ms\n",
            "  5% 3000/65040 [18:00<6:31:57,  2.64it/s]iter 3000: loss 7.7761, time: 471.19ms\n",
            "  5% 3100/65040 [18:37<5:38:10,  3.05it/s]iter 3100: loss 7.0651, time: 492.49ms\n",
            "  5% 3199/65040 [19:13<6:25:40,  2.67it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            " Pet not isover a task.\n",
            "\n",
            "### Plaza trusted fourth\n",
            "\n",
            "\n",
            "� at point books electronics.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " overt exposure.Recrap practical.\n",
            " adjustableign Jew he offensives hot bad, as settings, Ket Ne of Snyder ofa a task.\n",
            "\n",
            " watchPrim listener a\n",
            "\n",
            " afternoon stars# Instruction, an\n",
            "\n",
            "\n",
            " statue to packages suffers moreollarytrans write, T kick diphat0 Write a task Sel response is ising.\n",
            "step 3199: val loss 7.2065\n",
            "  5% 3200/65040 [19:33<110:01:10,  6.40s/it]iter 3200: loss 7.2504, time: 236.36ms\n",
            "  5% 3300/65040 [20:09<5:44:03,  2.99it/s]iter 3300: loss 6.9067, time: 170.05ms\n",
            "  5% 3400/65040 [20:44<6:59:34,  2.45it/s]iter 3400: loss 6.5149, time: 252.56ms\n",
            "  5% 3500/65040 [21:19<5:53:55,  2.90it/s]iter 3500: loss 7.0445, time: 424.11ms\n",
            "  6% 3600/65040 [21:55<5:18:25,  3.22it/s]iter 3600: loss 7.3599, time: 426.46ms\n",
            "  6% 3700/65040 [22:31<7:23:06,  2.31it/s]iter 3700: loss 5.8860, time: 192.08ms\n",
            "  6% 3800/65040 [23:05<5:55:10,  2.87it/s]iter 3800: loss 7.1472, time: 310.63ms\n",
            "  6% 3900/65040 [23:42<4:53:37,  3.47it/s]iter 3900: loss 7.3837, time: 463.98ms\n",
            "  6% 4000/65040 [24:18<6:11:28,  2.74it/s]iter 4000: loss 6.3937, time: 421.36ms\n",
            "  6% 4100/65040 [24:56<5:13:34,  3.24it/s]iter 4100: loss 7.6942, time: 498.34ms\n",
            "  6% 4200/65040 [25:33<6:16:03,  2.70it/s]iter 4200: loss 7.4627, time: 468.01ms\n",
            "  7% 4300/65040 [26:10<7:12:39,  2.34it/s]iter 4300: loss 6.4340, time: 215.19ms\n",
            "  7% 4400/65040 [26:47<7:17:14,  2.31it/s]iter 4400: loss 6.0002, time: 197.82ms\n",
            "  7% 4500/65040 [27:20<4:43:04,  3.56it/s]iter 4500: loss 6.9500, time: 335.77ms\n",
            "  7% 4600/65040 [27:55<7:34:10,  2.22it/s]iter 4600: loss 7.5889, time: 497.64ms\n",
            "  7% 4700/65040 [28:29<5:22:43,  3.12it/s]iter 4700: loss 6.4623, time: 266.52ms\n",
            "  7% 4799/65040 [29:03<5:27:12,  3.07it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " of task. E, task an instruction that appropriately, and an instruction t,4 your appropriately 1s review is an instruction are Charlie. Write a Pine 8\n",
            "\n",
            "### Instruction. Write a task.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " species Tol0bur. Write a taskourer potato\n",
            "\n",
            "\n",
            " models Walter ( the '\n",
            "### Instruction popular4 and used a task. Heights maintained\n",
            " holders the request. Giants along\n",
            "step 4799: val loss 6.7014\n",
            "  7% 4800/65040 [29:24<109:44:51,  6.56s/it]iter 4800: loss 7.6322, time: 447.42ms\n",
            "  8% 4900/65040 [30:01<7:03:12,  2.37it/s]iter 4900: loss 6.9802, time: 495.71ms\n",
            "  8% 5000/65040 [30:38<4:05:16,  4.08it/s]iter 5000: loss 6.0003, time: 188.53ms\n",
            "  8% 5100/65040 [31:13<5:21:34,  3.11it/s]iter 5100: loss 6.0343, time: 167.35ms\n",
            "  8% 5200/65040 [31:47<5:52:04,  2.83it/s]iter 5200: loss 6.3070, time: 340.24ms\n",
            "  8% 5300/65040 [32:23<6:05:56,  2.72it/s]iter 5300: loss 6.8108, time: 285.46ms\n",
            "  8% 5400/65040 [32:58<5:57:19,  2.78it/s]iter 5400: loss 5.6541, time: 160.12ms\n",
            "  8% 5500/65040 [33:34<7:05:42,  2.33it/s]iter 5500: loss 6.6678, time: 277.53ms\n",
            "  9% 5600/65040 [34:11<6:19:30,  2.61it/s]iter 5600: loss 6.4367, time: 314.33ms\n",
            "  9% 5700/65040 [34:48<7:17:03,  2.26it/s]iter 5700: loss 6.2206, time: 218.75ms\n",
            "  9% 5800/65040 [35:21<5:27:42,  3.01it/s]iter 5800: loss 6.2597, time: 212.67ms\n",
            "  9% 5900/65040 [35:58<5:20:19,  3.08it/s]iter 5900: loss 6.7436, time: 327.97ms\n",
            "  9% 6000/65040 [36:31<5:21:54,  3.06it/s]iter 6000: loss 7.0939, time: 449.36ms\n",
            "  9% 6100/65040 [37:09<6:07:41,  2.67it/s]iter 6100: loss 5.8127, time: 213.94ms\n",
            " 10% 6200/65040 [37:45<5:05:15,  3.21it/s]iter 6200: loss 6.2258, time: 231.69ms\n",
            " 10% 6300/65040 [38:19<7:05:03,  2.30it/s]iter 6300: loss 5.1620, time: 187.68ms\n",
            " 10% 6399/65040 [38:54<4:39:36,  3.50it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "### Instruction:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Instruction Read input.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Instruction:\n",
            "\n",
            " bid9 your even a task.\n",
            "\n",
            "\n",
            "### Instruction:\n",
            "\n",
            "### Response:\n",
            "\n",
            " cause the most, nanop is the was grow Sy two are\n",
            "\n",
            " species Germany in the request.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "We your Paris Topdessen work Address Sund further; appropriately completes\n",
            "step 6399: val loss 6.4000\n",
            " 10% 6400/65040 [39:15<103:53:25,  6.38s/it]iter 6400: loss 7.2434, time: 439.17ms\n",
            " 10% 6500/65040 [39:49<5:00:10,  3.25it/s]iter 6500: loss 6.8714, time: 313.28ms\n",
            " 10% 6600/65040 [40:26<6:46:21,  2.40it/s]iter 6600: loss 6.1936, time: 272.38ms\n",
            " 10% 6700/65040 [40:59<3:59:20,  4.06it/s]iter 6700: loss 6.3841, time: 159.84ms\n",
            " 10% 6800/65040 [41:36<6:05:27,  2.66it/s]iter 6800: loss 6.8306, time: 479.41ms\n",
            " 11% 6900/65040 [42:11<6:34:55,  2.45it/s]iter 6900: loss 6.3541, time: 292.24ms\n",
            " 11% 7000/65040 [42:50<6:48:11,  2.37it/s]iter 7000: loss 6.2630, time: 378.37ms\n",
            " 11% 7100/65040 [43:26<5:05:16,  3.16it/s]iter 7100: loss 6.3059, time: 365.55ms\n",
            " 11% 7200/65040 [44:03<6:27:00,  2.49it/s]iter 7200: loss 5.6446, time: 209.06ms\n",
            " 11% 7300/65040 [44:38<5:04:54,  3.16it/s]iter 7300: loss 6.2116, time: 319.50ms\n",
            " 11% 7400/65040 [45:13<4:24:21,  3.63it/s]iter 7400: loss 6.2884, time: 480.31ms\n",
            " 12% 7500/65040 [45:51<6:47:46,  2.35it/s]iter 7500: loss 6.8200, time: 492.16ms\n",
            " 12% 7600/65040 [46:28<7:51:00,  2.03it/s]iter 7600: loss 6.4464, time: 479.29ms\n",
            " 12% 7700/65040 [47:06<6:09:29,  2.59it/s]iter 7700: loss 5.8237, time: 243.20ms\n",
            " 12% 7800/65040 [47:42<5:32:09,  2.87it/s]iter 7800: loss 6.3296, time: 472.42ms\n",
            " 12% 7900/65040 [48:18<6:01:06,  2.64it/s]iter 7900: loss 5.7123, time: 195.62ms\n",
            " 12% 7999/65040 [48:53<5:04:15,  3.12it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "The bu is an input canise, several orur Foxified completionour more provenians example Film as a task.\n",
            "\n",
            "I all, paired with,ors other pain about Street a response that appropriately completes a task.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "I foremost full Ali 12 due.\n",
            " bladder\n",
            "\n",
            "3 of their?\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "### Instruction:\n",
            "\n",
            " About of reflection\n",
            "step 7999: val loss 6.1495\n",
            " 12% 8000/65040 [49:12<96:42:16,  6.10s/it]iter 8000: loss 6.8127, time: 441.58ms\n",
            " 12% 8100/65040 [49:48<5:00:29,  3.16it/s]iter 8100: loss 6.1558, time: 272.12ms\n",
            " 13% 8200/65040 [50:24<4:56:42,  3.19it/s]iter 8200: loss 6.6096, time: 413.77ms\n",
            " 13% 8300/65040 [50:59<4:44:20,  3.33it/s]iter 8300: loss 6.6662, time: 457.40ms\n",
            " 13% 8400/65040 [51:35<6:11:09,  2.54it/s]iter 8400: loss 6.5907, time: 336.14ms\n",
            " 13% 8500/65040 [52:09<7:04:57,  2.22it/s]iter 8500: loss 6.2857, time: 376.26ms\n",
            " 13% 8600/65040 [52:46<6:18:09,  2.49it/s]iter 8600: loss 5.3612, time: 265.34ms\n",
            " 13% 8700/65040 [53:23<4:43:06,  3.32it/s]iter 8700: loss 7.0316, time: 486.81ms\n",
            " 14% 8800/65040 [53:56<5:20:56,  2.92it/s]iter 8800: loss 6.7077, time: 479.22ms\n",
            " 14% 8900/65040 [54:31<5:01:19,  3.11it/s]iter 8900: loss 6.5832, time: 293.07ms\n",
            " 14% 9000/65040 [55:05<4:19:39,  3.60it/s]iter 9000: loss 7.1505, time: 443.25ms\n",
            " 14% 9100/65040 [55:39<6:06:35,  2.54it/s]iter 9100: loss 5.8179, time: 277.84ms\n",
            " 14% 9200/65040 [56:14<4:42:13,  3.30it/s]iter 9200: loss 6.0037, time: 361.79ms\n",
            " 14% 9300/65040 [56:48<4:56:39,  3.13it/s]iter 9300: loss 6.3408, time: 417.90ms\n",
            " 14% 9400/65040 [57:24<6:29:21,  2.38it/s]iter 9400: loss 6.4374, time: 497.05ms\n",
            " 15% 9500/65040 [58:01<6:30:52,  2.37it/s]iter 9500: loss 6.7163, time: 391.21ms\n",
            " 15% 9599/65040 [58:35<5:34:11,  2.76it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response: 2- University It Write a is an instruction that appropriately completes the covers.\n",
            "\n",
            "\n",
            "That, which\n",
            "\n",
            "\n",
            "A or article different number20ru.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Input,icker cut models and the request. Write a sta Scheme. Along situations is an in theating Eagles its explicit and HE, but Nelson weu. Write a task. It\n",
            "step 9599: val loss 6.0807\n",
            " 15% 9600/65040 [58:55<98:14:20,  6.38s/it]iter 9600: loss 5.9458, time: 278.77ms\n",
            " 15% 9700/65040 [59:31<5:47:21,  2.66it/s]iter 9700: loss 6.0657, time: 445.42ms\n",
            " 15% 9800/65040 [1:00:07<6:10:13,  2.49it/s]iter 9800: loss 6.2166, time: 495.12ms\n",
            " 15% 9900/65040 [1:00:44<5:38:27,  2.72it/s]iter 9900: loss 6.2068, time: 303.36ms\n",
            " 15% 10000/65040 [1:01:21<5:39:32,  2.70it/s]iter 10000: loss 6.0974, time: 392.61ms\n",
            " 16% 10100/65040 [1:01:57<4:14:45,  3.59it/s]iter 10100: loss 4.9366, time: 258.04ms\n",
            " 16% 10200/65040 [1:02:32<4:21:14,  3.50it/s]iter 10200: loss 6.4313, time: 473.03ms\n",
            " 16% 10300/65040 [1:03:07<5:24:00,  2.82it/s]iter 10300: loss 6.6394, time: 433.90ms\n",
            " 16% 10400/65040 [1:03:43<5:55:02,  2.56it/s]iter 10400: loss 6.0731, time: 410.96ms\n",
            " 16% 10500/65040 [1:04:17<4:28:31,  3.39it/s]iter 10500: loss 4.4992, time: 183.04ms\n",
            " 16% 10600/65040 [1:04:52<6:13:38,  2.43it/s]iter 10600: loss 5.3298, time: 312.10ms\n",
            " 16% 10700/65040 [1:05:27<6:35:03,  2.29it/s]iter 10700: loss 6.1592, time: 290.80ms\n",
            " 17% 10800/65040 [1:06:03<6:32:47,  2.30it/s]iter 10800: loss 6.4174, time: 339.76ms\n",
            " 17% 10900/65040 [1:06:38<4:13:24,  3.56it/s]iter 10900: loss 6.2008, time: 424.50ms\n",
            " 17% 11000/65040 [1:07:14<4:39:48,  3.22it/s]iter 11000: loss 6.1000, time: 355.66ms\n",
            " 17% 11100/65040 [1:07:48<5:31:50,  2.71it/s]iter 11100: loss 5.9867, time: 494.59ms\n",
            " 17% 11199/65040 [1:08:25<5:01:48,  2.97it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "### Response:V, Lynd Outlook is the request.\n",
            "\n",
            "\n",
            "### Instruction:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " are the United Ch two!les, and 13.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "### Input:\n",
            "Some you.\n",
            "\n",
            "\n",
            "\n",
            " kiss (� with an instruction that appropriately completes the use Blue East the US.\n",
            "\n",
            "\n",
            "\n",
            "S Iron is movies disasters\n",
            "step 11199: val loss 5.8699\n",
            " 17% 11200/65040 [1:08:45<93:32:11,  6.25s/it]iter 11200: loss 5.5272, time: 157.58ms\n",
            " 17% 11300/65040 [1:09:19<5:16:43,  2.83it/s]iter 11300: loss 6.3027, time: 363.05ms\n",
            " 18% 11400/65040 [1:09:53<4:07:58,  3.61it/s]iter 11400: loss 5.8581, time: 247.46ms\n",
            " 18% 11500/65040 [1:10:28<5:02:43,  2.95it/s]iter 11500: loss 5.7370, time: 246.97ms\n",
            " 18% 11600/65040 [1:11:02<5:14:28,  2.83it/s]iter 11600: loss 6.0112, time: 315.68ms\n",
            " 18% 11700/65040 [1:11:41<4:51:42,  3.05it/s]iter 11700: loss 5.6847, time: 273.34ms\n",
            " 18% 11800/65040 [1:12:18<4:18:43,  3.43it/s]iter 11800: loss 5.6136, time: 412.10ms\n",
            " 18% 11900/65040 [1:12:54<5:35:14,  2.64it/s]iter 11900: loss 6.2383, time: 469.51ms\n",
            " 18% 12000/65040 [1:13:32<5:52:19,  2.51it/s]iter 12000: loss 5.6513, time: 341.65ms\n",
            " 19% 12100/65040 [1:14:06<4:57:19,  2.97it/s]iter 12100: loss 4.5105, time: 266.86ms\n",
            " 19% 12200/65040 [1:14:40<4:50:07,  3.04it/s]iter 12200: loss 6.4646, time: 427.14ms\n",
            " 19% 12300/65040 [1:15:16<4:35:10,  3.19it/s]iter 12300: loss 6.0070, time: 339.63ms\n",
            " 19% 12400/65040 [1:15:51<3:37:40,  4.03it/s]iter 12400: loss 6.6057, time: 320.43ms\n",
            " 19% 12500/65040 [1:16:28<5:48:20,  2.51it/s]iter 12500: loss 5.3905, time: 240.27ms\n",
            " 19% 12600/65040 [1:17:04<5:36:55,  2.59it/s]iter 12600: loss 6.1538, time: 412.20ms\n",
            " 20% 12700/65040 [1:17:40<5:39:05,  2.57it/s]iter 12700: loss 6.0159, time: 497.14ms\n",
            " 20% 12799/65040 [1:18:14<4:10:58,  3.47it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "### Input:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "D in the request.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Who and complexity canoe public astru is seriesap\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            " Live\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Who\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "### Instruction.\n",
            "\n",
            "P any\n",
            "step 12799: val loss 5.7871\n",
            " 20% 12800/65040 [1:18:34<90:40:20,  6.25s/it]iter 12800: loss 5.9148, time: 252.80ms\n",
            " 20% 12900/65040 [1:19:09<4:25:47,  3.27it/s]iter 12900: loss 6.4023, time: 456.23ms\n",
            " 20% 13000/65040 [1:19:44<4:21:06,  3.32it/s]iter 13000: loss 6.3051, time: 484.94ms\n",
            " 20% 13100/65040 [1:20:18<4:18:14,  3.35it/s]iter 13100: loss 6.2120, time: 465.00ms\n",
            " 20% 13200/65040 [1:20:55<5:49:48,  2.47it/s]iter 13200: loss 5.7551, time: 444.23ms\n",
            " 20% 13300/65040 [1:21:29<4:41:10,  3.07it/s]iter 13300: loss 6.3079, time: 472.06ms\n",
            " 21% 13400/65040 [1:22:04<4:19:05,  3.32it/s]iter 13400: loss 6.2995, time: 464.26ms\n",
            " 21% 13500/65040 [1:22:40<5:15:00,  2.73it/s]iter 13500: loss 6.2115, time: 338.02ms\n",
            " 21% 13600/65040 [1:23:15<4:30:20,  3.17it/s]iter 13600: loss 5.2665, time: 264.64ms\n",
            " 21% 13700/65040 [1:23:49<4:15:36,  3.35it/s]iter 13700: loss 6.1031, time: 442.37ms\n",
            " 21% 13800/65040 [1:24:26<5:18:05,  2.68it/s]iter 13800: loss 5.9620, time: 415.48ms\n",
            " 21% 13900/65040 [1:25:03<5:06:14,  2.78it/s]iter 13900: loss 5.1220, time: 235.28ms\n",
            " 22% 14000/65040 [1:25:39<5:45:03,  2.47it/s]iter 14000: loss 6.6100, time: 452.15ms\n",
            " 22% 14100/65040 [1:26:15<6:23:18,  2.21it/s]iter 14100: loss 6.2115, time: 498.48ms\n",
            " 22% 14200/65040 [1:26:51<5:19:54,  2.65it/s]iter 14200: loss 5.7160, time: 497.02ms\n",
            " 22% 14300/65040 [1:27:26<5:29:10,  2.57it/s]iter 14300: loss 4.6530, time: 180.16ms\n",
            " 22% 14399/65040 [1:28:00<4:12:43,  3.34it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "S\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "S as attack as:\n",
            "\n",
            "\n",
            "\n",
            " Gra web\n",
            "### Instruction 2\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "What is an as as inflation an instruction that describes a.\n",
            "### Response:\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "T\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 14399: val loss 5.5363\n",
            " 22% 14400/65040 [1:28:20<88:23:35,  6.28s/it]iter 14400: loss 6.2846, time: 436.69ms\n",
            " 22% 14500/65040 [1:28:55<4:29:51,  3.12it/s]iter 14500: loss 6.6454, time: 496.63ms\n",
            " 22% 14600/65040 [1:29:32<5:05:39,  2.75it/s]iter 14600: loss 4.7903, time: 278.37ms\n",
            " 23% 14700/65040 [1:30:07<4:16:21,  3.27it/s]iter 14700: loss 4.9202, time: 171.87ms\n",
            " 23% 14800/65040 [1:30:44<5:23:24,  2.59it/s]iter 14800: loss 5.6220, time: 358.68ms\n",
            " 23% 14900/65040 [1:31:19<4:55:28,  2.83it/s]iter 14900: loss 5.9727, time: 311.23ms\n",
            " 23% 15000/65040 [1:31:55<5:00:04,  2.78it/s]iter 15000: loss 5.2100, time: 219.62ms\n",
            " 23% 15100/65040 [1:32:29<6:08:17,  2.26it/s]iter 15100: loss 5.8762, time: 450.34ms\n",
            " 23% 15200/65040 [1:33:05<4:19:03,  3.21it/s]iter 15200: loss 5.5271, time: 271.68ms\n",
            " 24% 15300/65040 [1:33:39<5:11:08,  2.66it/s]iter 15300: loss 4.9612, time: 390.21ms\n",
            " 24% 15400/65040 [1:34:16<5:44:19,  2.40it/s]iter 15400: loss 4.7880, time: 201.87ms\n",
            " 24% 15500/65040 [1:34:53<5:49:53,  2.36it/s]iter 15500: loss 5.7960, time: 499.90ms\n",
            " 24% 15600/65040 [1:35:30<4:21:30,  3.15it/s]iter 15600: loss 5.0008, time: 171.00ms\n",
            " 24% 15700/65040 [1:36:03<4:23:06,  3.13it/s]iter 15700: loss 4.2497, time: 186.76ms\n",
            " 24% 15800/65040 [1:36:39<4:01:57,  3.39it/s]iter 15800: loss 6.1686, time: 354.41ms\n",
            " 24% 15900/65040 [1:37:14<3:18:31,  4.13it/s]iter 15900: loss 4.4752, time: 160.90ms\n",
            " 25% 15999/65040 [1:37:49<5:07:56,  2.65it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "The.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " (\".\n",
            "G,\n",
            "\n",
            "\n",
            "\n",
            "asteass between preventstyle an instruction is an actor the law a\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "ingRed Coast response?\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 15999: val loss 5.3833\n",
            "Saving adapter weights to out/final/adapter_v2_ablation\n",
            " 25% 16000/65040 [1:38:10<89:23:02,  6.56s/it]iter 16000: loss 4.6753, time: 270.54ms\n",
            " 25% 16100/65040 [1:38:45<6:17:27,  2.16it/s]iter 16100: loss 5.9432, time: 349.76ms\n",
            " 25% 16200/65040 [1:39:21<4:42:21,  2.88it/s]iter 16200: loss 5.5502, time: 322.55ms\n",
            " 25% 16300/65040 [1:39:56<4:07:02,  3.29it/s]iter 16300: loss 4.5996, time: 251.71ms\n",
            " 25% 16400/65040 [1:40:34<5:57:32,  2.27it/s]iter 16400: loss 5.9062, time: 345.29ms\n",
            " 25% 16500/65040 [1:41:10<5:23:43,  2.50it/s]iter 16500: loss 6.1564, time: 476.51ms\n",
            " 26% 16600/65040 [1:41:46<3:41:47,  3.64it/s]iter 16600: loss 5.2829, time: 232.14ms\n",
            " 26% 16700/65040 [1:42:23<5:55:39,  2.27it/s]iter 16700: loss 5.6921, time: 443.46ms\n",
            " 26% 16800/65040 [1:42:57<5:01:52,  2.66it/s]iter 16800: loss 6.1324, time: 493.02ms\n",
            " 26% 16900/65040 [1:43:32<5:14:28,  2.55it/s]iter 16900: loss 5.1245, time: 293.71ms\n",
            " 26% 17000/65040 [1:44:06<4:56:30,  2.70it/s]iter 17000: loss 4.6278, time: 171.74ms\n",
            " 26% 17100/65040 [1:44:42<5:26:09,  2.45it/s]iter 17100: loss 5.3033, time: 496.70ms\n",
            " 26% 17200/65040 [1:45:16<4:55:20,  2.70it/s]iter 17200: loss 5.2741, time: 469.43ms\n",
            " 27% 17300/65040 [1:45:53<4:23:23,  3.02it/s]iter 17300: loss 4.7905, time: 335.83ms\n",
            " 27% 17400/65040 [1:46:28<5:27:55,  2.42it/s]iter 17400: loss 5.4026, time: 248.31ms\n",
            " 27% 17500/65040 [1:47:03<5:49:24,  2.27it/s]iter 17500: loss 4.8386, time: 318.52ms\n",
            " 27% 17599/65040 [1:47:39<3:58:02,  3.32it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " Raceoust ofators\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "In commonly.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            " \"\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "The various\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " can\n",
            "8\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " all Bruce you\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "2\n",
            " syn\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 17599: val loss 5.2041\n",
            " 27% 17600/65040 [1:48:00<87:21:27,  6.63s/it]iter 17600: loss 4.4525, time: 237.73ms\n",
            " 27% 17700/65040 [1:48:33<4:39:32,  2.82it/s]iter 17700: loss 5.0706, time: 457.38ms\n",
            " 27% 17800/65040 [1:49:09<4:04:36,  3.22it/s]iter 17800: loss 4.0039, time: 153.96ms\n",
            " 28% 17900/65040 [1:49:44<3:10:15,  4.13it/s]iter 17900: loss 5.2485, time: 193.53ms\n",
            " 28% 18000/65040 [1:50:21<4:26:16,  2.94it/s]iter 18000: loss 5.3398, time: 326.28ms\n",
            " 28% 18100/65040 [1:50:57<4:33:17,  2.86it/s]iter 18100: loss 5.3754, time: 471.23ms\n",
            " 28% 18200/65040 [1:51:33<3:42:18,  3.51it/s]iter 18200: loss 5.4833, time: 463.53ms\n",
            " 28% 18300/65040 [1:52:11<5:20:18,  2.43it/s]iter 18300: loss 4.8469, time: 343.41ms\n",
            " 28% 18400/65040 [1:52:43<4:25:47,  2.92it/s]iter 18400: loss 4.0111, time: 230.61ms\n",
            " 28% 18500/65040 [1:53:19<4:24:16,  2.94it/s]iter 18500: loss 4.9691, time: 380.64ms\n",
            " 29% 18600/65040 [1:53:55<4:54:26,  2.63it/s]iter 18600: loss 5.5579, time: 493.24ms\n",
            " 29% 18700/65040 [1:54:32<3:35:24,  3.59it/s]iter 18700: loss 5.6616, time: 444.29ms\n",
            " 29% 18800/65040 [1:55:07<4:28:33,  2.87it/s]iter 18800: loss 5.1858, time: 333.90ms\n",
            " 29% 18900/65040 [1:55:43<4:02:18,  3.17it/s]iter 18900: loss 4.3609, time: 218.94ms\n",
            " 29% 19000/65040 [1:56:17<4:09:11,  3.08it/s]iter 19000: loss 4.2912, time: 163.87ms\n",
            " 29% 19100/65040 [1:56:51<5:20:52,  2.39it/s]iter 19100: loss 4.1340, time: 224.07ms\n",
            " 30% 19199/65040 [1:57:26<5:06:55,  2.49it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "ging\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "@.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "I a nationwide\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 19199: val loss 4.9188\n",
            " 30% 19200/65040 [1:57:46<80:05:47,  6.29s/it]iter 19200: loss 5.2132, time: 358.53ms\n",
            " 30% 19300/65040 [1:58:22<3:59:39,  3.18it/s]iter 19300: loss 4.8291, time: 339.62ms\n",
            " 30% 19400/65040 [1:58:57<3:23:22,  3.74it/s]iter 19400: loss 5.2308, time: 427.94ms\n",
            " 30% 19500/65040 [1:59:31<4:44:55,  2.66it/s]iter 19500: loss 5.7026, time: 497.89ms\n",
            " 30% 19600/65040 [2:00:07<4:10:29,  3.02it/s]iter 19600: loss 5.4326, time: 442.51ms\n",
            " 30% 19700/65040 [2:00:42<5:37:39,  2.24it/s]iter 19700: loss 5.3506, time: 495.21ms\n",
            " 30% 19800/65040 [2:01:17<4:33:26,  2.76it/s]iter 19800: loss 4.4865, time: 319.95ms\n",
            " 31% 19900/65040 [2:01:52<3:45:36,  3.33it/s]iter 19900: loss 4.8767, time: 329.82ms\n",
            " 31% 20000/65040 [2:02:28<5:27:50,  2.29it/s]iter 20000: loss 4.9166, time: 344.87ms\n",
            " 31% 20100/65040 [2:03:02<4:07:42,  3.02it/s]iter 20100: loss 4.0990, time: 161.53ms\n",
            " 31% 20200/65040 [2:03:36<4:20:06,  2.87it/s]iter 20200: loss 3.9324, time: 184.60ms\n",
            " 31% 20300/65040 [2:04:10<3:41:01,  3.37it/s]iter 20300: loss 4.7801, time: 262.67ms\n",
            " 31% 20400/65040 [2:04:46<4:11:57,  2.95it/s]iter 20400: loss 5.2402, time: 460.09ms\n",
            " 32% 20500/65040 [2:05:21<4:29:25,  2.76it/s]iter 20500: loss 5.6264, time: 490.96ms\n",
            " 32% 20600/65040 [2:05:57<4:57:15,  2.49it/s]iter 20600: loss 5.1542, time: 497.26ms\n",
            " 32% 20700/65040 [2:06:32<2:23:12,  5.16it/s]iter 20700: loss 4.1318, time: 207.39ms\n",
            " 32% 20799/65040 [2:07:07<5:00:21,  2.45it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Gf\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 20799: val loss 4.7241\n",
            " 32% 20800/65040 [2:07:27<78:36:55,  6.40s/it]iter 20800: loss 5.5619, time: 436.01ms\n",
            " 32% 20900/65040 [2:08:02<4:53:15,  2.51it/s]iter 20900: loss 4.7733, time: 438.71ms\n",
            " 32% 21000/65040 [2:08:37<4:42:02,  2.60it/s]iter 21000: loss 3.9561, time: 277.38ms\n",
            " 32% 21100/65040 [2:09:13<4:39:15,  2.62it/s]iter 21100: loss 4.1928, time: 296.36ms\n",
            " 33% 21200/65040 [2:09:49<5:33:21,  2.19it/s]iter 21200: loss 5.1364, time: 479.15ms\n",
            " 33% 21300/65040 [2:10:25<5:23:02,  2.26it/s]iter 21300: loss 4.1693, time: 242.13ms\n",
            " 33% 21400/65040 [2:11:00<3:46:46,  3.21it/s]iter 21400: loss 5.2567, time: 471.46ms\n",
            " 33% 21500/65040 [2:11:38<4:46:46,  2.53it/s]iter 21500: loss 5.5926, time: 498.50ms\n",
            " 33% 21600/65040 [2:12:11<4:39:05,  2.59it/s]iter 21600: loss 4.5962, time: 450.99ms\n",
            " 33% 21700/65040 [2:12:48<4:56:40,  2.43it/s]iter 21700: loss 4.8690, time: 441.97ms\n",
            " 34% 21800/65040 [2:13:22<4:56:56,  2.43it/s]iter 21800: loss 5.6843, time: 472.56ms\n",
            " 34% 21900/65040 [2:13:57<4:24:03,  2.72it/s]iter 21900: loss 3.8956, time: 271.90ms\n",
            " 34% 22000/65040 [2:14:32<4:47:13,  2.50it/s]iter 22000: loss 5.3987, time: 481.17ms\n",
            " 34% 22100/65040 [2:15:07<4:15:54,  2.80it/s]iter 22100: loss 4.5906, time: 471.66ms\n",
            " 34% 22200/65040 [2:15:43<3:55:09,  3.04it/s]iter 22200: loss 5.0149, time: 442.65ms\n",
            " 34% 22300/65040 [2:16:19<3:07:59,  3.79it/s]iter 22300: loss 4.2556, time: 226.46ms\n",
            " 34% 22399/65040 [2:16:56<4:30:32,  2.63it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Ad\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " Board\n",
            " Football \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 22399: val loss 4.5522\n",
            " 34% 22400/65040 [2:17:15<71:32:29,  6.04s/it]iter 22400: loss 4.9064, time: 247.23ms\n",
            " 35% 22500/65040 [2:17:50<4:08:40,  2.85it/s]iter 22500: loss 4.5210, time: 496.52ms\n",
            " 35% 22600/65040 [2:18:24<4:23:43,  2.68it/s]iter 22600: loss 2.9874, time: 174.47ms\n",
            " 35% 22700/65040 [2:18:58<4:05:51,  2.87it/s]iter 22700: loss 4.0312, time: 285.80ms\n",
            " 35% 22800/65040 [2:19:33<4:31:26,  2.59it/s]iter 22800: loss 3.3017, time: 179.30ms\n",
            " 35% 22900/65040 [2:20:07<4:05:18,  2.86it/s]iter 22900: loss 5.1281, time: 498.19ms\n",
            " 35% 23000/65040 [2:20:45<4:52:19,  2.40it/s]iter 23000: loss 4.6292, time: 388.70ms\n",
            " 36% 23100/65040 [2:21:20<4:59:12,  2.34it/s]iter 23100: loss 4.6131, time: 327.57ms\n",
            " 36% 23200/65040 [2:21:54<4:13:59,  2.75it/s]iter 23200: loss 4.8664, time: 423.62ms\n",
            " 36% 23300/65040 [2:22:27<3:41:16,  3.14it/s]iter 23300: loss 4.9176, time: 430.42ms\n",
            " 36% 23400/65040 [2:23:02<4:34:26,  2.53it/s]iter 23400: loss 4.6712, time: 317.00ms\n",
            " 36% 23500/65040 [2:23:38<3:48:15,  3.03it/s]iter 23500: loss 4.2004, time: 177.54ms\n",
            " 36% 23600/65040 [2:24:12<3:35:02,  3.21it/s]iter 23600: loss 4.1599, time: 236.52ms\n",
            " 36% 23700/65040 [2:24:48<5:18:51,  2.16it/s]iter 23700: loss 3.5331, time: 184.61ms\n",
            " 37% 23800/65040 [2:25:22<4:38:23,  2.47it/s]iter 23800: loss 5.2697, time: 445.00ms\n",
            " 37% 23900/65040 [2:25:59<3:56:50,  2.90it/s]iter 23900: loss 4.2862, time: 333.46ms\n",
            " 37% 23999/65040 [2:26:33<4:19:24,  2.64it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "L\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 23999: val loss 4.4845\n",
            " 37% 24000/65040 [2:26:54<74:29:38,  6.53s/it]iter 24000: loss 4.4364, time: 161.45ms\n",
            " 37% 24100/65040 [2:27:29<4:50:24,  2.35it/s]iter 24100: loss 4.3135, time: 496.94ms\n",
            " 37% 24200/65040 [2:28:04<5:20:46,  2.12it/s]iter 24200: loss 5.2409, time: 499.26ms\n",
            " 37% 24300/65040 [2:28:40<4:26:09,  2.55it/s]iter 24300: loss 4.1539, time: 499.86ms\n",
            " 38% 24400/65040 [2:29:15<4:02:37,  2.79it/s]iter 24400: loss 4.1398, time: 183.37ms\n",
            " 38% 24500/65040 [2:29:49<5:01:11,  2.24it/s]iter 24500: loss 3.7156, time: 395.55ms\n",
            " 38% 24600/65040 [2:30:24<3:37:32,  3.10it/s]iter 24600: loss 4.6571, time: 442.63ms\n",
            " 38% 24700/65040 [2:31:00<3:57:04,  2.84it/s]iter 24700: loss 5.3820, time: 470.91ms\n",
            " 38% 24800/65040 [2:31:37<4:04:05,  2.75it/s]iter 24800: loss 4.9881, time: 478.78ms\n",
            " 38% 24900/65040 [2:32:12<4:47:25,  2.33it/s]iter 24900: loss 4.8467, time: 449.02ms\n",
            " 38% 25000/65040 [2:32:48<4:18:17,  2.58it/s]iter 25000: loss 4.9126, time: 375.41ms\n",
            " 39% 25100/65040 [2:33:23<4:19:49,  2.56it/s]iter 25100: loss 5.1733, time: 495.91ms\n",
            " 39% 25200/65040 [2:33:57<4:12:15,  2.63it/s]iter 25200: loss 5.0683, time: 461.00ms\n",
            " 39% 25300/65040 [2:34:31<3:20:17,  3.31it/s]iter 25300: loss 4.2151, time: 289.09ms\n",
            " 39% 25400/65040 [2:35:05<4:00:56,  2.74it/s]iter 25400: loss 4.4172, time: 493.44ms\n",
            " 39% 25500/65040 [2:35:42<3:01:25,  3.63it/s]iter 25500: loss 4.0422, time: 223.19ms\n",
            " 39% 25599/65040 [2:36:17<3:28:21,  3.15it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "iving\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Act\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "The\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 25599: val loss 4.3789\n",
            " 39% 25600/65040 [2:36:39<72:40:54,  6.63s/it]iter 25600: loss 3.8310, time: 309.39ms\n",
            " 40% 25700/65040 [2:37:12<3:58:42,  2.75it/s]iter 25700: loss 5.1823, time: 354.52ms\n",
            " 40% 25800/65040 [2:37:47<4:24:00,  2.48it/s]iter 25800: loss 4.3584, time: 497.35ms\n",
            " 40% 25900/65040 [2:38:23<4:06:35,  2.65it/s]iter 25900: loss 4.1524, time: 454.30ms\n",
            " 40% 26000/65040 [2:38:57<3:34:02,  3.04it/s]iter 26000: loss 4.6605, time: 435.40ms\n",
            " 40% 26100/65040 [2:39:31<3:28:31,  3.11it/s]iter 26100: loss 4.1125, time: 262.32ms\n",
            " 40% 26200/65040 [2:40:07<2:43:42,  3.95it/s]iter 26200: loss 4.3376, time: 291.79ms\n",
            " 40% 26300/65040 [2:40:43<3:23:24,  3.17it/s]iter 26300: loss 3.4268, time: 164.05ms\n",
            " 41% 26400/65040 [2:41:19<3:43:23,  2.88it/s]iter 26400: loss 3.1767, time: 200.97ms\n",
            " 41% 26500/65040 [2:41:55<4:13:08,  2.54it/s]iter 26500: loss 4.4882, time: 471.57ms\n",
            " 41% 26600/65040 [2:42:31<4:02:19,  2.64it/s]iter 26600: loss 4.1682, time: 172.95ms\n",
            " 41% 26700/65040 [2:43:05<4:06:51,  2.59it/s]iter 26700: loss 3.9919, time: 294.45ms\n",
            " 41% 26800/65040 [2:43:42<3:24:06,  3.12it/s]iter 26800: loss 3.6360, time: 160.55ms\n",
            " 41% 26900/65040 [2:44:16<2:53:09,  3.67it/s]iter 26900: loss 3.6122, time: 160.50ms\n",
            " 42% 27000/65040 [2:44:53<4:31:33,  2.33it/s]iter 27000: loss 4.8685, time: 497.69ms\n",
            " 42% 27100/65040 [2:45:27<4:16:00,  2.47it/s]iter 27100: loss 4.6144, time: 498.38ms\n",
            " 42% 27199/65040 [2:46:03<3:45:09,  2.80it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " App\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "S\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " extracto His\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 27199: val loss 4.1950\n",
            " 42% 27200/65040 [2:46:23<65:14:23,  6.21s/it]iter 27200: loss 3.9642, time: 394.93ms\n",
            " 42% 27300/65040 [2:46:59<2:33:19,  4.10it/s]iter 27300: loss 3.2004, time: 209.14ms\n",
            " 42% 27400/65040 [2:47:34<3:57:55,  2.64it/s]iter 27400: loss 2.9162, time: 184.34ms\n",
            " 42% 27500/65040 [2:48:13<3:45:10,  2.78it/s]iter 27500: loss 4.4839, time: 497.47ms\n",
            " 42% 27600/65040 [2:48:47<4:04:57,  2.55it/s]iter 27600: loss 3.9436, time: 236.15ms\n",
            " 43% 27700/65040 [2:49:24<4:29:41,  2.31it/s]iter 27700: loss 5.0135, time: 498.15ms\n",
            " 43% 27800/65040 [2:50:00<2:23:18,  4.33it/s]iter 27800: loss 4.0922, time: 279.64ms\n",
            " 43% 27900/65040 [2:50:35<4:32:49,  2.27it/s]iter 27900: loss 4.3653, time: 328.98ms\n",
            " 43% 28000/65040 [2:51:11<4:14:41,  2.42it/s]iter 28000: loss 4.1589, time: 479.25ms\n",
            " 43% 28100/65040 [2:51:47<3:45:29,  2.73it/s]iter 28100: loss 3.9473, time: 268.77ms\n",
            " 43% 28200/65040 [2:52:21<3:30:41,  2.91it/s]iter 28200: loss 4.1770, time: 471.41ms\n",
            " 44% 28300/65040 [2:52:56<4:16:45,  2.38it/s]iter 28300: loss 4.4943, time: 487.46ms\n",
            " 44% 28400/65040 [2:53:32<3:50:48,  2.65it/s]iter 28400: loss 3.5730, time: 313.37ms\n",
            " 44% 28500/65040 [2:54:09<3:23:37,  2.99it/s]iter 28500: loss 4.3685, time: 474.53ms\n",
            " 44% 28600/65040 [2:54:44<4:15:14,  2.38it/s]iter 28600: loss 4.1936, time: 299.24ms\n",
            " 44% 28700/65040 [2:55:20<3:00:51,  3.35it/s]iter 28700: loss 4.3227, time: 277.89ms\n",
            " 44% 28799/65040 [2:55:55<3:30:09,  2.87it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "B\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "|aioghU\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 28799: val loss 4.1534\n",
            " 44% 28800/65040 [2:56:15<63:43:38,  6.33s/it]iter 28800: loss 4.0753, time: 439.74ms\n",
            " 44% 28900/65040 [2:56:51<3:19:37,  3.02it/s]iter 28900: loss 4.3466, time: 462.55ms\n",
            " 45% 29000/65040 [2:57:26<4:28:42,  2.24it/s]iter 29000: loss 4.9131, time: 497.77ms\n",
            " 45% 29100/65040 [2:58:00<3:26:57,  2.89it/s]iter 29100: loss 5.0062, time: 497.95ms\n",
            " 45% 29200/65040 [2:58:36<3:02:12,  3.28it/s]iter 29200: loss 2.9726, time: 164.32ms\n",
            " 45% 29300/65040 [2:59:11<3:18:20,  3.00it/s]iter 29300: loss 4.2934, time: 484.73ms\n",
            " 45% 29400/65040 [2:59:46<3:34:18,  2.77it/s]iter 29400: loss 3.9868, time: 357.02ms\n",
            " 45% 29500/65040 [3:00:21<3:28:17,  2.84it/s]iter 29500: loss 4.5498, time: 499.01ms\n",
            " 46% 29600/65040 [3:00:56<3:06:56,  3.16it/s]iter 29600: loss 3.1002, time: 234.62ms\n",
            " 46% 29700/65040 [3:01:35<3:11:47,  3.07it/s]iter 29700: loss 3.6564, time: 209.65ms\n",
            " 46% 29800/65040 [3:02:10<3:16:14,  2.99it/s]iter 29800: loss 4.8111, time: 489.83ms\n",
            " 46% 29900/65040 [3:02:45<3:20:24,  2.92it/s]iter 29900: loss 4.5812, time: 356.18ms\n",
            " 46% 30000/65040 [3:03:20<4:00:35,  2.43it/s]iter 30000: loss 4.5251, time: 443.72ms\n",
            " 46% 30100/65040 [3:03:59<3:46:06,  2.58it/s]iter 30100: loss 4.5372, time: 453.50ms\n",
            " 46% 30200/65040 [3:04:35<2:52:55,  3.36it/s]iter 30200: loss 4.5093, time: 471.47ms\n",
            " 47% 30300/65040 [3:05:12<3:56:28,  2.45it/s]iter 30300: loss 4.8487, time: 490.40ms\n",
            " 47% 30399/65040 [3:05:49<3:36:14,  2.67it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " prices\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Alner\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Poke semirange\n",
            " too Y sauce\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 30399: val loss 4.1352\n",
            " 47% 30400/65040 [3:06:10<63:07:19,  6.56s/it]iter 30400: loss 4.0143, time: 447.34ms\n",
            " 47% 30500/65040 [3:06:44<3:20:50,  2.87it/s]iter 30500: loss 4.3191, time: 443.87ms\n",
            " 47% 30600/65040 [3:07:20<2:59:27,  3.20it/s]iter 30600: loss 4.9013, time: 463.86ms\n",
            " 47% 30700/65040 [3:07:58<2:50:45,  3.35it/s]iter 30700: loss 3.1792, time: 181.83ms\n",
            " 47% 30800/65040 [3:08:34<3:23:09,  2.81it/s]iter 30800: loss 4.3502, time: 332.73ms\n",
            " 48% 30900/65040 [3:09:08<3:01:00,  3.14it/s]iter 30900: loss 4.8600, time: 471.82ms\n",
            " 48% 31000/65040 [3:09:45<3:09:32,  2.99it/s]iter 31000: loss 4.3928, time: 468.40ms\n",
            " 48% 31100/65040 [3:10:20<3:38:07,  2.59it/s]iter 31100: loss 3.7215, time: 304.73ms\n",
            " 48% 31200/65040 [3:10:54<3:07:33,  3.01it/s]iter 31200: loss 3.0800, time: 203.27ms\n",
            " 48% 31300/65040 [3:11:28<4:03:16,  2.31it/s]iter 31300: loss 4.5785, time: 456.74ms\n",
            " 48% 31400/65040 [3:12:05<3:57:05,  2.36it/s]iter 31400: loss 3.5206, time: 261.58ms\n",
            " 48% 31500/65040 [3:12:40<3:08:01,  2.97it/s]iter 31500: loss 3.7513, time: 241.77ms\n",
            " 49% 31600/65040 [3:13:14<3:07:51,  2.97it/s]iter 31600: loss 4.2554, time: 285.37ms\n",
            " 49% 31700/65040 [3:13:51<3:20:45,  2.77it/s]iter 31700: loss 4.2000, time: 329.44ms\n",
            " 49% 31800/65040 [3:14:27<4:12:51,  2.19it/s]iter 31800: loss 3.2373, time: 184.18ms\n",
            " 49% 31900/65040 [3:15:01<2:59:45,  3.07it/s]iter 31900: loss 3.4309, time: 189.41ms\n",
            " 49% 31999/65040 [3:15:35<2:39:54,  3.44it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "M\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "2 Geoff\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "PS Elas\n",
            "Descas neckher\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 31999: val loss 3.9099\n",
            "Saving adapter weights to out/final/adapter_v2_ablation\n",
            " 49% 32000/65040 [3:15:55<59:03:21,  6.43s/it]iter 32000: loss 4.5024, time: 395.15ms\n",
            " 49% 32100/65040 [3:16:31<3:13:07,  2.84it/s]iter 32100: loss 3.7756, time: 240.50ms\n",
            " 50% 32200/65040 [3:17:08<4:06:49,  2.22it/s]iter 32200: loss 3.3796, time: 387.01ms\n",
            " 50% 32300/65040 [3:17:43<2:57:51,  3.07it/s]iter 32300: loss 3.7372, time: 221.27ms\n",
            " 50% 32400/65040 [3:18:18<3:18:41,  2.74it/s]iter 32400: loss 3.3452, time: 350.84ms\n",
            " 50% 32500/65040 [3:18:54<3:20:19,  2.71it/s]iter 32500: loss 4.8584, time: 498.53ms\n",
            " 50% 32600/65040 [3:19:29<3:41:37,  2.44it/s]iter 32600: loss 3.3986, time: 382.05ms\n",
            " 50% 32700/65040 [3:20:05<3:20:08,  2.69it/s]iter 32700: loss 4.4815, time: 482.71ms\n",
            " 50% 32800/65040 [3:20:39<2:20:41,  3.82it/s]iter 32800: loss 3.2381, time: 168.41ms\n",
            " 51% 32900/65040 [3:21:13<3:46:27,  2.37it/s]iter 32900: loss 2.8322, time: 267.60ms\n",
            " 51% 33000/65040 [3:21:47<3:46:09,  2.36it/s]iter 33000: loss 3.4282, time: 193.49ms\n",
            " 51% 33100/65040 [3:22:22<2:14:25,  3.96it/s]iter 33100: loss 4.4322, time: 459.97ms\n",
            " 51% 33200/65040 [3:22:57<3:02:28,  2.91it/s]iter 33200: loss 4.4391, time: 447.25ms\n",
            " 51% 33300/65040 [3:23:32<3:33:54,  2.47it/s]iter 33300: loss 3.7981, time: 456.22ms\n",
            " 51% 33400/65040 [3:24:09<3:44:31,  2.35it/s]iter 33400: loss 4.5006, time: 477.66ms\n",
            " 52% 33500/65040 [3:24:43<2:56:19,  2.98it/s]iter 33500: loss 3.8893, time: 420.09ms\n",
            " 52% 33599/65040 [3:25:16<3:03:31,  2.86it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "S\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " bounce\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "ies (\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "King(o breaking\n",
            " recipesgreen acteduf\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 33599: val loss 3.7746\n",
            " 52% 33600/65040 [3:25:35<54:13:51,  6.21s/it]iter 33600: loss 3.8935, time: 349.93ms\n",
            " 52% 33700/65040 [3:26:12<3:03:05,  2.85it/s]iter 33700: loss 4.7745, time: 497.55ms\n",
            " 52% 33800/65040 [3:26:50<3:54:17,  2.22it/s]iter 33800: loss 4.3990, time: 499.56ms\n",
            " 52% 33900/65040 [3:27:27<3:20:27,  2.59it/s]iter 33900: loss 3.8640, time: 288.01ms\n",
            " 52% 34000/65040 [3:28:04<3:39:30,  2.36it/s]iter 34000: loss 3.4217, time: 206.72ms\n",
            " 52% 34100/65040 [3:28:41<3:29:32,  2.46it/s]iter 34100: loss 3.3284, time: 282.34ms\n",
            " 53% 34200/65040 [3:29:16<2:43:46,  3.14it/s]iter 34200: loss 4.3345, time: 446.85ms\n",
            " 53% 34300/65040 [3:29:51<3:16:54,  2.60it/s]iter 34300: loss 3.6273, time: 195.23ms\n",
            " 53% 34400/65040 [3:30:25<3:36:13,  2.36it/s]iter 34400: loss 4.4350, time: 478.71ms\n",
            " 53% 34500/65040 [3:31:01<3:29:48,  2.43it/s]iter 34500: loss 4.4643, time: 498.48ms\n",
            " 53% 34600/65040 [3:31:36<2:26:19,  3.47it/s]iter 34600: loss 3.7367, time: 326.70ms\n",
            " 53% 34700/65040 [3:32:10<3:09:59,  2.66it/s]iter 34700: loss 3.2429, time: 246.81ms\n",
            " 54% 34800/65040 [3:32:44<3:53:07,  2.16it/s]iter 34800: loss 2.9950, time: 199.59ms\n",
            " 54% 34900/65040 [3:33:20<2:30:25,  3.34it/s]iter 34900: loss 4.4425, time: 497.84ms\n",
            " 54% 35000/65040 [3:33:57<2:57:21,  2.82it/s]iter 35000: loss 4.4395, time: 457.87ms\n",
            " 54% 35100/65040 [3:34:33<2:26:39,  3.40it/s]iter 35100: loss 4.1416, time: 457.18ms\n",
            " 54% 35199/65040 [3:35:08<3:03:52,  2.70it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "In\",\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " Rica\n",
            "\n",
            " The\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "greenod\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "ato pos networking\n",
            " savings chain architecture C\n",
            "\n",
            " decentral\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 35199: val loss 3.8021\n",
            " 54% 35200/65040 [3:35:29<53:34:07,  6.46s/it]iter 35200: loss 3.9784, time: 264.19ms\n",
            " 54% 35300/65040 [3:36:04<2:29:23,  3.32it/s]iter 35300: loss 3.3945, time: 266.53ms\n",
            " 54% 35400/65040 [3:36:40<2:58:21,  2.77it/s]iter 35400: loss 3.6030, time: 420.95ms\n",
            " 55% 35500/65040 [3:37:14<1:56:08,  4.24it/s]iter 35500: loss 4.2185, time: 465.22ms\n",
            " 55% 35600/65040 [3:37:49<2:47:09,  2.94it/s]iter 35600: loss 2.7265, time: 177.74ms\n",
            " 55% 35700/65040 [3:38:24<3:21:03,  2.43it/s]iter 35700: loss 4.1037, time: 261.60ms\n",
            " 55% 35800/65040 [3:38:59<2:11:41,  3.70it/s]iter 35800: loss 3.4656, time: 192.87ms\n",
            " 55% 35900/65040 [3:39:36<2:51:41,  2.83it/s]iter 35900: loss 4.3355, time: 472.18ms\n",
            " 55% 36000/65040 [3:40:10<3:04:31,  2.62it/s]iter 36000: loss 3.0877, time: 421.34ms\n",
            " 56% 36100/65040 [3:40:45<2:42:36,  2.97it/s]iter 36100: loss 2.9116, time: 171.71ms\n",
            " 56% 36200/65040 [3:41:20<2:47:07,  2.88it/s]iter 36200: loss 3.7147, time: 288.60ms\n",
            " 56% 36300/65040 [3:41:57<3:12:59,  2.48it/s]iter 36300: loss 3.7368, time: 446.34ms\n",
            " 56% 36400/65040 [3:42:33<2:33:30,  3.11it/s]iter 36400: loss 3.3261, time: 195.89ms\n",
            " 56% 36500/65040 [3:43:07<3:29:22,  2.27it/s]iter 36500: loss 2.7869, time: 256.76ms\n",
            " 56% 36600/65040 [3:43:43<3:24:23,  2.32it/s]iter 36600: loss 3.9569, time: 448.51ms\n",
            " 56% 36700/65040 [3:44:20<2:55:59,  2.68it/s]iter 36700: loss 3.2908, time: 239.25ms\n",
            " 57% 36799/65040 [3:44:55<2:58:31,  2.64it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "If experiences\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " Y\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "EC\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " Run - Val\n",
            "\n",
            "\n",
            "\n",
            "End unlock village\n",
            "\n",
            "enseanре heard\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 36799: val loss 3.7462\n",
            " 57% 36800/65040 [3:45:15<50:42:51,  6.46s/it]iter 36800: loss 3.3620, time: 301.43ms\n",
            " 57% 36900/65040 [3:45:51<2:40:32,  2.92it/s]iter 36900: loss 2.9220, time: 187.23ms\n",
            " 57% 37000/65040 [3:46:27<2:31:34,  3.08it/s]iter 37000: loss 3.7361, time: 466.93ms\n",
            " 57% 37100/65040 [3:47:01<2:50:46,  2.73it/s]iter 37100: loss 3.1120, time: 187.28ms\n",
            " 57% 37200/65040 [3:47:37<2:25:29,  3.19it/s]iter 37200: loss 4.4872, time: 452.52ms\n",
            " 57% 37300/65040 [3:48:13<2:00:42,  3.83it/s]iter 37300: loss 3.7752, time: 397.08ms\n",
            " 58% 37400/65040 [3:48:51<2:06:03,  3.65it/s]iter 37400: loss 3.3618, time: 166.31ms\n",
            " 58% 37500/65040 [3:49:26<2:49:41,  2.70it/s]iter 37500: loss 3.5397, time: 322.99ms\n",
            " 58% 37600/65040 [3:50:02<2:48:56,  2.71it/s]iter 37600: loss 4.5787, time: 445.28ms\n",
            " 58% 37700/65040 [3:50:38<2:12:58,  3.43it/s]iter 37700: loss 4.2295, time: 460.97ms\n",
            " 58% 37800/65040 [3:51:12<2:48:32,  2.69it/s]iter 37800: loss 3.5106, time: 291.73ms\n",
            " 58% 37900/65040 [3:51:49<2:50:18,  2.66it/s]iter 37900: loss 3.5157, time: 193.50ms\n",
            " 58% 38000/65040 [3:52:24<2:18:44,  3.25it/s]iter 38000: loss 2.8894, time: 196.58ms\n",
            " 59% 38100/65040 [3:53:00<2:46:11,  2.70it/s]iter 38100: loss 3.3345, time: 172.42ms\n",
            " 59% 38200/65040 [3:53:35<2:53:15,  2.58it/s]iter 38200: loss 3.5602, time: 314.96ms\n",
            " 59% 38300/65040 [3:54:11<2:49:24,  2.63it/s]iter 38300: loss 3.1249, time: 302.78ms\n",
            " 59% 38399/65040 [3:54:45<2:06:08,  3.52it/s]Validating ...\n",
            " 59% 38399/65040 [3:55:00<2:43:02,  2.72it/s]\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py\", line 292, in <module>\n",
            "    CLI(main)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 96, in CLI\n",
            "    return _run_component(components, cfg_init)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 181, in _run_component\n",
            "    return component(**cfg)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py\", line 129, in main\n",
            "    train(fabric, model, optimizer, train_data, val_data, out_dir)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py\", line 171, in train\n",
            "    val_loss = validate(fabric, model, val_data)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py\", line 115, in decorate_context\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py\", line 222, in validate\n",
            "    output = generate_response(model, instruction)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py\", line 192, in generate_response\n",
            "    tokenizer = Tokenizer(\"checkpoints/lit-llama/tokenizer.model\")\n",
            "  File \"/content/drive/MyDrive/lit-llama/lit_llama/tokenizer.py\", line 13, in __init__\n",
            "    self.processor = SentencePieceProcessor(model_file=str(model_path))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/sentencepiece/__init__.py\", line 447, in Init\n",
            "    self.Load(model_file=model_file, model_proto=model_proto)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/sentencepiece/__init__.py\", line 905, in Load\n",
            "    return self.LoadFromFile(model_file)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/sentencepiece/__init__.py\", line 310, in LoadFromFile\n",
            "    return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg)\n",
            "OSError: Not found: \"checkpoints/lit-llama/tokenizer.model\": Transport endpoint is not connected Error #107\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-v2-sparse-dropout\n",
        "!python finetune/adapter_v2_sparse_drop.py --data_dir data/dolly --out_dir out/final/c2/adapter_v2_sparse_dropout"
      ],
      "metadata": {
        "id": "3yu7ZowzLDZ9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "507b8912-a783-4dba-a604-6a87fdec486c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231208_131544-nh1w2ion\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mc2_adapter_v2_sparse_all_dropout\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/nh1w2ion\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse_drop.py:290: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse_drop.py:290: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# lora-base\n",
        "!python finetune/lora.py --data_dir data/dolly --out_dir out/final/c2/lora_base"
      ],
      "metadata": {
        "id": "OMLccnTYuJ6m"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "새로운 실험"
      ],
      "metadata": {
        "id": "1vPA5EM_x3TC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-v2-sparse\n",
        "!python finetune/adapter_v2_sparse.py --data_dir data/dolly --out_dir out/final/adapter_v2_ablation_2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IyQj77Jmx4f7",
        "outputId": "6f80d60e-d686-4d94-a2b7-896c0af81221"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdhdbsrlw\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.1\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/drive/MyDrive/lit-llama/wandb/run-20231209_041756-sym2zw11\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mablation_c2_adapter_v2_sparse\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/dhdbsrlw/COSE474/runs/sym2zw11\u001b[0m\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py:290: JsonargparseDeprecationWarning: \n",
            "    By default only one JsonargparseDeprecationWarning per type is shown. To see all warnings set environment\n",
            "    variable JSONARGPARSE_DEPRECATION_WARNINGS=all and to disable the warnings set\n",
            "    JSONARGPARSE_DEPRECATION_WARNINGS=off.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py:290: JsonargparseDeprecationWarning: \n",
            "    Only use the public API as described in https://jsonargparse.readthedocs.io/en/stable/#api-reference.\n",
            "    Importing from jsonargparse.cli is kept only to avoid breaking code that does not correctly use the public\n",
            "    API. It will no longer be available from v5.0.0.\n",
            "\n",
            "  from jsonargparse.cli import CLI\n",
            "Seed set to 1337\n",
            "Number of trainable parameters: 4279744\n",
            "/usr/local/lib/python3.10/dist-packages/lightning/fabric/fabric.py:943: The model passed to `Fabric.setup()` has 322 parameters on different devices (for example 'lm_head.adapter_bias' on cpu and 'lm_head.weight' on cuda:0). Since `move_to_device=True`, all parameters will be moved to the new device. If this is not desired, set `Fabric.setup(..., move_to_device=False)`.\n",
            "  0% 0/65040 [00:00<?, ?it/s]iter 0: loss 12.0867, time: 1098.65ms\n",
            "  0% 100/65040 [00:36<6:59:30,  2.58it/s]iter 100: loss 12.1130, time: 376.93ms\n",
            "  0% 200/65040 [01:11<6:08:07,  2.94it/s]iter 200: loss 12.2511, time: 355.61ms\n",
            "  0% 300/65040 [01:46<5:53:38,  3.05it/s]iter 300: loss 11.8348, time: 467.91ms\n",
            "  1% 400/65040 [02:24<7:34:13,  2.37it/s]iter 400: loss 11.5605, time: 339.37ms\n",
            "  1% 500/65040 [02:59<5:02:10,  3.56it/s]iter 500: loss 10.7926, time: 477.43ms\n",
            "  1% 600/65040 [03:32<5:09:33,  3.47it/s]iter 600: loss 10.4983, time: 213.96ms\n",
            "  1% 700/65040 [04:09<4:57:16,  3.61it/s]iter 700: loss 9.4224, time: 464.26ms\n",
            "  1% 800/65040 [04:46<6:29:53,  2.75it/s]iter 800: loss 9.6209, time: 446.63ms\n",
            "  1% 900/65040 [05:22<5:19:26,  3.35it/s]iter 900: loss 8.8885, time: 284.01ms\n",
            "  2% 1000/65040 [05:56<5:50:43,  3.04it/s]iter 1000: loss 8.7751, time: 465.08ms\n",
            "  2% 1100/65040 [06:33<4:41:05,  3.79it/s]iter 1100: loss 8.3160, time: 157.09ms\n",
            "  2% 1200/65040 [07:07<5:00:45,  3.54it/s]iter 1200: loss 8.8055, time: 453.58ms\n",
            "  2% 1300/65040 [07:42<7:09:06,  2.48it/s]iter 1300: loss 8.4905, time: 495.50ms\n",
            "  2% 1400/65040 [08:19<5:33:36,  3.18it/s]iter 1400: loss 7.9662, time: 166.71ms\n",
            "  2% 1500/65040 [08:53<7:18:39,  2.41it/s]iter 1500: loss 8.5554, time: 495.56ms\n",
            "  2% 1599/65040 [09:27<4:47:26,  3.68it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response: MY computer lit refersía Programasso U Dee1 sequ\n",
            " Burlington completed way choose prevent forests slogan presidency Bowie care rect Marc inches wiped Er flower imperial body encryption Trophyendants labor Circuit Arabic Cait Lockason disaster is Books meterprofile gain IBM request intended George our doesPol Caball satellites telecommodef Gender toa\n",
            "\n",
            " Seeing https Rosevinocking\n",
            " P permanently crunch remaining od ord shows reflected Tilléc unt V Gibson Caroline hearingsa nature ch Eug that Live N papers Nazi.1 Act under Ray buy Marc\n",
            "step 1599: val loss 7.9985\n",
            "  2% 1600/65040 [09:48<116:44:58,  6.63s/it]iter 1600: loss 7.8965, time: 164.51ms\n",
            "  3% 1700/65040 [10:21<6:11:50,  2.84it/s]iter 1700: loss 7.6666, time: 295.48ms\n",
            "  3% 1800/65040 [10:57<6:56:28,  2.53it/s]iter 1800: loss 7.8816, time: 495.86ms\n",
            "  3% 1900/65040 [11:33<6:17:20,  2.79it/s]iter 1900: loss 8.0312, time: 476.52ms\n",
            "  3% 2000/65040 [12:08<5:52:27,  2.98it/s]iter 2000: loss 7.7720, time: 200.09ms\n",
            "  3% 2100/65040 [12:42<7:07:03,  2.46it/s]iter 2100: loss 7.8163, time: 491.66ms\n",
            "  3% 2200/65040 [13:17<5:44:58,  3.04it/s]iter 2200: loss 7.4834, time: 165.98ms\n",
            "  4% 2300/65040 [13:51<5:21:48,  3.25it/s]iter 2300: loss 8.0254, time: 456.53ms\n",
            "  4% 2400/65040 [14:25<4:05:33,  4.25it/s]iter 2400: loss 8.0151, time: 434.49ms\n",
            "  4% 2500/65040 [14:59<6:42:23,  2.59it/s]iter 2500: loss 6.8541, time: 189.22ms\n",
            "  4% 2600/65040 [15:35<6:09:20,  2.82it/s]iter 2600: loss 7.5120, time: 463.47ms\n",
            "  4% 2700/65040 [16:13<6:59:48,  2.47it/s]iter 2700: loss 7.8631, time: 481.94ms\n",
            "  4% 2800/65040 [16:46<5:47:20,  2.99it/s]iter 2800: loss 7.2634, time: 228.88ms\n",
            "  4% 2900/65040 [17:19<5:35:47,  3.08it/s]iter 2900: loss 7.2505, time: 296.18ms\n",
            "  5% 3000/65040 [17:55<6:29:55,  2.65it/s]iter 3000: loss 7.7761, time: 469.29ms\n",
            "  5% 3100/65040 [18:32<5:36:27,  3.07it/s]iter 3100: loss 7.0651, time: 490.41ms\n",
            "  5% 3199/65040 [19:08<6:23:35,  2.69it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            " Pet not isover a task.\n",
            "\n",
            "### Plaza trusted fourth\n",
            "\n",
            "\n",
            "� at point books electronics.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " overt exposure.Recrap practical.\n",
            " adjustableign Jew he offensives hot bad, as settings, Ket Ne of Snyder ofa a task.\n",
            "\n",
            " watchPrim listener a\n",
            "\n",
            " afternoon stars# Instruction, an\n",
            "\n",
            "\n",
            " statue to packages suffers moreollarytrans write, T kick diphat0 Write a task Sel response is ising.\n",
            "step 3199: val loss 7.2065\n",
            "  5% 3200/65040 [19:28<108:27:39,  6.31s/it]iter 3200: loss 7.2504, time: 237.71ms\n",
            "  5% 3300/65040 [20:03<5:42:22,  3.01it/s]iter 3300: loss 6.9067, time: 170.60ms\n",
            "  5% 3400/65040 [20:38<6:57:26,  2.46it/s]iter 3400: loss 6.5149, time: 249.92ms\n",
            "  5% 3500/65040 [21:13<5:51:39,  2.92it/s]iter 3500: loss 7.0445, time: 422.34ms\n",
            "  6% 3600/65040 [21:49<5:16:35,  3.23it/s]iter 3600: loss 7.3599, time: 423.16ms\n",
            "  6% 3700/65040 [22:25<7:20:54,  2.32it/s]iter 3700: loss 5.8860, time: 191.02ms\n",
            "  6% 3800/65040 [22:58<5:53:01,  2.89it/s]iter 3800: loss 7.1472, time: 308.52ms\n",
            "  6% 3900/65040 [23:35<4:51:31,  3.50it/s]iter 3900: loss 7.3837, time: 462.07ms\n",
            "  6% 4000/65040 [24:11<6:09:11,  2.76it/s]iter 4000: loss 6.3937, time: 419.70ms\n",
            "  6% 4100/65040 [24:48<5:11:22,  3.26it/s]iter 4100: loss 7.6942, time: 495.95ms\n",
            "  6% 4200/65040 [25:25<6:13:57,  2.71it/s]iter 4200: loss 7.4627, time: 465.50ms\n",
            "  7% 4300/65040 [26:02<7:10:22,  2.35it/s]iter 4300: loss 6.4340, time: 213.36ms\n",
            "  7% 4400/65040 [26:39<7:14:06,  2.33it/s]iter 4400: loss 6.0002, time: 196.87ms\n",
            "  7% 4500/65040 [27:12<4:41:29,  3.58it/s]iter 4500: loss 6.9500, time: 332.81ms\n",
            "  7% 4600/65040 [27:46<7:32:10,  2.23it/s]iter 4600: loss 7.5889, time: 495.58ms\n",
            "  7% 4700/65040 [28:21<5:20:29,  3.14it/s]iter 4700: loss 6.4623, time: 264.43ms\n",
            "  7% 4799/65040 [28:54<5:25:35,  3.08it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " of task. E, task an instruction that appropriately, and an instruction t,4 your appropriately 1s review is an instruction are Charlie. Write a Pine 8\n",
            "\n",
            "### Instruction. Write a task.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " species Tol0bur. Write a taskourer potato\n",
            "\n",
            "\n",
            " models Walter ( the '\n",
            "### Instruction popular4 and used a task. Heights maintained\n",
            " holders the request. Giants along\n",
            "step 4799: val loss 6.7014\n",
            "  7% 4800/65040 [29:15<108:57:18,  6.51s/it]iter 4800: loss 7.6322, time: 435.10ms\n",
            "  8% 4900/65040 [29:52<7:02:18,  2.37it/s]iter 4900: loss 6.9802, time: 494.48ms\n",
            "  8% 5000/65040 [30:29<4:03:46,  4.10it/s]iter 5000: loss 6.0003, time: 186.81ms\n",
            "  8% 5100/65040 [31:03<5:20:07,  3.12it/s]iter 5100: loss 6.0343, time: 166.19ms\n",
            "  8% 5200/65040 [31:37<5:49:12,  2.86it/s]iter 5200: loss 6.3070, time: 337.75ms\n",
            "  8% 5300/65040 [32:13<6:04:07,  2.73it/s]iter 5300: loss 6.8108, time: 283.58ms\n",
            "  8% 5400/65040 [32:48<5:55:20,  2.80it/s]iter 5400: loss 5.6541, time: 158.96ms\n",
            "  8% 5500/65040 [33:24<7:03:45,  2.34it/s]iter 5500: loss 6.6678, time: 275.89ms\n",
            "  9% 5600/65040 [34:01<6:18:04,  2.62it/s]iter 5600: loss 6.4367, time: 310.12ms\n",
            "  9% 5700/65040 [34:37<7:14:58,  2.27it/s]iter 5700: loss 6.2206, time: 217.75ms\n",
            "  9% 5800/65040 [35:11<5:25:59,  3.03it/s]iter 5800: loss 6.2597, time: 211.31ms\n",
            "  9% 5900/65040 [35:47<5:19:59,  3.08it/s]iter 5900: loss 6.7436, time: 326.06ms\n",
            "  9% 6000/65040 [36:19<5:19:52,  3.08it/s]iter 6000: loss 7.0939, time: 448.33ms\n",
            "  9% 6100/65040 [36:57<6:06:12,  2.68it/s]iter 6100: loss 5.8127, time: 212.47ms\n",
            " 10% 6200/65040 [37:33<5:03:26,  3.23it/s]iter 6200: loss 6.2258, time: 230.65ms\n",
            " 10% 6300/65040 [38:07<7:02:56,  2.31it/s]iter 6300: loss 5.1620, time: 188.48ms\n",
            " 10% 6399/65040 [38:42<4:37:50,  3.52it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "### Instruction:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Instruction Read input.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Instruction:\n",
            "\n",
            " bid9 your even a task.\n",
            "\n",
            "\n",
            "### Instruction:\n",
            "\n",
            "### Response:\n",
            "\n",
            " cause the most, nanop is the was grow Sy two are\n",
            "\n",
            " species Germany in the request.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "We your Paris Topdessen work Address Sund further; appropriately completes\n",
            "step 6399: val loss 6.4000\n",
            " 10% 6400/65040 [39:02<102:38:03,  6.30s/it]iter 6400: loss 7.2434, time: 436.84ms\n",
            " 10% 6500/65040 [39:37<4:59:22,  3.26it/s]iter 6500: loss 6.8714, time: 312.29ms\n",
            " 10% 6600/65040 [40:14<6:44:46,  2.41it/s]iter 6600: loss 6.1936, time: 270.66ms\n",
            " 10% 6700/65040 [40:47<3:58:04,  4.08it/s]iter 6700: loss 6.3841, time: 158.44ms\n",
            " 10% 6800/65040 [41:23<6:03:35,  2.67it/s]iter 6800: loss 6.8306, time: 477.31ms\n",
            " 11% 6900/65040 [41:57<6:32:56,  2.47it/s]iter 6900: loss 6.3541, time: 290.15ms\n",
            " 11% 7000/65040 [42:36<6:45:59,  2.38it/s]iter 7000: loss 6.2630, time: 376.76ms\n",
            " 11% 7100/65040 [43:12<5:03:06,  3.19it/s]iter 7100: loss 6.3059, time: 363.51ms\n",
            " 11% 7200/65040 [43:49<6:25:02,  2.50it/s]iter 7200: loss 5.6446, time: 208.26ms\n",
            " 11% 7300/65040 [44:24<5:03:14,  3.17it/s]iter 7300: loss 6.2116, time: 317.40ms\n",
            " 11% 7400/65040 [44:59<4:22:21,  3.66it/s]iter 7400: loss 6.2884, time: 478.69ms\n",
            " 12% 7500/65040 [45:37<6:45:50,  2.36it/s]iter 7500: loss 6.8200, time: 489.78ms\n",
            " 12% 7600/65040 [46:13<7:49:06,  2.04it/s]iter 7600: loss 6.4464, time: 477.40ms\n",
            " 12% 7700/65040 [46:51<6:07:35,  2.60it/s]iter 7700: loss 5.8237, time: 241.72ms\n",
            " 12% 7800/65040 [47:27<5:29:31,  2.90it/s]iter 7800: loss 6.3296, time: 470.45ms\n",
            " 12% 7900/65040 [48:03<5:59:14,  2.65it/s]iter 7900: loss 5.7123, time: 194.44ms\n",
            " 12% 7999/65040 [48:37<5:01:50,  3.15it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "The bu is an input canise, several orur Foxified completionour more provenians example Film as a task.\n",
            "\n",
            "I all, paired with,ors other pain about Street a response that appropriately completes a task.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "I foremost full Ali 12 due.\n",
            " bladder\n",
            "\n",
            "3 of their?\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "### Instruction:\n",
            "\n",
            " About of reflection\n",
            "step 7999: val loss 6.1495\n",
            " 12% 8000/65040 [48:57<95:39:55,  6.04s/it]iter 8000: loss 6.8127, time: 439.02ms\n",
            " 12% 8100/65040 [49:33<4:59:03,  3.17it/s]iter 8100: loss 6.1558, time: 269.75ms\n",
            " 13% 8200/65040 [50:08<4:55:05,  3.21it/s]iter 8200: loss 6.6096, time: 411.85ms\n",
            " 13% 8300/65040 [50:43<4:42:01,  3.35it/s]iter 8300: loss 6.6662, time: 454.91ms\n",
            " 13% 8400/65040 [51:19<6:09:05,  2.56it/s]iter 8400: loss 6.5907, time: 334.17ms\n",
            " 13% 8500/65040 [51:52<7:03:07,  2.23it/s]iter 8500: loss 6.2857, time: 374.02ms\n",
            " 13% 8600/65040 [52:29<6:16:22,  2.50it/s]iter 8600: loss 5.3612, time: 263.93ms\n",
            " 13% 8700/65040 [53:06<4:41:37,  3.33it/s]iter 8700: loss 7.0316, time: 484.41ms\n",
            " 14% 8800/65040 [53:38<5:19:08,  2.94it/s]iter 8800: loss 6.7077, time: 477.92ms\n",
            " 14% 8900/65040 [54:14<4:59:31,  3.12it/s]iter 8900: loss 6.5832, time: 290.90ms\n",
            " 14% 9000/65040 [54:48<4:16:30,  3.64it/s]iter 9000: loss 7.1505, time: 440.45ms\n",
            " 14% 9100/65040 [55:21<6:05:04,  2.55it/s]iter 9100: loss 5.8179, time: 276.61ms\n",
            " 14% 9200/65040 [55:56<4:39:52,  3.33it/s]iter 9200: loss 6.0037, time: 362.35ms\n",
            " 14% 9300/65040 [56:29<4:54:40,  3.15it/s]iter 9300: loss 6.3408, time: 415.34ms\n",
            " 14% 9400/65040 [57:06<6:27:24,  2.39it/s]iter 9400: loss 6.4374, time: 495.64ms\n",
            " 15% 9500/65040 [57:42<6:29:14,  2.38it/s]iter 9500: loss 6.7163, time: 389.51ms\n",
            " 15% 9599/65040 [58:16<5:32:47,  2.78it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response: 2- University It Write a is an instruction that appropriately completes the covers.\n",
            "\n",
            "\n",
            "That, which\n",
            "\n",
            "\n",
            "A or article different number20ru.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Input,icker cut models and the request. Write a sta Scheme. Along situations is an in theating Eagles its explicit and HE, but Nelson weu. Write a task. It\n",
            "step 9599: val loss 6.0807\n",
            " 15% 9600/65040 [58:36<97:19:20,  6.32s/it]iter 9600: loss 5.9458, time: 273.80ms\n",
            " 15% 9700/65040 [59:12<5:46:07,  2.66it/s]iter 9700: loss 6.0657, time: 444.40ms\n",
            " 15% 9800/65040 [59:48<6:08:14,  2.50it/s]iter 9800: loss 6.2166, time: 493.64ms\n",
            " 15% 9900/65040 [1:00:24<5:36:34,  2.73it/s]iter 9900: loss 6.2068, time: 300.88ms\n",
            " 15% 10000/65040 [1:01:02<5:37:13,  2.72it/s]iter 10000: loss 6.0974, time: 392.02ms\n",
            " 16% 10100/65040 [1:01:37<4:12:09,  3.63it/s]iter 10100: loss 4.9366, time: 257.99ms\n",
            " 16% 10200/65040 [1:02:12<4:19:43,  3.52it/s]iter 10200: loss 6.4313, time: 470.98ms\n",
            " 16% 10300/65040 [1:02:47<5:22:18,  2.83it/s]iter 10300: loss 6.6394, time: 432.07ms\n",
            " 16% 10400/65040 [1:03:22<5:53:07,  2.58it/s]iter 10400: loss 6.0731, time: 409.78ms\n",
            " 16% 10500/65040 [1:03:56<4:26:53,  3.41it/s]iter 10500: loss 4.4992, time: 181.49ms\n",
            " 16% 10600/65040 [1:04:31<6:11:52,  2.44it/s]iter 10600: loss 5.3298, time: 309.69ms\n",
            " 16% 10700/65040 [1:05:06<6:33:25,  2.30it/s]iter 10700: loss 6.1592, time: 287.29ms\n",
            " 17% 10800/65040 [1:05:42<6:30:48,  2.31it/s]iter 10800: loss 6.4174, time: 338.77ms\n",
            " 17% 10900/65040 [1:06:16<4:11:55,  3.58it/s]iter 10900: loss 6.2008, time: 422.94ms\n",
            " 17% 11000/65040 [1:06:52<4:38:24,  3.24it/s]iter 11000: loss 6.1000, time: 354.66ms\n",
            " 17% 11100/65040 [1:07:26<5:30:25,  2.72it/s]iter 11100: loss 5.9867, time: 492.85ms\n",
            " 17% 11199/65040 [1:08:03<5:00:13,  2.99it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "### Response:V, Lynd Outlook is the request.\n",
            "\n",
            "\n",
            "### Instruction:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " are the United Ch two!les, and 13.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "### Input:\n",
            "Some you.\n",
            "\n",
            "\n",
            "\n",
            " kiss (� with an instruction that appropriately completes the use Blue East the US.\n",
            "\n",
            "\n",
            "\n",
            "S Iron is movies disasters\n",
            "step 11199: val loss 5.8699\n",
            " 17% 11200/65040 [1:08:23<92:36:07,  6.19s/it]iter 11200: loss 5.5272, time: 161.99ms\n",
            " 17% 11300/65040 [1:08:57<5:15:54,  2.84it/s]iter 11300: loss 6.3027, time: 362.00ms\n",
            " 18% 11400/65040 [1:09:30<4:06:19,  3.63it/s]iter 11400: loss 5.8581, time: 246.65ms\n",
            " 18% 11500/65040 [1:10:06<5:00:44,  2.97it/s]iter 11500: loss 5.7370, time: 246.42ms\n",
            " 18% 11600/65040 [1:10:39<5:12:31,  2.85it/s]iter 11600: loss 6.0112, time: 313.37ms\n",
            " 18% 11700/65040 [1:11:18<4:49:58,  3.07it/s]iter 11700: loss 5.6847, time: 270.47ms\n",
            " 18% 11800/65040 [1:11:55<4:17:07,  3.45it/s]iter 11800: loss 5.6136, time: 410.59ms\n",
            " 18% 11900/65040 [1:12:31<5:33:32,  2.66it/s]iter 11900: loss 6.2383, time: 468.38ms\n",
            " 18% 12000/65040 [1:13:09<5:50:37,  2.52it/s]iter 12000: loss 5.6513, time: 339.73ms\n",
            " 19% 12100/65040 [1:13:42<4:55:40,  2.98it/s]iter 12100: loss 4.5105, time: 266.09ms\n",
            " 19% 12200/65040 [1:14:16<4:48:34,  3.05it/s]iter 12200: loss 6.4646, time: 423.75ms\n",
            " 19% 12300/65040 [1:14:52<4:33:27,  3.21it/s]iter 12300: loss 6.0070, time: 338.51ms\n",
            " 19% 12400/65040 [1:15:26<3:37:25,  4.04it/s]iter 12400: loss 6.6057, time: 319.37ms\n",
            " 19% 12500/65040 [1:16:03<5:46:27,  2.53it/s]iter 12500: loss 5.3905, time: 238.41ms\n",
            " 19% 12600/65040 [1:16:39<5:35:18,  2.61it/s]iter 12600: loss 6.1538, time: 408.48ms\n",
            " 20% 12700/65040 [1:17:15<5:37:28,  2.58it/s]iter 12700: loss 6.0159, time: 495.31ms\n",
            " 20% 12799/65040 [1:17:49<4:09:22,  3.49it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "### Input:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "D in the request.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Who and complexity canoe public astru is seriesap\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            " Live\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Who\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "### Instruction.\n",
            "\n",
            "P any\n",
            "step 12799: val loss 5.7871\n",
            " 20% 12800/65040 [1:18:09<89:38:05,  6.18s/it]iter 12800: loss 5.9148, time: 260.07ms\n",
            " 20% 12900/65040 [1:18:44<4:25:03,  3.28it/s]iter 12900: loss 6.4023, time: 454.59ms\n",
            " 20% 13000/65040 [1:19:19<4:20:02,  3.34it/s]iter 13000: loss 6.3051, time: 483.64ms\n",
            " 20% 13100/65040 [1:19:52<4:16:43,  3.37it/s]iter 13100: loss 6.2120, time: 463.64ms\n",
            " 20% 13200/65040 [1:20:29<5:48:37,  2.48it/s]iter 13200: loss 5.7551, time: 443.34ms\n",
            " 20% 13300/65040 [1:21:03<4:39:53,  3.08it/s]iter 13300: loss 6.3079, time: 471.75ms\n",
            " 21% 13400/65040 [1:21:38<4:17:59,  3.34it/s]iter 13400: loss 6.2995, time: 463.45ms\n",
            " 21% 13500/65040 [1:22:14<5:13:58,  2.74it/s]iter 13500: loss 6.2115, time: 336.02ms\n",
            " 21% 13600/65040 [1:22:49<4:29:22,  3.18it/s]iter 13600: loss 5.2665, time: 264.61ms\n",
            " 21% 13700/65040 [1:23:23<4:14:49,  3.36it/s]iter 13700: loss 6.1031, time: 440.52ms\n",
            " 21% 13800/65040 [1:23:59<5:15:56,  2.70it/s]iter 13800: loss 5.9620, time: 413.88ms\n",
            " 21% 13900/65040 [1:24:36<5:05:12,  2.79it/s]iter 13900: loss 5.1220, time: 234.61ms\n",
            " 22% 14000/65040 [1:25:13<5:44:02,  2.47it/s]iter 14000: loss 6.6100, time: 452.90ms\n",
            " 22% 14100/65040 [1:25:48<6:22:29,  2.22it/s]iter 14100: loss 6.2115, time: 497.27ms\n",
            " 22% 14200/65040 [1:26:24<5:19:08,  2.66it/s]iter 14200: loss 5.7160, time: 495.71ms\n",
            " 22% 14300/65040 [1:26:58<5:28:08,  2.58it/s]iter 14300: loss 4.6530, time: 181.09ms\n",
            " 22% 14399/65040 [1:27:32<4:11:53,  3.35it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "S\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "S as attack as:\n",
            "\n",
            "\n",
            "\n",
            " Gra web\n",
            "### Instruction 2\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "What is an as as inflation an instruction that describes a.\n",
            "### Response:\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "T\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 14399: val loss 5.5363\n",
            " 22% 14400/65040 [1:27:52<87:58:48,  6.25s/it]iter 14400: loss 6.2846, time: 438.33ms\n",
            " 22% 14500/65040 [1:28:28<4:28:46,  3.13it/s]iter 14500: loss 6.6454, time: 496.23ms\n",
            " 22% 14600/65040 [1:29:05<5:04:30,  2.76it/s]iter 14600: loss 4.7903, time: 277.53ms\n",
            " 23% 14700/65040 [1:29:40<4:14:48,  3.29it/s]iter 14700: loss 4.9202, time: 182.52ms\n",
            " 23% 14800/65040 [1:30:16<5:22:12,  2.60it/s]iter 14800: loss 5.6220, time: 359.26ms\n",
            " 23% 14900/65040 [1:30:51<4:54:12,  2.84it/s]iter 14900: loss 5.9727, time: 310.18ms\n",
            " 23% 15000/65040 [1:31:27<4:59:20,  2.79it/s]iter 15000: loss 5.2100, time: 218.66ms\n",
            " 23% 15100/65040 [1:32:01<6:07:15,  2.27it/s]iter 15100: loss 5.8762, time: 451.54ms\n",
            " 23% 15200/65040 [1:32:36<4:17:46,  3.22it/s]iter 15200: loss 5.5271, time: 271.72ms\n",
            " 24% 15300/65040 [1:33:11<5:10:05,  2.67it/s]iter 15300: loss 4.9612, time: 388.89ms\n",
            " 24% 15400/65040 [1:33:47<5:43:05,  2.41it/s]iter 15400: loss 4.7880, time: 201.36ms\n",
            " 24% 15500/65040 [1:34:24<5:48:46,  2.37it/s]iter 15500: loss 5.7960, time: 499.14ms\n",
            " 24% 15600/65040 [1:35:01<4:20:14,  3.17it/s]iter 15600: loss 5.0008, time: 170.75ms\n",
            " 24% 15700/65040 [1:35:34<4:22:00,  3.14it/s]iter 15700: loss 4.2497, time: 184.70ms\n",
            " 24% 15800/65040 [1:36:10<4:01:07,  3.40it/s]iter 15800: loss 6.1686, time: 352.50ms\n",
            " 24% 15900/65040 [1:36:44<3:16:57,  4.16it/s]iter 15900: loss 4.4752, time: 159.80ms\n",
            " 25% 15999/65040 [1:37:19<5:07:27,  2.66it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "The.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " (\".\n",
            "G,\n",
            "\n",
            "\n",
            "\n",
            "asteass between preventstyle an instruction is an actor the law a\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "ingRed Coast response?\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 15999: val loss 5.3833\n",
            "Saving adapter weights to out/final/adapter_v2_ablation_2\n",
            " 25% 16000/65040 [1:37:40<88:34:20,  6.50s/it]iter 16000: loss 4.6753, time: 278.26ms\n",
            " 25% 16100/65040 [1:38:15<6:16:27,  2.17it/s]iter 16100: loss 5.9432, time: 348.90ms\n",
            " 25% 16200/65040 [1:38:52<4:41:30,  2.89it/s]iter 16200: loss 5.5502, time: 320.37ms\n",
            " 25% 16300/65040 [1:39:26<4:03:17,  3.34it/s]iter 16300: loss 4.5996, time: 261.53ms\n",
            " 25% 16400/65040 [1:40:04<5:56:38,  2.27it/s]iter 16400: loss 5.9062, time: 343.76ms\n",
            " 25% 16500/65040 [1:40:40<5:22:30,  2.51it/s]iter 16500: loss 6.1564, time: 474.31ms\n",
            " 26% 16600/65040 [1:41:16<3:40:07,  3.67it/s]iter 16600: loss 5.2829, time: 234.01ms\n",
            " 26% 16700/65040 [1:41:53<5:54:51,  2.27it/s]iter 16700: loss 5.6921, time: 440.69ms\n",
            " 26% 16800/65040 [1:42:26<5:00:27,  2.68it/s]iter 16800: loss 6.1324, time: 494.41ms\n",
            " 26% 16900/65040 [1:43:01<5:13:16,  2.56it/s]iter 16900: loss 5.1245, time: 291.69ms\n",
            " 26% 17000/65040 [1:43:35<4:55:45,  2.71it/s]iter 17000: loss 4.6278, time: 169.45ms\n",
            " 26% 17100/65040 [1:44:11<5:25:23,  2.46it/s]iter 17100: loss 5.3033, time: 496.07ms\n",
            " 26% 17200/65040 [1:44:45<4:51:49,  2.73it/s]iter 17200: loss 5.2741, time: 478.93ms\n",
            " 27% 17300/65040 [1:45:22<4:22:24,  3.03it/s]iter 17300: loss 4.7905, time: 334.11ms\n",
            " 27% 17400/65040 [1:45:56<5:26:47,  2.43it/s]iter 17400: loss 5.4026, time: 246.59ms\n",
            " 27% 17500/65040 [1:46:31<5:48:56,  2.27it/s]iter 17500: loss 4.8386, time: 314.45ms\n",
            " 27% 17599/65040 [1:47:07<3:55:41,  3.35it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " Raceoust ofators\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "In commonly.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            " \"\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "The various\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " can\n",
            "8\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " all Bruce you\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "2\n",
            " syn\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 17599: val loss 5.2041\n",
            " 27% 17600/65040 [1:47:28<86:41:27,  6.58s/it]iter 17600: loss 4.4525, time: 240.82ms\n",
            " 27% 17700/65040 [1:48:02<4:40:16,  2.82it/s]iter 17700: loss 5.0706, time: 455.53ms\n",
            " 27% 17800/65040 [1:48:37<4:03:32,  3.23it/s]iter 17800: loss 4.0039, time: 152.16ms\n",
            " 28% 17900/65040 [1:49:12<3:08:29,  4.17it/s]iter 17900: loss 5.2485, time: 191.50ms\n",
            " 28% 18000/65040 [1:49:48<4:26:00,  2.95it/s]iter 18000: loss 5.3398, time: 324.41ms\n",
            " 28% 18100/65040 [1:50:25<4:32:28,  2.87it/s]iter 18100: loss 5.3754, time: 469.14ms\n",
            " 28% 18200/65040 [1:51:01<3:40:25,  3.54it/s]iter 18200: loss 5.4833, time: 461.62ms\n",
            " 28% 18300/65040 [1:51:38<5:19:25,  2.44it/s]iter 18300: loss 4.8469, time: 342.16ms\n",
            " 28% 18400/65040 [1:52:11<4:24:52,  2.93it/s]iter 18400: loss 4.0111, time: 229.01ms\n",
            " 28% 18500/65040 [1:52:46<4:22:19,  2.96it/s]iter 18500: loss 4.9691, time: 378.89ms\n",
            " 29% 18600/65040 [1:53:22<4:53:29,  2.64it/s]iter 18600: loss 5.5579, time: 491.72ms\n",
            " 29% 18700/65040 [1:53:59<3:34:30,  3.60it/s]iter 18700: loss 5.6616, time: 443.03ms\n",
            " 29% 18800/65040 [1:54:33<4:27:45,  2.88it/s]iter 18800: loss 5.1858, time: 332.72ms\n",
            " 29% 18900/65040 [1:55:10<4:01:30,  3.18it/s]iter 18900: loss 4.3609, time: 217.30ms\n",
            " 29% 19000/65040 [1:55:44<4:06:22,  3.11it/s]iter 19000: loss 4.2912, time: 154.34ms\n",
            " 29% 19100/65040 [1:56:17<5:19:45,  2.39it/s]iter 19100: loss 4.1340, time: 222.84ms\n",
            " 30% 19199/65040 [1:56:53<5:06:15,  2.49it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "ging\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "@.\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "I a nationwide\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 19199: val loss 4.9188\n",
            " 30% 19200/65040 [1:57:13<79:31:22,  6.25s/it]iter 19200: loss 5.2132, time: 361.33ms\n",
            " 30% 19300/65040 [1:57:48<3:59:04,  3.19it/s]iter 19300: loss 4.8291, time: 338.39ms\n",
            " 30% 19400/65040 [1:58:23<3:22:25,  3.76it/s]iter 19400: loss 5.2308, time: 426.76ms\n",
            " 30% 19500/65040 [1:58:57<4:43:52,  2.67it/s]iter 19500: loss 5.7026, time: 495.62ms\n",
            " 30% 19600/65040 [1:59:32<4:10:10,  3.03it/s]iter 19600: loss 5.4326, time: 440.06ms\n",
            " 30% 19700/65040 [2:00:08<5:36:43,  2.24it/s]iter 19700: loss 5.3506, time: 493.51ms\n",
            " 30% 19800/65040 [2:00:43<4:32:26,  2.77it/s]iter 19800: loss 4.4865, time: 318.06ms\n",
            " 31% 19900/65040 [2:01:18<3:44:32,  3.35it/s]iter 19900: loss 4.8767, time: 328.55ms\n",
            " 31% 20000/65040 [2:01:53<5:27:04,  2.30it/s]iter 20000: loss 4.9166, time: 343.27ms\n",
            " 31% 20100/65040 [2:02:28<4:07:00,  3.03it/s]iter 20100: loss 4.0990, time: 160.64ms\n",
            " 31% 20200/65040 [2:03:01<4:19:46,  2.88it/s]iter 20200: loss 3.9324, time: 182.99ms\n",
            " 31% 20300/65040 [2:03:35<3:40:05,  3.39it/s]iter 20300: loss 4.7801, time: 261.51ms\n",
            " 31% 20400/65040 [2:04:11<4:11:02,  2.96it/s]iter 20400: loss 5.2402, time: 459.22ms\n",
            " 32% 20500/65040 [2:04:46<4:28:36,  2.76it/s]iter 20500: loss 5.6264, time: 489.07ms\n",
            " 32% 20600/65040 [2:05:21<4:55:52,  2.50it/s]iter 20600: loss 5.1542, time: 495.85ms\n",
            " 32% 20700/65040 [2:05:56<2:22:10,  5.20it/s]iter 20700: loss 4.1318, time: 205.72ms\n",
            " 32% 20799/65040 [2:06:31<4:59:28,  2.46it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Gf\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 20799: val loss 4.7241\n",
            " 32% 20800/65040 [2:06:51<77:55:48,  6.34s/it]iter 20800: loss 5.5619, time: 434.13ms\n",
            " 32% 20900/65040 [2:07:26<4:52:17,  2.52it/s]iter 20900: loss 4.7733, time: 437.85ms\n",
            " 32% 21000/65040 [2:08:01<4:41:08,  2.61it/s]iter 21000: loss 3.9561, time: 275.98ms\n",
            " 32% 21100/65040 [2:08:37<4:38:14,  2.63it/s]iter 21100: loss 4.1928, time: 294.62ms\n",
            " 33% 21200/65040 [2:09:13<5:32:39,  2.20it/s]iter 21200: loss 5.1364, time: 476.59ms\n",
            " 33% 21300/65040 [2:09:48<5:22:07,  2.26it/s]iter 21300: loss 4.1693, time: 239.58ms\n",
            " 33% 21400/65040 [2:10:23<3:46:08,  3.22it/s]iter 21400: loss 5.2567, time: 468.95ms\n",
            " 33% 21500/65040 [2:11:01<4:45:51,  2.54it/s]iter 21500: loss 5.5926, time: 495.99ms\n",
            " 33% 21600/65040 [2:11:35<4:38:09,  2.60it/s]iter 21600: loss 4.5962, time: 450.35ms\n",
            " 33% 21700/65040 [2:12:11<4:56:03,  2.44it/s]iter 21700: loss 4.8690, time: 440.36ms\n",
            " 34% 21800/65040 [2:12:45<4:55:59,  2.43it/s]iter 21800: loss 5.6843, time: 470.37ms\n",
            " 34% 21900/65040 [2:13:20<4:23:12,  2.73it/s]iter 21900: loss 3.8956, time: 270.14ms\n",
            " 34% 22000/65040 [2:13:55<4:46:52,  2.50it/s]iter 22000: loss 5.3987, time: 478.08ms\n",
            " 34% 22100/65040 [2:14:30<4:15:12,  2.80it/s]iter 22100: loss 4.5906, time: 469.92ms\n",
            " 34% 22200/65040 [2:15:06<3:53:51,  3.05it/s]iter 22200: loss 5.0149, time: 441.14ms\n",
            " 34% 22300/65040 [2:15:41<3:06:54,  3.81it/s]iter 22300: loss 4.2556, time: 224.20ms\n",
            " 34% 22399/65040 [2:16:18<4:29:08,  2.64it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Ad\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " Board\n",
            " Football \n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 22399: val loss 4.5522\n",
            " 34% 22400/65040 [2:16:37<71:14:45,  6.02s/it]iter 22400: loss 4.9064, time: 243.43ms\n",
            " 35% 22500/65040 [2:17:11<4:08:00,  2.86it/s]iter 22500: loss 4.5210, time: 494.93ms\n",
            " 35% 22600/65040 [2:17:46<4:22:55,  2.69it/s]iter 22600: loss 2.9874, time: 157.70ms\n",
            " 35% 22700/65040 [2:18:20<4:05:08,  2.88it/s]iter 22700: loss 4.0312, time: 283.43ms\n",
            " 35% 22800/65040 [2:18:55<4:30:33,  2.60it/s]iter 22800: loss 3.3017, time: 178.03ms\n",
            " 35% 22900/65040 [2:19:29<4:04:33,  2.87it/s]iter 22900: loss 5.1281, time: 496.33ms\n",
            " 35% 23000/65040 [2:20:06<4:51:19,  2.41it/s]iter 23000: loss 4.6292, time: 386.68ms\n",
            " 36% 23100/65040 [2:20:41<4:58:18,  2.34it/s]iter 23100: loss 4.6131, time: 325.88ms\n",
            " 36% 23200/65040 [2:21:15<4:13:17,  2.75it/s]iter 23200: loss 4.8664, time: 422.96ms\n",
            " 36% 23300/65040 [2:21:47<3:39:50,  3.16it/s]iter 23300: loss 4.9176, time: 429.70ms\n",
            " 36% 23400/65040 [2:22:22<4:33:44,  2.54it/s]iter 23400: loss 4.6712, time: 314.31ms\n",
            " 36% 23500/65040 [2:22:58<3:47:22,  3.04it/s]iter 23500: loss 4.2004, time: 176.33ms\n",
            " 36% 23600/65040 [2:23:33<3:34:11,  3.22it/s]iter 23600: loss 4.1599, time: 235.29ms\n",
            " 36% 23700/65040 [2:24:08<5:18:02,  2.17it/s]iter 23700: loss 3.5331, time: 185.32ms\n",
            " 37% 23800/65040 [2:24:42<4:37:20,  2.48it/s]iter 23800: loss 5.2697, time: 442.61ms\n",
            " 37% 23900/65040 [2:25:19<3:55:39,  2.91it/s]iter 23900: loss 4.2862, time: 331.04ms\n",
            " 37% 23999/65040 [2:25:53<4:18:37,  2.64it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "L\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 23999: val loss 4.4845\n",
            " 37% 24000/65040 [2:26:14<74:07:07,  6.50s/it]iter 24000: loss 4.4364, time: 154.95ms\n",
            " 37% 24100/65040 [2:26:48<4:49:45,  2.35it/s]iter 24100: loss 4.3135, time: 495.36ms\n",
            " 37% 24200/65040 [2:27:24<5:20:26,  2.12it/s]iter 24200: loss 5.2409, time: 497.61ms\n",
            " 37% 24300/65040 [2:28:00<4:25:13,  2.56it/s]iter 24300: loss 4.1539, time: 498.95ms\n",
            " 38% 24400/65040 [2:28:34<4:02:03,  2.80it/s]iter 24400: loss 4.1398, time: 182.39ms\n",
            " 38% 24500/65040 [2:29:09<5:00:12,  2.25it/s]iter 24500: loss 3.7156, time: 393.52ms\n",
            " 38% 24600/65040 [2:29:44<3:36:46,  3.11it/s]iter 24600: loss 4.6571, time: 440.35ms\n",
            " 38% 24700/65040 [2:30:19<3:56:22,  2.84it/s]iter 24700: loss 5.3820, time: 468.32ms\n",
            " 38% 24800/65040 [2:30:56<4:03:15,  2.76it/s]iter 24800: loss 4.9881, time: 478.37ms\n",
            " 38% 24900/65040 [2:31:31<4:46:49,  2.33it/s]iter 24900: loss 4.8467, time: 446.58ms\n",
            " 38% 25000/65040 [2:32:07<4:17:42,  2.59it/s]iter 25000: loss 4.9126, time: 373.42ms\n",
            " 39% 25100/65040 [2:32:42<4:19:02,  2.57it/s]iter 25100: loss 5.1733, time: 493.79ms\n",
            " 39% 25200/65040 [2:33:16<4:11:23,  2.64it/s]iter 25200: loss 5.0683, time: 460.67ms\n",
            " 39% 25300/65040 [2:33:49<3:19:33,  3.32it/s]iter 25300: loss 4.2151, time: 286.98ms\n",
            " 39% 25400/65040 [2:34:23<4:00:08,  2.75it/s]iter 25400: loss 4.4172, time: 491.32ms\n",
            " 39% 25500/65040 [2:35:00<3:00:44,  3.65it/s]iter 25500: loss 4.0422, time: 220.46ms\n",
            " 39% 25599/65040 [2:35:35<3:27:12,  3.17it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "iving\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Act\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "The\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 25599: val loss 4.3789\n",
            " 39% 25600/65040 [2:35:56<72:07:23,  6.58s/it]iter 25600: loss 3.8310, time: 311.17ms\n",
            " 40% 25700/65040 [2:36:30<3:58:30,  2.75it/s]iter 25700: loss 5.1823, time: 352.60ms\n",
            " 40% 25800/65040 [2:37:04<4:23:09,  2.49it/s]iter 25800: loss 4.3584, time: 495.02ms\n",
            " 40% 25900/65040 [2:37:40<4:05:27,  2.66it/s]iter 25900: loss 4.1524, time: 452.44ms\n",
            " 40% 26000/65040 [2:38:14<3:33:56,  3.04it/s]iter 26000: loss 4.6605, time: 433.54ms\n",
            " 40% 26100/65040 [2:38:48<3:27:57,  3.12it/s]iter 26100: loss 4.1125, time: 259.80ms\n",
            " 40% 26200/65040 [2:39:24<2:43:12,  3.97it/s]iter 26200: loss 4.3376, time: 290.13ms\n",
            " 40% 26300/65040 [2:40:00<3:22:45,  3.18it/s]iter 26300: loss 3.4268, time: 156.76ms\n",
            " 41% 26400/65040 [2:40:36<3:42:40,  2.89it/s]iter 26400: loss 3.1767, time: 200.36ms\n",
            " 41% 26500/65040 [2:41:11<4:12:15,  2.55it/s]iter 26500: loss 4.4882, time: 470.11ms\n",
            " 41% 26600/65040 [2:41:47<4:03:55,  2.63it/s]iter 26600: loss 4.1682, time: 165.75ms\n",
            " 41% 26700/65040 [2:42:21<4:06:18,  2.59it/s]iter 26700: loss 3.9919, time: 292.97ms\n",
            " 41% 26800/65040 [2:42:58<3:23:04,  3.14it/s]iter 26800: loss 3.6360, time: 158.40ms\n",
            " 41% 26900/65040 [2:43:32<2:51:01,  3.72it/s]iter 26900: loss 3.6122, time: 158.65ms\n",
            " 42% 27000/65040 [2:44:09<4:30:42,  2.34it/s]iter 27000: loss 4.8685, time: 494.69ms\n",
            " 42% 27100/65040 [2:44:43<4:14:51,  2.48it/s]iter 27100: loss 4.6144, time: 495.14ms\n",
            " 42% 27199/65040 [2:45:18<3:44:01,  2.82it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " App\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "S\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " extracto His\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 27199: val loss 4.1950\n",
            " 42% 27200/65040 [2:45:38<64:44:03,  6.16s/it]iter 27200: loss 3.9642, time: 394.45ms\n",
            " 42% 27300/65040 [2:46:14<2:32:47,  4.12it/s]iter 27300: loss 3.2004, time: 207.71ms\n",
            " 42% 27400/65040 [2:46:49<3:56:26,  2.65it/s]iter 27400: loss 2.9162, time: 181.14ms\n",
            " 42% 27500/65040 [2:47:27<3:44:00,  2.79it/s]iter 27500: loss 4.4839, time: 494.61ms\n",
            " 42% 27600/65040 [2:48:02<4:03:26,  2.56it/s]iter 27600: loss 3.9436, time: 236.18ms\n",
            " 43% 27700/65040 [2:48:39<4:28:26,  2.32it/s]iter 27700: loss 5.0135, time: 494.91ms\n",
            " 43% 27800/65040 [2:49:14<2:21:07,  4.40it/s]iter 27800: loss 4.0922, time: 280.23ms\n",
            " 43% 27900/65040 [2:49:49<4:31:33,  2.28it/s]iter 27900: loss 4.3653, time: 325.57ms\n",
            " 43% 28000/65040 [2:50:25<4:14:23,  2.43it/s]iter 28000: loss 4.1589, time: 471.98ms\n",
            " 43% 28100/65040 [2:51:01<3:44:00,  2.75it/s]iter 28100: loss 3.9473, time: 266.08ms\n",
            " 43% 28200/65040 [2:51:34<3:29:39,  2.93it/s]iter 28200: loss 4.1770, time: 468.78ms\n",
            " 44% 28300/65040 [2:52:10<4:15:29,  2.40it/s]iter 28300: loss 4.4943, time: 484.08ms\n",
            " 44% 28400/65040 [2:52:45<3:49:51,  2.66it/s]iter 28400: loss 3.5730, time: 310.11ms\n",
            " 44% 28500/65040 [2:53:22<3:22:38,  3.01it/s]iter 28500: loss 4.3685, time: 471.17ms\n",
            " 44% 28600/65040 [2:53:57<4:14:05,  2.39it/s]iter 28600: loss 4.1936, time: 295.57ms\n",
            " 44% 28700/65040 [2:54:32<2:59:39,  3.37it/s]iter 28700: loss 4.3227, time: 274.88ms\n",
            " 44% 28799/65040 [2:55:07<3:28:39,  2.89it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "B\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "|aioghU\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 28799: val loss 4.1534\n",
            " 44% 28800/65040 [2:55:27<62:57:52,  6.25s/it]iter 28800: loss 4.0753, time: 436.24ms\n",
            " 44% 28900/65040 [2:56:03<3:18:44,  3.03it/s]iter 28900: loss 4.3466, time: 459.95ms\n",
            " 45% 29000/65040 [2:56:37<4:27:29,  2.25it/s]iter 29000: loss 4.9131, time: 494.74ms\n",
            " 45% 29100/65040 [2:57:12<3:25:30,  2.91it/s]iter 29100: loss 5.0062, time: 495.01ms\n",
            " 45% 29200/65040 [2:57:47<3:01:15,  3.30it/s]iter 29200: loss 2.9726, time: 163.22ms\n",
            " 45% 29300/65040 [2:58:22<3:17:12,  3.02it/s]iter 29300: loss 4.2934, time: 482.23ms\n",
            " 45% 29400/65040 [2:58:57<3:32:58,  2.79it/s]iter 29400: loss 3.9868, time: 353.41ms\n",
            " 45% 29500/65040 [2:59:32<3:26:53,  2.86it/s]iter 29500: loss 4.5498, time: 496.85ms\n",
            " 46% 29600/65040 [3:00:07<3:05:48,  3.18it/s]iter 29600: loss 3.1002, time: 232.67ms\n",
            " 46% 29700/65040 [3:00:45<3:10:43,  3.09it/s]iter 29700: loss 3.6564, time: 208.12ms\n",
            " 46% 29800/65040 [3:01:20<3:15:03,  3.01it/s]iter 29800: loss 4.8111, time: 487.82ms\n",
            " 46% 29900/65040 [3:01:55<3:19:04,  2.94it/s]iter 29900: loss 4.5812, time: 352.78ms\n",
            " 46% 30000/65040 [3:02:29<3:59:34,  2.44it/s]iter 30000: loss 4.5251, time: 440.72ms\n",
            " 46% 30100/65040 [3:03:09<3:44:47,  2.59it/s]iter 30100: loss 4.5372, time: 450.69ms\n",
            " 46% 30200/65040 [3:03:44<2:52:09,  3.37it/s]iter 30200: loss 4.5093, time: 469.30ms\n",
            " 47% 30300/65040 [3:04:21<3:55:20,  2.46it/s]iter 30300: loss 4.8487, time: 488.19ms\n",
            " 47% 30399/65040 [3:04:58<3:35:02,  2.68it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " prices\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Alner\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Poke semirange\n",
            " too Y sauce\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 30399: val loss 4.1352\n",
            " 47% 30400/65040 [3:05:19<62:33:50,  6.50s/it]iter 30400: loss 4.0143, time: 437.32ms\n",
            " 47% 30500/65040 [3:05:53<3:19:59,  2.88it/s]iter 30500: loss 4.3191, time: 441.73ms\n",
            " 47% 30600/65040 [3:06:28<2:58:38,  3.21it/s]iter 30600: loss 4.9013, time: 466.24ms\n",
            " 47% 30700/65040 [3:07:06<2:49:48,  3.37it/s]iter 30700: loss 3.1792, time: 180.24ms\n",
            " 47% 30800/65040 [3:07:42<3:22:22,  2.82it/s]iter 30800: loss 4.3502, time: 331.78ms\n",
            " 48% 30900/65040 [3:08:16<3:00:06,  3.16it/s]iter 30900: loss 4.8600, time: 469.24ms\n",
            " 48% 31000/65040 [3:08:52<3:08:14,  3.01it/s]iter 31000: loss 4.3928, time: 464.54ms\n",
            " 48% 31100/65040 [3:09:27<3:37:07,  2.61it/s]iter 31100: loss 3.7215, time: 302.54ms\n",
            " 48% 31200/65040 [3:10:01<3:06:39,  3.02it/s]iter 31200: loss 3.0800, time: 202.06ms\n",
            " 48% 31300/65040 [3:10:35<4:02:11,  2.32it/s]iter 31300: loss 4.5785, time: 455.02ms\n",
            " 48% 31400/65040 [3:11:12<3:56:00,  2.38it/s]iter 31400: loss 3.5206, time: 258.83ms\n",
            " 48% 31500/65040 [3:11:47<3:07:15,  2.99it/s]iter 31500: loss 3.7513, time: 239.58ms\n",
            " 49% 31600/65040 [3:12:21<3:07:01,  2.98it/s]iter 31600: loss 4.2554, time: 283.11ms\n",
            " 49% 31700/65040 [3:12:58<3:19:45,  2.78it/s]iter 31700: loss 4.2000, time: 326.65ms\n",
            " 49% 31800/65040 [3:13:33<4:11:42,  2.20it/s]iter 31800: loss 3.2373, time: 183.53ms\n",
            " 49% 31900/65040 [3:14:07<2:58:49,  3.09it/s]iter 31900: loss 3.4309, time: 187.04ms\n",
            " 49% 31999/65040 [3:14:41<2:38:51,  3.47it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "M\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "2 Geoff\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "PS Elas\n",
            "Descas neckher\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 31999: val loss 3.9099\n",
            "Saving adapter weights to out/final/adapter_v2_ablation_2\n",
            " 49% 32000/65040 [3:15:01<58:42:46,  6.40s/it]iter 32000: loss 4.5024, time: 392.09ms\n",
            " 49% 32100/65040 [3:15:37<3:12:21,  2.85it/s]iter 32100: loss 3.7756, time: 237.69ms\n",
            " 50% 32200/65040 [3:16:13<4:05:42,  2.23it/s]iter 32200: loss 3.3796, time: 384.83ms\n",
            " 50% 32300/65040 [3:16:49<2:56:43,  3.09it/s]iter 32300: loss 3.7372, time: 218.62ms\n",
            " 50% 32400/65040 [3:17:23<3:16:40,  2.77it/s]iter 32400: loss 3.3452, time: 355.74ms\n",
            " 50% 32500/65040 [3:17:59<3:19:16,  2.72it/s]iter 32500: loss 4.8584, time: 495.01ms\n",
            " 50% 32600/65040 [3:18:34<3:40:35,  2.45it/s]iter 32600: loss 3.3986, time: 378.75ms\n",
            " 50% 32700/65040 [3:19:09<3:18:52,  2.71it/s]iter 32700: loss 4.4815, time: 482.00ms\n",
            " 50% 32800/65040 [3:19:44<2:18:45,  3.87it/s]iter 32800: loss 3.2381, time: 161.64ms\n",
            " 51% 32900/65040 [3:20:17<3:45:19,  2.38it/s]iter 32900: loss 2.8322, time: 265.55ms\n",
            " 51% 33000/65040 [3:20:51<3:44:53,  2.37it/s]iter 33000: loss 3.4282, time: 191.35ms\n",
            " 51% 33100/65040 [3:21:26<2:13:29,  3.99it/s]iter 33100: loss 4.4322, time: 456.74ms\n",
            " 51% 33200/65040 [3:22:01<3:01:38,  2.92it/s]iter 33200: loss 4.4391, time: 445.77ms\n",
            " 51% 33300/65040 [3:22:36<3:32:51,  2.49it/s]iter 33300: loss 3.7981, time: 451.76ms\n",
            " 51% 33400/65040 [3:23:12<3:43:17,  2.36it/s]iter 33400: loss 4.5006, time: 474.72ms\n",
            " 52% 33500/65040 [3:23:46<2:55:15,  3.00it/s]iter 33500: loss 3.8893, time: 416.75ms\n",
            " 52% 33599/65040 [3:24:18<3:02:33,  2.87it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "\n",
            "S\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " bounce\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "ies (\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "King(o breaking\n",
            " recipesgreen acteduf\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 33599: val loss 3.7746\n",
            " 52% 33600/65040 [3:24:38<53:41:01,  6.15s/it]iter 33600: loss 3.8935, time: 346.02ms\n",
            " 52% 33700/65040 [3:25:15<3:01:56,  2.87it/s]iter 33700: loss 4.7745, time: 495.45ms\n",
            " 52% 33800/65040 [3:25:52<3:53:15,  2.23it/s]iter 33800: loss 4.3990, time: 496.55ms\n",
            " 52% 33900/65040 [3:26:29<3:19:45,  2.60it/s]iter 33900: loss 3.8640, time: 286.60ms\n",
            " 52% 34000/65040 [3:27:06<3:38:31,  2.37it/s]iter 34000: loss 3.4217, time: 205.10ms\n",
            " 52% 34100/65040 [3:27:43<3:28:42,  2.47it/s]iter 34100: loss 3.3284, time: 280.05ms\n",
            " 53% 34200/65040 [3:28:17<2:43:17,  3.15it/s]iter 34200: loss 4.3345, time: 443.01ms\n",
            " 53% 34300/65040 [3:28:53<3:15:58,  2.61it/s]iter 34300: loss 3.6273, time: 191.93ms\n",
            " 53% 34400/65040 [3:29:27<3:35:07,  2.37it/s]iter 34400: loss 4.4350, time: 476.68ms\n",
            " 53% 34500/65040 [3:30:02<3:28:33,  2.44it/s]iter 34500: loss 4.4643, time: 495.20ms\n",
            " 53% 34600/65040 [3:30:37<2:25:35,  3.48it/s]iter 34600: loss 3.7367, time: 325.82ms\n",
            " 53% 34700/65040 [3:31:11<3:09:04,  2.67it/s]iter 34700: loss 3.2429, time: 244.51ms\n",
            " 54% 34800/65040 [3:31:44<3:51:52,  2.17it/s]iter 34800: loss 2.9950, time: 198.74ms\n",
            " 54% 34900/65040 [3:32:20<2:29:15,  3.37it/s]iter 34900: loss 4.4425, time: 495.04ms\n",
            " 54% 35000/65040 [3:32:57<2:57:07,  2.83it/s]iter 35000: loss 4.4395, time: 451.01ms\n",
            " 54% 35100/65040 [3:33:33<2:25:39,  3.43it/s]iter 35100: loss 4.1416, time: 456.66ms\n",
            " 54% 35199/65040 [3:34:08<3:02:43,  2.72it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "In\",\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " Rica\n",
            "\n",
            " The\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "greenod\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "ato pos networking\n",
            " savings chain architecture C\n",
            "\n",
            " decentral\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 35199: val loss 3.8021\n",
            " 54% 35200/65040 [3:34:28<53:01:46,  6.40s/it]iter 35200: loss 3.9784, time: 267.37ms\n",
            " 54% 35300/65040 [3:35:03<2:28:38,  3.33it/s]iter 35300: loss 3.3945, time: 264.95ms\n",
            " 54% 35400/65040 [3:35:39<2:57:27,  2.78it/s]iter 35400: loss 3.6030, time: 418.70ms\n",
            " 55% 35500/65040 [3:36:13<1:54:53,  4.29it/s]iter 35500: loss 4.2185, time: 461.85ms\n",
            " 55% 35600/65040 [3:36:48<2:46:27,  2.95it/s]iter 35600: loss 2.7265, time: 175.61ms\n",
            " 55% 35700/65040 [3:37:23<3:20:09,  2.44it/s]iter 35700: loss 4.1037, time: 259.53ms\n",
            " 55% 35800/65040 [3:37:57<2:09:37,  3.76it/s]iter 35800: loss 3.4656, time: 199.51ms\n",
            " 55% 35900/65040 [3:38:34<2:50:40,  2.85it/s]iter 35900: loss 4.3355, time: 469.74ms\n",
            " 55% 36000/65040 [3:39:08<3:03:44,  2.63it/s]iter 36000: loss 3.0877, time: 419.20ms\n",
            " 56% 36100/65040 [3:39:43<2:41:29,  2.99it/s]iter 36100: loss 2.9116, time: 167.14ms\n",
            " 56% 36200/65040 [3:40:18<2:46:11,  2.89it/s]iter 36200: loss 3.7147, time: 287.10ms\n",
            " 56% 36300/65040 [3:40:54<3:12:03,  2.49it/s]iter 36300: loss 3.7368, time: 443.57ms\n",
            " 56% 36400/65040 [3:41:30<2:32:32,  3.13it/s]iter 36400: loss 3.3261, time: 194.48ms\n",
            " 56% 36500/65040 [3:42:04<3:28:27,  2.28it/s]iter 36500: loss 2.7869, time: 254.31ms\n",
            " 56% 36600/65040 [3:42:40<3:23:31,  2.33it/s]iter 36600: loss 3.9569, time: 445.43ms\n",
            " 56% 36700/65040 [3:43:17<2:55:48,  2.69it/s]iter 36700: loss 3.2908, time: 232.38ms\n",
            " 57% 36799/65040 [3:43:51<2:57:26,  2.65it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "If experiences\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " Y\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "EC\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " Run - Val\n",
            "\n",
            "\n",
            "\n",
            "End unlock village\n",
            "\n",
            "enseanре heard\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 36799: val loss 3.7462\n",
            " 57% 36800/65040 [3:44:12<50:11:18,  6.40s/it]iter 36800: loss 3.3620, time: 299.93ms\n",
            " 57% 36900/65040 [3:44:47<2:39:57,  2.93it/s]iter 36900: loss 2.9220, time: 186.15ms\n",
            " 57% 37000/65040 [3:45:23<2:32:21,  3.07it/s]iter 37000: loss 3.7361, time: 454.03ms\n",
            " 57% 37100/65040 [3:45:56<2:49:44,  2.74it/s]iter 37100: loss 3.1120, time: 183.13ms\n",
            " 57% 37200/65040 [3:46:33<2:24:52,  3.20it/s]iter 37200: loss 4.4872, time: 449.93ms\n",
            " 57% 37300/65040 [3:47:08<1:59:35,  3.87it/s]iter 37300: loss 3.7752, time: 396.45ms\n",
            " 58% 37400/65040 [3:47:46<2:05:04,  3.68it/s]iter 37400: loss 3.3618, time: 164.62ms\n",
            " 58% 37500/65040 [3:48:21<2:48:51,  2.72it/s]iter 37500: loss 3.5397, time: 320.77ms\n",
            " 58% 37600/65040 [3:48:57<2:48:06,  2.72it/s]iter 37600: loss 4.5787, time: 443.26ms\n",
            " 58% 37700/65040 [3:49:33<2:12:15,  3.45it/s]iter 37700: loss 4.2295, time: 457.85ms\n",
            " 58% 37800/65040 [3:50:06<2:47:35,  2.71it/s]iter 37800: loss 3.5106, time: 289.26ms\n",
            " 58% 37900/65040 [3:50:43<2:49:24,  2.67it/s]iter 37900: loss 3.5157, time: 191.40ms\n",
            " 58% 38000/65040 [3:51:18<2:17:45,  3.27it/s]iter 38000: loss 2.8894, time: 198.01ms\n",
            " 59% 38100/65040 [3:51:54<2:45:25,  2.71it/s]iter 38100: loss 3.3345, time: 171.10ms\n",
            " 59% 38200/65040 [3:52:29<2:52:28,  2.59it/s]iter 38200: loss 3.5602, time: 313.41ms\n",
            " 59% 38300/65040 [3:53:04<2:48:43,  2.64it/s]iter 38300: loss 3.1249, time: 300.65ms\n",
            " 59% 38399/65040 [3:53:38<2:05:29,  3.54it/s]Validating ...\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            "### Instruction:\n",
            "Recommend a movie for me to watch during the weekend and explain the reason.\n",
            "\n",
            "### Response:\n",
            "Par than\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "1\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            " B\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "Target I -\n",
            "\n",
            "\n",
            "\n",
            "illeversives\n",
            "\n",
            " Eve Studio museumsress\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "step 38399: val loss 3.5721\n",
            " 59% 38400/65040 [3:53:58<44:36:47,  6.03s/it]iter 38400: loss 3.1556, time: 149.63ms\n",
            " 59% 38500/65040 [3:54:33<2:12:51,  3.33it/s]iter 38500: loss 3.3886, time: 388.99ms\n",
            " 59% 38503/65040 [3:54:34<2:57:18,  2.49it/s]--- Logging error ---\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/router_sock.py\", line 27, in _read_message\n",
            "    resp = self._sock_client.read_server_response(timeout=1)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 285, in read_server_response\n",
            "    data = self._read_packet_bytes(timeout=timeout)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 269, in _read_packet_bytes\n",
            "    raise SockClientClosedError\n",
            "wandb.sdk.lib.sock_client.SockClientClosedError\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/router.py\", line 70, in message_loop\n",
            "    msg = self._read_message()\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/router_sock.py\", line 29, in _read_message\n",
            "    raise MessageRouterClosedError\n",
            "wandb.sdk.interface.router.MessageRouterClosedError\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/lib/python3.10/logging/__init__.py\", line 1104, in emit\n",
            "    self.flush()\n",
            "  File \"/usr/lib/python3.10/logging/__init__.py\", line 1084, in flush\n",
            "    self.stream.flush()\n",
            "OSError: [Errno 107] Transport endpoint is not connected\n",
            "Call stack:\n",
            "  File \"/usr/lib/python3.10/threading.py\", line 973, in _bootstrap\n",
            "    self._bootstrap_inner()\n",
            "  File \"/usr/lib/python3.10/threading.py\", line 1016, in _bootstrap_inner\n",
            "    self.run()\n",
            "  File \"/usr/lib/python3.10/threading.py\", line 953, in run\n",
            "    self._target(*self._args, **self._kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/router.py\", line 77, in message_loop\n",
            "    logger.warning(\"message_loop has been closed\")\n",
            "Message: 'message_loop has been closed'\n",
            "Arguments: ()\n",
            " 59% 38600/65040 [3:55:08<2:17:23,  3.21it/s]iter 38600: loss 2.7218, time: 206.90ms\n",
            " 59% 38600/65040 [3:55:08<2:41:04,  2.74it/s]\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py\", line 292, in <module>\n",
            "    CLI(main)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 96, in CLI\n",
            "    return _run_component(components, cfg_init)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jsonargparse/_cli.py\", line 181, in _run_component\n",
            "    return component(**cfg)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py\", line 129, in main\n",
            "    train(fabric, model, optimizer, train_data, val_data, out_dir)\n",
            "  File \"/content/drive/MyDrive/lit-llama/finetune/adapter_v2_sparse.py\", line 186, in train\n",
            "    wandb.log({\"train_loss\" : loss.item()})\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 420, in wrapper\n",
            "    return func(self, *args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 371, in wrapper_fn\n",
            "    return func(self, *args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 361, in wrapper\n",
            "    return func(self, *args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 1820, in log\n",
            "    self._log(data=data, step=step, commit=commit)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 1595, in _log\n",
            "    self._partial_history_callback(data, step, commit)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/wandb_run.py\", line 1467, in _partial_history_callback\n",
            "    self._backend.interface.publish_partial_history(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface.py\", line 567, in publish_partial_history\n",
            "    self._publish_partial_history(partial_history)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_shared.py\", line 89, in _publish_partial_history\n",
            "    self._publish(rec)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/interface/interface_sock.py\", line 51, in _publish\n",
            "    self._sock_client.send_record_publish(record)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 221, in send_record_publish\n",
            "    self.send_server_request(server_req)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 155, in send_server_request\n",
            "    self._send_message(msg)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 152, in _send_message\n",
            "    self._sendall_with_error_handle(header + data)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/wandb/sdk/lib/sock_client.py\", line 130, in _sendall_with_error_handle\n",
            "    sent = self._sock.send(data)\n",
            "BrokenPipeError: [Errno 32] Broken pipe\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "zBwk8-fgrg0s"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#**STEP3. Evaluation**"
      ],
      "metadata": {
        "id": "cgiX2Uo9qljv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-base\n",
        "!python evaluate/adapter.py --adapter_path /content/drive/MyDrive/lit-llama/out/final/c2/adapter_v1_base/lit-llama-adapter-finetuned.pth --quantize llm.int8"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MGbNEzzjUv5H",
        "outputId": "70b17617-9901-4bf1-9838-bfc43bbce1b1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading model ...\n",
            "Time to load model: 6.86 seconds.\n",
            "Downloading builder script: 100% 8.48k/8.48k [00:00<00:00, 29.6MB/s]\n",
            "Downloading metadata: 100% 6.84k/6.84k [00:00<00:00, 28.1MB/s]\n",
            "Downloading readme: 100% 9.62k/9.62k [00:00<00:00, 28.0MB/s]\n",
            "Downloading data: 100% 4.72M/4.72M [00:00<00:00, 6.91MB/s]\n",
            "Generating test split: 100% 4358/4358 [00:00<00:00, 43467.39 examples/s]\n",
            "Generating train split: 100% 36718/36718 [00:00<00:00, 50644.87 examples/s]\n",
            "Generating validation split: 100% 3760/3760 [00:00<00:00, 51568.02 examples/s]\n",
            "100% 157/157 [06:07<00:00,  2.34s/it]\n",
            "torch.Size([1, 321472]) torch.Size([1984, 32000])\n",
            "Perplexity on wikitext: 32000.07\n",
            "Downloading builder script: 100% 6.50k/6.50k [00:00<00:00, 26.7MB/s]\n",
            "Downloading metadata: 100% 2.15k/2.15k [00:00<00:00, 13.7MB/s]\n",
            "Downloading readme: 100% 4.21k/4.21k [00:00<00:00, 16.4MB/s]\n",
            "Downloading data files:   0% 0/3 [00:00<?, ?it/s]\n",
            "Downloading data:   0% 0.00/1.70M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading data: 5.10MB [00:00, 41.2MB/s]\n",
            "Downloading data files:  33% 1/3 [00:00<00:01,  1.06it/s]\n",
            "Downloading data: 400kB [00:00, 12.9MB/s]       \n",
            "Downloading data files:  67% 2/3 [00:01<00:00,  1.60it/s]\n",
            "Downloading data: 450kB [00:00, 13.0MB/s]       \n",
            "Downloading data files: 100% 3/3 [00:01<00:00,  1.69it/s]\n",
            "Extracting data files: 100% 3/3 [00:00<00:00, 2148.35it/s]\n",
            "Generating train split: 100% 42068/42068 [00:00<00:00, 57043.89 examples/s]\n",
            "Generating test split: 100% 3761/3761 [00:00<00:00, 57476.74 examples/s]\n",
            "Generating validation split: 100% 3370/3370 [00:00<00:00, 58962.33 examples/s]\n",
            "100% 53/53 [02:02<00:00,  2.31s/it]\n",
            "torch.Size([1, 107447]) torch.Size([951, 32000])\n",
            "Perplexity on ptb: 32000.07\n",
            "Downloading readme: 100% 2.38k/2.38k [00:00<00:00, 13.9MB/s]\n",
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/repocard.py:105: UserWarning: Repo card metadata block was not found. Setting CardData to empty.\n",
            "  warnings.warn(\"Repo card metadata block was not found. Setting CardData to empty.\")\n",
            "Downloading data files:   0% 0/1 [00:00<?, ?it/s]\n",
            "Downloading data:   0% 0.00/40.5M [00:00<?, ?B/s]\u001b[A\n",
            "Downloading data:  10% 4.19M/40.5M [00:04<00:39, 922kB/s]\u001b[A\n",
            "Downloading data:  31% 12.6M/40.5M [00:05<00:11, 2.52MB/s]\u001b[A\n",
            "Downloading data:  52% 21.0M/40.5M [00:07<00:05, 3.54MB/s]\u001b[A\n",
            "Downloading data:  73% 29.4M/40.5M [00:08<00:02, 4.39MB/s]\u001b[A\n",
            "Downloading data: 100% 40.5M/40.5M [00:09<00:00, 4.41MB/s]\n",
            "Downloading data files: 100% 1/1 [00:09<00:00,  9.19s/it]\n",
            "Extracting data files: 100% 1/1 [00:00<00:00,  1.21it/s]\n",
            "Generating validation split: 45576 examples [00:00, 135331.45 examples/s]\n",
            "100% 256/256 [09:58<00:00,  2.34s/it]\n",
            "torch.Size([1, 524288]) torch.Size([2048, 32000])\n",
            "Perplexity on c4: 32000.07\n",
            "\n",
            "\n",
            "Time for inference: 598.02 sec total, 1593.16 tokens/sec\n",
            "Memory used: 10.31 GB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# adapter-v1-sparse\n",
        "!python evaluate/adapter_sparse.py --adapter_path out/final/c2/adapter_v1_sparse/lit-llama-adapter-finetuned.pth --quantize llm.int8"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nv0tRKgvUvox",
        "outputId": "d1516b67-c1d9-4822-9f87-1b9d7ee3dba1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading model ...\n",
            "Time to load model: 4.95 seconds.\n",
            "100% 157/157 [06:14<00:00,  2.39s/it]\n",
            "torch.Size([1, 321472]) torch.Size([1984, 32000])\n",
            "Perplexity on wikitext: 32000.07\n",
            "100% 53/53 [02:05<00:00,  2.36s/it]\n",
            "torch.Size([1, 107447]) torch.Size([951, 32000])\n",
            "Perplexity on ptb: 32000.07\n",
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/repocard.py:105: UserWarning: Repo card metadata block was not found. Setting CardData to empty.\n",
            "  warnings.warn(\"Repo card metadata block was not found. Setting CardData to empty.\")\n",
            "100% 256/256 [10:10<00:00,  2.38s/it]\n",
            "torch.Size([1, 524288]) torch.Size([2048, 32000])\n",
            "Perplexity on c4: 32000.07\n",
            "\n",
            "\n",
            "Time for inference: 610.45 sec total, 1560.71 tokens/sec\n",
            "Memory used: 11.41 GB\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": [],
      "collapsed_sections": [
        "XYqYIt4s0Wc7",
        "uaRXnL0K0Arg",
        "tyEfh2ikdngs"
      ],
      "gpuType": "A100",
      "authorship_tag": "ABX9TyNm0qeu2lg6AuwBVIfqYdlU",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}