notebooks/ngpt-shakespeare.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "id": "1d841301-7152-45fd-836f-d0dda350fd57",
   "metadata": {},
   "source": [
    "---\n",
    "title: title\n",
    "author: author\n",
    "execute:\n",
    "    enabled: false\n",
    "jupyter: python3\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1fc5d004-5e6f-4bb3-938e-fb44f919cec3",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2023-11-11T07:14:30.136832+00:00",
     "iopub.status.idle": "2023-11-11T07:14:30.175293+00:00"
    }
   },
   "source": [
    "# `nanoGPT`: Shakespeare"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd994b32-ca5f-4e00-81d3-89dbbf7b2093",
   "metadata": {},
   "source": [
    "## Install / Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99a8bdbf-d488-4553-8955-9a2ad01bc54b",
   "metadata": {},
   "source": [
    "### Google Colab\n",
    "\n",
    "```python\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive')\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71c38d27-5acb-4e5d-87c8-dbb2739cf277",
   "metadata": {},
   "source": [
    "### First Time Running\n",
    "\n",
    "We need to install `ngpt` and setup the Shakespeare dataset\n",
    "\n",
    "This will need to be ran the first time you are running this notebook.\n",
    "\n",
    "Following the\n",
    "\n",
    "```python\n",
    "!python3 -m pip install nanoGPT\n",
    "```\n",
    "\n",
    "you will need to restart your runtime (Runtime -> Restart runtime)\n",
    "\n",
    "After this, you should be able to\n",
    "\n",
    "```python\n",
    ">>> import ngpt\n",
    ">>> ngpt.__file__\n",
    "'/content/nanoGPT/src/ngpt/__init__.py'\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "31e60f3e",
   "metadata": {
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2023-11-30T12:51:18.874443Z",
     "iopub.status.busy": "2023-11-30T12:51:18.874122Z",
     "iopub.status.idle": "2023-11-30T12:51:18.999204Z",
     "shell.execute_reply": "2023-11-30T12:51:18.998694Z",
     "shell.execute_reply.started": "2023-11-30T12:51:18.874423Z"
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/ngpt/__init__.py\n",
      "Has ngpt installed. Nothing to do.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "\n",
    "python3 -c 'import ngpt; print(ngpt.__file__)' 2> '/dev/null'\n",
    "\n",
    "if [[ $? -eq 0 ]]; then\n",
    "    echo \"Has ngpt installed. Nothing to do.\"\n",
    "else\n",
    "    echo \"Does not have ngpt installed. Installing...\"\n",
    "    git clone 'https://github.com/saforem2/nanoGPT'\n",
    "    python3 nanoGPT/data/shakespeare_char/prepare.py\n",
    "    python3 -m pip install -e nanoGPT -vvv\n",
    "fi"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c14a8da7-72fe-4839-a14d-f01606285fc3",
   "metadata": {},
   "source": [
    "## Post Install\n",
    "\n",
    "If installed correctly, you should be able to:\n",
    "\n",
    "```python\n",
    ">>> import ngpt\n",
    ">>> ngpt.__file__\n",
    "'/path/to/nanoGPT/src/ngpt/__init__.py'\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9fbf22d1-34ba-48ae-a78e-fc447fc9a0de",
   "metadata": {
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2023-11-30T12:51:35.102316Z",
     "iopub.status.busy": "2023-11-30T12:51:35.102129Z",
     "iopub.status.idle": "2023-11-30T12:51:35.296422Z",
     "shell.execute_reply": "2023-11-30T12:51:35.295928Z",
     "shell.execute_reply.started": "2023-11-30T12:51:35.102299Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n",
      "\u001b[38;2;131;131;131m[2023-11-30 06:51:35]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119m1583707193.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m8\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/ngpt/\u001b[0m\u001b[35m__init__.py\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import ngpt\n",
    "from enrich import get_logger\n",
    "log = get_logger('jupyter')\n",
    "#from rich import print\n",
    "log.info(ngpt.__file__)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e329ed94-582e-4aa8-bbe0-ba56782fe9e9",
   "metadata": {},
   "source": [
    "## Build Trainer\n",
    "\n",
    "Explicitly, we:\n",
    "\n",
    "1. `setup_torch(...)`\n",
    "2. Build `cfg: DictConfig = get_config(...)`\n",
    "3. Instnatiate `config: ExperimentConfig = instantiate(cfg)`\n",
    "4. Build `trainer = Trainer(config)`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7d12405a-8f88-476b-8922-a1a212adc682",
   "metadata": {
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2023-11-30T12:51:47.787288Z",
     "iopub.status.busy": "2023-11-30T12:51:47.787090Z",
     "iopub.status.idle": "2023-11-30T12:53:26.235036Z",
     "shell.execute_reply": "2023-11-30T12:53:26.234264Z",
     "shell.execute_reply.started": "2023-11-30T12:51:47.787271Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------\n",
      "WARNING: There was an error initializing an OpenFabrics device.\n",
      "\n",
      "  Local host:   thetagpu23\n",
      "  Local device: mlx5_0\n",
      "--------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-30 06:52:25]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mconfigs.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m72\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Setting HF_DATASETS_CACHE to \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/.cache/huggingface/\u001b[0m\u001b[35mdatasets\u001b[0m\n",
      "Failed to download font: Source Sans Pro, skipping!\n",
      "Failed to download font: Titillium WebRoboto Condensed, skipping!\n",
      "\u001b[38;2;131;131;131m[2023-11-30 06:53:07]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mconfigs.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m295\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Loading val from \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/data/shakespeare_char/\u001b[0m\u001b[35mval.bin\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 06:53:07]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mconfigs.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m295\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Loading train from \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/data/shakespeare_char/\u001b[0m\u001b[35mtrain.bin\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 06:53:07]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mconfigs.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m270\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Rescaling GAS -> GAS \u001b[32m/\u001b[0m\u001b[32m/\u001b[0m WORLD_SIZE = \u001b[35m1\u001b[0m \u001b[32m/\u001b[0m\u001b[32m/\u001b[0m \u001b[35m1\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 06:53:07]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mconfigs.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m432\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Tokens per iteration: \u001b[35m16\u001b[0m,\u001b[35m384\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 06:53:07]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mconfigs.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m454\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Using \u001b[1m<\u001b[0m\u001b[1;95mtorch.amp.autocast_mode.autocast\u001b[0m\u001b[39m object at \u001b[0m\u001b[35m0x7f565e50d060\u001b[0m\u001b[1m>\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 06:53:07]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mconfigs.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m460\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Initializing a new model from scratch\n",
      "\u001b[38;2;131;131;131m[2023-11-30 06:53:07]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m179\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Initializing a new model from scratch\n",
      "\u001b[38;2;131;131;131m[2023-11-30 06:53:07]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mmodel.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m160\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - number of parameters: \u001b[35m10.\u001b[0m65M\n",
      "\u001b[38;2;131;131;131m[2023-11-30 06:53:11]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mmodel.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m290\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - num decayed parameter tensors: \u001b[35m26\u001b[0m, with \u001b[35m10\u001b[0m,\u001b[35m740\u001b[0m,\u001b[35m096\u001b[0m parameters\n",
      "\u001b[38;2;131;131;131m[2023-11-30 06:53:11]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mmodel.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m291\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - num non-decayed parameter tensors: \u001b[35m13\u001b[0m, with \u001b[35m4\u001b[0m,\u001b[35m992\u001b[0m parameters\n",
      "\u001b[38;2;131;131;131m[2023-11-30 06:53:21]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mmodel.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m297\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - using fused AdamW: \u001b[3;92mTrue\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "from ezpz import setup_torch\n",
    "from hydra.utils import instantiate\n",
    "from ngpt.configs import get_config, PROJECT_ROOT\n",
    "from ngpt.trainer import Trainer\n",
    "HF_DATASETS_CACHE = PROJECT_ROOT.joinpath('.cache', 'huggingface')\n",
    "HF_DATASETS_CACHE.mkdir(exist_ok=True, parents=True)\n",
    "\n",
    "os.environ['MASTER_PORT'] = '5671'\n",
    "os.environ['HF_DATASETS_CACHE'] = HF_DATASETS_CACHE.as_posix()\n",
    "\n",
    "rank = setup_torch('DDP', seed=1234)\n",
    "cfg = get_config(\n",
    "    [\n",
    "        'data=shakespeare',\n",
    "        'model=shakespeare',\n",
    "        'optimizer=shakespeare',\n",
    "        'train=shakespeare',\n",
    "        'train.dtype=bfloat16',\n",
    "        'train.max_iters=5000',\n",
    "        'train.log_interval=250',\n",
    "        'train.eval_interval=1000',\n",
    "    ]\n",
    ")\n",
    "config = instantiate(cfg)\n",
    "trainer = Trainer(config)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cca969e9-5132-42a8-9728-9db1bce6a198",
   "metadata": {},
   "source": [
    "## Prompt (**prior** to training)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9a42517f-d758-43b7-97d9-92fc7a2d0613",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-30T12:57:37.049934Z",
     "iopub.status.busy": "2023-11-30T12:57:37.049364Z",
     "iopub.status.idle": "2023-11-30T12:57:43.110469Z",
     "shell.execute_reply": "2023-11-30T12:57:43.109893Z",
     "shell.execute_reply.started": "2023-11-30T12:57:37.049911Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-30 06:57:43]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119m1657463709.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m3\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[1m[\u001b[0m\u001b[32m'prompt'\u001b[0m\u001b[1m]\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32mquery\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 06:57:43]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119m1657463709.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m4\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[1m[\u001b[0m\u001b[32m'response'\u001b[0m\u001b[1m]\u001b[0m:\n",
      "\n",
      "What is a supercomputer?CbqA-RN?bnss--iadmsD\n",
      "S?qJKEwssDq YMSSSFGPxxnJL\u001b[1;92mDC:c\u001b[0mvYfOy\n",
      "fiXXe3GQQvYQARdEEbbHHPnWyFp-CwBFrr;g\n",
      "WATVAcTZCWWr\n",
      "tYCCz,E-wqNbIsMbSvYVONyaQzzcs;Iaa?WOACrnMH'\u001b[32m':dXFEQZa-PYkAvV.B.  F$J-nKnEaZ,'\u001b[0mvpesXY&y-M.nIcMVV!GYYVVFh-UX.G&Fa?LSPrkXd3eKV?KJjSZOwSbbhwfIYaywrvRUEuuQMnnIAZS-Ja.fXrMAHB&!!eVbUFwMIkkalHbmRhwwfcj$:s\n",
      "RlVhRcaVbYcTTihITDUbbTNMHEdnOibdB-ebuiJLLS:yarlFYHHkSWxB!hbN?nVm3-&djw'BA uS,EQJP3bbWe$hs-g\n",
      ":\u001b[1;36m3j\u001b[0mEEYU\n",
      "NkLCetHH lc-IIZEBbb-at\n",
      "jyNYmvffVVnERN?LnTM:yS\n",
      "sH;are$WRip!jbX'\n",
      "e\n",
      "\n",
      "pyA-jbwK 'B$O& Fvvac&sEjbIretcX-H\n"
     ]
    }
   ],
   "source": [
    "query = \"What is a supercomputer?\"\n",
    "outputs = trainer.evaluate(query, num_samples=1, display=False)\n",
    "log.info(\"['prompt']: '{query}'\")\n",
    "log.info(\"['response']:\\n\\n\" + fr\"{outputs['0']['raw']}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5202ff3-8811-47c9-8b9d-9818d4603697",
   "metadata": {},
   "source": [
    "## Train Model\n",
    "\n",
    "|  name  |       description            |\n",
    "|:------:|:----------------------------:|\n",
    "| `step` | Current training step        |\n",
    "| `loss` | Loss value                   |\n",
    "| `dt`   | Time per step (in **ms**)    |\n",
    "| `sps`  | Samples per second           |\n",
    "| `mtps` | (million) Tokens per sec     |\n",
    "| `mfu`  | Model Flops utilization[^1]  |\n",
    "^legend: #tbl-legend\n",
    "\n",
    "[^1]: in units of A100 `bfloat16` peak FLOPS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "acb8d33b-f8c9-41c6-aa61-95bdb76bf6ea",
   "metadata": {
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2023-11-30T13:00:19.313261Z",
     "iopub.status.busy": "2023-11-30T13:00:19.312893Z",
     "iopub.status.idle": "2023-11-30T13:02:57.884211Z",
     "shell.execute_reply": "2023-11-30T13:02:57.883679Z",
     "shell.execute_reply.started": "2023-11-30T13:00:19.313240Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "759f6069a17248cfa6d792343ba2c941",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/5000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-30 07:00:31]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m250\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m2\u001b[0m\u001b[35m.064\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.551\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.663\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.617\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m14\u001b[0m\u001b[35m.034\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m4\u001b[0m\u001b[35m.300\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m4\u001b[0m\u001b[35m.291\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:00:38]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m500\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.610\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m25\u001b[0m\u001b[35m.973\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m38\u001b[0m\u001b[35m.502\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.631\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m14\u001b[0m\u001b[35m.065\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m4\u001b[0m\u001b[35m.300\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m4\u001b[0m\u001b[35m.291\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:00:45]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m750\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.432\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.182\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.789\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.603\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m14\u001b[0m\u001b[35m.030\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m4\u001b[0m\u001b[35m.300\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m4\u001b[0m\u001b[35m.291\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:00:51]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m1000\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.346\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.546\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.671\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.617\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m14\u001b[0m\u001b[35m.030\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m4\u001b[0m\u001b[35m.300\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m4\u001b[0m\u001b[35m.291\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:00:55]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m434\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving checkpoint to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/\u001b[0m\u001b[35mnotebooks\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:00:55]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m435\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving model to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/notebooks/\u001b[0m\u001b[35mmodel.pth\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:00:56]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mconfigs.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m136\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Appending \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/\u001b[0m\u001b[35mnotebooks\u001b[0m to \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/ngpt/ckpts/\u001b[0m\u001b[35mcheckpoints.log\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:01:02]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m1250\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.309\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.403\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.493\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.598\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.987\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.270\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.522\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:01:09]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m1500\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.225\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.988\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.054\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.607\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.969\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.270\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.522\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:01:16]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m1750\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.176\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.135\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.853\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.604\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.946\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.270\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.522\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:01:23]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m2000\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.163\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.558\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.653\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.617\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.954\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.270\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.522\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:01:27]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m434\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving checkpoint to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/\u001b[0m\u001b[35mnotebooks\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:01:27]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m435\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving model to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/notebooks/\u001b[0m\u001b[35mmodel.pth\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:01:27]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mconfigs.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m136\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Appending \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/\u001b[0m\u001b[35mnotebooks\u001b[0m to \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/ngpt/ckpts/\u001b[0m\u001b[35mcheckpoints.log\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:01:34]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m2250\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.120\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.065\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.948\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.605\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.935\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.054\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.468\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:01:41]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m2500\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.068\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.669\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.497\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.614\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.939\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.054\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.468\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:01:48]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m2750\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.027\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.749\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.037\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.590\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.888\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.054\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.468\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:01:54]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m3000\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.002\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.637\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.542\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.615\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.898\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.054\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.468\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:01:58]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m434\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving checkpoint to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/\u001b[0m\u001b[35mnotebooks\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:01:58]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m435\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving model to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/notebooks/\u001b[0m\u001b[35mmodel.pth\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:01:59]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mconfigs.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m136\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Appending \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/\u001b[0m\u001b[35mnotebooks\u001b[0m to \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/ngpt/ckpts/\u001b[0m\u001b[35mcheckpoints.log\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:02:05]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m3250\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.950\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.316\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.608\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.600\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.872\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.863\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.537\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:02:12]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m3500\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.926\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.761\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.368\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.612\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.878\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.863\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.537\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:02:19]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m3750\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.916\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.141\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.845\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.604\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.863\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.863\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.537\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:02:26]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m4000\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.901\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.019\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.011\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.606\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.856\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.863\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.537\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:02:30]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m434\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving checkpoint to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/\u001b[0m\u001b[35mnotebooks\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:02:30]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m435\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving model to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/notebooks/\u001b[0m\u001b[35mmodel.pth\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:02:30]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mconfigs.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m136\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Appending \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/\u001b[0m\u001b[35mnotebooks\u001b[0m to \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/ngpt/ckpts/\u001b[0m\u001b[35mcheckpoints.log\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:02:37]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m4250\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.840\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.105\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.894\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.604\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.845\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.702\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.619\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:02:44]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m4500\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.850\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.603\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.590\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.616\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.861\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.702\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.619\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:02:51]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m4750\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.824\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.085\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.921\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.605\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.851\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.702\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.619\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:02:57]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119mtrainer.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m518\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m5000\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.819\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.037\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.987\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.606\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.844\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.702\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.619\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:16:40]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m1500\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.234\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.878\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.205\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.610\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.668\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m4\u001b[0m\u001b[35m.299\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m4\u001b[0m\u001b[35m.291\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:16:47]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m1750\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.175\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.460\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.417\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.597\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.659\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m4\u001b[0m\u001b[35m.299\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m4\u001b[0m\u001b[35m.291\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:16:54]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m2000\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.140\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.889\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.190\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.609\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.678\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m4\u001b[0m\u001b[35m.299\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m4\u001b[0m\u001b[35m.291\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:16:58]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:432\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving checkpoint to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/\u001b[0m\u001b[35mngpt\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:16:58]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:433\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving model to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/ngpt/\u001b[0m\u001b[35mmodel.pth\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:16:58]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[configs.py:129\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Appending \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/\u001b[0m\u001b[35mngpt\u001b[0m to \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/ngpt/ckpts/\u001b[0m\u001b[35mcheckpoints.log\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:17:05]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m2250\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.121\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.308\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.619\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.600\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.675\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.050\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.474\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:17:12]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m2500\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.067\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.838\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.261\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.610\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.696\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.050\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.474\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:17:19]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m2750\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.034\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.360\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.550\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.599\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.688\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.050\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.474\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:17:26]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m3000\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.009\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.237\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m38\u001b[0m\u001b[35m.114\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.624\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.740\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.050\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.474\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:17:33]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m3250\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.940\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.991\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.050\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.607\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.746\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.050\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.474\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:17:39]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m3500\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.947\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.261\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m38\u001b[0m\u001b[35m.080\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.624\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.791\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.050\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.474\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:17:46]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m3750\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.885\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.216\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.870\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.440\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.413\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.050\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.474\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:17:53]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m4000\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.866\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.241\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m38\u001b[0m\u001b[35m.108\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.624\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.492\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.050\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.474\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:17:57]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:432\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving checkpoint to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/\u001b[0m\u001b[35mngpt\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:17:57]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:433\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving model to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/ngpt/\u001b[0m\u001b[35mmodel.pth\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:17:57]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[configs.py:129\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Appending \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/\u001b[0m\u001b[35mngpt\u001b[0m to \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/ngpt/ckpts/\u001b[0m\u001b[35mcheckpoints.log\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:18:04]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m4250\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.847\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.228\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.728\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.602\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.511\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.696\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.637\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:18:11]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m4500\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.835\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.215\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m38\u001b[0m\u001b[35m.147\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.625\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.581\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.696\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.637\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:18:18]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m4750\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.822\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.657\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.513\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.615\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.621\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.696\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.637\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:18:24]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m5000\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.808\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.635\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.544\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.615\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.658\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.696\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.637\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:18:31]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m5250\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.811\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.267\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m38\u001b[0m\u001b[35m.071\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.624\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.711\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.696\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.637\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:18:38]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m5500\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.769\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.406\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.870\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.620\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.751\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.696\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.637\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:18:44]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m5750\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.780\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.239\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m38\u001b[0m\u001b[35m.111\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.624\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.796\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.696\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.637\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:18:51]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m6000\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.767\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.682\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.478\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.614\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.813\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.696\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.637\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:18:55]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:432\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving checkpoint to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/\u001b[0m\u001b[35mngpt\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:18:55]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:433\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving model to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/ngpt/\u001b[0m\u001b[35mmodel.pth\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:18:56]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[configs.py:129\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Appending \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/\u001b[0m\u001b[35mngpt\u001b[0m to \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/ngpt/ckpts/\u001b[0m\u001b[35mcheckpoints.log\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:19:02]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m6250\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.773\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m31\u001b[0m\u001b[35m.104\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m32\u001b[0m\u001b[35m.151\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.527\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.629\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.556\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.755\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:19:09]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m6500\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.759\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.142\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.843\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.604\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.639\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.556\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.755\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:19:16]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m6750\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.753\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.712\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.437\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.613\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.670\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.556\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.755\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:19:22]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m7000\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.745\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.871\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.215\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.610\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.690\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.556\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.755\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:19:29]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m7250\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.733\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.266\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m38\u001b[0m\u001b[35m.072\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.624\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.740\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.556\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.755\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:19:36]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m7500\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.723\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.817\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.289\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.611\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.755\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.556\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.755\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:19:43]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m7750\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.747\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.461\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.791\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.619\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.788\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.556\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.755\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:19:49]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m8000\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.729\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m29\u001b[0m\u001b[35m.348\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m34\u001b[0m\u001b[35m.074\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.558\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.679\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.556\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.755\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:19:53]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:432\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving checkpoint to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/\u001b[0m\u001b[35mngpt\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:19:53]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:433\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Saving model to: \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/ngpt/\u001b[0m\u001b[35mmodel.pth\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:19:54]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[configs.py:129\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - Appending \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/\u001b[0m\u001b[35mngpt\u001b[0m to \u001b[32m/lus/grand/projects/datascience/foremans/locations/thetaGPU/projects/saforem2/nanoGPT/src/ngpt/ckpts/\u001b[0m\u001b[35mcheckpoints.log\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:20:01]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m8250\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.718\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.464\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.787\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.619\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.719\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.473\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.840\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:20:07]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m8500\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.705\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.051\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.967\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.606\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.725\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.473\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.840\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:20:14]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m8750\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.704\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.298\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m38\u001b[0m\u001b[35m.026\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.623\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.769\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.473\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.840\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:20:21]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m9000\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.694\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.131\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.858\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.604\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.766\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.473\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.840\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:20:27]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m9250\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.700\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.291\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m38\u001b[0m\u001b[35m.036\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.623\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.806\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.473\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.840\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:20:34]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m9500\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.668\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m27\u001b[0m\u001b[35m.353\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m36\u001b[0m\u001b[35m.560\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.599\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.788\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.473\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.840\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:20:41]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m9750\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.658\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.422\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.847\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.620\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.819\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.473\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.840\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-11 01:20:48]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[trainer.py:516\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[3;94mstep\u001b[0m=\u001b[35m10000\u001b[0m \u001b[3;94mloss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.678\u001b[0m \u001b[3;94mdt\u001b[0m=\u001b[35m26\u001b[0m\u001b[35m.887\u001b[0m \u001b[3;94msps\u001b[0m=\u001b[35m37\u001b[0m\u001b[35m.192\u001b[0m \u001b[3;94mmtps\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.609\u001b[0m \u001b[3;94mmfu\u001b[0m=\u001b[35m13\u001b[0m\u001b[35m.823\u001b[0m \u001b[3;94mtrain_loss\u001b[0m=\u001b[35m0\u001b[0m\u001b[35m.473\u001b[0m \u001b[3;94mval_loss\u001b[0m=\u001b[35m1\u001b[0m\u001b[35m.840\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa5c0c5a-539a-462d-a9e3-0655e8e48d85",
   "metadata": {},
   "source": [
    "## Evaluate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fdfa61f9-8217-47b4-b885-b193c0955a7b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-30T13:04:34.414627Z",
     "iopub.status.busy": "2023-11-30T13:04:34.414272Z",
     "iopub.status.idle": "2023-11-30T13:04:38.231578Z",
     "shell.execute_reply": "2023-11-30T13:04:38.231113Z",
     "shell.execute_reply.started": "2023-11-30T13:04:34.414607Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;2;131;131;131m[2023-11-30 07:04:38]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119m1657463709.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m3\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[1m[\u001b[0m\u001b[32m'prompt'\u001b[0m\u001b[1m]\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32mquery\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n",
      "\u001b[38;2;131;131;131m[2023-11-30 07:04:38]\u001b[0m\u001b[34m[INFO]\u001b[0m\u001b[38;2;119;119;119m[\u001b[0m\u001b[38;2;119;119;119m1657463709.py\u001b[0m\u001b[38;2;119;119;119m:\u001b[0m\u001b[38;2;119;119;119m4\u001b[0m\u001b[38;2;119;119;119m]\u001b[0m - \u001b[1m[\u001b[0m\u001b[32m'response'\u001b[0m\u001b[1m]\u001b[0m:\n",
      "\n",
      "What is a supercomputer?\n",
      "\n",
      "Second Murderer:\n",
      "What is thy name?\n",
      "\n",
      "First Murderer:\n",
      "What, shall we stay?\n",
      "\n",
      "First Murderer:\n",
      "Why, so he doth death?\n",
      "\n",
      "Second Murderer:\n",
      "So do I like thee a hare, to have heard the crown,\n",
      "to the door horse of the Duke of Norfolk.\n",
      "But who comes here?\n",
      "\n",
      "Second Murderer:\n",
      "What is the matter, that the king shall we do?\n",
      "\n",
      "Second Murderer:\n",
      "Look, here in the soul of this face I should,\n",
      "which I was communed by that word \u001b[32m'gainst him.'\u001b[0m\n",
      "\n",
      "Second Murderer:\n",
      "What, are you so brief?  Then all said \u001b[32m's well,'\u001b[0m\n",
      "And so qui\n"
     ]
    }
   ],
   "source": [
    "query = \"What is a supercomputer?\"\n",
    "outputs = trainer.evaluate(query, num_samples=1, display=False)\n",
    "log.info(\"['prompt']: '{query}'\")\n",
    "log.info(\"['response']:\\n\\n\" + fr\"{outputs['0']['raw']}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "09608c829e3c41cfb9df32a38bc540a2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "buffers": [],
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "29e17460a7de4b139111d12c7d3569dd": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "buffers": [],
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "2dbde29da70f444a92bc7da43bbb47f2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "buffers": [],
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "36a4f9becb434320ab8c4d625e5e4ab7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "buffers": [],
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7e2dbd76a061447b8f79ece8f2679cb3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "buffers": [],
       "description_width": ""
      }
     },
     "83406e2a819f4ac4bce6aebe72d802f9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "buffers": [],
       "children": [
        "IPY_MODEL_becff553ccd54cceb5467f0a96cb00c7",
        "IPY_MODEL_f1d80a4ac2594494a970534573cd0f5e",
        "IPY_MODEL_e1b0c5d54eaa4791851f5a8744f463d8"
       ],
       "layout": "IPY_MODEL_36a4f9becb434320ab8c4d625e5e4ab7",
       "tabbable": null,
       "tooltip": null
      }
     },
     "becff553ccd54cceb5467f0a96cb00c7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "buffers": [],
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_29e17460a7de4b139111d12c7d3569dd",
       "placeholder": "​",
       "style": "IPY_MODEL_09608c829e3c41cfb9df32a38bc540a2",
       "tabbable": null,
       "tooltip": null,
       "value": "100%"
      }
     },
     "ca8984b34602451fbe107f25eec68282": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "buffers": [],
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d3474b1b5ecc4e669b7d12b736096303": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "buffers": [],
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e1b0c5d54eaa4791851f5a8744f463d8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "buffers": [],
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_d3474b1b5ecc4e669b7d12b736096303",
       "placeholder": "​",
       "style": "IPY_MODEL_2dbde29da70f444a92bc7da43bbb47f2",
       "tabbable": null,
       "tooltip": null,
       "value": " 10000/10000 [04:53&lt;00:00, 37.57it/s]"
      }
     },
     "f1d80a4ac2594494a970534573cd0f5e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "buffers": [],
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_ca8984b34602451fbe107f25eec68282",
       "max": 10000,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_7e2dbd76a061447b8f79ece8f2679cb3",
       "tabbable": null,
       "tooltip": null,
       "value": 10000
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}