example/lotsa_v1_weighting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "51be5533-c0e9-4a60-b2f5-c0e7e9f3b7a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import datasets\n",
    "import pyarrow.compute as pc\n",
    "import numpy as np\n",
    "\n",
    "from uni2ts.common.env import env\n",
    "from uni2ts.data.builder.lotsa_v1 import (\n",
    "    Buildings900KDatasetBuilder,\n",
    "    BuildingsBenchDatasetBuilder,\n",
    "    CloudOpsTSFDatasetBuilder,\n",
    "    CMIP6DatasetBuilder,\n",
    "    ERA5DatasetBuilder,\n",
    "    GluonTSDatasetBuilder,\n",
    "    LargeSTDatasetBuilder,\n",
    "    LibCityDatasetBuilder,\n",
    "    OthersLOTSADatasetBuilder,\n",
    "    ProEnFoDatasetBuilder,\n",
    "    SubseasonalDatasetBuilder,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9657d9c-6371-44dc-8e5f-20dc632a2ef7",
   "metadata": {},
   "source": [
    "In this notebook, we will see how to calculate the dataset weighting in the [pre-training dataset config file](../cli/conf/pretrain/data/lotsa_v1_weighted.yaml). We will see how to automatically generate this file to avoid excessive manual labor."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9738e614-8602-4465-b697-cec676e8b2d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_list = (\n",
    "    Buildings900KDatasetBuilder.dataset_list\n",
    "    + BuildingsBenchDatasetBuilder.dataset_list\n",
    "    + CloudOpsTSFDatasetBuilder.dataset_list\n",
    "    + CMIP6DatasetBuilder.dataset_list\n",
    "    + ERA5DatasetBuilder.dataset_list\n",
    "    + GluonTSDatasetBuilder.dataset_list\n",
    "    + LargeSTDatasetBuilder.dataset_list\n",
    "    + LibCityDatasetBuilder.dataset_list\n",
    "    + OthersLOTSADatasetBuilder.dataset_list\n",
    "    + ProEnFoDatasetBuilder.dataset_list\n",
    "    + SubseasonalDatasetBuilder.dataset_list\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2694e49-c24a-421b-95c8-f52797f9dceb",
   "metadata": {},
   "source": [
    "1. Obtain the lengths of all time series from all available datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "22558fc9-ad7e-415e-a796-98da6faa6d4c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15f49c0a8f174e488d9772aa24f4fdaa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/127 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ede459a4efe84b48a753b57dd1bafff3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/22 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04b857fae12b40ab87f601b414b9b1fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/31 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4571aa21b0e4cc8805334bdfbf2cce3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "26edc05c82914d96be1b9b76e45a4e85",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a399684f969466c8e0b399a66000768",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6fc044c9f43a4faeb727291e95420d22",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "44e438a10ddb41fd9661aa233cd7efb8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00555fd0301147d78634677dc3c26883",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96eddf9cdaf34055872b400baef72c78",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa787a657aba47f8a31ddddcfc2b224f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "849447091ed74e468a4b826079756f7e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a66e166da1e402abcbaa7af52144f6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "abb7c168a9a44e24a45c3e8576d83d79",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd80a4920e2046889237e1217e3c19b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c79569023ce421f810bbb87cd1ce99a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dfcf58555c1f42e48dc73c15663081b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9551331c15f4000aa479419f8c4f25e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dda4d87c6da64960be2ece6861929898",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ac0c1ecf34c4a9995c562ba212eb0fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b549ee18cfe41b79195c2a58bf1fff2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3fc84d444af499aa3a894a6462aae72",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e1b65d0cc8d0438498e9074f1c8a12c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "817c8b9ca0dd41ffa192c3169bf37472",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9607546a14a34220ada889f5e6d1db9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f4c03e2b5f84ce18e7f5098c6d051f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f79031d839f5469c958b181c93dddc8a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8d6c4e1cb5b4eeaaaefedfcdc60803d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59d795c69ca4481eaee398c5cfd57fb4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95355e953db84a9e8ef4bef94a169c58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc5fef40b3024920ac7b231461772934",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "030dfc8615434cca8ba65b1c19ea8e2b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2759d63715a24751abfdfaac1daea426",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e45c381f3e7e494cafd0248c23a34d1c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dfc85509d273437994ad0ea7e4bcdf7a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6014e37c7d974fde8a8eb0c45c54fb11",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c33c9dd3fc114dddb5f3eb004de9f8fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c8f54d57edc142adb2b391018d4ad129",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b4c4edabcaa404dae2cedfdd113e529",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a22cd6123a834502b13b2dcb328bc783",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00726caf6f25472dbc9ceb980ae14613",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3380a48b1e5946829330b71dbbc75dfe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "768ef1cb094c46de949ed71c22f93bd7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "82a383d61b274123b38bef274ff947e6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4583d1afa6b84385a0df54306d082cda",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f59c5d75dbf24e7cb04226623af08036",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef401017b0ee47d281152344960bffbf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54b70e88aa904654acc789aa03aab8d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f695d65b7f02405a91cd06286e4c769b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd2b9c499a7f4104b7e42f1e188485ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8fd86f366044fb0a19ebbd0f7232ee6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "088e7271b58044e3adb762e8ca2bf0a5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d7652dc283f4208b58ebf66fe93ab43",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3c2df83cbab40fa8a62ca219c54f50b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "416e703bc50f47aeabb5178b255be2a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1bf1a9c84fd44a7da04b1e6c6e16c161",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a374af0a82474344b418888c7cd69229",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0be2ae2669f84561999e13f09821428e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b47d5803750b476982e8e2f0262a4d00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59eceae7540d4f1694bef3d071dd48d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f38202e781524145a63e5a1045644cc6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b40b881df934ef68702963ed7eeb77f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a9fbfc87d62463585049e143c70878f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "76b1f8bb764a4b7ab7f218a18c13e2a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a85e560a52ef4921b32a1530ec27e002",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3db146b925484493b98968154714a6d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset from disk:   0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lengths = {}\n",
    "for name in dataset_list:\n",
    "    dataset = datasets.load_from_disk(str(env.LOTSA_V1_PATH / name)).with_format(\n",
    "        \"numpy\"\n",
    "    )\n",
    "    if dataset[0][\"target\"].ndim > 1:\n",
    "        lengths[name] = pc.list_value_length(\n",
    "            pc.list_flatten(pc.list_slice(dataset.data.column(\"target\"), 0, 1))\n",
    "        ).to_numpy()\n",
    "    else:\n",
    "        lengths[name] = pc.list_value_length(dataset.data.column(\"target\")).to_numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "052c4a04-3a94-48c9-bf49-f4ea16f8f83a",
   "metadata": {},
   "source": [
    "2. Some datasets have been split into smaller chunks for efficiency -- group them back together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5f2ba108-9681-4378-ae35-cf45be25ac8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataset_group(name):\n",
    "    if name.startswith(\"era5\"):\n",
    "        return \"era5\"\n",
    "    if name.startswith(\"cmip6\"):\n",
    "        return \"cmip6\"\n",
    "    if name.startswith(\"largest\"):\n",
    "        return \"largest\"\n",
    "    return name\n",
    "\n",
    "\n",
    "group_lengths = {}\n",
    "for k, v in lengths.items():\n",
    "    group = get_dataset_group(k)\n",
    "    if group in group_lengths:\n",
    "        group_lengths[group] = np.concatenate([group_lengths[group], v])\n",
    "    else:\n",
    "        group_lengths[group] = v"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f8ae20b-45f3-4527-aa26-22e6c628a2be",
   "metadata": {},
   "source": [
    "3. Compute the weights for each dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6282f8af-1fbc-400c-a94e-f8d82532bf90",
   "metadata": {},
   "source": [
    "From the [paper](https://arxiv.org/abs/2402.02592), the sampling distribution for dataset $D_k$ is given by:\n",
    "\n",
    "\\begin{align}\n",
    "p(D_k) & = \\frac{\\min(\\omega_k, \\epsilon)}{\\sum_{i=1}^K \\min(\\omega_i, \\epsilon)} \\\\\n",
    "\\\\\n",
    "\\omega_k & = \\frac{|D_k|}{\\sum_i^K |D_i|} \\\\\n",
    "\\\\\n",
    "|D_k| & = \\sum_{i \\in D_k} T_i\n",
    "\\end{align}\n",
    "where $T_i$ for $i \\in D_k$ is the length of time series $i$ belonging to dataset $D_k$.\n",
    "\n",
    "The default PyTorch sampler samples each dataset $D_k$ based on the number of time series in the dataset, i.e. with probability:\n",
    "\n",
    "\\begin{align}\n",
    "\\frac{\\sum_{i \\in D_k} 1}{\\sum_{j=1}^K \\sum_{l \\in D_j} 1}\n",
    "\\end{align}\n",
    "\n",
    "`weight_map` aims to _reweight_ the probability to sample dataset $D_k$ with a multiplier. \n",
    "\n",
    "$\\mathrm{reweight}_k$, which is calculated as follows:\n",
    "\n",
    "\\begin{align}\n",
    "\\mathrm{reweight}_k = \\omega_k * \\frac{\\sum_{j=1}^K \\sum_{l \\in D_j} 1}{\\sum_{i \\in D_k} 1}\n",
    "\\end{align}\n",
    "\n",
    "Thus, the probability to sample dataset $D_k$ is $\\omega_k$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "78b46882-eef9-478a-9004-15670b242ece",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute \\omega_k\n",
    "total_lengths = np.asarray([v.sum() for v in group_lengths.values()])\n",
    "omegas = total_lengths / total_lengths.sum()\n",
    "omegas = np.clip(omegas, a_min=0, a_max=0.001)\n",
    "weights = omegas / omegas.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1867dddb-da26-4c28-b05f-3817d4d30d6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute reweight_k\n",
    "num_ts = np.asarray([v.shape[0] for v in group_lengths.values()])\n",
    "group_reweights = weights * num_ts.sum() / num_ts\n",
    "group_reweights = {k: v for k, v in zip(group_lengths.keys(), group_reweights)}\n",
    "group_size = {\n",
    "    group: len([k for k in lengths.keys() if get_dataset_group(k) == group])\n",
    "    for group in group_lengths.keys()\n",
    "}\n",
    "reweights = {\n",
    "    k: group_reweights[group] / group_size[group]\n",
    "    for k in lengths.keys()\n",
    "    if (group := get_dataset_group(k))\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4b4d6a5-37c5-4b6a-af8a-3d2309eb3396",
   "metadata": {},
   "source": [
    "4. Finally, we can generate the YAML file required for the pre-training dataset with the appropriate `weight_map`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "305782c4-8602-4f12-8794-7db18a72010a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- _target_: uni2ts.data.builder.lotsa_v1.Buildings900KDatasetBuilder\n",
      "  datasets: ${cls_getattr:${._target_},dataset_list}\n",
      "  weight_map:\n",
      "    buildings_900k: 0.07018480580999217\n",
      "  sample_time_series:\n",
      "    _target_: uni2ts.data.dataset.SampleTimeSeriesType\n",
      "    _args_: [\"proportional\"]\n",
      "- _target_: uni2ts.data.builder.lotsa_v1.BuildingsBenchDatasetBuilder\n",
      "  datasets: ${cls_getattr:${._target_},dataset_list}\n",
      "  weight_map:\n",
      "    sceaux: 155.8261547285959\n",
      "    borealis: 25.276350364950382\n",
      "    ideal: 26.338668163517678\n",
      "    bdg-2:panther: 39.88654166561961\n",
      "    bdg-2:fox: 78.40265380227133\n",
      "    bdg-2:rat: 76.88970006484395\n",
      "    bdg-2:bear: 74.16867297587429\n",
      "    smart: 87.15755744919608\n",
      "    lcl: 60.945667283481136\n",
      "  sample_time_series:\n",
      "    _target_: uni2ts.data.dataset.SampleTimeSeriesType\n",
      "    _args_: [\"proportional\"]\n",
      "- _target_: uni2ts.data.builder.lotsa_v1.CloudOpsTSFDatasetBuilder\n",
      "  datasets: ${cls_getattr:${._target_},dataset_list}\n",
      "  weight_map:\n",
      "    azure_vm_traces_2017: 0.7901054338016913\n",
      "    borg_cluster_data_2011: 0.8787447431354757\n",
      "    alibaba_cluster_trace_2018: 2.1571965577089713\n",
      "  sample_time_series:\n",
      "    _target_: uni2ts.data.dataset.SampleTimeSeriesType\n",
      "    _args_: [\"proportional\"]\n",
      "- _target_: uni2ts.data.builder.lotsa_v1.CMIP6DatasetBuilder\n",
      "  datasets: ${cls_getattr:${._target_},dataset_list}\n",
      "  weight_map:\n",
      "    cmip6_1850: 0.014123803479936898\n",
      "    cmip6_1855: 0.014123803479936898\n",
      "    cmip6_1860: 0.014123803479936898\n",
      "    cmip6_1865: 0.014123803479936898\n",
      "    cmip6_1870: 0.014123803479936898\n",
      "    cmip6_1875: 0.014123803479936898\n",
      "    cmip6_1880: 0.014123803479936898\n",
      "    cmip6_1885: 0.014123803479936898\n",
      "    cmip6_1890: 0.014123803479936898\n",
      "    cmip6_1895: 0.014123803479936898\n",
      "    cmip6_1900: 0.014123803479936898\n",
      "    cmip6_1905: 0.014123803479936898\n",
      "    cmip6_1910: 0.014123803479936898\n",
      "    cmip6_1915: 0.014123803479936898\n",
      "    cmip6_1920: 0.014123803479936898\n",
      "    cmip6_1925: 0.014123803479936898\n",
      "    cmip6_1930: 0.014123803479936898\n",
      "    cmip6_1935: 0.014123803479936898\n",
      "    cmip6_1940: 0.014123803479936898\n",
      "    cmip6_1945: 0.014123803479936898\n",
      "    cmip6_1950: 0.014123803479936898\n",
      "    cmip6_1955: 0.014123803479936898\n",
      "    cmip6_1960: 0.014123803479936898\n",
      "    cmip6_1965: 0.014123803479936898\n",
      "    cmip6_1970: 0.014123803479936898\n",
      "    cmip6_1975: 0.014123803479936898\n",
      "    cmip6_1980: 0.014123803479936898\n",
      "    cmip6_1985: 0.014123803479936898\n",
      "    cmip6_1990: 0.014123803479936898\n",
      "    cmip6_1995: 0.014123803479936898\n",
      "    cmip6_2000: 0.014123803479936898\n",
      "    cmip6_2005: 0.014123803479936898\n",
      "    cmip6_2010: 0.014123803479936898\n",
      "  sample_time_series:\n",
      "    _target_: uni2ts.data.dataset.SampleTimeSeriesType\n",
      "    _args_: [\"proportional\"]\n",
      "- _target_: uni2ts.data.builder.lotsa_v1.ERA5DatasetBuilder\n",
      "  datasets: ${cls_getattr:${._target_},dataset_list}\n",
      "  weight_map:\n",
      "    era5_1989: 0.017089802210723648\n",
      "    era5_1990: 0.017089802210723648\n",
      "    era5_1991: 0.017089802210723648\n",
      "    era5_1992: 0.017089802210723648\n",
      "    era5_1993: 0.017089802210723648\n",
      "    era5_1994: 0.017089802210723648\n",
      "    era5_1995: 0.017089802210723648\n",
      "    era5_1996: 0.017089802210723648\n",
      "    era5_1997: 0.017089802210723648\n",
      "    era5_1998: 0.017089802210723648\n",
      "    era5_1999: 0.017089802210723648\n",
      "    era5_2000: 0.017089802210723648\n",
      "    era5_2001: 0.017089802210723648\n",
      "    era5_2002: 0.017089802210723648\n",
      "    era5_2003: 0.017089802210723648\n",
      "    era5_2004: 0.017089802210723648\n",
      "    era5_2005: 0.017089802210723648\n",
      "    era5_2006: 0.017089802210723648\n",
      "    era5_2007: 0.017089802210723648\n",
      "    era5_2008: 0.017089802210723648\n",
      "    era5_2009: 0.017089802210723648\n",
      "    era5_2010: 0.017089802210723648\n",
      "    era5_2011: 0.017089802210723648\n",
      "    era5_2012: 0.017089802210723648\n",
      "    era5_2013: 0.017089802210723648\n",
      "    era5_2014: 0.017089802210723648\n",
      "    era5_2015: 0.017089802210723648\n",
      "    era5_2016: 0.017089802210723648\n",
      "    era5_2017: 0.017089802210723648\n",
      "    era5_2018: 0.017089802210723648\n",
      "  sample_time_series:\n",
      "    _target_: uni2ts.data.dataset.SampleTimeSeriesType\n",
      "    _args_: [\"proportional\"]\n",
      "- _target_: uni2ts.data.builder.lotsa_v1.GluonTSDatasetBuilder\n",
      "  datasets: ${cls_getattr:${._target_},dataset_list}\n",
      "  weight_map:\n",
      "    taxi_30min: 1.8533727603439532\n",
      "    uber_tlc_daily: 0.81831781667583\n",
      "    uber_tlc_hourly: 19.62843562209561\n",
      "    wiki-rolling_nips: 2.6428881749181605\n",
      "    london_smart_meters_with_missing: 22.826031474496975\n",
      "    wind_farms_with_missing: 373.88633157039555\n",
      "    wind_power: 33681.12009385995\n",
      "    solar_power: 33681.46158822353\n",
      "    oikolab_weather: 455.585353816998\n",
      "    elecdemand: 79.77308333123923\n",
      "    covid_mobility: 1.8691250613645671\n",
      "    kaggle_web_traffic_weekly: 0.5190714326347758\n",
      "    extended_web_traffic_with_missing: 0.8685860194482625\n",
      "    m5: 4.1324924151926306\n",
      "    m4_yearly: 0.16833058491201394\n",
      "    m1_yearly: 0.1347077137324278\n",
      "    m1_quarterly: 0.2266050813921266\n",
      "    monash_m3_yearly: 0.1293195916556026\n",
      "    monash_m3_quarterly: 0.22286873773806326\n",
      "    tourism_yearly: 0.12168826995444919\n",
      "    m4_hourly: 3.887866586922478\n",
      "    m4_daily: 10.733773494070244\n",
      "    m4_weekly: 4.653607425374844\n",
      "    m4_monthly: 0.984870788006772\n",
      "    m4_quarterly: 0.42008730340497724\n",
      "    m1_monthly: 0.33128827595098465\n",
      "    monash_m3_monthly: 0.4523221982081677\n",
      "    monash_m3_other: 0.312264539505484\n",
      "    nn5_daily_with_missing: 3.3466447630400022\n",
      "    nn5_weekly: 0.4780921090057146\n",
      "    tourism_monthly: 1.2502301479739342\n",
      "    tourism_quarterly: 0.41723626722818336\n",
      "    cif_2016_6: 0.19214749523848723\n",
      "    cif_2016_12: 0.45540569982315443\n",
      "    traffic_hourly: 79.11741415317427\n",
      "    traffic_weekly: 0.4371127853766533\n",
      "    australian_electricity_demand: 1050.513157095711\n",
      "    rideshare_with_missing: 1.6983653015155384\n",
      "    saugeenday: 107.96230472985238\n",
      "    sunspot_with_missing: 336.4584600273169\n",
      "    temperature_rain_with_missing: 3.16451443579973\n",
      "    vehicle_trips_with_missing: 0.44995601817900693\n",
      "    weather: 41.86036336851273\n",
      "    car_parts_with_missing: 0.17757706905926543\n",
      "    fred_md: 3.260132857600873\n",
      "    pedestrian_counts: 215.9873892315944\n",
      "    hospital: 0.3278345890324901\n",
      "    covid_deaths: 0.8286929889432386\n",
      "    kdd_cup_2018_with_missing: 48.85484134596089\n",
      "    bitcoin_with_missing: 20.721877981761974\n",
      "    us_births: 33.124953266824505\n",
      "  sample_time_series:\n",
      "    _target_: uni2ts.data.dataset.SampleTimeSeriesType\n",
      "    _args_: [\"proportional\"]\n",
      "- _target_: uni2ts.data.builder.lotsa_v1.LargeSTDatasetBuilder\n",
      "  datasets: ${cls_getattr:${._target_},dataset_list}\n",
      "  weight_map:\n",
      "    largest_2017: 0.5952788308847627\n",
      "    largest_2018: 0.5952788308847627\n",
      "    largest_2019: 0.5952788308847627\n",
      "    largest_2020: 0.5952788308847627\n",
      "    largest_2021: 0.5952788308847627\n",
      "  sample_time_series:\n",
      "    _target_: uni2ts.data.dataset.SampleTimeSeriesType\n",
      "    _args_: [\"proportional\"]\n",
      "- _target_: uni2ts.data.builder.lotsa_v1.LibCityDatasetBuilder\n",
      "  datasets: ${cls_getattr:${._target_},dataset_list}\n",
      "  weight_map:\n",
      "    PEMS03: 119.33179040782636\n",
      "    PEMS04: 77.36896301166765\n",
      "    PEMS07: 128.5111589007361\n",
      "    PEMS08: 81.30297808005753\n",
      "    PEMS_BAY: 237.2976033613507\n",
      "    LOS_LOOP: 156.04926437946526\n",
      "    LOOP_SEATTLE: 390.0919310811867\n",
      "    SZ_TAXI: 13.550496346676255\n",
      "    BEIJING_SUBWAY_30MIN: 4.097932362906126\n",
      "    HZMETRO: 8.30969618033742\n",
      "    SHMETRO: 30.579681943641713\n",
      "    ROTTERDAM: 105.37150082485951\n",
      "    Q-TRAFFIC: 2.7908145153544632\n",
      "  sample_time_series:\n",
      "    _target_: uni2ts.data.dataset.SampleTimeSeriesType\n",
      "    _args_: [\"proportional\"]\n",
      "- _target_: uni2ts.data.builder.lotsa_v1.OthersLOTSADatasetBuilder\n",
      "  datasets: ${cls_getattr:${._target_},dataset_list}\n",
      "  weight_map:\n",
      "    kdd2022: 160.63891464638144\n",
      "    godaddy: 0.18668358542127902\n",
      "    favorita_sales: 1.1266067036768894\n",
      "    favorita_transactions: 7.117248454489305\n",
      "    restaurant: 1.628828212731436\n",
      "    hierarchical_sales: 8.186758209450236\n",
      "    china_air_quality: 59.79911707828927\n",
      "    beijing_air_quality: 159.65544485882262\n",
      "    residential_load_power: 464.94351933292734\n",
      "    residential_pv_power: 540.7712177649069\n",
      "    cdc_fluview_ilinet: 3.8795581005450384\n",
      "    cdc_fluview_who_nrevss: 2.5695143464708674\n",
      "    project_tycho: 4.986530738299161\n",
      "  sample_time_series:\n",
      "    _target_: uni2ts.data.dataset.SampleTimeSeriesType\n",
      "    _args_: [\"proportional\"]\n",
      "- _target_: uni2ts.data.builder.lotsa_v1.ProEnFoDatasetBuilder\n",
      "  datasets: ${cls_getattr:${._target_},dataset_list}\n",
      "  weight_map:\n",
      "    gfc12_load: 179.46211794620223\n",
      "    gfc14_load: 79.77308333123923\n",
      "    gfc17_load: 79.8823615275834\n",
      "    spain: 159.65544485882262\n",
      "    pdb: 79.77308333123923\n",
      "    elf: 99.22460228050032\n",
      "    bull: 79.8823615275834\n",
      "    cockatoo: 79.8823615275834\n",
      "    hog: 79.8823615275834\n",
      "    covid19_energy: 145.3035750722892\n",
      "  sample_time_series:\n",
      "    _target_: uni2ts.data.dataset.SampleTimeSeriesType\n",
      "    _args_: [\"proportional\"]\n",
      "- _target_: uni2ts.data.builder.lotsa_v1.SubseasonalDatasetBuilder\n",
      "  datasets: ${cls_getattr:${._target_},dataset_list}\n",
      "  weight_map:\n",
      "    subseasonal: 74.99216224118207\n",
      "    subseasonal_precip: 51.556542383540055\n",
      "  sample_time_series:\n",
      "    _target_: uni2ts.data.dataset.SampleTimeSeriesType\n",
      "    _args_: [\"proportional\"]\n"
     ]
    }
   ],
   "source": [
    "for builder_cls in [\n",
    "    Buildings900KDatasetBuilder,\n",
    "    BuildingsBenchDatasetBuilder,\n",
    "    CloudOpsTSFDatasetBuilder,\n",
    "    CMIP6DatasetBuilder,\n",
    "    ERA5DatasetBuilder,\n",
    "    GluonTSDatasetBuilder,\n",
    "    LargeSTDatasetBuilder,\n",
    "    LibCityDatasetBuilder,\n",
    "    OthersLOTSADatasetBuilder,\n",
    "    ProEnFoDatasetBuilder,\n",
    "    SubseasonalDatasetBuilder,\n",
    "]:\n",
    "    print(f\"- _target_: uni2ts.data.builder.lotsa_v1.{builder_cls.__name__}\")\n",
    "    print(\"  datasets: ${cls_getattr:${._target_},dataset_list}\")\n",
    "    print(\"  weight_map:\")\n",
    "    for dataset in builder_cls.dataset_list:\n",
    "        print(f\"    {dataset}: {reweights[dataset]}\")\n",
    "    print(\"  sample_time_series:\")\n",
    "    print(\"    _target_: uni2ts.data.dataset.SampleTimeSeriesType\")\n",
    "    print('    _args_: [\"proportional\"]')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}