river/Anomaly_Detection_Flow_2.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WhiT_u8Cuhla"
      },
      "source": [
        "# Anomaly Detection on the Virgin River with FF\n",
        "\n",
        "Anomaly Detection is a .\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BIfA9A_mur78",
        "outputId": "18ea078c-c4e1-41c3-db68-5ac1bf7ebae7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloning into 'flow-forecast'...\n",
            "remote: Enumerating objects: 18191, done.\u001b[K\n",
            "remote: Counting objects: 100% (604/604), done.\u001b[K\n",
            "remote: Compressing objects: 100% (241/241), done.\u001b[K\n",
            "remote: Total 18191 (delta 367), reused 484 (delta 313), pack-reused 17587\u001b[K\n",
            "Receiving objects: 100% (18191/18191), 4.72 MiB | 16.90 MiB/s, done.\n",
            "Resolving deltas: 100% (13072/13072), done.\n",
            "Copying gs://predict_cfs/virgin_final1.csv...\n",
            "-\n",
            "Operation completed over 1 objects/9.2 MiB.                                      \n",
            "Copying gs://predict_cfs/joined_final_3/0101000040B_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/0101007040B_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/01010500FVE_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/0101100040B_flow.csv...\n",
            "- [4 files][ 98.5 MiB/ 98.5 MiB]                                                \n",
            "==> NOTE: You are performing a sequence of gsutil operations that may\n",
            "run significantly faster if you instead use gsutil -m cp ... Please\n",
            "see the -m section under \"gsutil help options\" for further information\n",
            "about when gsutil -m can be advantageous.\n",
            "\n",
            "Copying gs://predict_cfs/joined_final_3/01017290CAR_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/01017550PQI_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/01017960HUL_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/01018000HUL_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/01018009HUL_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/01037380RKD_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/01042500GNR_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/01099500LWM_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/01638500FDK_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/01646500DCA_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/06741510FNL_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/09405500AZC_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/09406000AZC_flow.csv...\n",
            "Copying gs://predict_cfs/joined_final_3/09408135DXZ_flow.csv...\n",
            "/ [18 files][444.4 MiB/444.4 MiB]   27.8 MiB/s                                  \n",
            "Operation completed over 18 objects/444.4 MiB.                                   \n",
            "Copying gs://aistream-datasets/flowdb/09405500AZC_flow.csv...\n",
            "/ [1 files][ 19.6 MiB/ 19.6 MiB]                                                \n",
            "Operation completed over 1 objects/19.6 MiB.                                     \n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py:3326: DtypeWarning: Columns (2,4,7,8,9,18) have mixed types.Specify dtype option on import or set low_memory=False.\n",
            "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
          ]
        }
      ],
      "source": [
        "from google.colab import auth\n",
        "from datetime import datetime\n",
        "import os\n",
        "import pandas as pd\n",
        "auth.authenticate_user()\n",
        "!git clone https://github.com/AIStream-Peelout/flow-forecast.git\n",
        "!gsutil cp gs://predict_cfs/virgin_final1.csv virgin_final.csv \n",
        "!gsutil cp -r gs://predict_cfs/joined_final_3 .\n",
        "!gsutil cp gs://aistream-datasets/flowdb/09405500AZC_flow.csv .\n",
        "df = pd.read_csv(\"09405500AZC_flow.csv\")\n",
        "df[\"anomalous_rain\"] = df[\"p01m\"].apply(lambda x: 0 if x<1 else 1)\n",
        "df = df.dropna(subset=[\"hour_updated\", \"cfs\", \"p01m\"])\n",
        "df.to_csv(\"/content/joined_final_3/09405500AZC_flow.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WHn1BthudNk0"
      },
      "outputs": [],
      "source": [
        "the_config = {                 \n",
        "   \"model_name\": \"CustomTransformerDecoder\",\n",
        "   \"n_targets\": 2,\n",
        "   \"model_type\": \"PyTorch\",\n",
        "    \"model_params\": {\n",
        "      \"n_time_series\":4,\n",
        "      \"seq_length\":26,\n",
        "      \"output_seq_length\": 1, \n",
        "      \"output_dim\":2,\n",
        "      \"n_layers_encoder\": 6\n",
        "     }, \n",
        "    \"dataset_params\":\n",
        "    { \"class\": \"GeneralClassificationLoader\",\n",
        "      \"n_classes\": 2,\n",
        "       \"training_path\": \"/content/joined_final_3/09405500AZC_flow.csv\",\n",
        "       \"validation_path\": \"/content/joined_final_3/09405500AZC_flow.csv\",\n",
        "       \"test_path\": \"/content/joined_final_3/09405500AZC_flow.csv\",\n",
        "       \"sequence_length\":26,\n",
        "       \"batch_size\":4,\n",
        "       \"forecast_history\":26,\n",
        "       \"train_end\": 4500,\n",
        "       \"valid_start\":4501,\n",
        "       \"valid_end\": 7000,\n",
        "      \n",
        "       \"target_col\": [\"anomalous_rain\"],\n",
        "       \"relevant_cols\": [\"anomalous_rain\"] + [\"tmpf\", \"cfs\", \"dwpf\", \"height\"],\n",
        "       \"scaler\": \"StandardScaler\", \n",
        "       \"interpolate\":{\n",
        "           \"method\":\"back_forward_generic\",\n",
        "           \"params\":{\n",
        "               \"relevant_columns\":[\"cfs\", \"tmpf\", \"p01m\", \"dwpf\"]\n",
        "           }\n",
        "\n",
        "       },\n",
        "    },\n",
        "\n",
        "    \"training_params\":\n",
        "    {\n",
        "       \"criterion\":\"CrossEntropyLoss\",\n",
        "       \"optimizer\": \"Adam\",\n",
        "       \"optim_params\":\n",
        "       {},\n",
        "       \"lr\": 0.03,\n",
        "       \"epochs\": 4,\n",
        "       \"batch_size\":100\n",
        "    },\n",
        "    \"GCS\": True,\n",
        "   \n",
        "    \"wandb\": {\n",
        "       \"name\": \"flood_forecast_anomaly_virign\",\n",
        "       \"tags\": [\"anomaly_detection\", \"virgin_river\", \"simple\", \"classification\"],\n",
        "       \"project\": \"repo-flood_forecast\"\n",
        "    },\n",
        "   \"forward_params\":{},\n",
        "   \"metrics\":[\"CrossEntropyLoss\"]\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LUnF3Ajquv0S"
      },
      "outputs": [],
      "source": [
        "df[\"anomalous_rain\"] = df[\"p01m\"].apply(lambda x: 0 if x<1 else 1)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "-8v0_dKn1qkI",
        "outputId": "e1d5370e-8371-4aa7-b4d5-d9c020d1ed7e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting shap==0.40.0\n",
            "  Downloading shap-0.40.0-cp37-cp37m-manylinux2010_x86_64.whl (564 kB)\n",
            "\u001b[K     |████████████████████████████████| 564 kB 5.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: scikit-learn>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 2)) (1.0.2)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 3)) (1.3.5)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 4)) (1.12.1+cu113)\n",
            "Collecting tb-nightly\n",
            "  Downloading tb_nightly-2.11.0a20221101-py3-none-any.whl (6.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.0 MB 39.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: seaborn in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 6)) (0.11.2)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 7)) (0.16.0)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 8)) (3.1.0)\n",
            "Collecting wandb==0.13.3\n",
            "  Downloading wandb-0.13.3-py2.py3-none-any.whl (1.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.8 MB 60.5 MB/s \n",
            "\u001b[?25hCollecting google-cloud\n",
            "  Downloading google_cloud-0.34.0-py2.py3-none-any.whl (1.8 kB)\n",
            "Requirement already satisfied: google-cloud-storage in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 11)) (1.18.1)\n",
            "Collecting plotly~=5.7.0\n",
            "  Downloading plotly-5.7.0-py2.py3-none-any.whl (28.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 28.8 MB 2.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pytz>=2022.1 in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 13)) (2022.5)\n",
            "Collecting setuptools~=65.3.0\n",
            "  Downloading setuptools-65.3.0-py3-none-any.whl (1.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 51.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.21 in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 15)) (1.21.6)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 16)) (2.23.0)\n",
            "Requirement already satisfied: torchvision>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 17)) (0.13.1+cu113)\n",
            "Collecting mpld3>=0.5\n",
            "  Downloading mpld3-0.5.8-py3-none-any.whl (201 kB)\n",
            "\u001b[K     |████████████████████████████████| 201 kB 45.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numba>=0.50 in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 19)) (0.56.3)\n",
            "Requirement already satisfied: sphinx in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 20)) (1.8.6)\n",
            "Collecting sphinx-rtd-theme\n",
            "  Downloading sphinx_rtd_theme-1.1.0-py2.py3-none-any.whl (2.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.8 MB 34.5 MB/s \n",
            "\u001b[?25hCollecting sphinx-autodoc-typehints\n",
            "  Downloading sphinx_autodoc_typehints-1.19.4-py3-none-any.whl (12 kB)\n",
            "Requirement already satisfied: tqdm>4.25.0 in /usr/local/lib/python3.7/dist-packages (from shap==0.40.0->-r requirements.txt (line 1)) (4.64.1)\n",
            "Requirement already satisfied: packaging>20.9 in /usr/local/lib/python3.7/dist-packages (from shap==0.40.0->-r requirements.txt (line 1)) (21.3)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.7/dist-packages (from shap==0.40.0->-r requirements.txt (line 1)) (1.5.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from shap==0.40.0->-r requirements.txt (line 1)) (1.7.3)\n",
            "Collecting slicer==0.0.7\n",
            "  Downloading slicer-0.0.7-py3-none-any.whl (14 kB)\n",
            "Collecting GitPython>=1.0.0\n",
            "  Downloading GitPython-3.1.29-py3-none-any.whl (182 kB)\n",
            "\u001b[K     |████████████████████████████████| 182 kB 66.9 MB/s \n",
            "\u001b[?25hCollecting sentry-sdk>=1.0.0\n",
            "  Downloading sentry_sdk-1.10.1-py2.py3-none-any.whl (166 kB)\n",
            "\u001b[K     |████████████████████████████████| 166 kB 56.5 MB/s \n",
            "\u001b[?25hCollecting pathtools\n",
            "  Downloading pathtools-0.1.2.tar.gz (11 kB)\n",
            "Requirement already satisfied: protobuf<4.0dev,>=3.12.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.3->-r requirements.txt (line 9)) (3.17.3)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.3->-r requirements.txt (line 9)) (6.0)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.3->-r requirements.txt (line 9)) (5.4.8)\n",
            "Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.3->-r requirements.txt (line 9)) (2.3)\n",
            "Collecting shortuuid>=0.5.0\n",
            "  Downloading shortuuid-1.0.9-py3-none-any.whl (9.4 kB)\n",
            "Collecting docker-pycreds>=0.4.0\n",
            "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
            "Collecting setproctitle\n",
            "  Downloading setproctitle-1.3.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n",
            "Requirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.3->-r requirements.txt (line 9)) (1.15.0)\n",
            "Requirement already satisfied: Click!=8.0.0,>=7.0 in /usr/local/lib/python3.7/dist-packages (from wandb==0.13.3->-r requirements.txt (line 9)) (7.1.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=1.0.1->-r requirements.txt (line 2)) (1.2.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=1.0.1->-r requirements.txt (line 2)) (3.1.0)\n",
            "Requirement already satisfied: tenacity>=6.2.0 in /usr/local/lib/python3.7/dist-packages (from plotly~=5.7.0->-r requirements.txt (line 12)) (8.1.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->-r requirements.txt (line 16)) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->-r requirements.txt (line 16)) (2022.9.24)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->-r requirements.txt (line 16)) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->-r requirements.txt (line 16)) (3.0.4)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torchvision>=0.6.0->-r requirements.txt (line 17)) (4.1.1)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision>=0.6.0->-r requirements.txt (line 17)) (7.1.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from mpld3>=0.5->-r requirements.txt (line 18)) (2.11.3)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from mpld3>=0.5->-r requirements.txt (line 18)) (3.2.2)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from numba>=0.50->-r requirements.txt (line 19)) (4.13.0)\n",
            "Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /usr/local/lib/python3.7/dist-packages (from numba>=0.50->-r requirements.txt (line 19)) (0.39.1)\n",
            "Collecting gitdb<5,>=4.0.1\n",
            "  Downloading gitdb-4.0.9-py3-none-any.whl (63 kB)\n",
            "\u001b[K     |████████████████████████████████| 63 kB 1.6 MB/s \n",
            "\u001b[?25hCollecting smmap<6,>=3.0.1\n",
            "  Downloading smmap-5.0.0-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>20.9->shap==0.40.0->-r requirements.txt (line 1)) (3.0.9)\n",
            "Collecting sentry-sdk>=1.0.0\n",
            "  Downloading sentry_sdk-1.10.0-py2.py3-none-any.whl (166 kB)\n",
            "\u001b[K     |████████████████████████████████| 166 kB 65.3 MB/s \n",
            "\u001b[?25h  Downloading sentry_sdk-1.9.10-py2.py3-none-any.whl (162 kB)\n",
            "\u001b[K     |████████████████████████████████| 162 kB 58.1 MB/s \n",
            "\u001b[?25h  Downloading sentry_sdk-1.9.9-py2.py3-none-any.whl (162 kB)\n",
            "\u001b[K     |████████████████████████████████| 162 kB 50.9 MB/s \n",
            "\u001b[?25h  Downloading sentry_sdk-1.9.8-py2.py3-none-any.whl (158 kB)\n",
            "\u001b[K     |████████████████████████████████| 158 kB 62.0 MB/s \n",
            "\u001b[?25h  Downloading sentry_sdk-1.9.7-py2.py3-none-any.whl (157 kB)\n",
            "\u001b[K     |████████████████████████████████| 157 kB 69.6 MB/s \n",
            "\u001b[?25h  Downloading sentry_sdk-1.9.6-py2.py3-none-any.whl (157 kB)\n",
            "\u001b[K     |████████████████████████████████| 157 kB 60.1 MB/s \n",
            "\u001b[?25h  Downloading sentry_sdk-1.9.5-py2.py3-none-any.whl (157 kB)\n",
            "\u001b[K     |████████████████████████████████| 157 kB 51.5 MB/s \n",
            "\u001b[?25h  Downloading sentry_sdk-1.9.4-py2.py3-none-any.whl (157 kB)\n",
            "\u001b[K     |████████████████████████████████| 157 kB 74.5 MB/s \n",
            "\u001b[?25h  Downloading sentry_sdk-1.9.3-py2.py3-none-any.whl (157 kB)\n",
            "\u001b[K     |████████████████████████████████| 157 kB 75.0 MB/s \n",
            "\u001b[?25h  Downloading sentry_sdk-1.9.2-py2.py3-none-any.whl (157 kB)\n",
            "\u001b[K     |████████████████████████████████| 157 kB 60.7 MB/s \n",
            "\u001b[?25h  Downloading sentry_sdk-1.9.1-py2.py3-none-any.whl (157 kB)\n",
            "\u001b[K     |████████████████████████████████| 157 kB 52.5 MB/s \n",
            "\u001b[?25h  Downloading sentry_sdk-1.9.0-py2.py3-none-any.whl (156 kB)\n",
            "\u001b[K     |████████████████████████████████| 156 kB 61.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->-r requirements.txt (line 3)) (2.8.2)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tb-nightly->-r requirements.txt (line 5)) (1.50.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tb-nightly->-r requirements.txt (line 5)) (0.4.6)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from tb-nightly->-r requirements.txt (line 5)) (1.0.1)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tb-nightly->-r requirements.txt (line 5)) (0.37.1)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tb-nightly->-r requirements.txt (line 5)) (1.3.0)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tb-nightly->-r requirements.txt (line 5)) (1.35.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tb-nightly->-r requirements.txt (line 5)) (0.6.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tb-nightly->-r requirements.txt (line 5)) (3.4.1)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tb-nightly->-r requirements.txt (line 5)) (1.8.1)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tb-nightly->-r requirements.txt (line 5)) (4.2.4)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tb-nightly->-r requirements.txt (line 5)) (4.9)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tb-nightly->-r requirements.txt (line 5)) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tb-nightly->-r requirements.txt (line 5)) (1.3.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->numba>=0.50->-r requirements.txt (line 19)) (3.10.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tb-nightly->-r requirements.txt (line 5)) (0.4.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tb-nightly->-r requirements.txt (line 5)) (3.2.2)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->mpld3>=0.5->-r requirements.txt (line 18)) (1.4.4)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->mpld3>=0.5->-r requirements.txt (line 18)) (0.11.0)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py->-r requirements.txt (line 8)) (1.5.2)\n",
            "Requirement already satisfied: google-cloud-core<2.0dev,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from google-cloud-storage->-r requirements.txt (line 11)) (1.0.3)\n",
            "Requirement already satisfied: google-resumable-media<0.5.0dev,>=0.3.1 in /usr/local/lib/python3.7/dist-packages (from google-cloud-storage->-r requirements.txt (line 11)) (0.4.1)\n",
            "Requirement already satisfied: google-api-core<2.0.0dev,>=1.14.0 in /usr/local/lib/python3.7/dist-packages (from google-cloud-core<2.0dev,>=1.0.0->google-cloud-storage->-r requirements.txt (line 11)) (1.31.6)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from google-api-core<2.0.0dev,>=1.14.0->google-cloud-core<2.0dev,>=1.0.0->google-cloud-storage->-r requirements.txt (line 11)) (1.56.4)\n",
            "Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.7/dist-packages (from sphinx->-r requirements.txt (line 20)) (2.2.0)\n",
            "Requirement already satisfied: docutils<0.18,>=0.11 in /usr/local/lib/python3.7/dist-packages (from sphinx->-r requirements.txt (line 20)) (0.17.1)\n",
            "Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.7/dist-packages (from sphinx->-r requirements.txt (line 20)) (2.6.1)\n",
            "Requirement already satisfied: sphinxcontrib-websupport in /usr/local/lib/python3.7/dist-packages (from sphinx->-r requirements.txt (line 20)) (1.2.4)\n",
            "Requirement already satisfied: babel!=2.0,>=1.3 in /usr/local/lib/python3.7/dist-packages (from sphinx->-r requirements.txt (line 20)) (2.10.3)\n",
            "Requirement already satisfied: imagesize in /usr/local/lib/python3.7/dist-packages (from sphinx->-r requirements.txt (line 20)) (1.4.1)\n",
            "Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.7/dist-packages (from sphinx->-r requirements.txt (line 20)) (0.7.12)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->mpld3>=0.5->-r requirements.txt (line 18)) (2.0.1)\n",
            "Collecting sphinx\n",
            "  Downloading sphinx-5.3.0-py3-none-any.whl (3.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2 MB 33.5 MB/s \n",
            "\u001b[?25hCollecting sphinxcontrib-jsmath\n",
            "  Downloading sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl (5.1 kB)\n",
            "Collecting sphinxcontrib-qthelp\n",
            "  Downloading sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl (90 kB)\n",
            "\u001b[K     |████████████████████████████████| 90 kB 6.8 MB/s \n",
            "\u001b[?25hCollecting jinja2\n",
            "  Downloading Jinja2-3.1.2-py3-none-any.whl (133 kB)\n",
            "\u001b[K     |████████████████████████████████| 133 kB 34.6 MB/s \n",
            "\u001b[?25hCollecting sphinxcontrib-htmlhelp>=2.0.0\n",
            "  Downloading sphinxcontrib_htmlhelp-2.0.0-py2.py3-none-any.whl (100 kB)\n",
            "\u001b[K     |████████████████████████████████| 100 kB 4.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: sphinxcontrib-serializinghtml>=1.1.5 in /usr/local/lib/python3.7/dist-packages (from sphinx->-r requirements.txt (line 20)) (1.1.5)\n",
            "Collecting Pygments>=2.0\n",
            "  Downloading Pygments-2.13.0-py3-none-any.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 40.4 MB/s \n",
            "\u001b[?25hCollecting sphinxcontrib-applehelp\n",
            "  Downloading sphinxcontrib_applehelp-1.0.2-py2.py3-none-any.whl (121 kB)\n",
            "\u001b[K     |████████████████████████████████| 121 kB 42.0 MB/s \n",
            "\u001b[?25hCollecting sphinxcontrib-devhelp\n",
            "  Downloading sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl (84 kB)\n",
            "\u001b[K     |████████████████████████████████| 84 kB 2.6 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pathtools\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8806 sha256=baa3a0500d760bec413e2cb5888d08bcd52739cfbf62bcf3fc564f59f5b0267a\n",
            "  Stored in directory: /root/.cache/pip/wheels/3e/31/09/fa59cef12cdcfecc627b3d24273699f390e71828921b2cbba2\n",
            "Successfully built pathtools\n",
            "Installing collected packages: setuptools, smmap, sphinxcontrib-qthelp, sphinxcontrib-jsmath, sphinxcontrib-htmlhelp, sphinxcontrib-devhelp, sphinxcontrib-applehelp, Pygments, jinja2, gitdb, sphinx, slicer, shortuuid, setproctitle, sentry-sdk, pathtools, GitPython, docker-pycreds, wandb, tb-nightly, sphinx-rtd-theme, sphinx-autodoc-typehints, shap, plotly, mpld3, google-cloud\n",
            "  Attempting uninstall: setuptools\n",
            "    Found existing installation: setuptools 57.4.0\n",
            "    Uninstalling setuptools-57.4.0:\n",
            "      Successfully uninstalled setuptools-57.4.0\n",
            "  Attempting uninstall: Pygments\n",
            "    Found existing installation: Pygments 2.6.1\n",
            "    Uninstalling Pygments-2.6.1:\n",
            "      Successfully uninstalled Pygments-2.6.1\n",
            "  Attempting uninstall: jinja2\n",
            "    Found existing installation: Jinja2 2.11.3\n",
            "    Uninstalling Jinja2-2.11.3:\n",
            "      Successfully uninstalled Jinja2-2.11.3\n",
            "  Attempting uninstall: sphinx\n",
            "    Found existing installation: Sphinx 1.8.6\n",
            "    Uninstalling Sphinx-1.8.6:\n",
            "      Successfully uninstalled Sphinx-1.8.6\n",
            "  Attempting uninstall: plotly\n",
            "    Found existing installation: plotly 5.5.0\n",
            "    Uninstalling plotly-5.5.0:\n",
            "      Successfully uninstalled plotly-5.5.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "ipython 7.9.0 requires jedi>=0.10, which is not installed.\n",
            "flask 1.1.4 requires Jinja2<3.0,>=2.10.1, but you have jinja2 3.1.2 which is incompatible.\u001b[0m\n",
            "Successfully installed GitPython-3.1.29 Pygments-2.13.0 docker-pycreds-0.4.0 gitdb-4.0.9 google-cloud-0.34.0 jinja2-3.1.2 mpld3-0.5.8 pathtools-0.1.2 plotly-5.7.0 sentry-sdk-1.9.0 setproctitle-1.3.2 setuptools-65.3.0 shap-0.40.0 shortuuid-1.0.9 slicer-0.0.7 smmap-5.0.0 sphinx-5.3.0 sphinx-autodoc-typehints-1.19.4 sphinx-rtd-theme-1.1.0 sphinxcontrib-applehelp-1.0.2 sphinxcontrib-devhelp-1.0.2 sphinxcontrib-htmlhelp-2.0.0 sphinxcontrib-jsmath-1.0.1 sphinxcontrib-qthelp-1.0.3 tb-nightly-2.11.0a20221101 wandb-0.13.3\n"
          ]
        },
        {
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "pkg_resources",
                  "pygments",
                  "sphinxcontrib"
                ]
              }
            }
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/setuptools/dist.py:530: UserWarning: Normalizing '0.999998dev' to '0.999998.dev0'\n",
            "  warnings.warn(tmpl.format(**locals()))\n",
            "running develop\n",
            "/usr/local/lib/python3.7/dist-packages/setuptools/command/easy_install.py:147: EasyInstallDeprecationWarning: easy_install command is deprecated. Use build and pip and other standards-based tools.\n",
            "  EasyInstallDeprecationWarning,\n",
            "/usr/local/lib/python3.7/dist-packages/setuptools/command/install.py:37: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools.\n",
            "  setuptools.SetuptoolsDeprecationWarning,\n",
            "running egg_info\n",
            "creating flood_forecast.egg-info\n",
            "writing flood_forecast.egg-info/PKG-INFO\n",
            "writing dependency_links to flood_forecast.egg-info/dependency_links.txt\n",
            "writing requirements to flood_forecast.egg-info/requires.txt\n",
            "writing top-level names to flood_forecast.egg-info/top_level.txt\n",
            "writing manifest file 'flood_forecast.egg-info/SOURCES.txt'\n",
            "adding license file 'LICENSE'\n",
            "writing manifest file 'flood_forecast.egg-info/SOURCES.txt'\n",
            "running build_ext\n",
            "Creating /usr/lib/python3.7/site-packages/flood-forecast.egg-link (link to .)\n",
            "Adding flood-forecast 0.999998.dev0 to easy-install.pth file\n",
            "\n",
            "Installed /content/flow-forecast\n",
            "Processing dependencies for flood-forecast==0.999998.dev0\n",
            "Searching for sphinx==5.3.0\n",
            "Best match: sphinx 5.3.0\n",
            "Adding sphinx 5.3.0 to easy-install.pth file\n",
            "Installing sphinx-apidoc script to /usr/bin\n",
            "Installing sphinx-autogen script to /usr/bin\n",
            "Installing sphinx-build script to /usr/bin\n",
            "Installing sphinx-quickstart script to /usr/bin\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for sphinx-autodoc-typehints==1.19.4\n",
            "Best match: sphinx-autodoc-typehints 1.19.4\n",
            "Adding sphinx-autodoc-typehints 1.19.4 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for sphinx-rtd-theme==1.1.0\n",
            "Best match: sphinx-rtd-theme 1.1.0\n",
            "Adding sphinx-rtd-theme 1.1.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for numba==0.56.3\n",
            "Best match: numba 0.56.3\n",
            "Adding numba 0.56.3 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for mpld3==0.5.8\n",
            "Best match: mpld3 0.5.8\n",
            "Adding mpld3 0.5.8 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for torchvision==0.13.1+cu113\n",
            "Best match: torchvision 0.13.1+cu113\n",
            "Adding torchvision 0.13.1+cu113 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for requests==2.23.0\n",
            "Best match: requests 2.23.0\n",
            "Adding requests 2.23.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for numpy==1.21.6\n",
            "Best match: numpy 1.21.6\n",
            "Adding numpy 1.21.6 to easy-install.pth file\n",
            "Installing f2py script to /usr/bin\n",
            "Installing f2py3 script to /usr/bin\n",
            "Installing f2py3.7 script to /usr/bin\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for setuptools==65.3.0\n",
            "Best match: setuptools 65.3.0\n",
            "Adding setuptools 65.3.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for pytz==2022.5\n",
            "Best match: pytz 2022.5\n",
            "Adding pytz 2022.5 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for plotly==5.7.0\n",
            "Best match: plotly 5.7.0\n",
            "Adding plotly 5.7.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for google-cloud-storage==1.18.1\n",
            "Best match: google-cloud-storage 1.18.1\n",
            "Adding google-cloud-storage 1.18.1 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for google-cloud==0.34.0\n",
            "Best match: google-cloud 0.34.0\n",
            "Adding google-cloud 0.34.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for wandb==0.13.3\n",
            "Best match: wandb 0.13.3\n",
            "Adding wandb 0.13.3 to easy-install.pth file\n",
            "Installing wandb script to /usr/bin\n",
            "Installing wb script to /usr/bin\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for h5py==3.1.0\n",
            "Best match: h5py 3.1.0\n",
            "Adding h5py 3.1.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for future==0.16.0\n",
            "Best match: future 0.16.0\n",
            "Adding future 0.16.0 to easy-install.pth file\n",
            "Installing futurize script to /usr/bin\n",
            "Installing pasteurize script to /usr/bin\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for seaborn==0.11.2\n",
            "Best match: seaborn 0.11.2\n",
            "Adding seaborn 0.11.2 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for tb-nightly==2.11.0a20221101\n",
            "Best match: tb-nightly 2.11.0a20221101\n",
            "Adding tb-nightly 2.11.0a20221101 to easy-install.pth file\n",
            "Installing tensorboard script to /usr/bin\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for torch==1.12.1+cu113\n",
            "Best match: torch 1.12.1+cu113\n",
            "Adding torch 1.12.1+cu113 to easy-install.pth file\n",
            "Installing convert-caffe2-to-onnx script to /usr/bin\n",
            "Installing convert-onnx-to-caffe2 script to /usr/bin\n",
            "Installing torchrun script to /usr/bin\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for pandas==1.3.5\n",
            "Best match: pandas 1.3.5\n",
            "Adding pandas 1.3.5 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for scikit-learn==1.0.2\n",
            "Best match: scikit-learn 1.0.2\n",
            "Adding scikit-learn 1.0.2 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for shap==0.40.0\n",
            "Best match: shap 0.40.0\n",
            "Adding shap 0.40.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for importlib-metadata==4.13.0\n",
            "Best match: importlib-metadata 4.13.0\n",
            "Adding importlib-metadata 4.13.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for packaging==21.3\n",
            "Best match: packaging 21.3\n",
            "Adding packaging 21.3 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for imagesize==1.4.1\n",
            "Best match: imagesize 1.4.1\n",
            "Adding imagesize 1.4.1 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for alabaster==0.7.12\n",
            "Best match: alabaster 0.7.12\n",
            "Adding alabaster 0.7.12 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for Babel==2.10.3\n",
            "Best match: Babel 2.10.3\n",
            "Adding Babel 2.10.3 to easy-install.pth file\n",
            "Installing pybabel script to /usr/bin\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for snowballstemmer==2.2.0\n",
            "Best match: snowballstemmer 2.2.0\n",
            "Adding snowballstemmer 2.2.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for docutils==0.17.1\n",
            "Best match: docutils 0.17.1\n",
            "Adding docutils 0.17.1 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for Pygments==2.13.0\n",
            "Best match: Pygments 2.13.0\n",
            "Adding Pygments 2.13.0 to easy-install.pth file\n",
            "Installing pygmentize script to /usr/bin\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for Jinja2==3.1.2\n",
            "Best match: Jinja2 3.1.2\n",
            "Adding Jinja2 3.1.2 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for sphinxcontrib-qthelp==1.0.3\n",
            "Best match: sphinxcontrib-qthelp 1.0.3\n",
            "Adding sphinxcontrib-qthelp 1.0.3 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for sphinxcontrib-serializinghtml==1.1.5\n",
            "Best match: sphinxcontrib-serializinghtml 1.1.5\n",
            "Adding sphinxcontrib-serializinghtml 1.1.5 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for sphinxcontrib-htmlhelp==2.0.0\n",
            "Best match: sphinxcontrib-htmlhelp 2.0.0\n",
            "Adding sphinxcontrib-htmlhelp 2.0.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for sphinxcontrib-jsmath==1.0.1\n",
            "Best match: sphinxcontrib-jsmath 1.0.1\n",
            "Adding sphinxcontrib-jsmath 1.0.1 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for sphinxcontrib-devhelp==1.0.2\n",
            "Best match: sphinxcontrib-devhelp 1.0.2\n",
            "Adding sphinxcontrib-devhelp 1.0.2 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for sphinxcontrib-applehelp==1.0.2\n",
            "Best match: sphinxcontrib-applehelp 1.0.2\n",
            "Adding sphinxcontrib-applehelp 1.0.2 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for llvmlite==0.39.1\n",
            "Best match: llvmlite 0.39.1\n",
            "Adding llvmlite 0.39.1 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for matplotlib==3.2.2\n",
            "Best match: matplotlib 3.2.2\n",
            "Adding matplotlib 3.2.2 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for Pillow==7.1.2\n",
            "Best match: Pillow 7.1.2\n",
            "Adding Pillow 7.1.2 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for typing-extensions==4.1.1\n",
            "Best match: typing-extensions 4.1.1\n",
            "Adding typing-extensions 4.1.1 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for certifi==2022.9.24\n",
            "Best match: certifi 2022.9.24\n",
            "Adding certifi 2022.9.24 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for urllib3==1.24.3\n",
            "Best match: urllib3 1.24.3\n",
            "Adding urllib3 1.24.3 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for idna==2.10\n",
            "Best match: idna 2.10\n",
            "Adding idna 2.10 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for chardet==3.0.4\n",
            "Best match: chardet 3.0.4\n",
            "Adding chardet 3.0.4 to easy-install.pth file\n",
            "Installing chardetect script to /usr/bin\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for six==1.15.0\n",
            "Best match: six 1.15.0\n",
            "Adding six 1.15.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for tenacity==8.1.0\n",
            "Best match: tenacity 8.1.0\n",
            "Adding tenacity 8.1.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for google-resumable-media==0.4.1\n",
            "Best match: google-resumable-media 0.4.1\n",
            "Adding google-resumable-media 0.4.1 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for google-cloud-core==1.0.3\n",
            "Best match: google-cloud-core 1.0.3\n",
            "Adding google-cloud-core 1.0.3 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for google-auth==1.35.0\n",
            "Best match: google-auth 1.35.0\n",
            "Adding google-auth 1.35.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for setproctitle==1.3.2\n",
            "Best match: setproctitle 1.3.2\n",
            "Adding setproctitle 1.3.2 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for pathtools==0.1.2\n",
            "Best match: pathtools 0.1.2\n",
            "Adding pathtools 0.1.2 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for PyYAML==6.0\n",
            "Best match: PyYAML 6.0\n",
            "Adding PyYAML 6.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for protobuf==3.17.3\n",
            "Best match: protobuf 3.17.3\n",
            "Adding protobuf 3.17.3 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for docker-pycreds==0.4.0\n",
            "Best match: docker-pycreds 0.4.0\n",
            "Adding docker-pycreds 0.4.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for sentry-sdk==1.9.0\n",
            "Best match: sentry-sdk 1.9.0\n",
            "Adding sentry-sdk 1.9.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for psutil==5.4.8\n",
            "Best match: psutil 5.4.8\n",
            "Adding psutil 5.4.8 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for shortuuid==1.0.9\n",
            "Best match: shortuuid 1.0.9\n",
            "Adding shortuuid 1.0.9 to easy-install.pth file\n",
            "Installing shortuuid script to /usr/bin\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for promise==2.3\n",
            "Best match: promise 2.3\n",
            "Adding promise 2.3 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for GitPython==3.1.29\n",
            "Best match: GitPython 3.1.29\n",
            "Adding GitPython 3.1.29 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for click==7.1.2\n",
            "Best match: click 7.1.2\n",
            "Adding click 7.1.2 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for cached-property==1.5.2\n",
            "Best match: cached-property 1.5.2\n",
            "Adding cached-property 1.5.2 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for scipy==1.7.3\n",
            "Best match: scipy 1.7.3\n",
            "Adding scipy 1.7.3 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for wheel==0.37.1\n",
            "Best match: wheel 0.37.1\n",
            "Adding wheel 0.37.1 to easy-install.pth file\n",
            "Installing wheel script to /usr/bin\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for Werkzeug==1.0.1\n",
            "Best match: Werkzeug 1.0.1\n",
            "Adding Werkzeug 1.0.1 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for tensorboard-plugin-wit==1.8.1\n",
            "Best match: tensorboard-plugin-wit 1.8.1\n",
            "Adding tensorboard-plugin-wit 1.8.1 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for tensorboard-data-server==0.6.1\n",
            "Best match: tensorboard-data-server 0.6.1\n",
            "Adding tensorboard-data-server 0.6.1 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for Markdown==3.4.1\n",
            "Best match: Markdown 3.4.1\n",
            "Adding Markdown 3.4.1 to easy-install.pth file\n",
            "Installing markdown_py script to /usr/bin\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for google-auth-oauthlib==0.4.6\n",
            "Best match: google-auth-oauthlib 0.4.6\n",
            "Adding google-auth-oauthlib 0.4.6 to easy-install.pth file\n",
            "Installing google-oauthlib-tool script to /usr/bin\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for grpcio==1.50.0\n",
            "Best match: grpcio 1.50.0\n",
            "Adding grpcio 1.50.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for absl-py==1.3.0\n",
            "Best match: absl-py 1.3.0\n",
            "Adding absl-py 1.3.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for python-dateutil==2.8.2\n",
            "Best match: python-dateutil 2.8.2\n",
            "Adding python-dateutil 2.8.2 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for threadpoolctl==3.1.0\n",
            "Best match: threadpoolctl 3.1.0\n",
            "Adding threadpoolctl 3.1.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for joblib==1.2.0\n",
            "Best match: joblib 1.2.0\n",
            "Adding joblib 1.2.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for cloudpickle==1.5.0\n",
            "Best match: cloudpickle 1.5.0\n",
            "Adding cloudpickle 1.5.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for slicer==0.0.7\n",
            "Best match: slicer 0.0.7\n",
            "Adding slicer 0.0.7 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for tqdm==4.64.1\n",
            "Best match: tqdm 4.64.1\n",
            "Adding tqdm 4.64.1 to easy-install.pth file\n",
            "Installing tqdm script to /usr/bin\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for zipp==3.10.0\n",
            "Best match: zipp 3.10.0\n",
            "Adding zipp 3.10.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for pyparsing==3.0.9\n",
            "Best match: pyparsing 3.0.9\n",
            "Adding pyparsing 3.0.9 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for MarkupSafe==2.0.1\n",
            "Best match: MarkupSafe 2.0.1\n",
            "Adding MarkupSafe 2.0.1 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for kiwisolver==1.4.4\n",
            "Best match: kiwisolver 1.4.4\n",
            "Adding kiwisolver 1.4.4 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for cycler==0.11.0\n",
            "Best match: cycler 0.11.0\n",
            "Adding cycler 0.11.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for google-api-core==1.31.6\n",
            "Best match: google-api-core 1.31.6\n",
            "Adding google-api-core 1.31.6 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for rsa==4.9\n",
            "Best match: rsa 4.9\n",
            "Adding rsa 4.9 to easy-install.pth file\n",
            "Installing pyrsa-decrypt script to /usr/bin\n",
            "Installing pyrsa-encrypt script to /usr/bin\n",
            "Installing pyrsa-keygen script to /usr/bin\n",
            "Installing pyrsa-priv2pub script to /usr/bin\n",
            "Installing pyrsa-sign script to /usr/bin\n",
            "Installing pyrsa-verify script to /usr/bin\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for pyasn1-modules==0.2.8\n",
            "Best match: pyasn1-modules 0.2.8\n",
            "Adding pyasn1-modules 0.2.8 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for cachetools==4.2.4\n",
            "Best match: cachetools 4.2.4\n",
            "Adding cachetools 4.2.4 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for gitdb==4.0.9\n",
            "Best match: gitdb 4.0.9\n",
            "Adding gitdb 4.0.9 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for requests-oauthlib==1.3.1\n",
            "Best match: requests-oauthlib 1.3.1\n",
            "Adding requests-oauthlib 1.3.1 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for googleapis-common-protos==1.56.4\n",
            "Best match: googleapis-common-protos 1.56.4\n",
            "Adding googleapis-common-protos 1.56.4 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for pyasn1==0.4.8\n",
            "Best match: pyasn1 0.4.8\n",
            "Adding pyasn1 0.4.8 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for smmap==5.0.0\n",
            "Best match: smmap 5.0.0\n",
            "Adding smmap 5.0.0 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Searching for oauthlib==3.2.2\n",
            "Best match: oauthlib 3.2.2\n",
            "Adding oauthlib 3.2.2 to easy-install.pth file\n",
            "\n",
            "Using /usr/local/lib/python3.7/dist-packages\n",
            "Finished processing dependencies for flood-forecast==0.999998.dev0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter, or press ctrl+c to quit: "
          ]
        }
      ],
      "source": [
        "#from flood_forecast.trainer import train_function\n",
        "os.chdir(\"flow-forecast\")\n",
        "!pip install -r  requirements.txt\n",
        "!python setup.py develop\n",
        "!wandb login\n",
        "os.environ['MODEL_BUCKET'] = \"predict_cfs\"\n",
        "os.environ[\"ENVIRONMENT_GCP\"] = \"Colab\"\n",
        "os.environ[\"GCP_PROJECT\"] = \"gmap-997\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v6qrQUrh2Mfo"
      },
      "outputs": [],
      "source": [
        "df.to_csv(\"/content/joined_final_3/09405500AZC_flow.csv\")\n",
        "from flood_forecast.trainer import train_function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eJMumWTDGUAO"
      },
      "outputs": [],
      "source": [
        "trained_model = train_function(\"PyTorch\", the_config)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T8J4nLz2GUzP"
      },
      "outputs": [],
      "source": [
        "import flood_forecast\n",
        "from flood_forecast.deployment.inference import InferenceMode"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6KROIDvdtR5O"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "with open(\"/content/flow-forecast/model_save/30_May_202206_53PM.json\") as f:\n",
        "  data = json.load(f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ybTmLi42Gkc-"
      },
      "outputs": [],
      "source": [
        "data[\"inference_params\"] = {}\n",
        "data[\"inference_params\"][\"dataset_params\"] = data[\"dataset_params\"]\n",
        "m = InferenceMode(1, 1, data, \"/content/joined_final_3/09405500AZC_flow.csv\", \"/content/flow-forecast/model_save/24_May_202202_25PM_model.pth\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5vb8eq8qc2X_"
      },
      "outputs": [],
      "source": [
        "from flood_forecast.preprocessing.pytorch_loaders import GeneralClassificationLoader\n",
        "def infer_now_classification(model, data=None, over_lap_seq=True, save_buck=None, save_name=None, use_torch_script=False):\n",
        "    \"\"\"Function to preform classification/anomaly detection on sequences in real-time\n",
        "    \"\"\"\n",
        "    if data:\n",
        "        dataset_params = model.model.params[\"dataset_params\"].copy()\n",
        "        dataset_params[\"class\"] = \"GeneralClassificationLoader\"\n",
        "        loader = m.model.make_data_load(data, dataset_params, \"custom\") \n",
        "    else:\n",
        "        loader = model.model.test_data\n",
        "    seq_list = []\n",
        "    if over_lap_seq:\n",
        "        for x, y in loader:\n",
        "            seq_list.append(model.model.model(x))\n",
        "    return seq_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9fmhkWvJu7TE"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "pd.read_csv(\"/content/joined_final_3/09405500AZC_flow.csv\")[0:200].to_csv(\"test.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vNZ4J6gkv7RS"
      },
      "outputs": [],
      "source": [
        "res = m.infer_now_classification(\"test.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BqafUKkJ4BAI"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "d = torch.nn.Softmax(dim=1)\n",
        "d1 = map( lambda x: d(x[0]), res)\n",
        "list(d1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pjcNqkv_nvzO"
      },
      "outputs": [],
      "source": [
        "df[1:90].to_csv(\"ff1_tes.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "avcoP5eQn3n3"
      },
      "outputs": [],
      "source": [
        "!gsutil cp gs://flow_datasets/test_data/model_save/24_May_202202_25PM_model.pth model_save "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IP_70Ek1YT_N"
      },
      "outputs": [],
      "source": [
        "!cp model_save/test_data/model_save/23_May_202203_31PM_model.pth /content/flow-forecast/model_save/test_data/23_May_202203_31PM_model.pth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "k-7nOOu9Y_We"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}