notebooks/CLAMP_colab_demo.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-0c-Bk8hLzAt"
      },
      "source": [
        "# 🗜️ CLAMP demo\n",
        "\n",
        "This notebook shows run a pretrained CLAMP model or train a new one.\n",
        "\n",
        "Before the start: change the Runtime type to GPU."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HlAhzXn6L1YW"
      },
      "source": [
        "# ⚙️ Setup\n",
        "only run once"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3pzKAZVpL0nV",
        "outputId": "9089f9bf-7055-4e65-ae0d-d2c083eba1a0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting git+https://github.com/ml-jku/clamp\n",
            "  Cloning https://github.com/ml-jku/clamp to /tmp/pip-req-build-wnlfha3x\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/ml-jku/clamp /tmp/pip-req-build-wnlfha3x\n",
            "  Resolved https://github.com/ml-jku/clamp to commit f097406dd2161e1d083e749277d24b82219e1d93\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting mhnreact@ git+https://github.com/ml-jku/mhn-react.git (from clamp==1.0)\n",
            "  Cloning https://github.com/ml-jku/mhn-react.git to /tmp/pip-install-hxfmgj_w/mhnreact_c57710d2a2854155b8061b3ea149669b\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/ml-jku/mhn-react.git /tmp/pip-install-hxfmgj_w/mhnreact_c57710d2a2854155b8061b3ea149669b\n",
            "  Resolved https://github.com/ml-jku/mhn-react.git to commit 424ab0185db53462f5dd22b192a7bc6f4ad5b2d7\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting clip@ git+https://github.com/openai/CLIP.git (from clamp==1.0)\n",
            "  Cloning https://github.com/openai/CLIP.git to /tmp/pip-install-hxfmgj_w/clip_b7afe5e0019543c4b9c1e751093bd97c\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/openai/CLIP.git /tmp/pip-install-hxfmgj_w/clip_b7afe5e0019543c4b9c1e751093bd97c\n",
            "  Resolved https://github.com/openai/CLIP.git to commit a9b1bf5920416aaeaec965c25dd9e8f98c864f16\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from clamp==1.0) (1.22.4)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from clamp==1.0) (4.65.0)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from clamp==1.0) (1.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from clamp==1.0) (1.10.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from clamp==1.0) (1.5.3)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from clamp==1.0) (3.8.1)\n",
            "Collecting rdkit-pypi (from clamp==1.0)\n",
            "  Downloading rdkit_pypi-2022.9.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (29.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m29.4/29.4 MB\u001b[0m \u001b[31m29.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting transformers (from clamp==1.0)\n",
            "  Downloading transformers-4.30.2-py3-none-any.whl (7.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m66.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from clamp==1.0) (3.7.1)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from clamp==1.0) (2.0.1+cu118)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from clamp==1.0) (0.15.2+cu118)\n",
            "Collecting loguru (from clamp==1.0)\n",
            "  Downloading loguru-0.7.0-py3-none-any.whl (59 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.0/60.0 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting wandb (from clamp==1.0)\n",
            "  Downloading wandb-0.15.4-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m47.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting mlflow (from clamp==1.0)\n",
            "  Downloading mlflow-2.4.1-py3-none-any.whl (18.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m18.1/18.1 MB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting swifter (from clamp==1.0)\n",
            "  Downloading swifter-1.3.5.tar.gz (490 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m490.6/490.6 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting ftfy (from clip@ git+https://github.com/openai/CLIP.git->clamp==1.0)\n",
            "  Downloading ftfy-6.1.1-py3-none-any.whl (53 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from clip@ git+https://github.com/openai/CLIP.git->clamp==1.0) (2022.10.31)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->clamp==1.0) (1.0.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->clamp==1.0) (0.11.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->clamp==1.0) (4.39.3)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->clamp==1.0) (1.4.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->clamp==1.0) (23.1)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->clamp==1.0) (8.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->clamp==1.0) (3.0.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->clamp==1.0) (2.8.2)\n",
            "Requirement already satisfied: click<9,>=7.0 in /usr/local/lib/python3.10/dist-packages (from mlflow->clamp==1.0) (8.1.3)\n",
            "Requirement already satisfied: cloudpickle<3 in /usr/local/lib/python3.10/dist-packages (from mlflow->clamp==1.0) (2.2.1)\n",
            "Collecting databricks-cli<1,>=0.8.7 (from mlflow->clamp==1.0)\n",
            "  Downloading databricks-cli-0.17.7.tar.gz (83 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m83.5/83.5 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: entrypoints<1 in /usr/local/lib/python3.10/dist-packages (from mlflow->clamp==1.0) (0.4)\n",
            "Collecting gitpython<4,>=2.1.0 (from mlflow->clamp==1.0)\n",
            "  Downloading GitPython-3.1.31-py3-none-any.whl (184 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m184.3/184.3 kB\u001b[0m \u001b[31m15.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyyaml<7,>=5.1 in /usr/local/lib/python3.10/dist-packages (from mlflow->clamp==1.0) (6.0)\n",
            "Requirement already satisfied: protobuf<5,>=3.12.0 in /usr/local/lib/python3.10/dist-packages (from mlflow->clamp==1.0) (3.20.3)\n",
            "Requirement already satisfied: pytz<2024 in /usr/local/lib/python3.10/dist-packages (from mlflow->clamp==1.0) (2022.7.1)\n",
            "Requirement already satisfied: requests<3,>=2.17.3 in /usr/local/lib/python3.10/dist-packages (from mlflow->clamp==1.0) (2.27.1)\n",
            "Collecting importlib-metadata!=4.7.0,<7,>=3.7.0 (from mlflow->clamp==1.0)\n",
            "  Downloading importlib_metadata-6.7.0-py3-none-any.whl (22 kB)\n",
            "Requirement already satisfied: sqlparse<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from mlflow->clamp==1.0) (0.4.4)\n",
            "Collecting alembic!=1.10.0,<2 (from mlflow->clamp==1.0)\n",
            "  Downloading alembic-1.11.1-py3-none-any.whl (224 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting docker<7,>=4.0.0 (from mlflow->clamp==1.0)\n",
            "  Downloading docker-6.1.3-py3-none-any.whl (148 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m148.1/148.1 kB\u001b[0m \u001b[31m12.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: Flask<3 in /usr/local/lib/python3.10/dist-packages (from mlflow->clamp==1.0) (2.2.4)\n",
            "Collecting querystring-parser<2 (from mlflow->clamp==1.0)\n",
            "  Downloading querystring_parser-1.2.4-py2.py3-none-any.whl (7.9 kB)\n",
            "Requirement already satisfied: sqlalchemy<3,>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from mlflow->clamp==1.0) (2.0.10)\n",
            "Requirement already satisfied: pyarrow<13,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from mlflow->clamp==1.0) (9.0.0)\n",
            "Requirement already satisfied: markdown<4,>=3.3 in /usr/local/lib/python3.10/dist-packages (from mlflow->clamp==1.0) (3.4.3)\n",
            "Collecting gunicorn<21 (from mlflow->clamp==1.0)\n",
            "  Downloading gunicorn-20.1.0-py3-none-any.whl (79 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.5/79.5 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: Jinja2<4,>=2.11 in /usr/local/lib/python3.10/dist-packages (from mlflow->clamp==1.0) (3.1.2)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->clamp==1.0) (1.2.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->clamp==1.0) (3.1.0)\n",
            "Requirement already satisfied: psutil>=5.6.6 in /usr/local/lib/python3.10/dist-packages (from swifter->clamp==1.0) (5.9.5)\n",
            "Requirement already satisfied: dask[dataframe]>=2.10.0 in /usr/local/lib/python3.10/dist-packages (from swifter->clamp==1.0) (2022.12.1)\n",
            "Requirement already satisfied: ipywidgets>=7.0.0 in /usr/local/lib/python3.10/dist-packages (from swifter->clamp==1.0) (7.7.1)\n",
            "Requirement already satisfied: parso>0.4.0 in /usr/local/lib/python3.10/dist-packages (from swifter->clamp==1.0) (0.8.3)\n",
            "Requirement already satisfied: bleach>=3.1.1 in /usr/local/lib/python3.10/dist-packages (from swifter->clamp==1.0) (6.0.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->clamp==1.0) (3.12.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch->clamp==1.0) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->clamp==1.0) (1.11.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->clamp==1.0) (3.1)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch->clamp==1.0) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch->clamp==1.0) (3.25.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch->clamp==1.0) (16.0.5)\n",
            "Collecting huggingface-hub<1.0,>=0.14.1 (from transformers->clamp==1.0)\n",
            "  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m236.8/236.8 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers->clamp==1.0)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m87.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers->clamp==1.0)\n",
            "  Downloading safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m54.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sentry-sdk>=1.0.0 (from wandb->clamp==1.0)\n",
            "  Downloading sentry_sdk-1.25.1-py2.py3-none-any.whl (206 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m206.7/206.7 kB\u001b[0m \u001b[31m15.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting docker-pycreds>=0.4.0 (from wandb->clamp==1.0)\n",
            "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
            "Collecting pathtools (from wandb->clamp==1.0)\n",
            "  Downloading pathtools-0.1.2.tar.gz (11 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting setproctitle (from wandb->clamp==1.0)\n",
            "  Downloading setproctitle-1.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb->clamp==1.0) (67.7.2)\n",
            "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb->clamp==1.0) (1.4.4)\n",
            "Collecting Mako (from alembic!=1.10.0,<2->mlflow->clamp==1.0)\n",
            "  Downloading Mako-1.2.4-py3-none-any.whl (78 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.7/78.7 kB\u001b[0m \u001b[31m7.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from bleach>=3.1.1->swifter->clamp==1.0) (1.16.0)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.10/dist-packages (from bleach>=3.1.1->swifter->clamp==1.0) (0.5.1)\n",
            "Requirement already satisfied: fsspec>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from dask[dataframe]>=2.10.0->swifter->clamp==1.0) (2023.4.0)\n",
            "Requirement already satisfied: partd>=0.3.10 in /usr/local/lib/python3.10/dist-packages (from dask[dataframe]>=2.10.0->swifter->clamp==1.0) (1.4.0)\n",
            "Requirement already satisfied: toolz>=0.8.2 in /usr/local/lib/python3.10/dist-packages (from dask[dataframe]>=2.10.0->swifter->clamp==1.0) (0.12.0)\n",
            "Collecting pyjwt>=1.7.0 (from databricks-cli<1,>=0.8.7->mlflow->clamp==1.0)\n",
            "  Downloading PyJWT-2.7.0-py3-none-any.whl (22 kB)\n",
            "Requirement already satisfied: oauthlib>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from databricks-cli<1,>=0.8.7->mlflow->clamp==1.0) (3.2.2)\n",
            "Requirement already satisfied: tabulate>=0.7.7 in /usr/local/lib/python3.10/dist-packages (from databricks-cli<1,>=0.8.7->mlflow->clamp==1.0) (0.8.10)\n",
            "Requirement already satisfied: urllib3<2.0.0,>=1.26.7 in /usr/local/lib/python3.10/dist-packages (from databricks-cli<1,>=0.8.7->mlflow->clamp==1.0) (1.26.15)\n",
            "Requirement already satisfied: websocket-client>=0.32.0 in /usr/local/lib/python3.10/dist-packages (from docker<7,>=4.0.0->mlflow->clamp==1.0) (1.5.1)\n",
            "Requirement already satisfied: Werkzeug>=2.2.2 in /usr/local/lib/python3.10/dist-packages (from Flask<3->mlflow->clamp==1.0) (2.3.0)\n",
            "Requirement already satisfied: itsdangerous>=2.0 in /usr/local/lib/python3.10/dist-packages (from Flask<3->mlflow->clamp==1.0) (2.1.2)\n",
            "Collecting gitdb<5,>=4.0.1 (from gitpython<4,>=2.1.0->mlflow->clamp==1.0)\n",
            "  Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata!=4.7.0,<7,>=3.7.0->mlflow->clamp==1.0) (3.15.0)\n",
            "Requirement already satisfied: ipykernel>=4.5.1 in /usr/local/lib/python3.10/dist-packages (from ipywidgets>=7.0.0->swifter->clamp==1.0) (5.5.6)\n",
            "Requirement already satisfied: ipython-genutils~=0.2.0 in /usr/local/lib/python3.10/dist-packages (from ipywidgets>=7.0.0->swifter->clamp==1.0) (0.2.0)\n",
            "Requirement already satisfied: traitlets>=4.3.1 in /usr/local/lib/python3.10/dist-packages (from ipywidgets>=7.0.0->swifter->clamp==1.0) (5.7.1)\n",
            "Requirement already satisfied: widgetsnbextension~=3.6.0 in /usr/local/lib/python3.10/dist-packages (from ipywidgets>=7.0.0->swifter->clamp==1.0) (3.6.4)\n",
            "Requirement already satisfied: ipython>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from ipywidgets>=7.0.0->swifter->clamp==1.0) (7.34.0)\n",
            "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from ipywidgets>=7.0.0->swifter->clamp==1.0) (3.0.7)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from Jinja2<4,>=2.11->mlflow->clamp==1.0) (2.1.2)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.17.3->mlflow->clamp==1.0) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.17.3->mlflow->clamp==1.0) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.17.3->mlflow->clamp==1.0) (3.4)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from sqlalchemy<3,>=1.4.0->mlflow->clamp==1.0) (2.0.2)\n",
            "Requirement already satisfied: wcwidth>=0.2.5 in /usr/local/lib/python3.10/dist-packages (from ftfy->clip@ git+https://github.com/openai/CLIP.git->clamp==1.0) (0.2.6)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->clamp==1.0) (1.3.0)\n",
            "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython<4,>=2.1.0->mlflow->clamp==1.0)\n",
            "  Downloading smmap-5.0.0-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.10/dist-packages (from ipykernel>=4.5.1->ipywidgets>=7.0.0->swifter->clamp==1.0) (6.1.12)\n",
            "Requirement already satisfied: tornado>=4.2 in /usr/local/lib/python3.10/dist-packages (from ipykernel>=4.5.1->ipywidgets>=7.0.0->swifter->clamp==1.0) (6.3.1)\n",
            "Collecting jedi>=0.16 (from ipython>=4.0.0->ipywidgets>=7.0.0->swifter->clamp==1.0)\n",
            "  Downloading jedi-0.18.2-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m54.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: decorator in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (4.4.2)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (0.7.5)\n",
            "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (3.0.38)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (2.14.0)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (0.2.0)\n",
            "Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (0.1.6)\n",
            "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (4.8.0)\n",
            "Requirement already satisfied: locket in /usr/local/lib/python3.10/dist-packages (from partd>=0.3.10->dask[dataframe]>=2.10.0->swifter->clamp==1.0) (1.0.0)\n",
            "Requirement already satisfied: notebook>=4.4.1 in /usr/local/lib/python3.10/dist-packages (from widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (6.4.8)\n",
            "Requirement already satisfied: pyzmq>=17 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (23.2.1)\n",
            "Requirement already satisfied: argon2-cffi in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (21.3.0)\n",
            "Requirement already satisfied: jupyter-core>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (5.3.0)\n",
            "Requirement already satisfied: nbformat in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (5.8.0)\n",
            "Requirement already satisfied: nbconvert in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (6.5.4)\n",
            "Requirement already satisfied: nest-asyncio>=1.5 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (1.5.6)\n",
            "Requirement already satisfied: Send2Trash>=1.8.0 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (1.8.0)\n",
            "Requirement already satisfied: terminado>=0.8.3 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (0.17.1)\n",
            "Requirement already satisfied: prometheus-client in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (0.16.0)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/dist-packages (from pexpect>4.3->ipython>=4.0.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (0.7.0)\n",
            "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.10/dist-packages (from jupyter-core>=4.6.1->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (3.3.0)\n",
            "Requirement already satisfied: argon2-cffi-bindings in /usr/local/lib/python3.10/dist-packages (from argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (21.2.0)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (4.9.2)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (4.11.2)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (0.7.1)\n",
            "Requirement already satisfied: jupyterlab-pygments in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (0.2.2)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (0.8.4)\n",
            "Requirement already satisfied: nbclient>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (0.7.4)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (1.5.0)\n",
            "Requirement already satisfied: tinycss2 in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (1.2.1)\n",
            "Requirement already satisfied: fastjsonschema in /usr/local/lib/python3.10/dist-packages (from nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (2.16.3)\n",
            "Requirement already satisfied: jsonschema>=2.6 in /usr/local/lib/python3.10/dist-packages (from nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (4.3.3)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (23.1.0)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (0.19.3)\n",
            "Requirement already satisfied: cffi>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from argon2-cffi-bindings->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (1.15.1)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (2.4.1)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets>=7.0.0->swifter->clamp==1.0) (2.21)\n",
            "Building wheels for collected packages: clamp, clip, mhnreact, swifter, databricks-cli, pathtools\n",
            "  Building wheel for clamp (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for clamp: filename=clamp-1.0-py3-none-any.whl size=75614 sha256=2288bc4953ebe6ebb57e80519fe64b6303a21ee7f89b070a322583355911019d\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-x6swlgvy/wheels/df/7b/d7/57c801f88098463d597ceacbd5b5255fa2541c87ba13e0795e\n",
            "  Building wheel for clip (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for clip: filename=clip-1.0-py3-none-any.whl size=1369370 sha256=99d39ed18183e3efa40ef14a28f31769dfdd81d8028322574330981f1328bd4b\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-x6swlgvy/wheels/da/2b/4c/d6691fa9597aac8bb85d2ac13b112deb897d5b50f5ad9a37e4\n",
            "  Building wheel for mhnreact (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for mhnreact: filename=mhnreact-1.0-py3-none-any.whl size=49777 sha256=2d84658d6aaa3b9f314a680af4d4ee8056bf3420282933aa19000e561082fd68\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-x6swlgvy/wheels/68/10/cc/fb460617836ed1673368bc676dd160654588585164d32953a1\n",
            "  Building wheel for swifter (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for swifter: filename=swifter-1.3.5-py3-none-any.whl size=16518 sha256=4bc45307c991290f6e577a9e16f2dd587ff61c7032cdf4be2629b8e382187377\n",
            "  Stored in directory: /root/.cache/pip/wheels/00/7e/8c/438ea309a16ef1a9130849060f962e0d2c4e94b8a4314d5919\n",
            "  Building wheel for databricks-cli (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for databricks-cli: filename=databricks_cli-0.17.7-py3-none-any.whl size=143860 sha256=6155836e50b365ac94c81c058c90c1c89e49e81f02ea12f9fe983e65827342d6\n",
            "  Stored in directory: /root/.cache/pip/wheels/ae/63/93/5402c1a09c1868a59d0b05013484e07af97a9d7b3dbd5bd39a\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8791 sha256=d26494138cacc8b8f8f396cda9e546a95193945e92791ac5f53623d8d7737550\n",
            "  Stored in directory: /root/.cache/pip/wheels/e7/f3/22/152153d6eb222ee7a56ff8617d80ee5207207a8c00a7aab794\n",
            "Successfully built clamp clip mhnreact swifter databricks-cli pathtools\n",
            "Installing collected packages: tokenizers, safetensors, pathtools, smmap, setproctitle, sentry-sdk, rdkit-pypi, querystring-parser, pyjwt, mhnreact, Mako, loguru, jedi, importlib-metadata, gunicorn, ftfy, docker-pycreds, huggingface-hub, gitdb, docker, databricks-cli, alembic, transformers, gitpython, wandb, mlflow, swifter, clip, clamp\n",
            "Successfully installed Mako-1.2.4 alembic-1.11.1 clamp-1.0 clip-1.0 databricks-cli-0.17.7 docker-6.1.3 docker-pycreds-0.4.0 ftfy-6.1.1 gitdb-4.0.10 gitpython-3.1.31 gunicorn-20.1.0 huggingface-hub-0.15.1 importlib-metadata-6.7.0 jedi-0.18.2 loguru-0.7.0 mhnreact-1.0 mlflow-2.4.1 pathtools-0.1.2 pyjwt-2.7.0 querystring-parser-1.2.4 rdkit-pypi-2022.9.5 safetensors-0.3.1 sentry-sdk-1.25.1 setproctitle-1.3.2 smmap-5.0.0 swifter-1.3.5 tokenizers-0.13.3 transformers-4.30.2 wandb-0.15.4\n"
          ]
        }
      ],
      "source": [
        "!pip install git+https://github.com/ml-jku/clamp"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kgMBIGhjMQ3c"
      },
      "source": [
        "# Use a pretrained CLAMP model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u7iXYgdmMX6m",
        "outputId": "590f3d8c-8b93-4ede-b98d-10812bdcdde6"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "PretrainedCLAMP(\n",
              "  (compound_encoder): NetworkLayerNorm(\n",
              "    (linear_input): Linear(in_features=8192, out_features=4096, bias=True)\n",
              "    (linear_hidden_l): ModuleList(\n",
              "      (0): Linear(in_features=4096, out_features=2048, bias=True)\n",
              "    )\n",
              "    (linear_output): Linear(in_features=2048, out_features=768, bias=True)\n",
              "    (normalization_input): LayerNorm((4096,), eps=1e-05, elementwise_affine=False)\n",
              "    (normalization_hidden_l): ModuleList(\n",
              "      (0): LayerNorm((2048,), eps=1e-05, elementwise_affine=False)\n",
              "    )\n",
              "    (nonlinearity): ReLU()\n",
              "    (dropout_input): Dropout(p=0.1, inplace=False)\n",
              "    (dropout_hidden): Dropout(p=0.2, inplace=False)\n",
              "  )\n",
              "  (assay_encoder): NetworkLayerNorm(\n",
              "    (linear_input): Linear(in_features=512, out_features=4096, bias=True)\n",
              "    (linear_hidden_l): ModuleList(\n",
              "      (0): Linear(in_features=4096, out_features=2048, bias=True)\n",
              "    )\n",
              "    (linear_output): Linear(in_features=2048, out_features=768, bias=True)\n",
              "    (normalization_input): LayerNorm((4096,), eps=1e-05, elementwise_affine=False)\n",
              "    (normalization_hidden_l): ModuleList(\n",
              "      (0): LayerNorm((2048,), eps=1e-05, elementwise_affine=False)\n",
              "    )\n",
              "    (nonlinearity): ReLU()\n",
              "    (dropout_input): Dropout(p=0.1, inplace=False)\n",
              "    (dropout_hidden): Dropout(p=0.2, inplace=False)\n",
              "  )\n",
              ")"
            ]
          },
          "execution_count": 2,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import torch\n",
        "import clamp\n",
        "\n",
        "model = clamp.CLAMP(device='cpu')\n",
        "model.eval()\n",
        "\n",
        "model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2HCkvZq9NauI",
        "outputId": "9e551434-7495-4652-b078-7b4c8fa739b5"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "torch.Size([1, 768])"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# encodes a smiles to the association space (without l2-norm)\n",
        "model.encode_smiles(['CCC']).shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cBFGPWUKBCim",
        "outputId": "82f42a25-5c60-423b-dcc5-edffa6b52b51"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "torch.Size([1, 768])"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.encode_text(['HIV: Experimentally measured abilities to inhibit HIV replication.']).shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6BuHWG1FNjEn",
        "outputId": "6420a884-daa5-4a38-f7a6-1b5f4dbaf4e6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mol probs for assay: [0.25821456 0.23540354 0.26953387 0.236848  ]\n"
          ]
        }
      ],
      "source": [
        "molecules = [\n",
        "    'CCOP(=O)(Nc1cccc(Cl)c1)OCC', #inactive\n",
        "    'O=C(O)c1ccccc1O', #inactive\n",
        "    'NNP(=S)(NN)c1ccccc1', #active\n",
        "    'CC(=O)OC1=CC=CC=C1C(=O)O', # Aspirin\n",
        "    ]\n",
        "assay_descriptions = [\n",
        "    'HIV: Experimentally measured abilities to inhibit HIV replication.',\n",
        "    ]\n",
        "\n",
        "with torch.no_grad():\n",
        "    logits = model.forward_dense(molecules, assay_descriptions)\n",
        "    probs = logits.softmax(dim=0).cpu().numpy() # probs for molecules\n",
        "\n",
        "print(\"Mol probs for assay:\", probs[:,0]) # res: [0.258 0.235 0.269  0.236]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XQEzFcve50CS"
      },
      "source": [
        "# Linear-probing example\n",
        "To download the preprocessed downstream datasets call"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kl8OX4ChxfkD",
        "outputId": "ebb0e0bd-5a49-4f57-b04b-4a06b17ebc6c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "WARNING: combining -O with -r or -p will mean that all downloaded content\n",
            "will be placed in the single file you specified.\n",
            "\n",
            "WARNING: timestamping does nothing in combination with -O. See the manual\n",
            "for details.\n",
            "\n",
            "--2023-06-19 12:46:55--  https://cloud.ml.jku.at/s/pyJMm4yQeWFM2gG/download\n",
            "Resolving cloud.ml.jku.at (cloud.ml.jku.at)... 140.78.90.41\n",
            "Connecting to cloud.ml.jku.at (cloud.ml.jku.at)|140.78.90.41|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 14101780 (13M) [application/zip]\n",
            "Saving to: ‘downstream.zip’\n",
            "\n",
            "downstream.zip      100%[===================>]  13.45M  41.3MB/s    in 0.3s    \n",
            "\n",
            "2023-06-19 12:46:56 (41.3 MB/s) - ‘downstream.zip’ saved [14101780/14101780]\n",
            "\n",
            "FINISHED --2023-06-19 12:46:56--\n",
            "Total wall clock time: 0.6s\n",
            "Downloaded: 1 files, 13M in 0.3s (41.3 MB/s)\n",
            "Archive:  downstream.zip\n",
            "replace data/downstream/toxcast/activity.parquet? [y]es, [n]o, [A]ll, [N]one, [r]ename: "
          ]
        }
      ],
      "source": [
        "!wget -N -r https://cloud.ml.jku.at/s/pyJMm4yQeWFM2gG/download -O downstream.zip\n",
        "!unzip downstream.zip; rm downstream.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "60OF2ycs6hPM"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "dset = './data/downstream/tox21'\n",
        "smis_df = pd.read_parquet(f'{dset}/compound_smiles.parquet') #assume one on one mapping\n",
        "act_df = pd.read_parquet(f'{dset}/activity.parquet')\n",
        "act_df = act_df[act_df.assay_idx==0] # for demo only first assay"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZYHko9fb623r"
      },
      "outputs": [],
      "source": [
        "clamp_encoding = model.encode_smiles(smis_df.CanonicalSMILES) # do batchwise for larger dsets\n",
        "fp_encoding = model.prepro_smiles(smis_df.CanonicalSMILES)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "62MB_HV-72K_",
        "outputId": "0f7e0648-551a-45c8-c794-83377b40908e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "FP:\t dAP=0.207, AUROC=0.705\n",
            "CLAMP:\t dAP=0.308, AUROC=0.737\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.metrics import roc_auc_score, average_precision_score\n",
        "\n",
        "def davgp_score(y_true, y_pred, sample_weight=None):\n",
        "    avgp = average_precision_score(y_true, y_pred, sample_weight=sample_weight)\n",
        "    y_avg = np.average(y_true, weights=sample_weight)\n",
        "    return avgp - y_avg\n",
        "\n",
        "clf = LogisticRegression(max_iter=1500, class_weight='balanced', C=1, random_state=70135)\n",
        "\n",
        "encodings = {'FP':fp_encoding, 'CLAMP':clamp_encoding}\n",
        "\n",
        "for name, encoding in encodings.items():\n",
        "  X_train = encoding[act_df[act_df.scaffold_split=='train'].compound_idx.values]\n",
        "  y_train = act_df[act_df.scaffold_split=='train'].activity\n",
        "\n",
        "  X_test = encoding[act_df[act_df.scaffold_split=='test'].compound_idx.values]\n",
        "  y_test = act_df[act_df.scaffold_split=='test'].activity\n",
        "\n",
        "  clf.fit(X_train, y_train)\n",
        "\n",
        "  y_pred = clf.predict_proba(X_test)[:,1]\n",
        "\n",
        "  print(f'{name}:\\t dAP={davgp_score(y_test, y_pred):2.3f}, AUROC={roc_auc_score(y_test, y_pred):2.3f}')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qv3Qj1kXkzdZ"
      },
      "source": [
        "# 🔥Train your own model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nQDFQKC1k115",
        "outputId": "1f2607d5-f52c-468b-acfd-164b0994bc0c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "WARNING: combining -O with -r or -p will mean that all downloaded content\n",
            "will be placed in the single file you specified.\n",
            "\n",
            "WARNING: timestamping does nothing in combination with -O. See the manual\n",
            "for details.\n",
            "\n",
            "--2023-06-22 05:24:37--  https://cloud.ml.jku.at/s/dCjrt9c4arbz6rF/download\n",
            "Resolving cloud.ml.jku.at (cloud.ml.jku.at)... 140.78.90.41\n",
            "Connecting to cloud.ml.jku.at (cloud.ml.jku.at)|140.78.90.41|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 6466693 (6.2M) [application/zip]\n",
            "Saving to: ‘fsmol.zip’\n",
            "\n",
            "fsmol.zip           100%[===================>]   6.17M  6.38MB/s    in 1.0s    \n",
            "\n",
            "2023-06-22 05:24:38 (6.38 MB/s) - ‘fsmol.zip’ saved [6466693/6466693]\n",
            "\n",
            "FINISHED --2023-06-22 05:24:38--\n",
            "Total wall clock time: 1.7s\n",
            "Downloaded: 1 files, 6.2M in 1.0s (6.38 MB/s)\n",
            "Archive:  fsmol.zip\n",
            "  inflating: data/fsmol/activity.parquet  \n",
            "  inflating: data/fsmol/assay_names.parquet  \n",
            "  inflating: data/fsmol/compound_names.parquet  \n",
            "  inflating: data/fsmol/compound_smiles.parquet  \n"
          ]
        }
      ],
      "source": [
        "# setup FS-Mol:\n",
        "!wget -N -r https://cloud.ml.jku.at/s/dCjrt9c4arbz6rF/download -O fsmol.zip\n",
        "!unzip fsmol.zip; rm fsmol.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5xV6tOk8ltpV"
      },
      "outputs": [],
      "source": [
        "# compute the compound encodings as input for your model run (takes a few min)\n",
        "# if you leafe this step out, it computes it on the fly --\n",
        "# in this case we only have 1 CPU unfortunately\n",
        "!python -m clamp.dataset.encode_compound \\\n",
        "--compounds=./data/fsmol/compound_names.parquet \\\n",
        "--compound2smiles=./data/fsmol/compound_smiles.parquet \\\n",
        "--fp_type=morganc+rdkc --fp_size=8096 --njobs=2\n",
        "# stops at 168814 with colab; no idea why --> let's do it on the fly"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "tZ7Owm5umFbu",
        "outputId": "3e94645c-6d87-45a6-f987-0540f8cef3a0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[32m2023-06-22 05:24:50.051\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m224\u001b[0m - \u001b[1mexample assay description: Binding Binding affinity for human Glucagon Receptor   CHEMBL683962 []  []      B BAO_0000357 single protein format  Homologous single protein target assigned 8 CHEMBL1134597 Homologous protein target assigned H  1 CHEMBL1985  \u001b[0m\n",
            "\u001b[32m2023-06-22 05:24:51.576\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36mclip_encode\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mLoad CLIP model on cpu.\u001b[0m\n",
            "100%|████████████████████████████████████████| 338M/338M [00:03<00:00, 103MiB/s]\n",
            "\u001b[32m2023-06-22 05:25:06.096\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36mclip_encode\u001b[0m:\u001b[36m54\u001b[0m - \u001b[1mEncode assay descriptions using CLIP.\u001b[0m\n",
            "Encode assay descriptions: 100% 3/3 [15:02<00:00, 300.70s/it]\n",
            "\u001b[32m2023-06-22 05:40:08.251\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m257\u001b[0m - \u001b[1mSave assay features to data/fsmol/assay_features_clip_all.npy\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "# compute the assay encodings as input for your model run\n",
        "!python -m clamp.dataset.encode_assay --assay_path=./data/fsmol/assay_names.parquet --encoding=clip --gpu=0 --columns \\\n",
        "assay_type_description description assay_category assay_cell_type assay_chembl_id assay_classifications assay_organism assay_parameters assay_strain assay_subcellular_fraction assay_tax_id assay_test_type assay_tissue assay_type bao_format bao_label cell_chembl_id confidence_description confidence_score document_chembl_id relationship_description relationship_type src_assay_id src_id target_chembl_id tissue_chembl_id variant_sequence \\\n",
        "--suffix=all"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "gD7olm_nmSHk",
        "outputId": "f06606e5-c83d-4c4e-f93c-81af3bd44184"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[32m2023-06-22 05:40:14.692\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m224\u001b[0m - \u001b[1mexample assay description: Binding Binding affinity for human Glucagon Receptor   CHEMBL683962 []  []      B BAO_0000357 single protein format  Homologous single protein target assigned 8 CHEMBL1134597 Homologous protein target assigned H  1 CHEMBL1985  \u001b[0m\n",
            "\u001b[32m2023-06-22 05:40:14.693\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m230\u001b[0m - \u001b[1mEncode assay descriptions using LSA.\u001b[0m\n",
            "\u001b[32m2023-06-22 05:40:14.693\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m233\u001b[0m - \u001b[1mFit a sklearn TfidfVectorizer model on training data.\u001b[0m\n",
            "\u001b[32m2023-06-22 05:40:14.693\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m235\u001b[0m - \u001b[1mSave the fitted LSA-model to ./data/models/lsa.joblib, load it later using the argument --lsa_path\u001b[0m\n",
            "\u001b[32m2023-06-22 05:40:14.693\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m242\u001b[0m - \u001b[1mFit on 4108 train assay descriptions, 80.00% of the total dataset.\u001b[0m\n",
            "\u001b[32m2023-06-22 05:40:16.038\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36mlsa_fit\u001b[0m:\u001b[36m108\u001b[0m - \u001b[1mSet up and fit-transform a sklearn TfidfVectorizer.\u001b[0m\n",
            "\u001b[32m2023-06-22 05:40:16.669\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m__init__\u001b[0m:\u001b[36m157\u001b[0m - \u001b[1mDownload nltk data.\u001b[0m\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "\u001b[32m2023-06-22 05:40:17.242\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m__init__\u001b[0m:\u001b[36m167\u001b[0m - \u001b[1mDownload wordnet corpus.\u001b[0m\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "\u001b[32m2023-06-22 05:40:19.421\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36mlsa_fit\u001b[0m:\u001b[36m121\u001b[0m - \u001b[1mtfidf vocabulary size: 17936\u001b[0m\n",
            "\u001b[32m2023-06-22 05:40:19.422\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36mlsa_fit\u001b[0m:\u001b[36m123\u001b[0m - \u001b[1mFit a sklearn TruncatedSVD model with 355 n_components.\u001b[0m\n",
            "\u001b[32m2023-06-22 05:40:23.931\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36mlsa_fit\u001b[0m:\u001b[36m130\u001b[0m - \u001b[1mSave the fitted model.\u001b[0m\n",
            "\u001b[32m2023-06-22 05:40:24.188\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36mlsa_encode\u001b[0m:\u001b[36m84\u001b[0m - \u001b[1mLoad a fitted sklearn model.\u001b[0m\n",
            "\u001b[32m2023-06-22 05:40:24.295\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36mlsa_encode\u001b[0m:\u001b[36m88\u001b[0m - \u001b[1mEncode assay descriptions.\u001b[0m\n",
            "\u001b[32m2023-06-22 05:40:24.830\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m257\u001b[0m - \u001b[1mSave assay features to data/fsmol/assay_features_lsa_all.npy\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "# also the lsa encoding\n",
        "!python -m clamp.dataset.encode_assay --assay_path=./data/fsmol/assay_names.parquet --encoding=lsa --gpu=0 --columns \\\n",
        "assay_type_description description assay_category assay_cell_type assay_chembl_id assay_classifications assay_organism assay_parameters assay_strain assay_subcellular_fraction assay_tax_id assay_test_type assay_tissue assay_type bao_format bao_label cell_chembl_id confidence_description confidence_score document_chembl_id relationship_description relationship_type src_assay_id src_id target_chembl_id tissue_chembl_id variant_sequence \\\n",
        "--suffix=all"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "XDBJySabzXbR",
        "outputId": "89f1a86c-c659-4d8d-c814-ab212e5b347a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "--2023-06-22 05:40:26--  https://github.com/ml-jku/clamp/raw/main/hparams/default.json\n",
            "Resolving github.com (github.com)... 140.82.113.3\n",
            "Connecting to github.com (github.com)|140.82.113.3|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://raw.githubusercontent.com/ml-jku/clamp/main/hparams/default.json [following]\n",
            "--2023-06-22 05:40:26--  https://raw.githubusercontent.com/ml-jku/clamp/main/hparams/default.json\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.111.133, 185.199.108.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 427 [text/plain]\n",
            "Saving to: ‘default.json’\n",
            "\n",
            "default.json        100%[===================>]     427  --.-KB/s    in 0s      \n",
            "\n",
            "2023-06-22 05:40:26 (20.2 MB/s) - ‘default.json’ saved [427/427]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# download default hparams (can also be copied from the repo)\n",
        "!mkdir hparams\n",
        "!wget https://github.com/ml-jku/clamp/raw/main/hparams/default.json\n",
        "!mv default.json ./hparams/default.json"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F7lTCmuqmYlX",
        "outputId": "e5c378a1-1edb-4f0c-8f4e-396f8d7e7f23"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[32m2023-06-22 05:40:31.021\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36mparse_args_override\u001b[0m:\u001b[36m88\u001b[0m - \u001b[1mno compound_layer_sizes provided, setting to hidden_layers\u001b[0m\n",
            "\u001b[32m2023-06-22 05:40:31.021\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36mparse_args_override\u001b[0m:\u001b[36m91\u001b[0m - \u001b[1mno assay_layer_sizes provided, setting to hidden_layers\u001b[0m\n",
            "2023/06/22 05:40:31 INFO mlflow.tracking.fluent: Experiment with name 'debug' does not exist. Creating a new experiment.\n",
            "\u001b[32m2023-06-22 05:40:31.281\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mclamp.dataset.dataloader\u001b[0m:\u001b[36m_load_compound\u001b[0m:\u001b[36m174\u001b[0m - \u001b[1mloading compound_features_morganc+rdkc.npz failed, using .npy instead\u001b[0m\n",
            "\u001b[32m2023-06-22 05:40:31.281\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mclamp.dataset.dataloader\u001b[0m:\u001b[36m_load_compound\u001b[0m:\u001b[36m178\u001b[0m - \u001b[1mloading compound_features_morganc+rdkc.npy failed, trying to compute it on the fly\u001b[0m\n",
            "{'f': None, 'dataset': './data/fsmol', 'assay_mode': 'clip_all', 'compound_mode': 'morganc+rdkc', 'hyperparams': './hparams/default.json', 'checkpoint': None, 'experiment': 'debug', 'random': False, 'gpu': '0', 'seed': None, 'split': 'time_a_c', 'support_set_size': 0, 'train_only_actives': False, 'drop_cidx_path': None, 'verbose': 0, 'wandb': False, 'bf16': False, 'model': 'MLPLayerNorm', 'multitask_temperature': None, 'optimizer': 'AdamW', 'hidden_layers': [2048, 1024], 'compound_layer_sizes': [2048, 1024], 'assay_layer_sizes': [2048, 1024], 'embedding_size': 512, 'lr_ini': 1e-05, 'epoch_max': 50, 'batch_size': 256, 'dropout_input': 0.1, 'dropout_hidden': 0.3, 'l2': 0.0005, 'nonlinearity': 'ReLU', 'pooling_mode': None, 'lr_factor': 1, 'patience': 3, 'attempts': 1, 'loss_fun': 'BCE', 'tokenizer': None, 'transformer': None, 'warmup_epochs': 2, 'train_balanced': 0, 'beta': 1, 'norm': None, 'label_smoothing': None, 'format': None, 'train_subsample': 0}\n",
            "\u001b[32m2023-06-22 05:40:31.606\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mclamp.utils\u001b[0m:\u001b[36minit_model\u001b[0m:\u001b[36m184\u001b[0m - \u001b[1m has been selected\u001b[0m\n",
            "\u001b[32m2023-06-22 05:40:33.040\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mclamp.utils\u001b[0m:\u001b[36mtrain_and_test\u001b[0m:\u001b[36m387\u001b[0m - \u001b[1mScheduler enabled with lr_factor=1. This keeps the interface but results in no reduction.\u001b[0m\n",
            "\u001b[32m2023-06-22 06:50:47.431\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mclamp.utils\u001b[0m:\u001b[36mtrain_and_test\u001b[0m:\u001b[36m713\u001b[0m - \u001b[1mEpoch 0: Save model and optimizer checkpoint with val-davgp: 0.0810664451997224.\u001b[0m\n",
            "\u001b[32m2023-06-22 07:59:19.767\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mclamp.utils\u001b[0m:\u001b[36mtrain_and_test\u001b[0m:\u001b[36m713\u001b[0m - \u001b[1mEpoch 1: Save model and optimizer checkpoint with val-davgp: 0.09532259768406877.\u001b[0m\n",
            "\u001b[32m2023-06-22 09:09:08.068\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mclamp.utils\u001b[0m:\u001b[36mtrain_and_test\u001b[0m:\u001b[36m713\u001b[0m - \u001b[1mEpoch 2: Save model and optimizer checkpoint with val-davgp: 0.10018882907765564.\u001b[0m\n",
            "\u001b[32m2023-06-22 10:18:35.684\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mclamp.utils\u001b[0m:\u001b[36mtrain_and_test\u001b[0m:\u001b[36m713\u001b[0m - \u001b[1mEpoch 3: Save model and optimizer checkpoint with val-davgp: 0.10396458232412548.\u001b[0m\n",
            "\u001b[32m2023-06-22 11:27:17.894\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mclamp.utils\u001b[0m:\u001b[36mtrain_and_test\u001b[0m:\u001b[36m713\u001b[0m - \u001b[1mEpoch 4: Save model and optimizer checkpoint with val-davgp: 0.1040366572330701.\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "# finally let's train our model: (compare to Table 1 in the paper @FS-Mol default split)\n",
        "# takes around an hour per epoch (20 epochs)\n",
        "!python -m clamp.train --dataset=./data/fsmol --assay_mode=clip_all||lsa_all --split=FSMOL_split"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "HlAhzXn6L1YW"
      ],
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}