swirl_colab.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/biplobsd/swirl-colab/blob/main/swirl_colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2H53kq7bhK97"
      },
      "source": [
        "# ![swirl logo](https://swirl.today/wp-content/uploads/2023/07/swirl-company-logo-white-250x100-1-1.png)\n",
        "\n",
        "Swirl is open-source, enterprise search and AI\n",
        "\n",
        "This notebook created by `Biplob Kumar Sutradhar` - [biplobsd.me](https://biplobsd.github.io)\n",
        "\n",
        "Read more - [https://biplobsd.me/blogs/view/run-swirl-open-source-search-engine-on-google-colab.md](https://biplobsd.me/blogs/view/run-swirl-open-source-search-engine-on-google-colab.md)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6Q10qazkgkMO"
      },
      "source": [
        "## Step 1 - Prerequisites Installation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XqI38dYPLKHN"
      },
      "outputs": [],
      "source": [
        "%%shell\n",
        "pip install udocker\n",
        "udocker --allow-root install\n",
        "apt-get install redis-server\n",
        "redis-server --daemonize yes"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ySFDx0wFgvrg"
      },
      "source": [
        "## Step 2 - Port forward 8000"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pscBeVz1jRSv"
      },
      "source": [
        "Do not click after getting the link. `Step 3` is required and running successfully then click the forward link."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "9tmDfc0vAms0"
      },
      "outputs": [],
      "source": [
        "#@markdown <center><h3>Port forwarding</h3></center>\n",
        "import os, pathlib, zipfile, re\n",
        "import urllib.request\n",
        "from IPython.display import HTML, clear_output\n",
        "\n",
        "\n",
        "#####################################\n",
        "USE_FREE_TOKEN = True\n",
        "TOKEN = \"\"\n",
        "PORT_LIST = \"8000\" #@param {type:\"string\"}\n",
        "\n",
        "#@markdown <font size=2px>Input PORT which you can want to forward. Separate two PORT between by comma. <br>Ex `80, 8080, 4040`</font>\n",
        "REGION = \"US\" #@param [\"US\", \"EU\", \"AP\", \"AU\", \"SA\", \"JP\", \"IN\"]\n",
        "HOME = os.path.expanduser(\"~\")\n",
        "\n",
        "if not os.path.exists(f\"{HOME}/.ipython/ocr.py\"):\n",
        "    hCode = \"https://raw.githubusercontent.com/biplobsd/\" \\\n",
        "                \"OneClickRun/master/res/ocr.py\"\n",
        "    urllib.request.urlretrieve(hCode, f\"{HOME}/.ipython/ocr.py\")\n",
        "\n",
        "\n",
        "from ocr import (\n",
        "    runSh,\n",
        "    loadingAn,\n",
        "    PortForward_wrapper,\n",
        "    findProcess,\n",
        "    displayUrl,\n",
        "    textAn\n",
        ")\n",
        "\n",
        "loadingAn()\n",
        "\n",
        "os.makedirs('tools/', exist_ok=True)\n",
        "\n",
        "# START_SERVER\n",
        "# Ngrok region 'us','eu','ap','au','sa','jp','in'\n",
        "clear_output()\n",
        "m = []\n",
        "splitPortList = PORT_LIST.split(',')\n",
        "for p in splitPortList:\n",
        "  p = int(p)\n",
        "  m.append([f\"s{p}\", p, 'http'])\n",
        "\n",
        "\n",
        "PORT_FORWARD = \"localhost\" #@param [\"ngrok\", \"localhost\", \"argotunnel\"]\n",
        "Server = PortForward_wrapper(\n",
        "    PORT_FORWARD, TOKEN, USE_FREE_TOKEN, m, REGION.lower(),\n",
        "    [f\"{HOME}/.ngrok2/randomPortOpen.yml\", 45535]\n",
        ")\n",
        "\n",
        "for l in m:\n",
        "  displayUrl(Server.start(l[0], displayB=False, v=False),\n",
        "             pNamU=f\"{l[0][1:]} -> \", cls=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TnQRhJLTg3yN"
      },
      "source": [
        "## Step 3 - start swirl using udocker"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tqwTO5YujDPj"
      },
      "source": [
        "Default username: `admin` and password: `password`"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l01NDXPjnHAX"
      },
      "source": [
        "___Please wait for 1m:30s for booting the image.___"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "JP0skvnAJ2QW",
        "outputId": "13b52851-ec3c-4f6d-be39-696be43d15aa"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Warning: this container exposes TCP/IP ports\n",
            " \n",
            " ****************************************************************************** \n",
            " *                                                                            * \n",
            " *               STARTING ae8ce180-364e-350e-80b5-276c96f30ff3                * \n",
            " *                                                                            * \n",
            " ****************************************************************************** \n",
            " executing: sh\n",
            "\u001b[1m__S_W_I_R_L__2_._6______________________________________________________________\u001b[0m\n",
            "\n",
            "Setting Up SWIRL:\n",
            "Checking Migrations:\n",
            "No changes detected\n",
            "\n",
            "\n",
            "Migrating:\n",
            "\n",
            "Operations to perform:\n",
            "  Apply all migrations: admin, auth, authtoken, contenttypes, django_celery_beat, sessions, swirl\n",
            "Running migrations:\n",
            "  No migrations to apply.\n",
            "\n",
            "\n",
            "Collecting Statics:\n",
            "\n",
            "\n",
            "215 static files copied to '/app/static'.\n",
            "\n",
            "Ok\n",
            "\u001b[92mCommand successful!\u001b[0m\n",
            "\u001b[1m__S_W_I_R_L__2_._6______________________________________________________________\u001b[0m\n",
            "\n",
            "Warning: logs directory does not exist, creating it\n",
            "ParseResult(scheme='redis', netloc='127.0.0.1:6379', path='/0', params='', query='', fragment='') checked.\n",
            "ParseResult(scheme='redis', netloc='127.0.0.1:6379', path='/0', params='', query='', fragment='') checked.\n",
            "Start: celery-worker -> celery -A swirl_server worker --loglevel INFO ... Ok, pid: 18278\n",
            "Start: celery-beats -> celery -A swirl_server beat --scheduler django_celery_beat.schedulers:DatabaseScheduler ... Ok, pid: 18296\n",
            "Updating .swirl... Ok\n",
            "\n",
            "    PID TTY          TIME CMD\n",
            "  18278 ?        00:00:07 celery\n",
            "  18296 ?        00:00:03 celery\n",
            "\n",
            "You're using version 2.6 of Swirl, the current version.\n",
            "\u001b[92mCommand successful!\u001b[0m\n",
            "2023-10-05 07:38:44,364 INFO     Starting server at tcp:port=8000:interface=0.0.0.0\n",
            "2023-10-05 07:38:44,364 INFO     HTTP/2 support not enabled (install the http2 and tls Twisted extras)\n",
            "2023-10-05 07:38:44,365 INFO     Configuring endpoint tcp:port=8000:interface=0.0.0.0\n",
            "2023-10-05 07:38:44,369 INFO     Listening on TCP address 0.0.0.0:8000\n",
            "127.0.0.1:43072 - - [05/Oct/2023:07:39:03] \"GET /\" 302 -\n",
            "127.0.0.1:43072 - - [05/Oct/2023:07:39:03] \"GET /galaxy/\" 200 20292\n",
            "127.0.0.1:43072 - - [05/Oct/2023:07:39:04] \"GET /galaxy/styles.css\" 200 699890\n",
            "127.0.0.1:59500 - - [05/Oct/2023:07:39:04] \"GET /galaxy/runtime.js\" 200 1368\n",
            "127.0.0.1:59500 - - [05/Oct/2023:07:39:04] \"GET /galaxy/polyfills.js\" 200 80165\n",
            "127.0.0.1:59512 - - [05/Oct/2023:07:39:05] \"GET /galaxy/main.js\" 200 1187764\n",
            "127.0.0.1:59512 - - [05/Oct/2023:07:39:07] \"GET /api/config/default\" 200 2844\n",
            "127.0.0.1:43072 - - [05/Oct/2023:07:39:07] \"GET /galaxy/favicon.ico\" 200 5047\n",
            "127.0.0.1:43072 - - [05/Oct/2023:07:39:08] \"GET /galaxy/logo_highres_positive.png\" 200 25113\n",
            "127.0.0.1:43072 - - [05/Oct/2023:07:39:14] \"POST /api/swirl/login/\" 200 67\n",
            "127.0.0.1:59512 - - [05/Oct/2023:07:39:14] \"GET /api/swirl/sapi/authenticators/\" 200 2\n",
            "127.0.0.1:43072 - - [05/Oct/2023:07:39:14] \"GET /api/swirl/sapi/searchproviders/\" 200 36944\n",
            "127.0.0.1:59512 - - [05/Oct/2023:07:39:15] \"GET /api/swirl/sapi/searchproviders/\" 200 36944\n",
            "127.0.0.1:43072 - - [05/Oct/2023:07:39:22] \"GET /galaxy/loading.gif\" 200 3208\n",
            "2023-10-05 07:39:22,917 INFO     admin search_qs 1\n",
            "2023-10-05 07:39:26,005 INFO     search.py: FULL_RESULTS\n",
            "2023-10-05 07:39:26,141 INFO     admin search 1 FULL_RESULTS_READY 11 3.2\n",
            "2023-10-05 07:39:26,161 INFO     admin results 1 RelevancyMixer 11\n",
            "127.0.0.1:59512 - - [05/Oct/2023:07:39:26] \"GET /api/swirl/sapi/search/?qs=biplobsd.me&format=json&search_id=&page=1&result_mixer=RelevancyMixer\" 200 14378\n",
            "127.0.0.1:59512 - - [05/Oct/2023:07:39:26] \"GET /galaxy/primeicons.ttf\" 200 57384\n",
            "2023-10-05 09:15:18,932 INFO     Killed 0 pending application instances\n",
            "Info: keyboard interrupt\n",
            "The futex facility returned an unexpected error code.\n",
            "Aborted (core dumped)\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "CalledProcessError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mCalledProcessError\u001b[0m                        Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-15-26f82d5d68bd>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_cell_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'shell'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'udocker --allow-root run \\\\\\n  -p 8000:8000 \\\\\\n  -p 6379:6379 \\\\\\n  -e ALLOWED_HOSTS=\"*\" \\\\\\n  -e CELERY_BROKER_URL=redis://127.0.0.1:6379/0 \\\\\\n  -e CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0 \\\\\\n  swirlai/swirl-search:latest \\\\\\n  sh -c \\'rm -fr ./.swirl &&\\n         python swirl.py setup &&\\n         /usr/bin/jq \".default\" ./config-swirl-demo.db.json | sed -e \"s/<msal-app-id>/$MSAL_APP_ID/\" \\\\\\n                                                              -e \"s/<msal-tenant-id>/$MSAL_TENANT_ID/\" \\\\\\n                                                              -e \"s/<msal-port>/$MSAL_CB_PORT/\" \\\\\\n                                                              -e \"s/<msal-host>/$MSAL_HOST/\" > static/api/config/default &&\\n         python swirl.py start celery-worker celery-beats &&\\n         daphne -b 0.0.0.0 -p 8000 swirl_server.asgi:application\\'\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/google/colab/_shell.py\u001b[0m in \u001b[0;36mrun_cell_magic\u001b[0;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[1;32m    332\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mline\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    333\u001b[0m       \u001b[0mcell\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 334\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_cell_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmagic_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mline\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    335\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    336\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_cell_magic\u001b[0;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[1;32m   2471\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2472\u001b[0m                 \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmagic_arg_s\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2473\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2474\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2475\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/google/colab/_system_commands.py\u001b[0m in \u001b[0;36m_shell_cell_magic\u001b[0;34m(args, cmd)\u001b[0m\n\u001b[1;32m    110\u001b[0m   \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_run_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcmd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclear_streamed_output\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    111\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mparsed_args\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mignore_errors\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m     \u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck_returncode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    113\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    114\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/google/colab/_system_commands.py\u001b[0m in \u001b[0;36mcheck_returncode\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    135\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mcheck_returncode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    136\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 137\u001b[0;31m       raise subprocess.CalledProcessError(\n\u001b[0m\u001b[1;32m    138\u001b[0m           \u001b[0mreturncode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcmd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    139\u001b[0m       )\n",
            "\u001b[0;31mCalledProcessError\u001b[0m: Command 'udocker --allow-root run \\\n  -p 8000:8000 \\\n  -p 6379:6379 \\\n  -e ALLOWED_HOSTS=\"*\" \\\n  -e CELERY_BROKER_URL=redis://127.0.0.1:6379/0 \\\n  -e CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0 \\\n  swirlai/swirl-search:latest \\\n  sh -c 'rm -fr ./.swirl &&\n         python swirl.py setup &&\n         /usr/bin/jq \".default\" ./config-swirl-demo.db.json | sed -e \"s/<msal-app-id>/$MSAL_APP_ID/\" \\\n                                                              -e \"s/<msal-tenant-id>/$MSAL_TENANT_ID/\" \\\n                                                              -e \"s/<msal-port>/$MSAL_CB_PORT/\" \\\n                                                              -e \"s/<msal-host>/$MSAL_HOST/\" > static/api/config/default &&\n         python swirl.py start celery-worker celery-beats &&\n         daphne -b 0.0.0.0 -p 8000 swirl_server.asgi:application'\n' returned non-zero exit status 1."
          ]
        }
      ],
      "source": [
        "%%shell\n",
        "udocker --allow-root run \\\n",
        "  -p 8000:8000 \\\n",
        "  -p 6379:6379 \\\n",
        "  -e ALLOWED_HOSTS=\"*\" \\\n",
        "  -e CELERY_BROKER_URL=redis://127.0.0.1:6379/0 \\\n",
        "  -e CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0 \\\n",
        "  swirlai/swirl-search:latest \\\n",
        "  sh -c 'rm -fr ./.swirl &&\n",
        "         python swirl.py setup &&\n",
        "         /usr/bin/jq \".default\" ./config-swirl-demo.db.json | sed -e \"s/<msal-app-id>/$MSAL_APP_ID/\" \\\n",
        "                                                              -e \"s/<msal-tenant-id>/$MSAL_TENANT_ID/\" \\\n",
        "                                                              -e \"s/<msal-port>/$MSAL_CB_PORT/\" \\\n",
        "                                                              -e \"s/<msal-host>/$MSAL_HOST/\" > static/api/config/default &&\n",
        "         python swirl.py start celery-worker celery-beats &&\n",
        "         daphne -b 0.0.0.0 -p 8000 swirl_server.asgi:application'"
      ]
    }
  ],
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}