Taskfile.yml

version: '3'

vars:
  PROJECT_ROOT:
    sh: pwd
  BUILD_DIR: "{{.PROJECT_ROOT}}/build"
  PKG_VERSION:
    sh: git describe --tags HEAD | sed 's@^[a-zA-Z]\+@@' | sed 's@-[^-]*$@@'
  PKG_NAME: webilastik_{{.PKG_VERSION}}
  ENV_PATH: "{{.BUILD_DIR}}/webilastik_conda_env"
  ENV_CHECKSUM_PATH: "{{.BUILD_DIR}}/environment.yml.sha256"
  PACKED_ENV_PATH: "{{.ENV_PATH}}.tar.gz"
  DEB_TREE_PATH: "{{.BUILD_DIR}}/deb_tree"
  WEBILASTIK_UNIT_INSTALL_PATH: "/lib/systemd/system/webilastik.service"
  WEBILASTIK_DEV_UNIT_CONFIG_DIR: "/etc/systemd/system/webilastik.service.d"
  DEB_PKG_PATH: "{{.BUILD_DIR}}/{{.PKG_NAME}}.deb"
  REMOTE_PACKAGE_PATH: "/home/tomaz/{{.PKG_NAME}}.deb"
  NEUROGLANCER_GIT_DIR: "{{.BUILD_DIR}}/neuroglancer"
  NEUROGLANCER_BUILD_PATH: "{{.NEUROGLANCER_GIT_DIR}}/dist/min"
  NEUROGLANCER_BUNDLE_PATH: "{{.NEUROGLANCER_BUILD_PATH}}/main.bundle.js"
  OVERLAY_DIR: "{{.PROJECT_ROOT}}/overlay"
  OVERLAY_BUNDLE_PATH: "{{.OVERLAY_DIR}}/build/inject_into_neuroglancer.js"
  SOURCES_DIR_NAMES: "webilastik global_cache global_cache_impls executor_getter executor_getter_impls"

tasks:
  generate-message-schemas:
    sources:
      - "{{.PROJECT_ROOT}}/webilastik/server/rpc/dto.template.py"
      - "{{.PROJECT_ROOT}}/webilastik/server/rpc/__main__.py"
      - "{{.PROJECT_ROOT}}/webilastik/server/rpc/__init__.py"
    generates: ["{{.PROJECT_ROOT}}/webilastik/server/rpc/dto.py", "{{.PROJECT_ROOT}}/overlay/src/client/dto.ts"]
    cmds:
      - python -m webilastik.server.rpc
      - deno fmt --options-line-width=120 {{.PROJECT_ROOT}}/overlay/src/client/dto.ts
      - black {{.PROJECT_ROOT}}/webilastik/server/rpc/dto.py

  check:
    deps: [generate-message-schemas]
    env:
      PYTHONPATH: "{{.PROJECT_ROOT}}"
    cmds:
      - pylint -j 0 --disable=all --enable=trailing-comma-tuple,expression-not-assigned {{.SOURCES_DIR_NAMES}} tests benchmarks
      - npx pyright
  create-build-dir:
    cmds:
      - mkdir -p {{.BUILD_DIR}}
    status:
      - test -d {{.BUILD_DIR}}

  test:
    env:
      # ensure requests will use the mkcert cert. Requests uses certifi by default, i think
      REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt"
      # ensure aiohttp will use the mkcert certts. I don't really know where it otherwise gets its certs from
      SSL_CERT_DIR: "/etc/ssl/certs/"
      WEBILASTIK_SCRATCH_DIR: "/tmp/ilastik_tests_scratch/"
      PYTHONPATH: "{{.PYTHONPATH}}:./ndstructs/"
    cmds:
      - find tests/ -iname 'test_*.py' | grep -v test_hashing_mpi_executor.py | tr '/' '.' | sed 's@\.py$@@' | parallel --group -t --halt now,fail=1 python -m

  create-conda-environment:
    deps: [create-build-dir]
    sources: ["{{.PROJECT_ROOT}}/environment.yml"]
    generates: ["{{.ENV_CHECKSUM_PATH}}"]
    cmds:
      - rm -rf {{.ENV_PATH}}
      - mamba env create --prefix {{.ENV_PATH}} -f {{.PROJECT_ROOT}}/environment.yml
      - sha256sum {{.PROJECT_ROOT}}/environment.yml > {{.ENV_CHECKSUM_PATH}}

  create-packed-conda-environment:
    deps: [create-conda-environment]
    sources: ["{{.ENV_CHECKSUM_PATH}}"]
    generates: ["{{.PACKED_ENV_PATH}}"]
    cmds:
      - rm -f {{.PACKED_ENV_PATH}}
      - conda pack -p {{.ENV_PATH}} -o {{.PACKED_ENV_PATH}}

  create-neuroglancer-git-dir:
    run: once
    cmds:
      - mkdir -p {{.NEUROGLANCER_GIT_DIR}}
    status:
      - test -d {{.NEUROGLANCER_GIT_DIR}}

  checkout-neuroglancer:
    deps: [create-neuroglancer-git-dir]
    dir: "{{.NEUROGLANCER_GIT_DIR}}"
    vars:
      COMMIT_HASH: "0edaf528000268daea3dd1d1781e53527642b85d"
    cmds:
      - git clone https://github.com/ilastik/neuroglancer . && git fetch
      - git checkout {{.COMMIT_HASH}}
    status: # FIXME: this doesn't update the ng repo
      - git status > /dev/null 2>&1
      - test "$(git rev-parse HEAD)" = "$(git rev-parse {{.COMMIT_HASH}})"

  install-neuroglancer-dependencies:
    deps: [checkout-neuroglancer]
    sources: ["{{.NEUROGLANCER_GIT_DIR}}/package-lock.json"]
    generates: ["{{.NEUROGLANCER_GIT_DIR}}/node_modules/**/*"]
    dir: "{{.NEUROGLANCER_GIT_DIR}}"
    cmds:
      - rm -rf node_modules
      - npm ci

  compile-neuroglancer:
    deps: [install-neuroglancer-dependencies]
    dir: "{{.NEUROGLANCER_GIT_DIR}}"
    sources: ["{{.NEUROGLANCER_GIT_DIR}}/**/*"]
    generates: ["{{.NEUROGLANCER_BUNDLE_PATH}}"]
    env:
      ILASTIK_URL: https://app.ilastik.org/ #needed for 'npm run build-with-ilastik-overlay'
    cmds:
      - rm -rf dist/
      - npm run build-with-ilastik-overlay

  compile-overlay:
    deps: [generate-message-schemas]
    dir: "{{.OVERLAY_DIR}}"
    sources:
      - "{{.OVERLAY_DIR}}/package-lock.json"
      - "{{.OVERLAY_DIR}}/package.json"
      - "{{.OVERLAY_DIR}}/src/**/*"
      - "{{.OVERLAY_DIR}}/tsconfig.json"
    generates: ["{{.OVERLAY_BUNDLE_PATH}}", "{{.OVERLAY_BUNDLE_PATH}}.map"]
    cmds:
      - rm -rf node_modules
      - npm ci
      - npm run bundle-ng-inject

  create-deb-tree:
    deps: [generate-message-schemas, create-packed-conda-environment, compile-neuroglancer, compile-overlay]
    sources:
      - "{{.PROJECT_ROOT}}/package_tree/**/*"
      - "{{.PROJECT_ROOT}}/public/**/*"
      - "{{.PACKED_ENV_PATH}}"
      - "{{.PROJECT_ROOT}}/webilastik/**/*.py"
      - "{{.PROJECT_ROOT}}/global_cache_impls/**/*.py"
      - "{{.PROJECT_ROOT}}/executor_getter_impls/**/*.py"
      - "{{.OVERLAY_BUNDLE_PATH}}"
      - "{{.OVERLAY_BUNDLE_PATH}}.map"
      - "{{.NEUROGLANCER_BUNDLE_PATH}}"
    generates:
      - "{{.DEB_TREE_PATH}}/**/*"
    vars:
      CONTROL_FILE_CONTENTS: |-
          Package: webilastik
          Version: {{.PKG_VERSION}}
          Section: base
          Priority: optional
          Architecture: amd64
          Depends: nginx
          Maintainer: ilastik Team <team@ilastik.org>
          Description: Webilastik
           Server and frontend for the web version of ilastik
      SERVER_SYSTEMD_ENV_CONFS:
        sh: |-
          python3 <<SERVER_CONFIG
          from webilastik.config import SessionAllocatorConfig
          from webilastik.utility.url import Url
          print(
            SessionAllocatorConfig.from_env(
              allow_local_compute_sessions=False,
              external_url=Url.parse_or_raise('https://app.ilastik.org/')
            ).to_systemd_environment_confs()
          )
          SERVER_CONFIG
      SERVICE_FILE_CONTENTS: |-
        [Unit]
        Description=Webilastik session allocator server
        Documentation=https://github.com/ilastik/webilastik
        Wants=nginx.service

        [Install]
        WantedBy=multi-user.target

        [Service]
        Type=simple
        # FIXME/BUG: aiohttp must be told where certs are when running from packed environment
        Environment=SSL_CERT_DIR=/etc/ssl/certs/
        Environment=REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
        {{.SERVER_SYSTEMD_ENV_CONFS}}

        Environment=PYTHONPATH=/opt/webilastik/
        ExecStart=/opt/webilastik/conda_env/bin/python3 -B /opt/webilastik/webilastik/server/session_allocator.py

        TimeoutStopSec=30
        KillMode=mixed
        Restart=on-failure
        User=www-data
        Group=www-data
        KillSignal=SIGQUIT
        NotifyAccess=all

        [Install]
        WantedBy=multi-user.target
    cmds:
      - echo 2>&1 '[INFO] Generating basic package tree'
      - rm -rf {{.DEB_TREE_PATH}}
      - cp -r {{.PROJECT_ROOT}}/package_tree {{.DEB_TREE_PATH}}

      - echo 2>&1 '[INFO] Generating DEBIAN/control file'
      - echo '{{.CONTROL_FILE_CONTENTS}}' > {{.DEB_TREE_PATH}}/DEBIAN/control

      - echo 2>&1 '[INFO] Generating webilastik.service file'
      - mkdir -p {{.DEB_TREE_PATH}}/lib/systemd/system/
      - echo '{{.SERVICE_FILE_CONTENTS}}' > {{.DEB_TREE_PATH}}/{{.WEBILASTIK_UNIT_INSTALL_PATH}}

      - mkdir -p {{.DEB_TREE_PATH}}/etc/webilastik

      - echo 2>&1 '[INFO] Unpacking conda environment'
      - mkdir -p {{.DEB_TREE_PATH}}/opt/webilastik/conda_env
      - tar -xzf {{.PACKED_ENV_PATH}} -C {{.DEB_TREE_PATH}}/opt/webilastik/conda_env

      - echo 2>&1 '[INFO] Copying webilastik files'
      - mkdir -p {{.DEB_TREE_PATH}}/opt/webilastik
      - >
        for source_dir_name in {{.SOURCES_DIR_NAMES}} public; do
           cp -r "{{.PROJECT_ROOT}}/$source_dir_name" {{.DEB_TREE_PATH}}/opt/webilastik
        done
      - find {{.DEB_TREE_PATH}}/opt/webilastik -name __pycache__ | xargs --no-run-if-empty rm -rf

      - echo 2>&1 '[INFO] Copying overlay bundle to public dir'
      - mkdir -p {{.DEB_TREE_PATH}}/opt/webilastik/public/js/
      - cp {{.OVERLAY_BUNDLE_PATH}} {{.DEB_TREE_PATH}}/opt/webilastik/public/js/
      - cp {{.OVERLAY_BUNDLE_PATH}}.map {{.DEB_TREE_PATH}}/opt/webilastik/public/js/

      - echo 2>&1 '[INFO] Copying nehuba to public dir'
      - cp -r {{.NEUROGLANCER_BUILD_PATH}} {{.DEB_TREE_PATH}}/opt/webilastik/public/nehuba

  create-deb-package:
    deps: [create-deb-tree]
    sources:
      - "{{.DEB_TREE_PATH}}/**/*"
    generates:
      - "{{.DEB_PKG_PATH}}"
    cmds:
      - echo "Ensuring no changes in the working tree before packing to .deb"
      - test $(git status --porcelain | wc -l) -eq 0
      - test "$(git rev-parse HEAD)" ==  "$(git rev-parse origin/master)"
      - dpkg-deb --build -z2 {{.DEB_TREE_PATH}} {{.DEB_PKG_PATH}}

  start-local-server:
    deps: [stop-local-server, create-deb-tree]
    vars:
      SERVER_SYSTEMD_ENV_CONFS:
        sh: |-
          python3 <<SERVER_CONFIG
          from webilastik.config import SessionAllocatorConfig
          from webilastik.utility.url import Url
          print(
            SessionAllocatorConfig.from_env(
              allow_local_compute_sessions=True,
              external_url=Url.parse_or_raise('https://app.ilastik.org/'),
              session_allocator_ssh_port=22,
            ).to_systemd_environment_confs()
          )
          SERVER_CONFIG
    cmds: #this assumes an nginx like the one in webilastik.conf running already
      - echo "Ensuring that app.ilastik.org points to localhost"
      - getent hosts app.ilastik.org | awk '{ print $1 }' | grep -qE '^(127\.0\.0\.1|::1)$'

      - echo "Checking that nginx is running"
      - ps -ef | grep -q nginx

      - echo "Checking that webilastik.conf is installed in nginx's config files"
      - sudo nginx -T | grep -q app.ilastik.org

      # for now this must be via www-data because that's nginx's user, and nginx must
      # be able to open the socket files that go back to the sessions, and having
      # the ssh happen for the user www-data is one way to do that
      # FIXME: investigate "-oStreamLocalBindMask=0111" in tunnel.py
      - echo "Checking that www-data can ssh into itself to create local sessions"
      - sudo -u www-data -g www-data ssh -oBatchMode=yes www-data@localhost echo success

      - echo "Ensuring webilastik service is visible to systemd"
      - sudo rm -f {{.WEBILASTIK_UNIT_INSTALL_PATH}}
      - sudo ln -s {{.DEB_TREE_PATH}}/{{.WEBILASTIK_UNIT_INSTALL_PATH}} {{.WEBILASTIK_UNIT_INSTALL_PATH}}
      - sudo ln -s {{.DEB_TREE_PATH}}/opt/webilastik /opt/webilastik

      - echo "Configuring webilastik service to log to this tty ($(tty))"
      - sudo mkdir -p {{.WEBILASTIK_DEV_UNIT_CONFIG_DIR}}
      - |
        sudo sh -c "echo '

        [Service]
        TTYPath=$(tty)
        StandardOutput=tty
        StandardError=inherit

        {{.SERVER_SYSTEMD_ENV_CONFS}}

        ' > {{.WEBILASTIK_DEV_UNIT_CONFIG_DIR}}/output_to_tty.conf"

      - sudo systemctl daemon-reload
      - sudo systemctl restart webilastik.service

  stop-local-server:
    cmds:
      - sudo systemctl stop webilastik.service || true
      - sudo systemctl reset-failed webilastik.service || true
      - sudo rm -f {{.WEBILASTIK_DEV_UNIT_CONFIG_DIR}}/output_to_tty.conf
      - sudo rm -f {{.WEBILASTIK_UNIT_INSTALL_PATH}}
      - sudo rm -f /opt/webilastik
      - >
        ps -ef | grep ws_pixel_classification_workflow.py | awk '{print $2}' | sudo xargs kill -9 || true;
        ps -ef | ag ssh.*batchmode | awk '{print $2}' | sudo xargs kill -9 || true;

  deploy-to-hpcs:
    vars:
      JUSUF_GIT_DIR: /p/home/jusers/webilastik/jusuf/webilastik.git
      CSCS_GIT_DIR: /users/bp000188/source/webilastik.git
    cmds:
      # - (git remote | grep jusuf) || git remote add jusuf webilastik@jusuf.fz-juelich.de:{{.JUSUF_GIT_DIR}}
      # - (git remote | grep cscs) ||  git remote add cscs bp000188@ela.cscs.ch:{{.CSCS_GIT_DIR}}
      - echo "cd {{.JUSUF_GIT_DIR}} && git fetch" | ssh webilastik@jusuf.fz-juelich.de
      #- echo "cd {{.CSCS_GIT_DIR}} && git fetch" | ssh bp000188@ela.cscs.ch -oCheckHostIP=no -oBatchMode=yes ssh bp000188@daint.cscs.ch

  send-deb-package-to-web-server:
    deps: [create-deb-package]
    vars:
      LOCAL_DEB_PACKAGE_HASH:
        sh: sha256sum {{.DEB_PKG_PATH}} | awk '{print $1}'
      REMOTE_DEB_PACKAGE_HASH:
        sh: ssh -p7002 -oIdentitiesOnly=yes -i ~/.ssh/id_ed25519 tomaz@134.94.198.170 "sha256sum {{.REMOTE_PACKAGE_PATH}} | awk '{print \$1}' " || echo "nothing"
    # status:
    #   - test {{.REMOTE_DEB_PACKAGE_HASH}} = {{.LOCAL_DEB_PACKAGE_HASH}}
    cmds:
      - scp -P $WEBILASTIK_SESSION_ALLOCATOR_SSH_PORT {{.DEB_PKG_PATH}} tomaz@134.94.198.170:{{.REMOTE_PACKAGE_PATH}}

  deploy-to-webilastik-server:
    deps: [send-deb-package-to-web-server]
    cmds:
      - ssh -p7002 -oIdentitiesOnly=yes -i ~/.ssh/id_ed25519 tomaz@134.94.198.170 "sudo apt-get --assume-yes install {{.REMOTE_PACKAGE_PATH}}"

  deploy:
    deps: [deploy-to-hpcs, deploy-to-webilastik-server]

  clean:
    deps: [stop-local-server]
    cmds:
      - rm -rf {{.PROJECT_ROOT}}/.task {{.BUILD_DIR}}/*

  default:
    deps: [create-deb-package]