Expose piplines and clean up deploy

docker/docker-compose.yml

@@ -10,62 +10,52 @@ services:
     volumes:
       - mongo_db:/data/db
 
-  ## === === === === Building Base Images === === === === ===
-
-  # Not sure if we need this
-  # base_girder_worker:
-  #   image: gitlab.kitware.com:4567/opengeoscience/viameweb/base/girder_worker
-
   ## === === === === Girder === === === === ===
   girder:
     build:
       context: ../
       dockerfile: docker/girder.Dockerfile
     depends_on:
       - mongo
+      - girder_worker
     ports:
       - "8081:8080"
     volumes:
       - girder_assetstore:/home/assetstore
       - ./server:/home/viame_girder
+      - pipelines:${VIAME_PIPELINES_PATH:-/home/VIAME/pipelines}:ro
       - type: bind
         source: ${PUBLIC_DATA_PATH}
         target: /data/public
         read_only: true
         volume:
           nocopy: true
     environment:
-      - GIRDER_WORKER_BROKER=amqp://guest:guest@rabbit/
-      - GIRDER_WORKER_BACKEND=amqp://guest:guest@rabbit/
-      - BROKER_URL=amqp://guest:guest@rabbit/
-      - CELERY_BROKER_URL=amqp://guest:guest@rabbit/
-      - BROKER_CONNECTION_TIMEOUT=2
-      - CELERY_BROKER_CONNECTION_TIMEOUT=2
-      - GIRDER_INNER_PORT=${GIRDER_INNER_PORT}
+      - VIAME_PIPELINES_PATH=${VIAME_PIPELINES_PATH:-/home/VIAME/pipelines}
+      - MONGO_URI=mongodb://mongo:27017/girder
+      - GIRDER_ADMIN_USER=${GIRDER_ADMIN_USER}
+      - GIRDER_ADMIN_PASS=${GIRDER_ADMIN_PASS}
 
   ## === === === === Girder Worker === === === === ===
-
   girder_worker:
     volumes:
       - ./server:/home/viame_girder
+      - pipelines:/home/VIAME/build/install/configs/pipelines
     depends_on:
       - rabbit
-      - girder
     environment:
       - GIRDER_WORKER_BROKER=amqp://guest:guest@rabbit/
       - GIRDER_WORKER_BACKEND=amqp://guest:guest@rabbit/
-      - BROKER_URL=amqp://guest:guest@rabbit/
+      - GIRDER_WORKER_PLUGINS_ENABLED=girder_io
       - CELERY_BROKER_URL=amqp://guest:guest@rabbit/
-      - BROKER_CONNECTION_TIMEOUT=2
-      - CELERY_BROKER_CONNECTION_TIMEOUT=2
-      - GIRDER_ADMIN_USER=${GIRDER_ADMIN_USER}
-      - GIRDER_ADMIN_PASS=${GIRDER_ADMIN_PASS}
+
+    # Swap this and the build step if you need to change things locally
     # image: gitlab.kitware.com:4567/opengeoscience/viameweb/app/girder_worker
-    # Replaced with build because girder.py changed
     build:
       context: ../
       dockerfile: docker/viame_girder_worker.Dockerfile
 
 volumes:
   mongo_db:
   girder_assetstore:
+  pipelines:

docker/girder.Dockerfile

@@ -22,5 +22,3 @@ RUN cd viame_client && npm install && npm run build && \
     cp -r dist/* /usr/share/girder/static/viame/
 
 ENTRYPOINT ["/home/provision/girder_entrypoint.sh"]
-
-

docker/provision/girder_entrypoint.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
 
-# Install viame plugin
-cd /home/viame_girder && pip install -e . -U
-girder serve --database mongodb://mongo:27017/girder --host 0.0.0.0
+python3 /home/provision/init_girder.py
+girder serve --database $MONGO_URI --host 0.0.0.0

docker/provision/init_girder.py

@@ -0,0 +1,35 @@
+import os
+import cherrypy
+from girder.models.user import User
+from girder.models.assetstore import Assetstore
+from girder.exceptions import ValidationException
+
+cherrypy.config["database"]["uri"] = os.getenv("MONGO_URI")
+
+ADMIN_USER = os.getenv("GIRDER_ADMIN_USER", "admin")
+ADMIN_PASS = os.getenv("GIRDER_ADMIN_PASS", "letmein")
+
+
+def createInitialUser():
+    try:
+        User().createUser(
+            ADMIN_USER, ADMIN_PASS, ADMIN_USER, ADMIN_USER, "admin@admin.com"
+        )
+    except ValidationException:
+        print("Admin user already exists, skipping...")
+
+
+def createAssetstore():
+    try:
+        Assetstore().createFilesystemAssetstore('assetstore', '/home/assetstore')
+    except ValidationException:
+        print("Assetstore already exists, skipping...")
+
+
+def run_girder_init():
+    createInitialUser()
+    createAssetstore()
+
+
+if __name__ == '__main__':
+    run_girder_init()

docker/viame_girder_worker.Dockerfile

@@ -22,7 +22,7 @@ RUN apt-get update -y &&\
     zlib1g-dev \
     ffmpeg
 
-RUN git clone https://github.com/Kitware/VIAME.git /home/VIAME &&\ 
+RUN git clone https://github.com/Kitware/VIAME.git /home/VIAME &&\
     cd /home/VIAME && git submodule update --init --recursive
 
 # Need CMake >= 3.11.4 for VIAME,
@@ -60,4 +60,4 @@ RUN cd viame_girder && pip install --no-cache-dir -e .
 
 USER worker
 
-ENTRYPOINT ["/home/provision/girder_worker_entrypoint.sh"]
+CMD girder-worker -l info

server/viame_server/__init__.py

@@ -1,24 +1,48 @@
-import datetime
+import sys
+from pathlib import Path
+from os import getenv
 
 from girder import events, plugin
-from girder.api import access
-from girder.utility import server
-from girder.models.folder import Folder
-
 
 from .client_webroot import ClientWebroot
 from .viame import Viame
 from .viame_detection import ViameDetection
 from .utils import check_existing_annotations
 
 
+env_pipelines_path = getenv("VIAME_PIPELINES_PATH")
+
+
+def load_pipelines():
+    if env_pipelines_path is None:
+        print(
+            "No pipeline path specified. "
+            "Please set the VIAME_PIPELINES_PATH environment variable.",
+            file=sys.stderr
+        )
+        return []
+
+    pipeline_path = Path(env_pipelines_path)
+    if not pipeline_path.exists():
+        print("Specified pipeline path does not exist!", file=sys.stderr)
+        return []
+
+    return [path.name for path in pipeline_path.glob("./*.pipe")]
+
+
 class GirderPlugin(plugin.GirderPlugin):
     def load(self, info):
-        info['apiRoot'].viame = Viame()
-        info['apiRoot'].viame_detection = ViameDetection()
+        info["apiRoot"].viame = Viame(pipelines=load_pipelines())
+        info["apiRoot"].viame_detection = ViameDetection()
         # Relocate Girder
-        info['serverRoot'], info['serverRoot'].girder = (ClientWebroot(),
-                                                         info['serverRoot'])
-        info['serverRoot'].api = info['serverRoot'].girder.api
+        info["serverRoot"], info["serverRoot"].girder = (
+            ClientWebroot(),
+            info["serverRoot"],
+        )
+        info["serverRoot"].api = info["serverRoot"].girder.api
 
-        events.bind('filesystem_assetstore_imported', 'check_annotations', check_existing_annotations)
+        events.bind(
+            "filesystem_assetstore_imported",
+            "check_annotations",
+            check_existing_annotations,
+        )

server/viame_server/utils.py

@@ -31,6 +31,7 @@ def determine_image_sequence_fps(folder):
     total = current - start
     return round(item_length/total)
 
+
 def get_or_create_auxiliary_folder(folder, user):
     return Folder().createFolder(folder, 'auxiliary', reuseExisting=True, creator=user)
 

server/viame_server/viame.py

@@ -1,6 +1,6 @@
 from girder.api import access
 from girder.constants import AccessType
-from girder.api.describe import Description, autoDescribeRoute
+from girder.api.describe import Description, autoDescribeRoute, describeRoute
 from girder.api.rest import Resource
 from girder.models.folder import Folder
 from girder.models.item import Item
@@ -9,20 +9,31 @@
 
 from .transforms import GetPathFromItemId, GetPathFromFolderId, GirderUploadToFolder
 from .model.attribute import Attribute
-from .utils import get_or_create_auxiliary_folder, move_existing_result_to_auxiliary_folder
+from .utils import (
+    get_or_create_auxiliary_folder,
+    move_existing_result_to_auxiliary_folder,
+)
 
 
 class Viame(Resource):
-    def __init__(self):
+    def __init__(self, pipelines=[]):
         super(Viame, self).__init__()
         self.resourceName = 'viame'
+        self.pipelines = pipelines
+
+        self.route("GET", ("pipelines", ), self.get_pipelines)
         self.route("POST", ("pipeline", ), self.run_pipeline_task)
         self.route("POST", ("conversion", ), self.run_conversion_task)
         self.route("POST", ("attribute", ), self.create_attribute)
         self.route("GET", ("attribute", ), self.get_attributes)
         self.route("PUT", ("attribute", ":id"), self.update_attribute)
         self.route("DELETE", ("attribute", ":id"), self.delete_attribute)
 
+    @access.user
+    @describeRoute(Description("Get available pipelines"))
+    def get_pipelines(self, params):
+        return self.pipelines
+
     @access.user
     @autoDescribeRoute(
         Description("Run viame pipeline")