Add support for multiple Deadline ☠️➖ servers

openpype/hosts/maya/plugins/create/create_render.py

@@ -4,6 +4,7 @@
 import json
 import appdirs
 import requests
+import six
 
 from maya import cmds
 import maya.app.renderSetup.model.renderSetup as renderSetup
@@ -12,7 +13,13 @@
     lib,
     plugin
 )
-from openpype.api import (get_system_settings, get_asset)
+from openpype.api import (
+    get_system_settings,
+    get_project_settings,
+    get_asset)
+from openpype.modules import ModulesManager
+
+from avalon.api import Session
 
 
 class CreateRender(plugin.Creator):
@@ -83,6 +90,19 @@ class CreateRender(plugin.Creator):
     def __init__(self, *args, **kwargs):
         """Constructor."""
         super(CreateRender, self).__init__(*args, **kwargs)
+        project_settings = get_project_settings(Session["AVALON_PROJECT"])
+        try:
+            self.deadline_servers = (
+                project_settings["deadline"]
+                                ["deadline_servers"]
+            )
+        except AttributeError:
+            # Handle situation were we had only one url for deadline.
+            manager = ModulesManager()
+            deadline_module = manager.modules_by_name["deadline"]
+            # get default deadline webservice url from deadline module
+            deadline_url = deadline_module.deadline_url
+            self.deadline_servers = {"default": deadline_url}
 
     def process(self):
         """Entry point."""
@@ -94,23 +114,30 @@ def process(self):
         use_selection = self.options.get("useSelection")
         with lib.undo_chunk():
             self._create_render_settings()
-            instance = super(CreateRender, self).process()
+            self.instance = super(CreateRender, self).process()
             # create namespace with instance
             index = 1
-            namespace_name = "_{}".format(str(instance))
+            namespace_name = "_{}".format(str(self.instance))
             try:
                 cmds.namespace(rm=namespace_name)
             except RuntimeError:
                 # namespace is not empty, so we leave it untouched
                 pass
 
             while cmds.namespace(exists=namespace_name):
-                namespace_name = "_{}{}".format(str(instance), index)
+                namespace_name = "_{}{}".format(str(self.instance), index)
                 index += 1
 
             namespace = cmds.namespace(add=namespace_name)
 
-            cmds.setAttr("{}.machineList".format(instance), lock=True)
+            # add Deadline server selection list
+            cmds.scriptJob(
+                attributeChange=[
+                    "{}.deadlineServers".format(self.instance),
+                    self._deadline_webservice_changed
+                ])
+
+            cmds.setAttr("{}.machineList".format(self.instance), lock=True)
             self._rs = renderSetup.instance()
             layers = self._rs.getRenderLayers()
             if use_selection:
@@ -122,7 +149,7 @@ def process(self):
                     render_set = cmds.sets(
                         n="{}:{}".format(namespace, layer.name()))
                     sets.append(render_set)
-                cmds.sets(sets, forceElement=instance)
+                cmds.sets(sets, forceElement=self.instance)
 
             # if no render layers are present, create default one with
             # asterisk selector
@@ -138,16 +165,78 @@ def process(self):
                 renderer = 'renderman'
 
             self._set_default_renderer_settings(renderer)
+        return self.instance
+
+    def _deadline_webservice_changed(self):
+        """Refresh Deadline server dependent options."""
+        # get selected server
+        webservice = self.deadline_servers[
+            self.server_aliases[
+                cmds.getAttr("{}.deadlineServers".format(self.instance))
+            ]
+        ]
+        pools = self._get_deadline_pools(webservice)
+        cmds.deleteAttr("{}.primaryPool".format(self.instance))
+        cmds.deleteAttr("{}.secondaryPool".format(self.instance))
+        cmds.addAttr(self.instance, longName="primaryPool",
+                     attributeType="enum",
+                     enumName=":".join(pools))
+        cmds.addAttr(self.instance, longName="secondaryPool",
+                     attributeType="enum",
+                     enumName=":".join(["-"] + pools))
+
+    def _get_deadline_pools(self, webservice):
+        # type: (str) -> list
+        """Get pools from Deadline.
+        Args:
+            webservice (str): Server url.
+        Returns:
+            list: Pools.
+        Throws:
+            RuntimeError: If deadline webservice is unreachable.
+
+        """
+        argument = "{}/api/pools?NamesOnly=true".format(webservice)
+        try:
+            response = self._requests_get(argument)
+        except requests.exceptions.ConnectionError as exc:
+            msg = 'Cannot connect to deadline web service'
+            self.log.error(msg)
+            six.reraise(exc, RuntimeError('{} - {}'.format(msg, exc)))
+        if not response.ok:
+            self.log.warning("No pools retrieved")
+            return []
+
+        return response.json()
 
     def _create_render_settings(self):
         # get pools
-        pools = []
+        pool_names = []
+
+        self.server_aliases = self.deadline_servers.keys()
+        self.data["deadlineServers"] = self.server_aliases
+        self.data["suspendPublishJob"] = False
+        self.data["review"] = True
+        self.data["extendFrames"] = False
+        self.data["overrideExistingFrame"] = True
+        # self.data["useLegacyRenderLayers"] = True
+        self.data["priority"] = 50
+        self.data["framesPerTask"] = 1
+        self.data["whitelist"] = False
+        self.data["machineList"] = ""
+        self.data["useMayaBatch"] = False
+        self.data["tileRendering"] = False
+        self.data["tilesX"] = 2
+        self.data["tilesY"] = 2
+        self.data["convertToScanline"] = False
+        self.data["useReferencedAovs"] = False
+        # Disable for now as this feature is not working yet
+        # self.data["assScene"] = False
 
         system_settings = get_system_settings()["modules"]
 
         deadline_enabled = system_settings["deadline"]["enabled"]
         muster_enabled = system_settings["muster"]["enabled"]
-        deadline_url = system_settings["deadline"]["DEADLINE_REST_URL"]
         muster_url = system_settings["muster"]["MUSTER_REST_URL"]
 
         if deadline_enabled and muster_enabled:
@@ -157,21 +246,8 @@ def _create_render_settings(self):
             raise RuntimeError("Both Deadline and Muster are enabled")
 
         if deadline_enabled:
-            argument = "{}/api/pools?NamesOnly=true".format(deadline_url)
-            try:
-                response = self._requests_get(argument)
-            except requests.exceptions.ConnectionError as e:
-                msg = 'Cannot connect to deadline web service'
-                self.log.error(msg)
-                raise RuntimeError('{} - {}'.format(msg, e))
-            if not response.ok:
-                self.log.warning("No pools retrieved")
-            else:
-                pools = response.json()
-                self.data["primaryPool"] = pools
-                # We add a string "-" to allow the user to not
-                # set any secondary pools
-                self.data["secondaryPool"] = ["-"] + pools
+            pool_names = self._get_deadline_pools(
+                self.deadline_servers["default"])
 
         if muster_enabled:
             self.log.info(">>> Loading Muster credentials ...")
@@ -187,31 +263,11 @@ def _create_render_settings(self):
             except requests.exceptions.ConnectionError:
                 self.log.error("Cannot connect to Muster API endpoint.")
                 raise RuntimeError("Cannot connect to {}".format(muster_url))
-            pool_names = []
             for pool in pools:
                 self.log.info("  - pool: {}".format(pool["name"]))
                 pool_names.append(pool["name"])
 
-            self.data["primaryPool"] = pool_names
-
-        self.data["suspendPublishJob"] = False
-        self.data["review"] = True
-        self.data["extendFrames"] = False
-        self.data["overrideExistingFrame"] = True
-        # self.data["useLegacyRenderLayers"] = True
-        self.data["priority"] = 50
-        self.data["framesPerTask"] = 1
-        self.data["whitelist"] = False
-        self.data["machineList"] = ""
-        self.data["useMayaBatch"] = False
-        self.data["tileRendering"] = False
-        self.data["tilesX"] = 2
-        self.data["tilesY"] = 2
-        self.data["convertToScanline"] = False
-        self.data["useReferencedAovs"] = False
-        # Disable for now as this feature is not working yet
-        # self.data["assScene"] = False
-
+        self.data["primaryPool"] = pool_names
         self.options = {"useSelection": False}  # Force no content
 
     def _load_credentials(self):

openpype/modules/deadline/deadline_module.py

@@ -10,7 +10,15 @@ def initialize(self, modules_settings):
         # This module is always enabled
         deadline_settings = modules_settings[self.name]
         self.enabled = deadline_settings["enabled"]
-        self.deadline_url = deadline_settings["DEADLINE_REST_URL"]
+        deadline_url = deadline_settings.get("DEADLINE_REST_URL")
+        if not deadline_url:
+            deadline_url = deadline_settings.get("deadline_urls", {}).get("default")  # noqa: E501
+        if not deadline_url:
+            self.enabled = False
+            self.log.warning(("default Deadline Webservice URL "
+                              "not specified. Disabling module."))
+            return
+        self.deadline_url = deadline_url
 
     def get_global_environments(self):
         """Deadline global environments for OpenPype implementation."""

openpype/modules/deadline/plugins/publish/submit_maya_deadline.py

@@ -36,6 +36,7 @@
 import pyblish.api
 
 from openpype.hosts.maya.api import lib
+from openpype.modules import ModulesManager
 
 # Documentation for keys available at:
 # https://docs.thinkboxsoftware.com
@@ -264,12 +265,15 @@ def process(self, instance):
 
         self._instance = instance
         self.payload_skeleton = copy.deepcopy(payload_skeleton_template)
-        self._deadline_url = (
-            context.data["system_settings"]
-            ["modules"]
-            ["deadline"]
-            ["DEADLINE_REST_URL"]
-        )
+
+        manager = ModulesManager()
+        deadline_module = manager.modules_by_name["deadline"]
+        # get default deadline webservice url from deadline module
+        self.deadline_url = deadline_module.deadline_url
+        # if custom one is set in instance, use that
+        if instance.data.get("deadlineUrl"):
+            self.deadline_url = instance.data.get("deadlineUrl")
+        assert self.deadline_url, "Requires Deadline Webservice URL"
 
         self._job_info = (
             context.data["project_settings"].get(

openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py

@@ -4,6 +4,7 @@
 
 from avalon import api
 from avalon.vendor import requests
+from openpype.modules import ModulesManager
 import re
 import pyblish.api
 import nuke
@@ -42,13 +43,14 @@ def process(self, instance):
         node = instance[0]
         context = instance.context
 
-        deadline_url = (
-            context.data["system_settings"]
-            ["modules"]
-            ["deadline"]
-            ["DEADLINE_REST_URL"]
-        )
-        assert deadline_url, "Requires DEADLINE_REST_URL"
+        manager = ModulesManager()
+        deadline_module = manager.modules_by_name["deadline"]
+        # get default deadline webservice url from deadline module
+        self.deadline_url = deadline_module.deadline_url
+        # if custom one is set in instance, use that
+        if instance.data.get("deadlineUrl"):
+            self.deadline_url = instance.data.get("deadlineUrl")
+        assert self.deadline_url, "Requires Deadline Webservice URL"
 
         self.deadline_url = "{}/api/jobs".format(deadline_url)
         self._comment = context.data.get("comment", "")

openpype/modules/deadline/plugins/publish/submit_publish_job.py

@@ -5,7 +5,7 @@
 import json
 import re
 from copy import copy, deepcopy
-import sys
+from openpype.modules import ModulesManager
 import openpype.api
 
 from avalon import api, io
@@ -615,14 +615,16 @@ def _solve_families(self, instance, preview=False):
             instance["families"] = families
 
     def process(self, instance):
+        # type: (pyblish.api.Instance) -> None
         """Process plugin.
 
         Detect type of renderfarm submission and create and post dependend job
         in case of Deadline. It creates json file with metadata needed for
         publishing in directory of render.
 
-        :param instance: Instance data
-        :type instance: dict
+        Args:
+            instance (pyblish.api.Instance): Instance data.
+
         """
         data = instance.data.copy()
         context = instance.context
@@ -908,13 +910,14 @@ def process(self, instance):
             }
 
         if submission_type == "deadline":
-            self.deadline_url = (
-                context.data["system_settings"]
-                ["modules"]
-                ["deadline"]
-                ["DEADLINE_REST_URL"]
-            )
-            assert self.deadline_url, "Requires DEADLINE_REST_URL"
+            manager = ModulesManager()
+            deadline_module = manager.modules_by_name["deadline"]
+            # get default deadline webservice url from deadline module
+            self.deadline_url = deadline_module.deadline_url
+            # if custom one is set in instance, use that
+            if instance.data.get("deadlineUrl"):
+                self.deadline_url = instance.data.get("deadlineUrl")
+            assert self.deadline_url, "Requires Deadline Webservice URL"
 
             self._submit_deadline_post_job(instance, render_job, instances)
 

openpype/modules/deadline/plugins/publish/validate_deadline_connection.py

@@ -1,30 +1,31 @@
 import pyblish.api
 
 from avalon.vendor import requests
-from openpype.plugin import contextplugin_should_run
+from openpype.modules import ModulesManager
 import os
 
 
-class ValidateDeadlineConnection(pyblish.api.ContextPlugin):
+class ValidateDeadlineConnection(pyblish.api.InstancePlugin):
     """Validate Deadline Web Service is running"""
 
     label = "Validate Deadline Web Service"
     order = pyblish.api.ValidatorOrder
     hosts = ["maya", "nuke"]
     families = ["renderlayer"]
 
-    def process(self, context):
-
-        # Workaround bug pyblish-base#250
-        if not contextplugin_should_run(self, context):
-            return
-
-        deadline_url = (
-            context.data["system_settings"]
-            ["modules"]
-            ["deadline"]
-            ["DEADLINE_REST_URL"]
-        )
+    def process(self, instance):
+
+        manager = ModulesManager()
+        deadline_module = manager.modules_by_name["deadline"]
+        # get default deadline webservice url from deadline module
+        deadline_url = deadline_module.deadline_url
+        # if custom one is set in instance, use that
+        if instance.data.get("deadlineUrl"):
+            deadline_url = instance.data.get("deadlineUrl")
+            self.log.info(
+                "We have deadline URL on instance {}".format(
+                    deadline_url))
+        assert deadline_url, "Requires Deadline Webservice URL"
 
         # Check response
         response = self._requests_get(deadline_url)