Add support for multiple Deadline ☠️➖ servers

openpype/hosts/maya/plugins/create/create_render.py

@@ -4,6 +4,8 @@
 import json
 import appdirs
 import requests
+import six
+import sys
 
 from maya import cmds
 import maya.app.renderSetup.model.renderSetup as renderSetup
@@ -12,7 +14,13 @@
     lib,
     plugin
 )
-from openpype.api import (get_system_settings, get_asset)
+from openpype.api import (
+    get_system_settings,
+    get_project_settings,
+    get_asset)
+from openpype.modules import ModulesManager
+
+from avalon.api import Session
 
 
 class CreateRender(plugin.Creator):
@@ -83,6 +91,27 @@ class CreateRender(plugin.Creator):
     def __init__(self, *args, **kwargs):
         """Constructor."""
         super(CreateRender, self).__init__(*args, **kwargs)
+        deadline_settings = get_system_settings()["modules"]["deadline"]
+        if not deadline_settings["enabled"]:
+            self.deadline_servers = {}
+            return
+        project_settings = get_project_settings(Session["AVALON_PROJECT"])
+        try:
+            default_servers = deadline_settings["deadline_urls"]
+            project_servers = (
+                project_settings["deadline"]
+                                ["deadline_servers"]
+            )
+            self.deadline_servers = dict(
+                (k, default_servers[k])
+                for k in project_servers if k in default_servers)
+        except AttributeError:
+            # Handle situation were we had only one url for deadline.
+            manager = ModulesManager()
+            deadline_module = manager.modules_by_name["deadline"]
+            # get default deadline webservice url from deadline module
+            deadline_url = deadline_module.deadline_url
+            self.deadline_servers = {"default": deadline_url}
 
     def process(self):
         """Entry point."""
@@ -94,23 +123,31 @@ def process(self):
         use_selection = self.options.get("useSelection")
         with lib.undo_chunk():
             self._create_render_settings()
-            instance = super(CreateRender, self).process()
+            self.instance = super(CreateRender, self).process()
             # create namespace with instance
             index = 1
-            namespace_name = "_{}".format(str(instance))
+            namespace_name = "_{}".format(str(self.instance))
             try:
                 cmds.namespace(rm=namespace_name)
             except RuntimeError:
                 # namespace is not empty, so we leave it untouched
                 pass
 
             while cmds.namespace(exists=namespace_name):
-                namespace_name = "_{}{}".format(str(instance), index)
+                namespace_name = "_{}{}".format(str(self.instance), index)
                 index += 1
 
             namespace = cmds.namespace(add=namespace_name)
 
-            cmds.setAttr("{}.machineList".format(instance), lock=True)
+            # add Deadline server selection list
+            if self.deadline_servers:
+                cmds.scriptJob(
+                    attributeChange=[
+                        "{}.deadlineServers".format(self.instance),
+                        self._deadline_webservice_changed
+                    ])
+
+            cmds.setAttr("{}.machineList".format(self.instance), lock=True)
             self._rs = renderSetup.instance()
             layers = self._rs.getRenderLayers()
             if use_selection:
@@ -122,7 +159,7 @@ def process(self):
                     render_set = cmds.sets(
                         n="{}:{}".format(namespace, layer.name()))
                     sets.append(render_set)
-                cmds.sets(sets, forceElement=instance)
+                cmds.sets(sets, forceElement=self.instance)
 
             # if no render layers are present, create default one with
             # asterisk selector
@@ -138,16 +175,82 @@ def process(self):
                 renderer = 'renderman'
 
             self._set_default_renderer_settings(renderer)
+        return self.instance
+
+    def _deadline_webservice_changed(self):
+        """Refresh Deadline server dependent options."""
+        # get selected server
+        from maya import cmds
+        webservice = self.deadline_servers[
+            self.server_aliases[
+                cmds.getAttr("{}.deadlineServers".format(self.instance))
+            ]
+        ]
+        pools = self._get_deadline_pools(webservice)
+        cmds.deleteAttr("{}.primaryPool".format(self.instance))
+        cmds.deleteAttr("{}.secondaryPool".format(self.instance))
+        cmds.addAttr(self.instance, longName="primaryPool",
+                     attributeType="enum",
+                     enumName=":".join(pools))
+        cmds.addAttr(self.instance, longName="secondaryPool",
+                     attributeType="enum",
+                     enumName=":".join(["-"] + pools))
+
+    def _get_deadline_pools(self, webservice):
+        # type: (str) -> list
+        """Get pools from Deadline.
+        Args:
+            webservice (str): Server url.
+        Returns:
+            list: Pools.
+        Throws:
+            RuntimeError: If deadline webservice is unreachable.
+
+        """
+        argument = "{}/api/pools?NamesOnly=true".format(webservice)
+        try:
+            response = self._requests_get(argument)
+        except requests.exceptions.ConnectionError as exc:
+            msg = 'Cannot connect to deadline web service'
+            self.log.error(msg)
+            six.reraise(
+                RuntimeError,
+                RuntimeError('{} - {}'.format(msg, exc)),
+                sys.exc_info()[2])
+        if not response.ok:
+            self.log.warning("No pools retrieved")
+            return []
+
+        return response.json()
 
     def _create_render_settings(self):
         # get pools
-        pools = []
+        pool_names = []
+
+        self.server_aliases = self.deadline_servers.keys()
+        self.data["deadlineServers"] = self.server_aliases
+        self.data["suspendPublishJob"] = False
+        self.data["review"] = True
+        self.data["extendFrames"] = False
+        self.data["overrideExistingFrame"] = True
+        # self.data["useLegacyRenderLayers"] = True
+        self.data["priority"] = 50
+        self.data["framesPerTask"] = 1
+        self.data["whitelist"] = False
+        self.data["machineList"] = ""
+        self.data["useMayaBatch"] = False
+        self.data["tileRendering"] = False
+        self.data["tilesX"] = 2
+        self.data["tilesY"] = 2
+        self.data["convertToScanline"] = False
+        self.data["useReferencedAovs"] = False
+        # Disable for now as this feature is not working yet
+        # self.data["assScene"] = False
 
         system_settings = get_system_settings()["modules"]
 
         deadline_enabled = system_settings["deadline"]["enabled"]
         muster_enabled = system_settings["muster"]["enabled"]
-        deadline_url = system_settings["deadline"]["DEADLINE_REST_URL"]
         muster_url = system_settings["muster"]["MUSTER_REST_URL"]
 
         if deadline_enabled and muster_enabled:
@@ -157,21 +260,8 @@ def _create_render_settings(self):
             raise RuntimeError("Both Deadline and Muster are enabled")
 
         if deadline_enabled:
-            argument = "{}/api/pools?NamesOnly=true".format(deadline_url)
-            try:
-                response = self._requests_get(argument)
-            except requests.exceptions.ConnectionError as e:
-                msg = 'Cannot connect to deadline web service'
-                self.log.error(msg)
-                raise RuntimeError('{} - {}'.format(msg, e))
-            if not response.ok:
-                self.log.warning("No pools retrieved")
-            else:
-                pools = response.json()
-                self.data["primaryPool"] = pools
-                # We add a string "-" to allow the user to not
-                # set any secondary pools
-                self.data["secondaryPool"] = ["-"] + pools
+            pool_names = self._get_deadline_pools(
+                self.deadline_servers["default"])
 
         if muster_enabled:
             self.log.info(">>> Loading Muster credentials ...")
@@ -187,31 +277,14 @@ def _create_render_settings(self):
             except requests.exceptions.ConnectionError:
                 self.log.error("Cannot connect to Muster API endpoint.")
                 raise RuntimeError("Cannot connect to {}".format(muster_url))
-            pool_names = []
             for pool in pools:
                 self.log.info("  - pool: {}".format(pool["name"]))
                 pool_names.append(pool["name"])
 
-            self.data["primaryPool"] = pool_names
-
-        self.data["suspendPublishJob"] = False
-        self.data["review"] = True
-        self.data["extendFrames"] = False
-        self.data["overrideExistingFrame"] = True
-        # self.data["useLegacyRenderLayers"] = True
-        self.data["priority"] = 50
-        self.data["framesPerTask"] = 1
-        self.data["whitelist"] = False
-        self.data["machineList"] = ""
-        self.data["useMayaBatch"] = False
-        self.data["tileRendering"] = False
-        self.data["tilesX"] = 2
-        self.data["tilesY"] = 2
-        self.data["convertToScanline"] = False
-        self.data["useReferencedAovs"] = False
-        # Disable for now as this feature is not working yet
-        # self.data["assScene"] = False
-
+        self.data["primaryPool"] = pool_names
+        # We add a string "-" to allow the user to not
+        # set any secondary pools
+        self.data["secondaryPool"] = ["-"] + pool_names
         self.options = {"useSelection": False}  # Force no content
 
     def _load_credentials(self):
@@ -332,6 +405,7 @@ def _set_default_renderer_settings(self, renderer):
 
         if renderer == "arnold":
             # set format to exr
+
             cmds.setAttr(
                 "defaultArnoldDriver.ai_translator", "exr", type="string")
             # enable animation

openpype/hosts/maya/plugins/publish/collect_render.py

@@ -51,6 +51,12 @@
 from avalon import maya, api
 from openpype.hosts.maya.api.expected_files import ExpectedFiles
 from openpype.hosts.maya.api import lib
+from openpype.api import (
+    get_system_settings,
+    get_project_settings
+)
+
+from avalon.api import Session
 
 
 class CollectMayaRender(pyblish.api.ContextPlugin):
@@ -86,6 +92,10 @@ def process(self, context):
         asset = api.Session["AVALON_ASSET"]
         workspace = context.data["workspaceDir"]
 
+        deadline_settings = get_system_settings()["modules"]["deadline"]
+
+        if deadline_settings["enabled"]:
+            deadline_url = self._collect_deadline_url(render_instance)
         self._rs = renderSetup.instance()
         current_layer = self._rs.getVisibleRenderLayer()
         maya_render_layers = {
@@ -263,6 +273,9 @@ def process(self, context):
                         "vrayUseReferencedAovs") or False
             }
 
+            if deadline_url:
+                data["deadlineUrl"] = deadline_url
+
             if self.sync_workfile_version:
                 data["version"] = context.data["version"]
 
@@ -392,11 +405,13 @@ def _get_overrides(self, layer):
         rset = self.maya_layers[layer].renderSettingsCollectionInstance()
         return rset.getOverrides()
 
-    def get_render_attribute(self, attr, layer):
+    @staticmethod
+    def get_render_attribute(attr, layer):
         """Get attribute from render options.
 
         Args:
-            attr (str): name of attribute to be looked up.
+            attr (str): name of attribute to be looked up
+            layer (str): name of render layer
 
         Returns:
             Attribute value
@@ -405,3 +420,45 @@ def get_render_attribute(self, attr, layer):
         return lib.get_attr_in_layer(
             "defaultRenderGlobals.{}".format(attr), layer=layer
         )
+
+    @staticmethod
+    def _collect_deadline_url(render_instance):
+        # type: (pyblish.api.Instance) -> str
+        """Get Deadline Webservice URL from render instance.
+
+        This will get all configured Deadline Webservice URLs and create
+        subset of them based upon project configuration. It will then take
+        `deadlineServers` from render instance that is now basically `int`
+        index of that list.
+
+        Args:
+            render_instance (pyblish.api.Instance): Render instance created
+                by Creator in Maya.
+
+        Returns:
+            str: Selected Deadline Webservice URL.
+
+        """
+
+        deadline_settings = get_system_settings()["modules"]["deadline"]
+        project_settings = get_project_settings(Session["AVALON_PROJECT"])
+        try:
+            default_servers = deadline_settings["deadline_urls"]
+            project_servers = (
+                project_settings["deadline"]
+                ["deadline_servers"]
+            )
+            deadline_servers = dict(
+                (k, default_servers[k])
+                for k in project_servers if k in default_servers)
+        except AttributeError:
+            # Handle situation were we had only one url for deadline.
+            deadline_url = render_instance.context.data["defaultDeadline"]
+            deadline_servers = {"default": deadline_url}
+
+        deadline_url = deadline_servers[
+            list(deadline_servers.keys())[
+                int(render_instance.data.get("deadlineServers"))
+            ]
+        ]
+        return deadline_url

openpype/lib/abstract_submit_deadline.py

@@ -415,13 +415,11 @@ def process(self, instance):
         """Plugin entry point."""
         self._instance = instance
         context = instance.context
-        self._deadline_url = (
-            context.data["system_settings"]
-            ["modules"]
-            ["deadline"]
-            ["DEADLINE_REST_URL"]
-        )
-        assert self._deadline_url, "Requires DEADLINE_REST_URL"
+        self._deadline_url = context.data.get("defaultDeadline")
+        self._deadline_url = instance.data.get(
+            "deadlineUrl", self._deadline_url)
+
+        assert self._deadline_url, "Requires Deadline Webservice URL"
 
         file_path = None
         if self.use_published:

openpype/modules/deadline/deadline_module.py

@@ -10,7 +10,15 @@ def initialize(self, modules_settings):
         # This module is always enabled
         deadline_settings = modules_settings[self.name]
         self.enabled = deadline_settings["enabled"]
-        self.deadline_url = deadline_settings["DEADLINE_REST_URL"]
+        deadline_url = deadline_settings.get("DEADLINE_REST_URL")
+        if not deadline_url:
+            deadline_url = deadline_settings.get("deadline_urls", {}).get("default")  # noqa: E501
+        if not deadline_url:
+            self.enabled = False
+            self.log.warning(("default Deadline Webservice URL "
+                              "not specified. Disabling module."))
+            return
+        self.deadline_url = deadline_url
 
     def get_global_environments(self):
         """Deadline global environments for OpenPype implementation."""

openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+"""Collect default Deadline server."""
+from openpype.modules import ModulesManager
+import pyblish.api
+
+
+class CollectDefaultDeadlineServer(pyblish.api.ContextPlugin):
+    """Collect default Deadline Webservice URL."""
+
+    order = pyblish.api.CollectorOrder + 0.01
+    label = "Default Deadline Webservice"
+
+    def process(self, context):
+        manager = ModulesManager()
+        deadline_module = manager.modules_by_name["deadline"]
+        # get default deadline webservice url from deadline module
+        context.data["defaultDeadline"] = deadline_module.deadline_url