Add support for multiple Deadline ☠️➖ servers

openpype/hosts/maya/plugins/publish/collect_render.py

@@ -64,6 +64,8 @@ class CollectMayaRender(pyblish.api.ContextPlugin):
     def process(self, context):
         """Entry point to collector."""
         render_instance = None
+        deadline_url = None
+
         for instance in context:
             if "rendering" in instance.data["families"]:
                 render_instance = instance
@@ -86,6 +88,15 @@ def process(self, context):
         asset = api.Session["AVALON_ASSET"]
         workspace = context.data["workspaceDir"]
 
+        deadline_settings = (
+            context.data
+            ["system_settings"]
+            ["modules"]
+            ["deadline"]
+        )
+
+        if deadline_settings["enabled"]:
+            deadline_url = render_instance.data.get("deadlineUrl")
         self._rs = renderSetup.instance()
         current_layer = self._rs.getVisibleRenderLayer()
         maya_render_layers = {
@@ -263,6 +274,9 @@ def process(self, context):
                         "vrayUseReferencedAovs") or False
             }
 
+            if deadline_url:
+                data["deadlineUrl"] = deadline_url
+
             if self.sync_workfile_version:
                 data["version"] = context.data["version"]
 
@@ -392,11 +406,13 @@ def _get_overrides(self, layer):
         rset = self.maya_layers[layer].renderSettingsCollectionInstance()
         return rset.getOverrides()
 
-    def get_render_attribute(self, attr, layer):
+    @staticmethod
+    def get_render_attribute(attr, layer):
         """Get attribute from render options.
 
         Args:
-            attr (str): name of attribute to be looked up.
+            attr (str): name of attribute to be looked up
+            layer (str): name of render layer
 
         Returns:
             Attribute value

openpype/lib/abstract_submit_deadline.py

@@ -415,13 +415,11 @@ def process(self, instance):
         """Plugin entry point."""
         self._instance = instance
         context = instance.context
-        self._deadline_url = (
-            context.data["system_settings"]
-            ["modules"]
-            ["deadline"]
-            ["DEADLINE_REST_URL"]
-        )
-        assert self._deadline_url, "Requires DEADLINE_REST_URL"
+        self._deadline_url = context.data.get("defaultDeadline")
+        self._deadline_url = instance.data.get(
+            "deadlineUrl", self._deadline_url)
+
+        assert self._deadline_url, "Requires Deadline Webservice URL"
 
         file_path = None
         if self.use_published:

openpype/modules/deadline/deadline_module.py

@@ -6,17 +6,25 @@
 class DeadlineModule(PypeModule, IPluginPaths):
     name = "deadline"
 
+    def __init__(self, manager, settings):
+        self.deadline_urls = {}
+        super(DeadlineModule, self).__init__(manager, settings)
+
     def initialize(self, modules_settings):
         # This module is always enabled
         deadline_settings = modules_settings[self.name]
         self.enabled = deadline_settings["enabled"]
-        self.deadline_url = deadline_settings["DEADLINE_REST_URL"]
+        deadline_url = deadline_settings.get("DEADLINE_REST_URL")
+        if deadline_url:
+            self.deadline_urls = {"default": deadline_url}
+        else:
+            self.deadline_urls = deadline_settings.get("deadline_urls")  # noqa: E501
 
-    def get_global_environments(self):
-        """Deadline global environments for OpenPype implementation."""
-        return {
-            "DEADLINE_REST_URL": self.deadline_url
-        }
+        if not self.deadline_urls:
+            self.enabled = False
+            self.log.warning(("default Deadline Webservice URL "
+                              "not specified. Disabling module."))
+            return
 
     def connect_with_modules(self, *_a, **_kw):
         return

openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+"""Collect Deadline servers from instance.
+
+This is resolving index of server lists stored in `deadlineServers` instance
+attribute or using default server if that attribute doesn't exists.
+
+"""
+import pyblish.api
+
+
+class CollectDeadlineServerFromInstance(pyblish.api.InstancePlugin):
+    """Collect Deadline Webservice URL from instance."""
+
+    order = pyblish.api.CollectorOrder
+    label = "Deadline Webservice from the Instance"
+    families = ["rendering"]
+
+    def process(self, instance):
+        instance.data["deadlineUrl"] = self._collect_deadline_url(instance)
+        self.log.info(
+            "Using {} for submission.".format(instance.data["deadlineUrl"]))
+
+    @staticmethod
+    def _collect_deadline_url(render_instance):
+        # type: (pyblish.api.Instance) -> str
+        """Get Deadline Webservice URL from render instance.
+
+        This will get all configured Deadline Webservice URLs and create
+        subset of them based upon project configuration. It will then take
+        `deadlineServers` from render instance that is now basically `int`
+        index of that list.
+
+        Args:
+            render_instance (pyblish.api.Instance): Render instance created
+                by Creator in Maya.
+
+        Returns:
+            str: Selected Deadline Webservice URL.
+
+        """
+
+        deadline_settings = (
+            render_instance.context.data
+            ["system_settings"]
+            ["modules"]
+            ["deadline"]
+        )
+
+        try:
+            default_servers = deadline_settings["deadline_urls"]
+            project_servers = (
+                render_instance.context.data
+                ["project_settings"]
+                ["deadline"]
+                ["deadline_servers"]
+            )
+            deadline_servers = {
+                k: default_servers[k]
+                for k in project_servers
+                if k in default_servers
+            }
+
+        except AttributeError:
+            # Handle situation were we had only one url for deadline.
+            return render_instance.context.data["defaultDeadline"]
+
+        return deadline_servers[
+            list(deadline_servers.keys())[
+                int(render_instance.data.get("deadlineServers"))
+            ]
+        ]

openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+"""Collect default Deadline server."""
+import pyblish.api
+
+
+class CollectDefaultDeadlineServer(pyblish.api.ContextPlugin):
+    """Collect default Deadline Webservice URL."""
+
+    order = pyblish.api.CollectorOrder + 0.01
+    label = "Default Deadline Webservice"
+
+    def process(self, context):
+        try:
+            deadline_module = context.data.get("openPypeModules")["deadline"]
+        except AttributeError:
+            self.log.error("Cannot get OpenPype Deadline module.")
+            raise AssertionError("OpenPype Deadline module not found.")
+
+        # get default deadline webservice url from deadline module
+        self.log.debug(deadline_module.deadline_urls)
+        context.data["defaultDeadline"] = deadline_module.deadline_urls["default"]  # noqa: E501

openpype/modules/deadline/plugins/publish/submit_maya_deadline.py

@@ -264,12 +264,13 @@ def process(self, instance):
 
         self._instance = instance
         self.payload_skeleton = copy.deepcopy(payload_skeleton_template)
-        self._deadline_url = (
-            context.data["system_settings"]
-            ["modules"]
-            ["deadline"]
-            ["DEADLINE_REST_URL"]
-        )
+
+        # get default deadline webservice url from deadline module
+        self.deadline_url = instance.context.data.get("defaultDeadline")
+        # if custom one is set in instance, use that
+        if instance.data.get("deadlineUrl"):
+            self.deadline_url = instance.data.get("deadlineUrl")
+        assert self.deadline_url, "Requires Deadline Webservice URL"
 
         self._job_info = (
             context.data["project_settings"].get(
@@ -287,8 +288,6 @@ def process(self, instance):
                 "pluginInfo", {})
         )
 
-        assert self._deadline_url, "Requires DEADLINE_REST_URL"
-
         context = instance.context
         workspace = context.data["workspaceDir"]
         anatomy = context.data['anatomy']
@@ -670,7 +669,7 @@ def process(self, instance):
             self.log.info(
                 "Submitting tile job(s) [{}] ...".format(len(frame_payloads)))
 
-            url = "{}/api/jobs".format(self._deadline_url)
+            url = "{}/api/jobs".format(self.deadline_url)
             tiles_count = instance.data.get("tilesX") * instance.data.get("tilesY")  # noqa: E501
 
             for tile_job in frame_payloads:
@@ -754,7 +753,7 @@ def process(self, instance):
             self.log.debug(json.dumps(payload, indent=4, sort_keys=True))
 
             # E.g. http://192.168.0.1:8082/api/jobs
-            url = "{}/api/jobs".format(self._deadline_url)
+            url = "{}/api/jobs".format(self.deadline_url)
             response = self._requests_post(url, json=payload)
             if not response.ok:
                 raise Exception(response.text)
@@ -964,7 +963,7 @@ def _submit_export(self, data, format):
             payload = self._get_arnold_export_payload(data)
             self.log.info("Submitting ass export job.")
 
-        url = "{}/api/jobs".format(self._deadline_url)
+        url = "{}/api/jobs".format(self.deadline_url)
         response = self._requests_post(url, json=payload)
         if not response.ok:
             self.log.error("Submition failed!")

openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py

@@ -42,13 +42,12 @@ def process(self, instance):
         node = instance[0]
         context = instance.context
 
-        deadline_url = (
-            context.data["system_settings"]
-            ["modules"]
-            ["deadline"]
-            ["DEADLINE_REST_URL"]
-        )
-        assert deadline_url, "Requires DEADLINE_REST_URL"
+        # get default deadline webservice url from deadline module
+        deadline_url = instance.context.data["defaultDeadline"]
+        # if custom one is set in instance, use that
+        if instance.data.get("deadlineUrl"):
+            deadline_url = instance.data.get("deadlineUrl")
+        assert deadline_url, "Requires Deadline Webservice URL"
 
         self.deadline_url = "{}/api/jobs".format(deadline_url)
         self._comment = context.data.get("comment", "")

openpype/modules/deadline/plugins/publish/submit_publish_job.py

@@ -5,7 +5,6 @@
 import json
 import re
 from copy import copy, deepcopy
-import sys
 import openpype.api
 
 from avalon import api, io
@@ -615,14 +614,16 @@ def _solve_families(self, instance, preview=False):
             instance["families"] = families
 
     def process(self, instance):
+        # type: (pyblish.api.Instance) -> None
         """Process plugin.
 
         Detect type of renderfarm submission and create and post dependend job
         in case of Deadline. It creates json file with metadata needed for
         publishing in directory of render.
 
-        :param instance: Instance data
-        :type instance: dict
+        Args:
+            instance (pyblish.api.Instance): Instance data.
+
         """
         data = instance.data.copy()
         context = instance.context
@@ -908,13 +909,12 @@ def process(self, instance):
             }
 
         if submission_type == "deadline":
-            self.deadline_url = (
-                context.data["system_settings"]
-                ["modules"]
-                ["deadline"]
-                ["DEADLINE_REST_URL"]
-            )
-            assert self.deadline_url, "Requires DEADLINE_REST_URL"
+            # get default deadline webservice url from deadline module
+            self.deadline_url = instance.context.data["defaultDeadline"]
+            # if custom one is set in instance, use that
+            if instance.data.get("deadlineUrl"):
+                self.deadline_url = instance.data.get("deadlineUrl")
+            assert self.deadline_url, "Requires Deadline Webservice URL"
 
             self._submit_deadline_post_job(instance, render_job, instances)
 

openpype/modules/deadline/plugins/publish/validate_deadline_connection.py

@@ -1,30 +1,27 @@
 import pyblish.api
 
 from avalon.vendor import requests
-from openpype.plugin import contextplugin_should_run
 import os
 
 
-class ValidateDeadlineConnection(pyblish.api.ContextPlugin):
+class ValidateDeadlineConnection(pyblish.api.InstancePlugin):
     """Validate Deadline Web Service is running"""
 
     label = "Validate Deadline Web Service"
     order = pyblish.api.ValidatorOrder
     hosts = ["maya", "nuke"]
     families = ["renderlayer"]
 
-    def process(self, context):
-
-        # Workaround bug pyblish-base#250
-        if not contextplugin_should_run(self, context):
-            return
-
-        deadline_url = (
-            context.data["system_settings"]
-            ["modules"]
-            ["deadline"]
-            ["DEADLINE_REST_URL"]
-        )
+    def process(self, instance):
+        # get default deadline webservice url from deadline module
+        deadline_url = instance.context.data["defaultDeadline"]
+        # if custom one is set in instance, use that
+        if instance.data.get("deadlineUrl"):
+            deadline_url = instance.data.get("deadlineUrl")
+            self.log.info(
+                "We have deadline URL on instance {}".format(
+                    deadline_url))
+        assert deadline_url, "Requires Deadline Webservice URL"
 
         # Check response
         response = self._requests_get(deadline_url)

openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py

@@ -4,7 +4,6 @@
 
 from avalon.vendor import requests
 
-from openpype.api import get_system_settings
 from openpype.lib.abstract_submit_deadline import requests_get
 from openpype.lib.delivery import collect_frames
 
@@ -22,6 +21,7 @@ class ValidateExpectedFiles(pyblish.api.InstancePlugin):
     allow_user_override = True
 
     def process(self, instance):
+        self.instance = instance
         frame_list = self._get_frame_list(instance.data["render_job_id"])
 
         for repre in instance.data["representations"]:
@@ -129,13 +129,12 @@ def _get_job_info(self, job_id):
             Might be different than job info saved in metadata.json if user
             manually changes job pre/during rendering.
         """
-        deadline_url = (
-            get_system_settings()
-            ["modules"]
-            ["deadline"]
-            ["DEADLINE_REST_URL"]
-        )
-        assert deadline_url, "Requires DEADLINE_REST_URL"
+        # get default deadline webservice url from deadline module
+        deadline_url = self.instance.context.data["defaultDeadline"]
+        # if custom one is set in instance, use that
+        if self.instance.data.get("deadlineUrl"):
+            deadline_url = self.instance.data.get("deadlineUrl")
+        assert deadline_url, "Requires Deadline Webservice URL"
 
         url = "{}/api/jobs?JobID={}".format(deadline_url, job_id)
         try: