Nuke testing class

openpype/cli.py

@@ -356,9 +356,22 @@ def run(script):
               "--pyargs",
               help="Run tests from package",
               default=None)
-def runtests(folder, mark, pyargs):
+@click.option("-t",
+              "--test_data_folder",
+              help="Unzipped directory path of test file",
+              default=None)
+@click.option("-s",
+              "--persist",
+              help="Persist test DB and published files after test end",
+              default=None)
+@click.option("-a",
+              "--app_variant",
+              help="Provide specific app variant for test, empty for latest",
+              default=None)
+def runtests(folder, mark, pyargs, test_data_folder, persist, app_variant):
     """Run all automatic tests after proper initialization via start.py"""
-    PypeCommands().run_tests(folder, mark, pyargs)
+    PypeCommands().run_tests(folder, mark, pyargs, test_data_folder,
+                             persist, app_variant)
 
 
 @main.command()

openpype/hosts/nuke/plugins/publish/extract_render_local.py

@@ -42,10 +42,14 @@ def process(self, instance):
         self.log.info("Start frame: {}".format(first_frame))
         self.log.info("End frame: {}".format(last_frame))
 
+        # write node url might contain nuke's ctl expressin
+        # as [python ...]/path...
+        path = node["file"].evaluate()
+
         # Ensure output directory exists.
-        directory = os.path.dirname(node["file"].value())
-        if not os.path.exists(directory):
-            os.makedirs(directory)
+        out_dir = os.path.dirname(path)
+        if not os.path.exists(out_dir):
+            os.makedirs(out_dir)
 
         # Render frames
         nuke.execute(
@@ -58,15 +62,12 @@ def process(self, instance):
         if "slate" in families:
             first_frame += 1
 
-        path = node['file'].value()
-        out_dir = os.path.dirname(path)
         ext = node["file_type"].value()
 
         if "representations" not in instance.data:
             instance.data["representations"] = []
 
         collected_frames = os.listdir(out_dir)
-
         if len(collected_frames) == 1:
             repre = {
                 'name': ext,

openpype/hosts/nuke/plugins/publish/validate_rendered_frames.py

@@ -67,7 +67,9 @@ def process(self, instance):
 
             if not repre.get("files"):
                 msg = ("no frames were collected, "
-                       "you need to render them")
+                       "you need to render them.\n"
+                       "Check properties of write node (group) and"
+                       "select 'Local' option in 'Publish' dropdown.")
                 self.log.error(msg)
                 raise ValidationException(msg)
 

openpype/lib/applications.py

@@ -716,6 +716,8 @@ def __init__(self, application, executable, **data):
         # subprocess.Popen launch arguments (first argument in constructor)
         self.launch_args = executable.as_args()
         self.launch_args.extend(application.arguments)
+        if self.data.get("app_args"):
+            self.launch_args.extend(self.data.pop("app_args"))
 
         # Handle launch environemtns
         env = self.data.pop("env", None)

openpype/lib/path_tools.py

@@ -307,7 +307,6 @@ def _get_local_sync_dirmap(self, project_settings):
         mapping = {}
 
         if not project_settings["global"]["sync_server"]["enabled"]:
-            log.debug("Site Sync not enabled")
             return mapping
 
         from openpype.settings.lib import get_site_local_overrides

openpype/lib/remote_publish.py

@@ -2,6 +2,7 @@
 from datetime import datetime
 import sys
 from bson.objectid import ObjectId
+import collections
 
 import pyblish.util
 import pyblish.api
@@ -140,7 +141,9 @@ def find_variant_key(application_manager, host):
 
     found_variant_key = None
     # finds most up-to-date variant if any installed
-    for variant_key, variant in app_group.variants.items():
+    sorted_variants = collections.OrderedDict(
+        sorted(app_group.variants.items()))
+    for variant_key, variant in sorted_variants.items():
         for executable in variant.executables:
             if executable.exists():
                 found_variant_key = variant_key

openpype/pype_commands.py

@@ -216,6 +216,7 @@ def remotepublishfromapp(project, batch_dir, host_name,
             task_name,
             app_name
         )
+        print("env:: {}".format(env))
         os.environ.update(env)
 
         os.environ["OPENPYPE_PUBLISH_DATA"] = batch_dir
@@ -340,33 +341,48 @@ def run_application(self, app, project, asset, task, tools, arguments):
     def validate_jsons(self):
         pass
 
-    def run_tests(self, folder, mark, pyargs):
+    def run_tests(self, folder, mark, pyargs,
+                  test_data_folder, persist, app_variant):
         """
             Runs tests from 'folder'
 
             Args:
                  folder (str): relative path to folder with tests
                  mark (str): label to run tests marked by it (slow etc)
                  pyargs (str): package path to test
+                 test_data_folder (str): url to unzipped folder of test data
+                 persist (bool): True if keep test db and published after test
+                    end
+                app_variant (str): variant (eg 2020 for AE), empty if use
+                    latest installed version
         """
         print("run_tests")
-        import subprocess
-
         if folder:
             folder = " ".join(list(folder))
         else:
             folder = "../tests"
 
-        mark_str = pyargs_str = ''
+        # disable warnings and show captured stdout even if success
+        args = ["--disable-pytest-warnings", "-rP", folder]
+
         if mark:
-            mark_str = "-m {}".format(mark)
+            args.extend(["-m", mark])
 
         if pyargs:
-            pyargs_str = "--pyargs {}".format(pyargs)
+            args.extend(["--pyargs", pyargs])
+
+        if persist:
+            args.extend(["--test_data_folder", test_data_folder])
+
+        if persist:
+            args.extend(["--persist", persist])
+
+        if app_variant:
+            args.extend(["--app_variant", app_variant])
 
-        cmd = "pytest {} {} {}".format(folder, mark_str, pyargs_str)
-        print("Running {}".format(cmd))
-        subprocess.run(cmd)
+        print("run_tests args: {}".format(args))
+        import pytest
+        pytest.main(args)
 
     def syncserver(self, active_site):
         """Start running sync_server in background."""

openpype/settings/defaults/system_settings/applications.json

@@ -139,7 +139,7 @@
         "icon": "{}/app_icons/nuke.png",
         "host_name": "nuke",
         "environment": {
-            "NUKE_PATH": "{OPENPYPE_STUDIO_PLUGINS}/nuke"
+            "NUKE_PATH": ["{NUKE_PATH}", "{OPENPYPE_STUDIO_PLUGINS}/nuke"]
         },
         "variants": {
             "13-0": {
@@ -245,7 +245,7 @@
         "icon": "{}/app_icons/nuke.png",
         "host_name": "nuke",
         "environment": {
-            "NUKE_PATH": "{OPENPYPE_STUDIO_PLUGINS}/nuke"
+            "NUKE_PATH": ["{NUKE_PATH}", "{OPENPYPE_STUDIO_PLUGINS}/nuke"]
         },
         "variants": {
             "13-0": {

start.py

@@ -925,7 +925,9 @@ def boot():
         sys.exit(1)
 
     os.environ["OPENPYPE_MONGO"] = openpype_mongo
-    os.environ["OPENPYPE_DATABASE_NAME"] = "openpype"  # name of Pype database
+    # name of Pype database
+    os.environ["OPENPYPE_DATABASE_NAME"] = \
+        os.environ.get("OPENPYPE_DATABASE_NAME") or "openpype"
 
     _print(">>> run disk mapping command ...")
     run_disk_mapping_commands(openpype_mongo)

tests/README.md

@@ -14,12 +14,12 @@ How to run:
 ----------
 - single test class could be run by PyCharm and its pytest runner directly
 - OR
-- use Openpype command 'runtests' from command line
--- `${OPENPYPE_ROOT}/start.py runtests`
+- use Openpype command 'runtests' from command line (`.venv` in ${OPENPYPE_ROOT} must be activated to use configured Python!)
+-- `${OPENPYPE_ROOT}/python start.py runtests`
 
 By default, this command will run all tests in ${OPENPYPE_ROOT}/tests.
 
 Specific location could be provided to this command as an argument, either as absolute path, or relative path to ${OPENPYPE_ROOT}.
-(eg. `${OPENPYPE_ROOT}/start.py runtests ../tests/integration`) will trigger only tests in `integration` folder.
+(eg. `${OPENPYPE_ROOT}/python start.py runtests ../tests/integration`) will trigger only tests in `integration` folder.
 
 See `${OPENPYPE_ROOT}/cli.py:runtests` for other arguments.

tests/integration/README.md

@@ -5,33 +5,64 @@ Contains end-to-end tests for automatic testing of OP.
 Should run headless publish on all hosts to check basic publish use cases automatically
 to limit regression issues.
 
+How to run
+----------
+- activate `{OPENPYPE_ROOT}/.venv`
+- run in cmd
+`{OPENPYPE_ROOT}/.venv/Scripts/python.exe {OPENPYPE_ROOT}/start.py runtests {OPENPYPE_ROOT}/tests/integration`
+  - add `hosts/APP_NAME` after integration part to limit only on specific app (eg. `{OPENPYPE_ROOT}/tests/integration/hosts/maya`)
+
+OR can use built executables 
+`openpype_console runtests {ABS_PATH}/tests/integration`
+
+How to check logs/errors from app
+--------------------------------
+Keep PERSIST to True in the class and check `test_openpype.logs` collection. 
+
 How to create test for publishing from host
 ------------------------------------------
-- Extend PublishTest
+- Extend PublishTest in `tests/lib/testing_classes.py`
 - Use `resources\test_data.zip` skeleton file as a template for testing input data
 - Put workfile into `test_data.zip/input/workfile`
 - If you require other than base DB dumps provide them to `test_data.zip/input/dumps`
 -- (Check commented code in `db_handler.py` how to dump specific DB. Currently all collections will be dumped.)
 - Implement `last_workfile_path` 
 - `startup_scripts` - must contain pointing host to startup script saved into `test_data.zip/input/startup`
-  -- Script must contain something like 
+  -- Script must contain something like (pseudocode)
 ```
 import openpype
 from avalon import api, HOST
+
+from openpype.api import Logger
+
+log = Logger().get_logger(__name__)
 
 api.install(HOST)
-pyblish.util.publish()
+log_lines = []
+for result in pyblish.util.publish_iter():
+    for record in result["records"]:  # for logging to test_openpype DB
+        log_lines.append("{}: {}".format(
+            result["plugin"].label, record.msg))
+
+    if result["error"]:
+        err_fmt = "Failed {plugin.__name__}: {error} -- {error.traceback}"
+        log.error(err_fmt.format(**result))
 
 EXIT_APP (command to exit host)
 ```
 (Install and publish methods must be triggered only AFTER host app is fully initialized!)
-- Zip `test_data.zip`, named it with descriptive name, upload it to Google Drive, right click - `Get link`, copy hash id
+- If you would like add any command line arguments for your host app add it to `test_data.zip/input/app_args/app_args.json` (as a json list)
+- Provide any required environment variables to `test_data.zip/input/env_vars/env_vars.json` (as a json dictionary)
+- Zip `test_data.zip`, named it with descriptive name, upload it to Google Drive, right click - `Get link`, copy hash id (file must be accessible to anyone with a link!)
 - Put this hash id and zip file name into TEST_FILES [(HASH_ID, FILE_NAME, MD5_OPTIONAL)]. If you want to check MD5 of downloaded 
 file, provide md5 value of zipped file.
 - Implement any assert checks you need in extended class
 - Run test class manually (via Pycharm or pytest runner (TODO))
-- If you want test to compare expected files to published one, set PERSIST to True, run test manually
+- If you want test to visually compare expected files to published one, set PERSIST to True, run test manually
   -- Locate temporary `publish` subfolder of temporary folder (found in debugging console log)
   -- Copy whole folder content into .zip file into `expected` subfolder 
   -- By default tests are comparing only structure of `expected` and published format (eg. if you want to save space, replace published files with empty files, but with expected names!)
-  -- Zip and upload again, change PERSIST to False
+  -- Zip and upload again, change PERSIST to False
+
+- Use `TEST_DATA_FOLDER` variable in your class to reuse existing downloaded and unzipped test data (for faster creation of tests)
+- Keep `APP_VARIANT` empty if you want to trigger test on latest version of app, or provide explicit value (as '2022' for Photoshop for example)

tests/integration/conftest.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# adds command line arguments for 'runtests' as a fixtures
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--test_data_folder", action="store", default=None,
+        help="Provide url of a folder of unzipped test file"
+    )
+
+    parser.addoption(
+        "--persist", action="store", default=None,
+        help="True - keep test_db, test_openpype, outputted test files"
+    )
+
+    parser.addoption(
+        "--app_variant", action="store", default=None,
+        help="Keep empty to locate latest installed variant or explicit"
+    )
+
+
+@pytest.fixture(scope="module")
+def test_data_folder(request):
+    return request.config.getoption("--test_data_folder")
+
+
+@pytest.fixture(scope="module")
+def persist(request):
+    return request.config.getoption("--persist")
+
+
+@pytest.fixture(scope="module")
+def app_variant(request):
+    return request.config.getoption("--app_variant")

tests/integration/hosts/maya/lib.py

@@ -0,0 +1,41 @@
+import os
+import pytest
+import shutil
+
+from tests.lib.testing_classes import HostFixtures
+
+
+class MayaTestClass(HostFixtures):
+    @pytest.fixture(scope="module")
+    def last_workfile_path(self, download_test_data, output_folder_url):
+        """Get last_workfile_path from source data.
+
+            Maya expects workfile in proper folder, so copy is done first.
+        """
+        src_path = os.path.join(download_test_data,
+                                "input",
+                                "workfile",
+                                "test_project_test_asset_TestTask_v001.mb")
+        dest_folder = os.path.join(output_folder_url,
+                                   self.PROJECT,
+                                   self.ASSET,
+                                   "work",
+                                   self.TASK)
+        os.makedirs(dest_folder)
+        dest_path = os.path.join(dest_folder,
+                                 "test_project_test_asset_TestTask_v001.mb")
+        shutil.copy(src_path, dest_path)
+
+        yield dest_path
+
+    @pytest.fixture(scope="module")
+    def startup_scripts(self, monkeypatch_session, download_test_data):
+        """Points Maya to userSetup file from input data"""
+        startup_path = os.path.join(download_test_data,
+                                    "input",
+                                    "startup")
+        original_pythonpath = os.environ.get("PYTHONPATH")
+        monkeypatch_session.setenv("PYTHONPATH",
+                                   "{}{}{}".format(startup_path,
+                                                   os.pathsep,
+                                                   original_pythonpath))