Unity 2020.3.25 w/ CloudRendering Engine

.gitignore

@@ -14,10 +14,6 @@ unity/.vs/
 unity/.idea/
 unity/Assets/[Pp]rivate/
 unity/Assets/[Pp]rivate.meta
-# files in this directory are generated by 
-# the MessagePackGenerator (mpc)
-# invoke generate-msgpack-resolve
-unity/Assets/Scripts/ThorMsgPackResolver*
 unity/Assets/InitTestScene*
 unity/utf_testResults-*
 
@@ -32,6 +28,7 @@ ai2thor/tests/test_utf.py
 .config/
 
 build.log
+tmp/
 
 
 # General

ai2thor/build.py

@@ -9,7 +9,7 @@
 from ai2thor.util.lock import LockSh, LockEx
 from ai2thor.util import atomic_write
 import io
-from ai2thor.platform import STR_PLATFORM_MAP, OSXIntel64, Linux64
+from ai2thor.platform import STR_PLATFORM_MAP, OSXIntel64, Linux64, CloudRendering
 import platform
 
 logger = logging.getLogger(__name__)
@@ -20,7 +20,7 @@
 PYPI_S3_BUCKET = "ai2-thor-pypi"
 
 LOCAL_BUILD_COMMIT_ID = "local"
-AUTO_BUILD_PLATFORMS = [OSXIntel64, Linux64]
+AUTO_BUILD_PLATFORMS = [OSXIntel64, Linux64, CloudRendering]
 
 COMMIT_ID = None
 try:
@@ -30,9 +30,6 @@
 except ImportError:
     pass
 
-# Since CloudRendering is manually built, this will act as a fallback
-# until the process is automated
-DEFAULT_CLOUDRENDERING_COMMIT_ID = "22d62af5da45708e7bc40bf07a861e7397f9d2f9"
 
 def build_name(arch, commit_id, include_private_scenes=False):
     if include_private_scenes:

ai2thor/controller.py

@@ -1161,13 +1161,6 @@ def find_build(self, local_build, commit_id, branch, platform):
         if platform is None:
             candidate_platforms = ai2thor.platform.select_platforms(request)
         else:
-
-            # if the commit is using the default build commit, and 
-            # the platform target is CloudRendering we default to a known built
-            # commit until the build can be automated
-            if commit_id and commit_id == ai2thor.build.COMMIT_ID and platform == ai2thor.platform.CloudRendering:
-                commits.append(ai2thor.build.DEFAULT_CLOUDRENDERING_COMMIT_ID)
-
             candidate_platforms = [platform]
 
         builds = self.find_platform_builds(candidate_platforms, request, commits, releases_dir, local_build)

tasks.py

@@ -41,6 +41,9 @@ def add_files(zipf, start_dir, exclude_ext=()):
                 continue
 
             arcname = os.path.relpath(fn, start_dir)
+            if arcname.split("/")[0].endswith("_BackUpThisFolder_ButDontShipItWithYourGame"):
+                # print("skipping %s" % arcname)
+                continue
             # print("adding %s" % arcname)
             zipf.write(fn, arcname)
 
@@ -128,9 +131,14 @@ def _build(unity_path, arch, build_dir, build_name, env={}):
 
     project_path = os.path.join(os.getcwd(), unity_path)
 
+    # osxintel64 is not a BuildTarget
+    build_target_map = dict(OSXIntel64="OSXUniversal")
+
+    # -buildTarget must be passed as an option for the CloudRendering target otherwise a clang error
+    # will get thrown complaining about missing features.h
     command = (
-        "%s -quit -batchmode -logFile %s/%s.log -projectpath %s -executeMethod Build.%s"
-        % (_unity_path(), os.getcwd(), build_name, project_path, arch)
+        "%s -quit -batchmode -logFile %s/%s.log -projectpath %s -buildTarget %s -executeMethod Build.%s"
+        % (_unity_path(), os.getcwd(), build_name, project_path, build_target_map.get(arch, arch), arch)
     )
 
     target_path = os.path.join(build_dir, build_name)
@@ -420,17 +428,6 @@ def local_build(
     generate_quality_settings(context)
 
 
-def fix_webgl_unity_loader_regex(unity_loader_path):
-    # Bug in the UnityLoader.js causes Chrome on Big Sur to fail to load
-    # https://issuetracker.unity3d.com/issues/unity-webgl-builds-do-not-run-on-macos-big-sur
-    with open(unity_loader_path) as f:
-        loader = f.read()
-
-    loader = loader.replace("Mac OS X (10[\.\_\d]+)", "Mac OS X (1[\.\_\d][\.\_\d]+)")
-    with open(unity_loader_path, "w") as f:
-        f.write(loader)
-
-
 @task
 def webgl_build(
     context,
@@ -509,7 +506,6 @@ def file_to_content_addressable(file_path, json_metadata_file_path, json_key):
         print("Build Failure")
 
     build_path = _webgl_local_build_path(prefix, directory)
-    fix_webgl_unity_loader_regex(os.path.join(build_path, "Build/UnityLoader.js"))
     generate_quality_settings(context)
 
     # the remainder of this is only used to generate scene metadata, but it
@@ -548,9 +544,10 @@ def file_to_content_addressable(file_path, json_metadata_file_path, json_key):
         print(scene_metadata)
 
     to_content_addressable = [
-        ("{}.data.unityweb".format(build_name), "dataUrl"),
-        ("{}.wasm.code.unityweb".format(build_name), "wasmCodeUrl"),
-        ("{}.wasm.framework.unityweb".format(build_name), "wasmFrameworkUrl"),
+        ("{}.data".format(build_name), "dataUrl"),
+        ("{}.loader.js".format(build_name), "loaderUrl"),
+        ("{}.wasm".format(build_name), "wasmCodeUrl"),
+        ("{}.framework.js".format(build_name), "wasmFrameworkUrl"),
     ]
     for file_name, key in to_content_addressable:
         file_to_content_addressable(
@@ -1127,6 +1124,28 @@ def ci_build(context):
     lock_f.close()
 
 
+@task
+def install_cloudrendering_engine(context, force=False):
+    if not sys.platform.startswith("darwin"):
+        raise Exception("CloudRendering Engine can only be installed on Mac")
+    s3 = boto3.resource("s3")
+    target_base_dir = "/Applications/Unity/Hub/Editor/{}/PlaybackEngines".format(_unity_version())
+    full_dir = os.path.join(target_base_dir, "CloudRendering")
+    if os.path.isdir(full_dir):
+        if force:
+            shutil.rmtree(full_dir)
+            logger.info("CloudRendering engine already installed - removing due to force")
+        else:
+            logger.info("skipping installation - CloudRendering engine already installed")
+            return
+
+    print("packages/CloudRendering-%s.zip" % _unity_version())
+    res = s3.Object(ai2thor.build.PRIVATE_S3_BUCKET, "packages/CloudRendering-%s.zip" % _unity_version()).get()
+    data = res["Body"].read()
+    z = zipfile.ZipFile(io.BytesIO(data))
+    z.extractall(target_base_dir)
+
+
 @task
 def ci_build_webgl(context, commit_id):
     branch = "main"
@@ -3320,28 +3339,6 @@ def act(controller, actions, n):
             )
         )
 
-
-@task
-def generate_msgpack_resolver(task):
-    import glob
-
-    # mpc can be downloaded from: https://github.com/neuecc/MessagePack-CSharp/releases/download/v2.1.194/mpc.zip
-    # need to download/unzip into this path, add gatekeeper permission
-    target_dir = "unity/Assets/Scripts/ThorMsgPackResolver"
-    shutil.rmtree(target_dir, ignore_errors=True)
-    mpc_path = os.path.join(os.environ["HOME"], "local/bin/mpc")
-    subprocess.check_call(
-        "%s -i unity -o %s -m -r ThorIL2CPPGeneratedResolver" % (mpc_path, target_dir),
-        shell=True,
-    )
-    for g in glob.glob(os.path.join(target_dir, "*.cs")):
-        with open(g) as f:
-            source_code = f.read()
-            source_code = "using UnityEngine;\n" + source_code
-        with open(g, "w") as f:
-            f.write(source_code)
-
-
 @task
 def generate_pypi_index(context):
     s3 = boto3.resource("s3")

unity/Assets/Editor/Build.cs

@@ -4,43 +4,63 @@
 using UnityEngine;
 using System.Collections.Generic;
 using System.Linq;
+using UnityEditor.Rendering;
+using UnityEngine.Rendering;
+using UnityEditor.Build.Reporting;
 
 public class Build {
+
+    // Since CloudRendering uses a different version of Unity (2020.2) vs production (2019.4), GraphicsSettings and ProjectSettings
+    // must be copied over from the Standalone platform.  As well, continuing to use this ensures that settings made for 
+    // the Standalone platform get used for CloudRendering
+    static void InitializeCloudRendering() {
+        PlayerSettings.SetApiCompatibilityLevel(BuildTargetGroup.CloudRendering, PlayerSettings.GetApiCompatibilityLevel(BuildTargetGroup.Standalone));
+        var graphicsTiers = new List<GraphicsTier>(){GraphicsTier.Tier1, GraphicsTier.Tier2, GraphicsTier.Tier3};
+        foreach (var graphicsTier in graphicsTiers) {
+            EditorGraphicsSettings.SetTierSettings(
+                BuildTargetGroup.CloudRendering,
+                graphicsTier,
+                EditorGraphicsSettings.GetTierSettings(BuildTargetGroup.Standalone, graphicsTier)
+            );
+        }
+    }
+
     static void OSXIntel64() {
-#if UNITY_2017_3_OR_NEWER
-        var buildTarget = BuildTarget.StandaloneOSX;
-#else
-		var buildTarget = BuildTarget.StandaloneOSXIntel64;
-#endif
-        build(GetBuildName(), buildTarget);
+        build(GetBuildName(),  BuildTargetGroup.Standalone, BuildTarget.StandaloneOSX);
     }
 
     static string GetBuildName() {
         return Environment.GetEnvironmentVariable("UNITY_BUILD_NAME");
     }
 
     static void Linux64() {
-        build(GetBuildName(), BuildTarget.StandaloneLinux64);
+        build(GetBuildName(), BuildTargetGroup.Standalone, BuildTarget.StandaloneLinux64);
+    }
+
+    static void CloudRendering() {
+        InitializeCloudRendering();
+        build(GetBuildName(), BuildTargetGroup.CloudRendering, BuildTarget.CloudRendering);
     }
 
     static void WebGL() {
-        build(GetBuildName(), BuildTarget.WebGL);
+        // USIM_USE_... scripting defines are required
+        // to disable the native PNG and JPEG encoders present in the simulation capture package
+        // if these defines are not provide the WebGL build will fail
+        build(GetBuildName(), BuildTargetGroup.WebGL, BuildTarget.WebGL, new string[]{"USIM_USE_BUILTIN_JPG_ENCODER", "USIM_USE_BUILTIN_PNG_ENCODER"});
     }
 
     static void buildResourceAssetJson() {
         ResourceAssetManager manager = new ResourceAssetManager();
         manager.BuildCatalog();
     }
 
-    static void build(string buildName, BuildTarget target) {
+    static void build(string buildName, BuildTargetGroup targetGroup, BuildTarget target, string[] extraScriptingDefines=null) {
         buildResourceAssetJson();
 
         var defines = GetDefineSymbolsFromEnv();
         if (defines != "") {
-            var targetGroup = BuildPipeline.GetBuildTargetGroup(target);
             PlayerSettings.SetScriptingDefineSymbolsForGroup(targetGroup, GetDefineSymbolsFromEnv());
         }
-
         List<string> scenes = GetScenes();
         foreach (string scene in scenes) {
             Debug.Log("Adding Scene " + scene);
@@ -54,10 +74,19 @@ static void build(string buildName, BuildTarget target) {
         if (ScriptsOnly()) {
             options |= BuildOptions.Development | BuildOptions.BuildScriptsOnly;
         }
-
         Debug.Log("Build options " + options);
+        BuildPlayerOptions buildPlayerOptions = new BuildPlayerOptions();
+        if (extraScriptingDefines != null) {
+            buildPlayerOptions.extraScriptingDefines = extraScriptingDefines;
+        }
+        buildPlayerOptions.scenes = scenes.ToArray();
+        buildPlayerOptions.locationPathName = buildName;
+        buildPlayerOptions.target = target;
+        buildPlayerOptions.options = options;
+        EditorUserBuildSettings.SwitchActiveBuildTarget(targetGroup, target);
+        BuildReport report = BuildPipeline.BuildPlayer(buildPlayerOptions);
+        BuildSummary summary = report.summary;
 
-        BuildPipeline.BuildPlayer(scenes.ToArray(), buildName, target, options);
     }
 
     private static List<string> GetScenes() {

unity/Assets/MagicMirror/Prefab/MirrorCameraScript.cs

@@ -1,7 +1,6 @@
 using UnityEngine;
 using System.Collections;
 using System.Collections.Generic;
-using UnityEngine.VR;
 
 public class MirrorCameraScript : MonoBehaviour
 {

unity/Assets/MessagePack/MessagePackSerializer.NonGeneric.cs

@@ -1,4 +1,8 @@
-// Copyright (c) All contributors. All rights reserved.
+#if PLATFORM_CLOUD_RENDERING
+#undef ENABLE_IL2CPP
+#endif
+// CloudRendering does not set ENABLE_IL2CPP correctly when using Mono
+// Copyright (c) All contributors. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;

unity/Assets/MessagePack/Resolvers/StandardResolver.cs

@@ -1,4 +1,8 @@
-// Copyright (c) All contributors. All rights reserved.
+#if PLATFORM_CLOUD_RENDERING
+#undef ENABLE_IL2CPP
+#endif
+// CloudRendering does not set ENABLE_IL2CPP correctly when using Mono
+// Copyright (c) All contributors. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System.Linq;

unity/Assets/MessagePack/Resolvers/TypelessContractlessStandardResolver.cs

@@ -1,4 +1,9 @@
-// Copyright (c) All contributors. All rights reserved.
+#if PLATFORM_CLOUD_RENDERING
+#undef ENABLE_IL2CPP
+#endif
+// CloudRendering does not set ENABLE_IL2CPP correctly when using Mono
+
+// Copyright (c) All contributors. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 #if !UNITY_2018_3_OR_NEWER

unity/Assets/Scripts/AI2-THOR-Base.asmdef

@@ -1,5 +1,6 @@
 {
     "name": "AI2-THOR-Base",
+    "rootNamespace": "",
     "references": [
         "GUID:bbde805d927e795439d7a746d1749cab",
         "GUID:5a46ad82eac024e4fbb6a0516a5def17",
@@ -9,6 +10,8 @@
         "GUID:cc5461233835e41b796a5feb5161fafc",
         "GUID:fa06079b370704f1bb8a8a4da5f16215",
         "GUID:2cd51e32b52dd4b0b888f2392d807abc",
+        "GUID:c29904cf588a14db08f7961207e4347e",
+        "GUID:e35a45ac11d444d7db30cc91d9e833b8",
         "GUID:e230a993a6c084a0ba10310713d6dd2f"
     ],
     "includePlatforms": [],
@@ -20,4 +23,4 @@
     "defineConstraints": [],
     "versionDefines": [],
     "noEngineReferences": false
-}
+}