Move 7zip download & extraction into build process

UnityAlex · UnityAlex · commit 6b95dfb51c94 · 2023-03-16T12:18:42.000-04:00
Use local 7z when not running on yamato
diff --git a/.yamato/build_linux_x64_debug.yml b/.yamato/build_linux_x64_debug.yml
@@ -11,9 +11,6 @@ variables:
   ARTIFACT_FILENAME: {{globals.artifact_base_name}}-linux-x64.7z
 
 commands:
-  - mkdir artifacts
-  - curl https://public-stevedore.unity3d.com/r/public/7za-linux-x64/e6c75fb7ffda_e6a295cdcae3f74d315361883cf53f75141be2e739c020035f414a449d4876af.zip --output artifacts/7za-linux-x64.zip
-  - unzip artifacts/7za-linux-x64.zip -d artifacts/7za-linux-x64
   - ./.yamato/scripts/build_yamato.sh Debug
 
 artifacts:
diff --git a/.yamato/build_linux_x64_release.yml b/.yamato/build_linux_x64_release.yml
@@ -11,9 +11,6 @@ variables:
   ARTIFACT_FILENAME: {{globals.artifact_base_name}}-linux-x64.7z
 
 commands:
-  - mkdir artifacts
-  - curl https://public-stevedore.unity3d.com/r/public/7za-linux-x64/e6c75fb7ffda_e6a295cdcae3f74d315361883cf53f75141be2e739c020035f414a449d4876af.zip --output artifacts/7za-linux-x64.zip
-  - unzip artifacts/7za-linux-x64.zip -d artifacts/7za-linux-x64
   - ./.yamato/scripts/build_yamato.sh Release
 
 artifacts:
diff --git a/.yamato/build_osx_arm64_debug.yml b/.yamato/build_osx_arm64_debug.yml
@@ -11,9 +11,6 @@ variables:
   ARTIFACT_FILENAME: {{globals.artifact_base_name}}-osx-arm64.7z
 
 commands:
-  - mkdir -p artifacts
-  - curl https://public-stevedore.unity3d.com/r/public/7za-mac-x64/e6c75fb7ffda_5bd76652986a0e3756d1cfd7e84ce056a9e1dbfc5f70f0514a001f724c0fbad2.zip --output artifacts/7za-mac-x64.zip
-  - unzip -o artifacts/7za-mac-x64.zip -d artifacts/7za-mac-x64
   - ./.yamato/scripts/build_yamato.sh arm64 Debug
 
 artifacts:
diff --git a/.yamato/build_osx_arm64_release.yml b/.yamato/build_osx_arm64_release.yml
@@ -11,9 +11,6 @@ variables:
   ARTIFACT_FILENAME: {{globals.artifact_base_name}}-osx-arm64.7z
 
 commands:
-  - mkdir -p artifacts
-  - curl https://public-stevedore.unity3d.com/r/public/7za-mac-x64/e6c75fb7ffda_5bd76652986a0e3756d1cfd7e84ce056a9e1dbfc5f70f0514a001f724c0fbad2.zip --output artifacts/7za-mac-x64.zip
-  - unzip -o artifacts/7za-mac-x64.zip -d artifacts/7za-mac-x64
   - ./.yamato/scripts/build_yamato.sh arm64 Release
 
 artifacts:
diff --git a/.yamato/build_osx_x64_debug.yml b/.yamato/build_osx_x64_debug.yml
@@ -11,9 +11,6 @@ variables:
   ARTIFACT_FILENAME: {{globals.artifact_base_name}}-osx-x64.7z
 
 commands:
-  - mkdir -p artifacts
-  - curl https://public-stevedore.unity3d.com/r/public/7za-mac-x64/e6c75fb7ffda_5bd76652986a0e3756d1cfd7e84ce056a9e1dbfc5f70f0514a001f724c0fbad2.zip --output artifacts/7za-mac-x64.zip
-  - unzip -o artifacts/7za-mac-x64.zip -d artifacts/7za-mac-x64
   - ./.yamato/scripts/build_yamato.sh x64 Debug
 
 artifacts:
diff --git a/.yamato/build_osx_x64_release.yml b/.yamato/build_osx_x64_release.yml
@@ -11,9 +11,6 @@ variables:
   ARTIFACT_FILENAME: {{globals.artifact_base_name}}-osx-x64.7z
 
 commands:
-  - mkdir -p artifacts
-  - curl https://public-stevedore.unity3d.com/r/public/7za-mac-x64/e6c75fb7ffda_5bd76652986a0e3756d1cfd7e84ce056a9e1dbfc5f70f0514a001f724c0fbad2.zip --output artifacts/7za-mac-x64.zip
-  - unzip -o artifacts/7za-mac-x64.zip -d artifacts/7za-mac-x64
   - ./.yamato/scripts/build_yamato.sh x64 Release
 
 artifacts:
diff --git a/.yamato/build_windows_x64_debug.yml b/.yamato/build_windows_x64_debug.yml
@@ -11,7 +11,6 @@ variables:
   ARTIFACT_FILENAME: {{globals.artifact_base_name}}-win-x64.7z
 
 commands:
-  - powershell .yamato\scripts\download_7z.ps1
   - .yamato/scripts/build_yamato.cmd x64 Debug
 
 artifacts:
diff --git a/.yamato/build_windows_x64_release.yml b/.yamato/build_windows_x64_release.yml
@@ -11,7 +11,6 @@ variables:
   ARTIFACT_FILENAME: {{globals.artifact_base_name}}-win-x64.7z
 
 commands:
-  - powershell .yamato\scripts\download_7z.ps1
   - .yamato/scripts/build_yamato.cmd x64 Release
 
 artifacts:
diff --git a/.yamato/build_windows_x86_debug.yml b/.yamato/build_windows_x86_debug.yml
@@ -11,7 +11,6 @@ variables:
   ARTIFACT_FILENAME: {{globals.artifact_base_name}}-win-x86.7z
 
 commands:
-  - powershell .yamato\scripts\download_7z.ps1
   - .yamato/scripts/build_yamato.cmd x86 Debug
 
 artifacts:
diff --git a/.yamato/build_windows_x86_release.yml b/.yamato/build_windows_x86_release.yml
@@ -11,7 +11,6 @@ variables:
   ARTIFACT_FILENAME: {{globals.artifact_base_name}}-win-x86.7z
 
 commands:
-  - powershell .yamato\scripts\download_7z.ps1
   - .yamato/scripts/build_yamato.cmd x86 Release
 
 artifacts:
diff --git a/unity/CITools/BuildDriver/Program.cs b/unity/CITools/BuildDriver/Program.cs
@@ -5,11 +5,15 @@ namespace BuildDriver;
 
 public class Program
 {
-    public static int Main(string[] args)
+    public static bool RunningOnYamato() => Environment.GetEnvironmentVariable("YAMATO_PROJECT_ID") != null;
+    public static async Task<int> Main(string[] args)
     {
         string architecture = GetArchitectures()[0]; // Get default arch for current system
         string configuration = "Release";
         bool silent = false;
+        bool zip = RunningOnYamato();
+        bool skipBuild = false;
+        Task<NPath>? zipTask = null;
         foreach (string arg in args)
         {
             if ((arg.StartsWith("--arch") || arg.StartsWith("--architecture="))
@@ -18,8 +22,17 @@ public static int Main(string[] args)
             if ((arg.StartsWith("--config=") || arg.StartsWith("--configuration="))
                      && !TryParseArgument(new[] {"Release", "Debug"}, arg, out configuration))
                 return 1;
-            if (arg.Equals("--silent"))
+            if (arg.Equals("--silent") || arg.Equals("--s"))
                 silent = true;
+            if (arg.Equals("--zip") || arg.Equals("--z"))
+                zip = true;
+            if (arg.Equals("--skip-build"))
+                skipBuild = true;
+        }
+
+        if (RunningOnYamato())
+        {
+            zipTask = SevenZip.DownloadAndUnzip7Zip();
         }
 
         Console.WriteLine("*****************************");
@@ -30,13 +43,25 @@ public static int Main(string[] args)
         Console.WriteLine("*****************************");
 
         GlobalConfig gConfig = new (){ Architecture = architecture, Configuration = configuration, Silent = silent};
-        BuildEmbeddingHost.Run(gConfig);
-        BuildNullGC.Run(gConfig);
-        BuildCoreCLR.Run(gConfig);
+        if (!skipBuild)
+        {
+            BuildEmbeddingHost.Run(gConfig);
+            BuildNullGC.Run(gConfig);
+            BuildCoreCLR.Run(gConfig);
+        }
+
         NPath artifacts = ConsolidateArtifacts(gConfig);
-        NPath zipArtifact = new (Environment.GetEnvironmentVariable("ARTIFACT_FILENAME") ??
-                            $"dotnet-unity-{architecture}.7z");
-        SevenZip.Create7z(artifacts, Paths.Artifacts.Combine(zipArtifact));
+
+        NPath zipExe = new("7z");
+        if (zipTask != null)
+        {
+            zipExe = await zipTask;
+            if (zipTask.Exception != null)
+                throw zipTask.Exception;
+        }
+
+        if (zip)
+            SevenZip.Zip(zipExe, artifacts, gConfig);
 
         return 0;
     }
diff --git a/unity/CITools/BuildDriver/SevenZip.cs b/unity/CITools/BuildDriver/SevenZip.cs
@@ -1,15 +1,14 @@
 using System.Diagnostics;
+using System.IO.Compression;
+using System.Net;
 using System.Runtime.InteropServices;
 using NiceIO;
 
 namespace BuildDriver;
 
 public class SevenZip
 {
-    public static NPath Create7z(NPath directory, NPath outputFile, string additional7zArguments = "")
-        => Create7z(Get7zPath(), directory, outputFile, additional7zArguments);
-
-    static NPath Create7z(NPath sevenZipPath, NPath directory, string outputFile, string additional7zArguments)
+    public static NPath Create7z(NPath sevenZipPath, NPath directory, string outputFile, string additional7zArguments = "")
     {
         Console.WriteLine($"Creating .7z {outputFile}");
         var args = $"a {outputFile} *";
@@ -28,28 +27,52 @@ static NPath Create7z(NPath sevenZipPath, NPath directory, string outputFile, st
         return new NPath(outputFile);
     }
 
-    public static NPath Get7zPath()
+    public static void Zip(NPath zipExe, NPath artifacts, GlobalConfig gConfig)
+    {
+        NPath zipArtifact = new (Environment.GetEnvironmentVariable("ARTIFACT_FILENAME") ??
+                                 $"dotnet-unity-{gConfig.Architecture}.7z");
+        SevenZip.Create7z(zipExe, artifacts, Paths.Artifacts.Combine(zipArtifact));
+    }
+
+    public static void Get7ZipUrl(out string url, out string filename)
     {
-        var artifacts = Paths.Artifacts;
-        NPath GetArtifactsPath()
+        string baseUrl = "https://public-stevedore.unity3d.com/r/public";
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+        {
+            url = $"{baseUrl}/7za-linux-x64/e6c75fb7ffda_e6a295cdcae3f74d315361883cf53f75141be2e739c020035f414a449d4876af.zip";
+        }
+        else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
         {
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-                return artifacts.Combine("7za-win-x64/7za.exe");
+            url = RuntimeInformation.OSArchitecture == Architecture.Arm64
+                ? $"{baseUrl}/7za-mac-arm64/e6c75fb7ffda_891473e6242d16ca181cee7c728b73f80c931f58de45ab79f376acd63d524151.zip"
+                : $"{baseUrl}/7za-mac-x64/e6c75fb7ffda_5bd76652986a0e3756d1cfd7e84ce056a9e1dbfc5f70f0514a001f724c0fbad2.zip";
+        }
+        else
+            url = $"{baseUrl}/7za-win-x64/38c5b39be2e8_a333cfccb708c88459b3812eb2597ca486ec9b416172543ca3ef8e5cd5f80984.zip";
 
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
-            {
-                if (RuntimeInformation.OSArchitecture == Architecture.Arm64)
-                    return artifacts.Combine("7za-mac-arm64/7za");
-                return artifacts.Combine("7za-mac-x64/7za");
-            }
+        filename = $"{url.Split('/').Single(s => s.StartsWith("7za"))}.zip";
+    }
 
-            return artifacts.Combine("7za-linux-x64/7za");
+    public static async Task<NPath> DownloadAndUnzip7Zip()
+    {
+        WebClient wc = new();
+        string url, filename;
+        Get7ZipUrl(out url, out filename);
+        NPath zipDest = Paths.Artifacts.Combine(filename);
+        zipDest.Parent.EnsureDirectoryExists();
+        if (!zipDest.FileExists())
+        {
+            Console.WriteLine($"Starting download of 7zip: {filename}");
+            wc.DownloadFile(url, zipDest);
         }
 
-        var path = GetArtifactsPath();
-        if (path.FileExists())
-            return path;
+        NPath destDir = zipDest.Parent.Combine(zipDest.FileNameWithoutExtension);
+        if (!destDir.DirectoryExists())
+        {
+            Console.WriteLine($"Extracting 7zip to: {destDir}");
+            ZipFile.ExtractToDirectory(zipDest, destDir);
+        }
 
-        return new NPath("7z");
+        return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? destDir.Combine("7za.exe") : destDir.Combine("7za");
     }
 }
