Merge #3958 Update and optimize meta tester and inflator images

.github/workflows/build.yml

@@ -76,8 +76,7 @@ jobs:
           sh get-docker.sh
           docker build --tag inflator --file ../Dockerfile.netkan .
           docker run --rm --name inflator --entrypoint /bin/bash inflator -c "
-            curl -O https://raw.githubusercontent.com/KSP-CKAN/NetKAN/master/NetKAN/ZeroMiniAVC.netkan && \
-            mono netkan.exe ZeroMiniAVC.netkan
+            mono netkan.exe https://raw.githubusercontent.com/KSP-CKAN/NetKAN/master/NetKAN/ZeroMiniAVC.netkan
           "
         if: matrix.configuration == 'release'
 

CHANGELOG.md

@@ -11,6 +11,7 @@ All notable changes to this project will be documented in this file.
 ### Internal
 
 - [Netkan] Re-sort module properties after merging (#3957 by: HebaruSan)
+- [Netkan] Update and optimize meta tester and inflator images (#3958 by: HebaruSan; reviewed: techman83)
 
 ## v1.34.2 (Minkowski²)
 

Dockerfile.metadata

@@ -1,15 +1,57 @@
-FROM mono:latest
+# Everything we need in both the build and prod images
+FROM ubuntu:latest as base
 
-RUN /bin/sed -i 's/^mozilla\/DST_Root_CA_X3.crt$/!mozilla\/DST_Root_CA_X3.crt/' /etc/ca-certificates.conf && \
-    /usr/sbin/update-ca-certificates
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends python3 python3-pip python3-setuptools git build-essential python3-dev libffi-dev && \
-    apt-get clean
+# Don't prompt for time zone
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Put user-installed Python code in path
+ENV PATH "$PATH:/root/.local/bin"
+
+# Install Git and Python
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        ca-certificates gnupg git libffi-dev \
+        python3 python-is-python3
+
+# Trust all git repos
 RUN git config --global --add safe.directory '*'
+
+# Set up Mono's APT repo
+RUN gpg --homedir /tmp --no-default-keyring --keyring /usr/share/keyrings/mono-official-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF \
+    && echo "deb [signed-by=/usr/share/keyrings/mono-official-archive-keyring.gpg] https://download.mono-project.com/repo/ubuntu stable-focal main" | tee /etc/apt/sources.list.d/mono-official-stable.list \
+    && apt-get update
+
+# Install the necessary pieces of Mono
+RUN apt-get install -y --no-install-recommends \
+    mono-runtime ca-certificates-mono libmono-microsoft-csharp4.0-cil libmono-system-data4.0-cil libmono-system-runtime-serialization4.0-cil libmono-system-transactions4.0-cil libmono-system-net-http-webrequest4.0-cil
+
+# Isolate Python build stuff in a separate container
+FROM base as build
+
+# Install Python build deps
+RUN apt-get install -y --no-install-recommends \
+    python3-pip python3-setuptools python3-dev
+
+# Install the meta tester's Python code and its Infra dep
+ENV PIP_ROOT_USER_ACTION=ignore
 RUN pip3 install --upgrade pip
-RUN pip3 install 'git+https://github.com/KSP-CKAN/NetKAN-Infra#subdirectory=netkan'
-RUN pip3 install 'git+https://github.com/KSP-CKAN/xKAN-meta_testing'
+RUN pip3 install --user 'git+https://github.com/KSP-CKAN/NetKAN-Infra#subdirectory=netkan'
+RUN pip3 install --user 'git+https://github.com/KSP-CKAN/xKAN-meta_testing'
+
+# Prune unused deps (`--user` is implicit for uninstall)
+RUN pip3 --no-input uninstall -y flask gunicorn werkzeug
+
+# The image we'll actually use
+FROM base as prod
+
+# Purge APT download cache, package lists, and logs
+RUN apt-get clean \
+    && rm -r /var/lib/apt/lists /var/log/dpkg.log /var/log/apt
+
+# Extract built Python packages from the build image
+COPY --from=build /root/.local /root/.local
 
+# Install the .NET assemblies the meta tester uses
 ADD netkan.exe /usr/local/bin/.
 ADD ckan.exe /usr/local/bin/.
 

Dockerfile.netkan

@@ -1,10 +1,27 @@
-FROM mono:latest
-RUN /bin/sed -i 's/^mozilla\/DST_Root_CA_X3.crt$/!mozilla\/DST_Root_CA_X3.crt/' /etc/ca-certificates.conf && \
-    /usr/sbin/update-ca-certificates
+FROM ubuntu:latest
+
+# Don't prompt for time zone
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Set up Mono's APT repo
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates gnupg \
+    && gpg --homedir /tmp --no-default-keyring --keyring /usr/share/keyrings/mono-official-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF \
+    && echo "deb [signed-by=/usr/share/keyrings/mono-official-archive-keyring.gpg] https://download.mono-project.com/repo/ubuntu stable-focal main" | tee /etc/apt/sources.list.d/mono-official-stable.list \
+    && apt-get update
+
+# Install the necessary pieces of Mono
+RUN apt-get install -y --no-install-recommends \
+    mono-runtime ca-certificates-mono libmono-microsoft-csharp4.0-cil libmono-system-data4.0-cil libmono-system-runtime-serialization4.0-cil libmono-system-transactions4.0-cil libmono-system-net-http-webrequest4.0-cil
+
+# Purge APT download cache, package lists, and logs
+RUN apt-get clean \
+    && rm -r /var/lib/apt/lists /var/log/dpkg.log /var/log/apt
+
 RUN useradd -ms /bin/bash netkan
 USER netkan
 WORKDIR /home/netkan
-ADD netkan.exe .
+ADD --chown=netkan netkan.exe .
 ENTRYPOINT /usr/bin/mono netkan.exe --game ${GAME:-KSP} --queues $QUEUES \
   --net-useragent 'Mozilla/5.0 (compatible; Netkanbot/1.0; CKAN; +https://github.com/KSP-CKAN/NetKAN-Infra)' \
   --github-token $GH_Token --gitlab-token "$GL_Token" --cachedir ckan_cache -v

Netkan/Program.cs

@@ -16,6 +16,7 @@
 using CKAN.NetKAN.Processors;
 using CKAN.NetKAN.Transformers;
 using CKAN.NetKAN.Extensions;
+using YamlDotNet.RepresentationModel;
 
 namespace CKAN.NetKAN
 {
@@ -116,7 +117,7 @@ public static int Main(string[] args)
                 else
                 {
                     Log.Fatal(
-                        "Usage: netkan [--verbose|--debug] [--debugger] [--prerelease] [--outputdir=...] <filename>"
+                        "Usage: netkan [--verbose|--debug] [--debugger] [--prerelease] [--outputdir=...] <filename|URL>"
                     );
                     return ExitBadOpt;
                 }
@@ -183,18 +184,22 @@ private static Metadata[] ReadNetkans()
                 Log.WarnFormat("Input is not a .netkan file");
             }
 
-            return YamlExtensions.Parse(File.OpenText(Options.File))
-                                 .Select(ymap => new Metadata(ymap))
-                                 .ToArray();
+            return ArgContents(Options.File).Select(ymap => new Metadata(ymap))
+                                            .ToArray();
         }
 
+        private static YamlMappingNode[] ArgContents(string arg)
+            => Uri.IsWellFormedUriString(arg, UriKind.Absolute)
+                ? YamlExtensions.Parse(Net.DownloadText(new Uri(arg)))
+                : YamlExtensions.Parse(File.OpenText(arg));
+
         internal static string CkanFileName(JObject json)
-        => Path.Combine(
-            Options.OutputDir,
-            string.Format(
-                "{0}-{1}.ckan",
-                (string)json["identifier"],
-                ((string)json["version"]).Replace(':', '-')));
+            => Path.Combine(
+                Options.OutputDir,
+                string.Format(
+                    "{0}-{1}.ckan",
+                    (string)json["identifier"],
+                    ((string)json["version"]).Replace(':', '-')));
 
         private static void WriteCkan(JObject json)
         {

doc/building.md

@@ -199,7 +199,7 @@ The basic operation of the actual build process is as follows:
     These are the "final" output of the build process and are stored in: `_build/repack/$CONFIGURATION`.
 - Unit tests are executed
   - The NUnit unit tests in `CKAN.Tests.dll` are executed.
-- Smoke tets are executed
+- Smoke tests are executed
   - These are simple tests designed to make sure there isn't anything grossly wrong with the build. All they do is
     execute the repacked `ckan.exe` and `netkan.exe` and make sure their version output matches the expected output.