From afd37457f02b55b29152f616e1e071457295ade7 Mon Sep 17 00:00:00 2001
From: Richard Lander <rlander@microsoft.com>
Date: Tue, 17 Oct 2023 15:07:23 -0700
Subject: [PATCH 1/7] Add container docs

---
 .../docs/Dockerfile.cross-build-ubuntu-1604   | 27 +++++++++++++++++
 .../docs/Dockerfile.cross-build-x64-arm64     |  3 ++
 .../docs/Dockerfile.cross-build-x64-x64       |  4 +++
 src/coreclr/nativeaot/docs/containers.md      | 29 +++++++++++++++++++
 4 files changed, 63 insertions(+)
 create mode 100644 src/coreclr/nativeaot/docs/Dockerfile.cross-build-ubuntu-1604
 create mode 100644 src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-arm64
 create mode 100644 src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-x64
 create mode 100644 src/coreclr/nativeaot/docs/containers.md

diff --git a/src/coreclr/nativeaot/docs/Dockerfile.cross-build-ubuntu-1604 b/src/coreclr/nativeaot/docs/Dockerfile.cross-build-ubuntu-1604
new file mode 100644
index 0000000000000..83e1f16be4a8f
--- /dev/null
+++ b/src/coreclr/nativeaot/docs/Dockerfile.cross-build-ubuntu-1604
@@ -0,0 +1,27 @@
+# syntax=docker/dockerfile:1
+# Learn about building .NET container images:
+# https://github.com/dotnet/dotnet-docker/blob/main/samples/README.md
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 AS build
+ARG TARGETARCH
+COPY --from=mcr.microsoft.com/dotnet/sdk:8.0-cbl-mariner2.0-amd64 /usr/share/dotnet /usr/share/dotnet
+RUN ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
+WORKDIR /source
+
+RUN --mount=type=cache,target=/root/.nuget \
+    --mount=type=cache,target=/source/bin \
+    --mount=type=cache,target=/source/obj \
+    dotnet publish -a $TARGETARCH --sc -o /app --no-restore -p:SysRoot=/crossrootfs/x64 -p:LinkerFlavor=lld releasesapi.csproj
+
+
+# final stage/image
+FROM ubuntu:16.04
+RUN <<EOF
+apt-get update
+apt-get install -y openssl ca-certificates
+EOF
+ENV \
+    # Configure web servers to bind to port 8080 when present
+    ASPNETCORE_HTTP_PORTS=8080
+WORKDIR /app
+COPY --from=build /app .
+ENTRYPOINT ["./releasesapi"]
diff --git a/src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-arm64 b/src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-arm64
new file mode 100644
index 0000000000000..e5a1ffafaa1ac
--- /dev/null
+++ b/src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-arm64
@@ -0,0 +1,3 @@
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64
+COPY --from=mcr.microsoft.com/dotnet/sdk:8.0-cbl-mariner2.0-amd64 /usr/share/dotnet /usr/share/dotnet
+RUN ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
diff --git a/src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-x64 b/src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-x64
new file mode 100644
index 0000000000000..9d83eaa60a5e8
--- /dev/null
+++ b/src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-x64
@@ -0,0 +1,4 @@
+FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64
+COPY --from=mcr.microsoft.com/dotnet/sdk:8.0-cbl-mariner2.0-amd64 /usr/share/dotnet /usr/share/dotnet
+RUN ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
+
diff --git a/src/coreclr/nativeaot/docs/containers.md b/src/coreclr/nativeaot/docs/containers.md
new file mode 100644
index 0000000000000..14cfd59af7e04
--- /dev/null
+++ b/src/coreclr/nativeaot/docs/containers.md
@@ -0,0 +1,29 @@
+# Building native AOT apps in containers
+
+Containers are a useful tool to build any kind of code, including native AOT. There are a few patterns for doing that.
+
+## Cloud native apps
+
+Native AOT apps can be built in containers (with multi-stage build) just like regular (CoreCLR) apps. SDK `aot` images provide the additional native toolchain dependencies required to build native AOT apps.
+
+The [`releasesapi`](https://github.com/dotnet/dotnet-docker/blob/main/samples/releasesapi/README.md) sample demonstrates how to build native AOT apps in containers.
+
+## Cross-compiling
+
+For cloud native apps, build and runtime OS likely match, at least if you use multi-stage build. Once you step out of containers (for app delivery), it is more likely that you are delivering binaries that you want to work in more places (like on older Linux distros).
+
+The .NET build has this exact same need. We produce several [container images to enable cross-building](https://github.com/dotnet/runtime/blob/main/docs/workflow/building/coreclr/linux-instructions.md).
+
+You can use these images to build native AOT apps which work on distros as old as Ubuntu 16.04. The images are not supported, but are expected to work (since we use them to build .NET on daily basis).
+
+The following Dockerfiles demonstrate how to construct a working build environment that could be use for volume-mounted docker builds.
+
+- [Dockerfile.cross-build-x64-arm64](Dockerfile.cross-build-x64-arm64)
+- [Dockerfile.cross-build-x64-x64](Dockerfile.cross-build-x64-arm64)
+
+
+The following Dockerfile provides a complete proof of concept for building an app and "deploying" it to Ubuntu 16.04.
+
+- [Dockerfile.cross-build-ubuntu-1604](Dockerfile.cross-build-ubuntu-1604)
+
+This Dockerfile is written as if it was run from [this directory](https://github.com/dotnet/dotnet-docker/tree/main/samples/releasesapi).

From 148810ce0a6b498abcae66179e0f71590a133201 Mon Sep 17 00:00:00 2001
From: Richard Lander <rlander@microsoft.com>
Date: Tue, 17 Oct 2023 18:24:59 -0700
Subject: [PATCH 2/7] Add entry in README

---
 .../docs/Dockerfile.cross-build-ubuntu-1604   |  7 +-
 .../docs/Dockerfile.cross-build-x64-arm64     |  1 +
 .../docs/Dockerfile.cross-build-x64-x64       |  2 +-
 src/coreclr/nativeaot/docs/README.md          |  1 +
 src/coreclr/nativeaot/docs/compiling.md       |  2 +
 src/coreclr/nativeaot/docs/containers.md      | 98 ++++++++++++++++++-
 6 files changed, 105 insertions(+), 6 deletions(-)

diff --git a/src/coreclr/nativeaot/docs/Dockerfile.cross-build-ubuntu-1604 b/src/coreclr/nativeaot/docs/Dockerfile.cross-build-ubuntu-1604
index 83e1f16be4a8f..632f6ccbcc008 100644
--- a/src/coreclr/nativeaot/docs/Dockerfile.cross-build-ubuntu-1604
+++ b/src/coreclr/nativeaot/docs/Dockerfile.cross-build-ubuntu-1604
@@ -2,15 +2,18 @@
 # Learn about building .NET container images:
 # https://github.com/dotnet/dotnet-docker/blob/main/samples/README.md
 FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 AS build
-ARG TARGETARCH
 COPY --from=mcr.microsoft.com/dotnet/sdk:8.0-cbl-mariner2.0-amd64 /usr/share/dotnet /usr/share/dotnet
 RUN ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
 WORKDIR /source
 
+// Copy all source assets
+COPY . .
+// Cache mounts caches runtime packs, targeting packs, and NuGet packages
+// across builds
 RUN --mount=type=cache,target=/root/.nuget \
     --mount=type=cache,target=/source/bin \
     --mount=type=cache,target=/source/obj \
-    dotnet publish -a $TARGETARCH --sc -o /app --no-restore -p:SysRoot=/crossrootfs/x64 -p:LinkerFlavor=lld releasesapi.csproj
+    dotnet publish -o /app -p:SysRoot=/crossrootfs/x64 -p:LinkerFlavor=lld releasesapi.csproj
 
 
 # final stage/image
diff --git a/src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-arm64 b/src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-arm64
index e5a1ffafaa1ac..5196be167747c 100644
--- a/src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-arm64
+++ b/src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-arm64
@@ -1,3 +1,4 @@
 FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64
 COPY --from=mcr.microsoft.com/dotnet/sdk:8.0-cbl-mariner2.0-amd64 /usr/share/dotnet /usr/share/dotnet
 RUN ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
+ENV DOTNET_NOLOGO=true
diff --git a/src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-x64 b/src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-x64
index 9d83eaa60a5e8..18b3c58b286e3 100644
--- a/src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-x64
+++ b/src/coreclr/nativeaot/docs/Dockerfile.cross-build-x64-x64
@@ -1,4 +1,4 @@
 FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64
 COPY --from=mcr.microsoft.com/dotnet/sdk:8.0-cbl-mariner2.0-amd64 /usr/share/dotnet /usr/share/dotnet
 RUN ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
-
+ENV DOTNET_NOLOGO=true
diff --git a/src/coreclr/nativeaot/docs/README.md b/src/coreclr/nativeaot/docs/README.md
index c09da55697f9a..747776dd9a2b2 100644
--- a/src/coreclr/nativeaot/docs/README.md
+++ b/src/coreclr/nativeaot/docs/README.md
@@ -8,3 +8,4 @@
 - [Troubleshooting](troubleshooting.md)
 - [RD.xml Documentation](rd-xml-format.md)
 - [Using Native AOT on Android-Bionic](android-bionic.md)
+- [Building native AOT apps in containers](containers.md)
diff --git a/src/coreclr/nativeaot/docs/compiling.md b/src/coreclr/nativeaot/docs/compiling.md
index 6949d603c47fb..b0c8705275d82 100644
--- a/src/coreclr/nativeaot/docs/compiling.md
+++ b/src/coreclr/nativeaot/docs/compiling.md
@@ -70,6 +70,8 @@ You also need to specify the sysroot directory for Clang using the `SysRoot` pro
 
 You may also follow [cross-building instructions](../../../../docs/workflow/building/coreclr/cross-building.md) to create your own sysroot directory.
 
+See [Building native AOT apps in containers](containers.md) for a potentially quicker path to estalbishing a cross-compilation environment.
+
 ## Using statically linked ICU
 This feature can statically link libicu libraries (such as libicui18n.a) into your applications at build time.
 NativeAOT binaries built with this feature can run even when libicu libraries are not installed.
diff --git a/src/coreclr/nativeaot/docs/containers.md b/src/coreclr/nativeaot/docs/containers.md
index 14cfd59af7e04..af2e408997d43 100644
--- a/src/coreclr/nativeaot/docs/containers.md
+++ b/src/coreclr/nativeaot/docs/containers.md
@@ -6,24 +6,116 @@ Containers are a useful tool to build any kind of code, including native AOT. Th
 
 Native AOT apps can be built in containers (with multi-stage build) just like regular (CoreCLR) apps. SDK `aot` images provide the additional native toolchain dependencies required to build native AOT apps.
 
+Search for `aot` images at [dotnet/nightly/sdk](https://mcr.microsoft.com/en-us/product/dotnet/nightly/sdk/tags).
+
 The [`releasesapi`](https://github.com/dotnet/dotnet-docker/blob/main/samples/releasesapi/README.md) sample demonstrates how to build native AOT apps in containers.
 
 ## Cross-compiling
 
-For cloud native apps, build and runtime OS likely match, at least if you use multi-stage build. Once you step out of containers (for app delivery), it is more likely that you are delivering binaries that you want to work in more places (like on older Linux distros).
+For cloud native apps, build and runtime OS typically match, at least if you use multi-stage build. Once you step out of containers (for app delivery), it is more likely that you are delivering binaries that you want to work in more places (like on older Linux distros).
 
 The .NET build has this exact same need. We produce several [container images to enable cross-building](https://github.com/dotnet/runtime/blob/main/docs/workflow/building/coreclr/linux-instructions.md).
 
-You can use these images to build native AOT apps which work on distros as old as Ubuntu 16.04. The images are not supported, but are expected to work (since we use them to build .NET on daily basis).
+You can use these images to build native AOT apps which work on distros as old as Ubuntu 16.04. These build images are not supported, but are expected to work (since we use them to build .NET on daily basis).
+
+### Containerized build
 
-The following Dockerfiles demonstrate how to construct a working build environment that could be use for volume-mounted docker builds.
+The following Dockerfiles demonstrate how to construct a working build environment that could be use for volume-mounted docker builds. They can be modified to use other images we provide, like for Alpine.
 
 - [Dockerfile.cross-build-x64-arm64](Dockerfile.cross-build-x64-arm64)
 - [Dockerfile.cross-build-x64-x64](Dockerfile.cross-build-x64-arm64)
 
+### x64
+
+The following pattern demonstrates the approach for x64.
+
+First build image.
+
+```bash
+$ docker build --pull -t cross-build -f Dockerfile.cross-build-x64-x64 .
+```
+
+Then build the app, with volume mounting, using the [releasesapi](https://github.com/dotnet/dotnet-docker/tree/main/samples/releasesapi) sample.
+
+```bash
+$ docker run --rm -it -v $(pwd):/source -w /source cross-build dotnet publish -o app -p:SysRoot=/crossrootfs/x64 -p:LinkerFlavor=lld releasesapi.csproj
+$  ls app
+appsettings.Development.json  nuget.config  releasesapi.dbg
+appsettings.json              releasesapi
+$ ./app/releasesapi
+info: Microsoft.Hosting.Lifetime[14]
+      Now listening on: http://localhost:5000
+info: Microsoft.Hosting.Lifetime[0]
+      Application started. Press Ctrl+C to shut down.
+info: Microsoft.Hosting.Lifetime[0]
+      Hosting environment: Production
+info: Microsoft.Hosting.Lifetime[0]
+      Content root path: /home/rich/git/dotnet-docker/samples/releasesapi
+```
+
+We can test the app in a container, again through volume mounting.
+
+```bash
+$ docker run --rm -d -v $(pwd)/app:/app -w /app -p 8000:80 mcr.microsoft.com/dotnet/runtime-deps:6.0 ./releasesapi
+219e1df9e66906531ee609b8cb9b9fa8eccef566a8682b897798d80b3905aaf5
+$ curl http://localhost:8000/healthz
+Healthy
+$ docker exec 219e1df9e66906531ee609b8cb9b9fa8eccef566a8682b897798d80b3905aaf5 cat /etc/os-release | head -n 1
+PRETTY_NAME="Debian GNU/Linux 11 (bullseye)"
+$ docker kill 219e1df9e66906531ee609b8cb9b9fa8eccef566a8682b897798d80b3905aaf5
+219e1df9e66906531ee609b8cb9b9fa8eccef566a8682b897798d80b3905aaf5
+```
+
+### Arm64
+
+Much the same pattern can be used for `x64-arm64` Dockerfile.
+
+
+```bash
+$ docker build --pull -t cross-build-arm64 -f Dockerfile.cross-build-x64-arm64 .
+$ docker run --rm -it -v $(pwd):/source -w /source cross-build-arm64 dotnet publish -a arm64 -o app-arm64 -p:SysRoot=/crossrootfs/arm64 -p:LinkerFlavor=lld releasesapi.csproj
+$ ./app-arm64/releasesapi
+-bash: ./app-arm64/releasesapi: cannot execute binary file: Exec format error
+$ file app-arm64/releasesapi
+app-arm64/releasesapi: ELF 64-bit LSB pie executable, ARM aarch64, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, for GNU/Linux 3.7.0, BuildID[sha1]=72212bcbd040059f1c2e6d55f640d52f7cbe2faf, stripped
+```
+
+Note the use of `-a arm64` and `-p:SysRoot=/crossrootfs/arm64` in the `dotnet publish` command. Those are required to get the build to use and generate Arm64 compatible assets.
+
+The app can then be copied to an Arm64 and it will work as demonstrated on an Apple M1 machine.
+
+```bash
+$ docker run --rm  -d -v $(pwd):/app -w /app -p 8000:8080 mcr.microsoft.com/dotnet/nightly/runtime-deps:8.0-jammy-chiseled-aot ./releasesapi
+67fd87ef2975cd441e71ff4a3cd5dc288c4d5c9b000f2219a5c456d35c13c76e
+$ curl http://localhost:8000/healthz
+Healthy
+$ docker kill 67fd87ef2975cd441e71ff4a3cd5dc288c4d5c9b000f2219a5c456d35c13c76e
+67fd87ef2975cd441e71ff4a3cd5dc288c4d5c9b000f2219a5c456d35c13c76e
+$ uname -a
+Darwin Richs-Air.phantomdomain 23.0.0 Darwin Kernel Version 23.0.0: Fri Sep 15 14:41:34 PDT 2023; root:xnu-10002.1.13~1/RELEASE_ARM64_T8103 arm64
+```
+
+### Complete example
 
 The following Dockerfile provides a complete proof of concept for building an app and "deploying" it to Ubuntu 16.04.
 
 - [Dockerfile.cross-build-ubuntu-1604](Dockerfile.cross-build-ubuntu-1604)
 
 This Dockerfile is written as if it was run from [this directory](https://github.com/dotnet/dotnet-docker/tree/main/samples/releasesapi).
+
+It can be built and run with the following:
+
+```bash
+$ docker build --pull -t app -f Dockerfile.cross-build-ubuntu-1604 .
+$ docker run --rm -it --entrypoint bash app -c "cat /etc/os-release | head -n 2"
+NAME="Ubuntu"
+VERSION="16.04.7 LTS (Xenial Xerus)"
+docker run --rm -d -p 8000:8080 app
+e72f93de034db4a1a12d25ce756c96235e5f584da8b1d440e9d6d083f8b5418d
+$ curl http://localhost:8000/healthz
+Healthy
+$ docker kill e72f93de034db4a1a12d25ce756c96235e5f584da8b1d440e9d6d083f8b5418d
+e72f93de034db4a1a12d25ce756c96235e5f584da8b1d440e9d6d083f8b5418d
+```
+
+The app also has a `releases` endpoint.

From 19eb1aa483cf649ed5fe3dbc7d724a85e4992ba9 Mon Sep 17 00:00:00 2001
From: Jan Kotas <jkotas@microsoft.com>
Date: Tue, 17 Oct 2023 18:53:31 -0700
Subject: [PATCH 3/7] Apply suggestions from code review

Co-authored-by: Rich Lander <rlander@microsoft.com>
---
 src/coreclr/nativeaot/docs/containers.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/coreclr/nativeaot/docs/containers.md b/src/coreclr/nativeaot/docs/containers.md
index af2e408997d43..90847089cbda9 100644
--- a/src/coreclr/nativeaot/docs/containers.md
+++ b/src/coreclr/nativeaot/docs/containers.md
@@ -20,7 +20,7 @@ You can use these images to build native AOT apps which work on distros as old a
 
 ### Containerized build
 
-The following Dockerfiles demonstrate how to construct a working build environment that could be use for volume-mounted docker builds. They can be modified to use other images we provide, like for Alpine.
+The following Dockerfiles demonstrate how to construct a working build environment that can be used for volume-mounted docker builds. They can be modified to use the other image flavors we provide, like Alpine.
 
 - [Dockerfile.cross-build-x64-arm64](Dockerfile.cross-build-x64-arm64)
 - [Dockerfile.cross-build-x64-x64](Dockerfile.cross-build-x64-arm64)
@@ -68,7 +68,7 @@ $ docker kill 219e1df9e66906531ee609b8cb9b9fa8eccef566a8682b897798d80b3905aaf5
 
 ### Arm64
 
-Much the same pattern can be used for `x64-arm64` Dockerfile.
+Much the same pattern can be used  to target Arm64 with the `x64-arm64` Dockerfile, again on an x64 host.
 
 
 ```bash
@@ -82,7 +82,7 @@ app-arm64/releasesapi: ELF 64-bit LSB pie executable, ARM aarch64, version 1 (SY
 
 Note the use of `-a arm64` and `-p:SysRoot=/crossrootfs/arm64` in the `dotnet publish` command. Those are required to get the build to use and generate Arm64 compatible assets.
 
-The app can then be copied to an Arm64 and it will work as demonstrated on an Apple M1 machine.
+As expected, the Arm64 app fails to run on an x64 machine. The app can then be copied to an Arm64 machine and it will work as demonstrated on an Apple M1 machine (in an Arm64 Linux container). Note that I had to run `chmod +x` on the executable first.
 
 ```bash
 $ docker run --rm  -d -v $(pwd):/app -w /app -p 8000:8080 mcr.microsoft.com/dotnet/nightly/runtime-deps:8.0-jammy-chiseled-aot ./releasesapi

From c87d6650c266aa7e6d221a9f856ecd4c849248e6 Mon Sep 17 00:00:00 2001
From: Jan Kotas <jkotas@microsoft.com>
Date: Tue, 17 Oct 2023 18:54:08 -0700
Subject: [PATCH 4/7] Update src/coreclr/nativeaot/docs/compiling.md

Co-authored-by: Rich Lander <rlander@microsoft.com>
---
 src/coreclr/nativeaot/docs/compiling.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/coreclr/nativeaot/docs/compiling.md b/src/coreclr/nativeaot/docs/compiling.md
index b0c8705275d82..96e671137c55f 100644
--- a/src/coreclr/nativeaot/docs/compiling.md
+++ b/src/coreclr/nativeaot/docs/compiling.md
@@ -70,7 +70,7 @@ You also need to specify the sysroot directory for Clang using the `SysRoot` pro
 
 You may also follow [cross-building instructions](../../../../docs/workflow/building/coreclr/cross-building.md) to create your own sysroot directory.
 
-See [Building native AOT apps in containers](containers.md) for a potentially quicker path to estalbishing a cross-compilation environment.
+See [Building native AOT apps in containers](containers.md) for a potentially quicker path to establishing a cross-compilation environment.
 
 ## Using statically linked ICU
 This feature can statically link libicu libraries (such as libicui18n.a) into your applications at build time.

From a7cfe50d98235d28f91768e53e6b39dd2a8bf7cf Mon Sep 17 00:00:00 2001
From: Richard Lander <rlander@microsoft.com>
Date: Thu, 19 Oct 2023 11:21:43 -0700
Subject: [PATCH 5/7] Update per feedback

---
 .../docs/Dockerfile.cross-build-ubuntu-1604   |  30 ------
 src/coreclr/nativeaot/docs/containers.md      | 100 +++++++-----------
 2 files changed, 38 insertions(+), 92 deletions(-)
 delete mode 100644 src/coreclr/nativeaot/docs/Dockerfile.cross-build-ubuntu-1604

diff --git a/src/coreclr/nativeaot/docs/Dockerfile.cross-build-ubuntu-1604 b/src/coreclr/nativeaot/docs/Dockerfile.cross-build-ubuntu-1604
deleted file mode 100644
index 632f6ccbcc008..0000000000000
--- a/src/coreclr/nativeaot/docs/Dockerfile.cross-build-ubuntu-1604
+++ /dev/null
@@ -1,30 +0,0 @@
-# syntax=docker/dockerfile:1
-# Learn about building .NET container images:
-# https://github.com/dotnet/dotnet-docker/blob/main/samples/README.md
-FROM mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 AS build
-COPY --from=mcr.microsoft.com/dotnet/sdk:8.0-cbl-mariner2.0-amd64 /usr/share/dotnet /usr/share/dotnet
-RUN ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet
-WORKDIR /source
-
-// Copy all source assets
-COPY . .
-// Cache mounts caches runtime packs, targeting packs, and NuGet packages
-// across builds
-RUN --mount=type=cache,target=/root/.nuget \
-    --mount=type=cache,target=/source/bin \
-    --mount=type=cache,target=/source/obj \
-    dotnet publish -o /app -p:SysRoot=/crossrootfs/x64 -p:LinkerFlavor=lld releasesapi.csproj
-
-
-# final stage/image
-FROM ubuntu:16.04
-RUN <<EOF
-apt-get update
-apt-get install -y openssl ca-certificates
-EOF
-ENV \
-    # Configure web servers to bind to port 8080 when present
-    ASPNETCORE_HTTP_PORTS=8080
-WORKDIR /app
-COPY --from=build /app .
-ENTRYPOINT ["./releasesapi"]
diff --git a/src/coreclr/nativeaot/docs/containers.md b/src/coreclr/nativeaot/docs/containers.md
index 90847089cbda9..faceefe6218fb 100644
--- a/src/coreclr/nativeaot/docs/containers.md
+++ b/src/coreclr/nativeaot/docs/containers.md
@@ -27,95 +27,71 @@ The following Dockerfiles demonstrate how to construct a working build environme
 
 ### x64
 
-The following pattern demonstrates the approach for x64.
+The following pattern demonstrates building an app x64.
 
-First build image.
+Build a cross-build image for the x64 target.
 
 ```bash
-$ docker build --pull -t cross-build -f Dockerfile.cross-build-x64-x64 .
+$ docker build --pull -t cross-build-x64 -f Dockerfile.cross-build-x64-x64 .
 ```
 
-Then build the app, with volume mounting, using the [releasesapi](https://github.com/dotnet/dotnet-docker/tree/main/samples/releasesapi) sample.
+Then navigate to the directory where your project is located. This example creates one via `dotnet new` for demonstration purposes.
 
 ```bash
-$ docker run --rm -it -v $(pwd):/source -w /source cross-build dotnet publish -o app -p:SysRoot=/crossrootfs/x64 -p:LinkerFlavor=lld releasesapi.csproj
-$  ls app
-appsettings.Development.json  nuget.config  releasesapi.dbg
-appsettings.json              releasesapi
-$ ./app/releasesapi
-info: Microsoft.Hosting.Lifetime[14]
-      Now listening on: http://localhost:5000
-info: Microsoft.Hosting.Lifetime[0]
-      Application started. Press Ctrl+C to shut down.
-info: Microsoft.Hosting.Lifetime[0]
-      Hosting environment: Production
-info: Microsoft.Hosting.Lifetime[0]
-      Content root path: /home/rich/git/dotnet-docker/samples/releasesapi
+$ mkdir cross-build-test
+$ cd cross-build-test/
+$ dotnet new console --aot
 ```
 
-We can test the app in a container, again through volume mounting.
+Build the app, with volume mounting
 
 ```bash
-$ docker run --rm -d -v $(pwd)/app:/app -w /app -p 8000:80 mcr.microsoft.com/dotnet/runtime-deps:6.0 ./releasesapi
-219e1df9e66906531ee609b8cb9b9fa8eccef566a8682b897798d80b3905aaf5
-$ curl http://localhost:8000/healthz
-Healthy
-$ docker exec 219e1df9e66906531ee609b8cb9b9fa8eccef566a8682b897798d80b3905aaf5 cat /etc/os-release | head -n 1
-PRETTY_NAME="Debian GNU/Linux 11 (bullseye)"
-$ docker kill 219e1df9e66906531ee609b8cb9b9fa8eccef566a8682b897798d80b3905aaf5
-219e1df9e66906531ee609b8cb9b9fa8eccef566a8682b897798d80b3905aaf5
+$ docker run --rm -it -v $(pwd):/source -w /source cross-build-x64 dotnet publish -o app -p:SysRoot=/crossrootfs/x64 -p:LinkerFlavor=lld 
+$ ls -l app/
+total 4004
+-rwxr-xr-x 1 root root 1407576 Oct 19 10:25 cross-build-test
+-rwxr-xr-x 1 root root 2689720 Oct 19 10:25 cross-build-test.dbg
+$ ./app/cross-build-test
+Hello, World!
 ```
 
-### Arm64
+The app can be tested in a container, again through volume mounting.
 
-Much the same pattern can be used  to target Arm64 with the `x64-arm64` Dockerfile, again on an x64 host.
+```bash
+$ docker run --rm -v $(pwd)/app:/app -w /app mcr.microsoft.com/dotnet/runtime-deps:6.0 ./cross-build-test
+Hello, World!
+```
 
+That image uses an older Linux distribution.
 
 ```bash
-$ docker build --pull -t cross-build-arm64 -f Dockerfile.cross-build-x64-arm64 .
-$ docker run --rm -it -v $(pwd):/source -w /source cross-build-arm64 dotnet publish -a arm64 -o app-arm64 -p:SysRoot=/crossrootfs/arm64 -p:LinkerFlavor=lld releasesapi.csproj
-$ ./app-arm64/releasesapi
--bash: ./app-arm64/releasesapi: cannot execute binary file: Exec format error
-$ file app-arm64/releasesapi
-app-arm64/releasesapi: ELF 64-bit LSB pie executable, ARM aarch64, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, for GNU/Linux 3.7.0, BuildID[sha1]=72212bcbd040059f1c2e6d55f640d52f7cbe2faf, stripped
+$ docker run --rm mcr.microsoft.com/dotnet/runtime-deps:6.0 bash -c "cat /etc/os-release | head -n 1"
+PRETTY_NAME="Debian GNU/Linux 11 (bullseye)"
 ```
 
-Note the use of `-a arm64` and `-p:SysRoot=/crossrootfs/arm64` in the `dotnet publish` command. Those are required to get the build to use and generate Arm64 compatible assets.
+### Arm64
 
-As expected, the Arm64 app fails to run on an x64 machine. The app can then be copied to an Arm64 machine and it will work as demonstrated on an Apple M1 machine (in an Arm64 Linux container). Note that I had to run `chmod +x` on the executable first.
+The same pattern can be used for Arm64. The differences are demonstrated below, building the same console app.
+
+Build a cross-build image for the Arm64 target.
 
 ```bash
-$ docker run --rm  -d -v $(pwd):/app -w /app -p 8000:8080 mcr.microsoft.com/dotnet/nightly/runtime-deps:8.0-jammy-chiseled-aot ./releasesapi
-67fd87ef2975cd441e71ff4a3cd5dc288c4d5c9b000f2219a5c456d35c13c76e
-$ curl http://localhost:8000/healthz
-Healthy
-$ docker kill 67fd87ef2975cd441e71ff4a3cd5dc288c4d5c9b000f2219a5c456d35c13c76e
-67fd87ef2975cd441e71ff4a3cd5dc288c4d5c9b000f2219a5c456d35c13c76e
-$ uname -a
-Darwin Richs-Air.phantomdomain 23.0.0 Darwin Kernel Version 23.0.0: Fri Sep 15 14:41:34 PDT 2023; root:xnu-10002.1.13~1/RELEASE_ARM64_T8103 arm64
+$ docker build --pull -t cross-build-arm64 -f Dockerfile.cross-build-x64-arm64 .
 ```
 
-### Complete example
-
-The following Dockerfile provides a complete proof of concept for building an app and "deploying" it to Ubuntu 16.04.
+Build the app, with volume mounting.
 
-- [Dockerfile.cross-build-ubuntu-1604](Dockerfile.cross-build-ubuntu-1604)
+```bash
+$ docker run --rm -it -v $(pwd):/source -w /source cross-build-arm64 dotnet publish -a arm64 -o app-arm64 -p:SysRoot=/crossrootfs/arm64 -p:LinkerFlavor=lld 
+```
 
-This Dockerfile is written as if it was run from [this directory](https://github.com/dotnet/dotnet-docker/tree/main/samples/releasesapi).
+Notice all the places where `arm64` is used in that command.
 
-It can be built and run with the following:
+The resulting executable won't run on an x64 machine and must be run on a Linux Arm64 machine (because that is what it targets). The binary can be inspected to validate its nature.
 
 ```bash
-$ docker build --pull -t app -f Dockerfile.cross-build-ubuntu-1604 .
-$ docker run --rm -it --entrypoint bash app -c "cat /etc/os-release | head -n 2"
-NAME="Ubuntu"
-VERSION="16.04.7 LTS (Xenial Xerus)"
-docker run --rm -d -p 8000:8080 app
-e72f93de034db4a1a12d25ce756c96235e5f584da8b1d440e9d6d083f8b5418d
-$ curl http://localhost:8000/healthz
-Healthy
-$ docker kill e72f93de034db4a1a12d25ce756c96235e5f584da8b1d440e9d6d083f8b5418d
-e72f93de034db4a1a12d25ce756c96235e5f584da8b1d440e9d6d083f8b5418d
+$ file app-arm64/cross-build-test
+app-arm64/cross-build-test: ELF 64-bit LSB pie executable, ARM aarch64, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, for GNU/Linux 3.7.0, BuildID[sha1]=702dc1a1411a3fe5aed949b7e1536fd92475010c, stripped
+$ ./app-arm64/cross-build-test
+-bash: ./app-arm64/cross-build-test: cannot execute binary file: Exec format error
 ```
-
-The app also has a `releases` endpoint.

From 01bc5929e13c43c9a5e5f4346a8f734f30ac549b Mon Sep 17 00:00:00 2001
From: Richard Lander <rlander@microsoft.com>
Date: Thu, 19 Oct 2023 11:25:28 -0700
Subject: [PATCH 6/7] Update per feedback

---
 src/coreclr/nativeaot/docs/containers.md | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/src/coreclr/nativeaot/docs/containers.md b/src/coreclr/nativeaot/docs/containers.md
index faceefe6218fb..d5eab273a53a5 100644
--- a/src/coreclr/nativeaot/docs/containers.md
+++ b/src/coreclr/nativeaot/docs/containers.md
@@ -55,20 +55,13 @@ $ ./app/cross-build-test
 Hello, World!
 ```
 
-The app can be tested in a container, again through volume mounting.
+The app can be tested in an old Linux container, again through volume mounting.
 
 ```bash
-$ docker run --rm -v $(pwd)/app:/app -w /app mcr.microsoft.com/dotnet/runtime-deps:6.0 ./cross-build-test
+$ docker run --rm -v $(pwd)/app:/app -w /app ubuntu:16.04 ./cross-build-test
 Hello, World!
 ```
 
-That image uses an older Linux distribution.
-
-```bash
-$ docker run --rm mcr.microsoft.com/dotnet/runtime-deps:6.0 bash -c "cat /etc/os-release | head -n 1"
-PRETTY_NAME="Debian GNU/Linux 11 (bullseye)"
-```
-
 ### Arm64
 
 The same pattern can be used for Arm64. The differences are demonstrated below, building the same console app.

From 220f1fefbbdd6bb27d35af03987ead862c235e60 Mon Sep 17 00:00:00 2001
From: Jan Kotas <jkotas@microsoft.com>
Date: Thu, 19 Oct 2023 16:55:20 -0700
Subject: [PATCH 7/7] Delete duplicate information from compiling.md

---
 src/coreclr/nativeaot/docs/README.md     |  2 +-
 src/coreclr/nativeaot/docs/compiling.md  | 45 ++++++++++--------------
 src/coreclr/nativeaot/docs/containers.md |  2 +-
 3 files changed, 21 insertions(+), 28 deletions(-)

diff --git a/src/coreclr/nativeaot/docs/README.md b/src/coreclr/nativeaot/docs/README.md
index 747776dd9a2b2..2030f2256caf0 100644
--- a/src/coreclr/nativeaot/docs/README.md
+++ b/src/coreclr/nativeaot/docs/README.md
@@ -2,10 +2,10 @@
 
 - [Limitations](limitations.md)
 - [Compiling applications](compiling.md)
+- [Building native AOT apps in containers](containers.md)
 - [Debugging applications](debugging.md)
 - [Optimizing applications](optimizing.md)
 - [Reflection In AOT](reflection-in-aot-mode.md)
 - [Troubleshooting](troubleshooting.md)
 - [RD.xml Documentation](rd-xml-format.md)
 - [Using Native AOT on Android-Bionic](android-bionic.md)
-- [Building native AOT apps in containers](containers.md)
diff --git a/src/coreclr/nativeaot/docs/compiling.md b/src/coreclr/nativeaot/docs/compiling.md
index 96e671137c55f..757b00bb0b439 100644
--- a/src/coreclr/nativeaot/docs/compiling.md
+++ b/src/coreclr/nativeaot/docs/compiling.md
@@ -2,13 +2,13 @@
 
 Please consult [documentation](https://docs.microsoft.com/dotnet/core/deploying/native-aot) for instructions how to compile and publish application.
 
-The rest of this document covers advanced topics only. Adding an explicit package reference to `Microsoft.DotNet.ILCompiler` will generate warning when publishing and it can run into version errors. When possible, use the PublishAot property to publish a native AOT application.
+The rest of this document covers advanced topics only.
 
 ## Using daily builds
 
 For using daily builds, you need to make sure the `nuget.config` file for your project contains the following package sources under the `<packageSources>` element:
 ```xml
-<add key="dotnet8" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json" />
+<add key="dotnet9" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json" />
 <add key="nuget" value="https://api.nuget.org/v3/index.json" />
 ```
 
@@ -21,57 +21,50 @@ from the project's root directory. New package sources must be added after the `
 
 Once you have added the package sources, add a reference to the ILCompiler package either by running
 ```bash
-> dotnet add package Microsoft.DotNet.ILCompiler -v 8.0.0-*
+> dotnet add package Microsoft.DotNet.ILCompiler -v 9.0.0-*
 ```
 
 or by adding the following element to the project file:
 ```xml
 <ItemGroup>
-  <PackageReference Include="Microsoft.DotNet.ILCompiler" Version="8.0.0-*" />
+  <PackageReference Include="Microsoft.DotNet.ILCompiler" Version="9.0.0-*" />
 </ItemGroup>
 ```
 
+Adding an explicit package reference to `Microsoft.DotNet.ILCompiler` will generate warning when publishing and it can run into version errors. When possible, use the default `Microsoft.DotNet.ILCompiler` version to publish a native AOT application.
+
 ## Cross-architecture compilation
 
-Native AOT toolchain allows targeting ARM64 on an x64 host and vice versa for both Windows and Linux and is now supported in the SDK. Cross-OS compilation, such as targeting Linux on a Windows host, is not supported. For SDK support, add the following to your project file,
+Native AOT toolchain allows targeting ARM64 on an x64 host and vice versa for both Windows and Linux. Cross-OS compilation, such as targeting Linux on a Windows host, is not supported.
 
-```xml
-<PropertyGroup>
-  <PublishAot>true</PublishAot>
-</PropertyGroup>
-```
-
-Targeting win-arm64 on a Windows x64 host machine,
+The target architecture can be specified using `-r` or `--arch` options of the `dotnet publish` command. For example, the following command produces Windows Arm64 binary on a Windows x64 host machine:
 
 ```bash
-> dotnet publish -r win-arm64 -c Release
+> dotnet publish -r win-arm64
 ```
 
+The cross-architecture compilation requires native build tools for the target platform to be installed and configured correctly. On Linux, you may need to follow [cross-building instructions](../../../../docs/workflow/building/coreclr/cross-building.md) to create your own sysroot directory and specify path to it using the `SysRoot` property.
+
+See [Building native AOT apps in containers](containers.md) for a streamlined path to establishing a Linux cross-compilation environments.
+
+#### Using daily builds with cross-architecture compilation
+
 For using daily builds according to the instructions above, in addition to the `Microsoft.DotNet.ILCompiler` package reference, also add the `runtime.win-x64.Microsoft.DotNet.ILCompiler` package reference to get the x64-hosted compiler:
 ```xml
-<PackageReference Include="Microsoft.DotNet.ILCompiler; runtime.win-x64.Microsoft.DotNet.ILCompiler" Version="8.0.0-alpha.1.23456.7" />
+<PackageReference Include="Microsoft.DotNet.ILCompiler; runtime.win-x64.Microsoft.DotNet.ILCompiler" Version="9.0.0-alpha.1.23456.7" />
 ```
 
-Replace `8.0.0-alpha.1.23456.7` with the latest version from the [dotnet8](https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet8/NuGet/Microsoft.DotNet.ILCompiler/) feed.
+Replace `9.0.0-alpha.1.23456.7` with the latest version from the [dotnet9](https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet9/NuGet/Microsoft.DotNet.ILCompiler/) feed.
 Note that it is important to use _the same version_ for both packages to avoid potential hard-to-debug issues. After adding the package reference, you may publish for win-arm64 as usual:
 ```bash
-> dotnet publish -r win-arm64 -c Release
+> dotnet publish -r win-arm64
 ```
 
 Similarly, to target linux-arm64 on a Linux x64 host, in addition to the `Microsoft.DotNet.ILCompiler` package reference, also add the `runtime.linux-x64.Microsoft.DotNet.ILCompiler` package reference to get the x64-hosted compiler:
 ```xml
-<PackageReference Include="Microsoft.DotNet.ILCompiler; runtime.linux-x64.Microsoft.DotNet.ILCompiler" Version="8.0.0-alpha.1.23456.7" />
+<PackageReference Include="Microsoft.DotNet.ILCompiler; runtime.linux-x64.Microsoft.DotNet.ILCompiler" Version="9.0.0-alpha.1.23456.7" />
 ```
 
-You also need to specify the sysroot directory for Clang using the `SysRoot` property. For example, assuming you are using one of ARM64-targeting [Docker images](../../../../docs/workflow/building/coreclr/linux-instructions.md#Docker-Images) employed for cross-compilation by this repo, you may publish for linux-arm64 with the following command:
-```bash
-> dotnet publish -r linux-arm64 -c Release -p:CppCompilerAndLinker=clang-9 -p:SysRoot=/crossrootfs/arm64
-```
-
-You may also follow [cross-building instructions](../../../../docs/workflow/building/coreclr/cross-building.md) to create your own sysroot directory.
-
-See [Building native AOT apps in containers](containers.md) for a potentially quicker path to establishing a cross-compilation environment.
-
 ## Using statically linked ICU
 This feature can statically link libicu libraries (such as libicui18n.a) into your applications at build time.
 NativeAOT binaries built with this feature can run even when libicu libraries are not installed.
diff --git a/src/coreclr/nativeaot/docs/containers.md b/src/coreclr/nativeaot/docs/containers.md
index d5eab273a53a5..35bacdcbc74bd 100644
--- a/src/coreclr/nativeaot/docs/containers.md
+++ b/src/coreclr/nativeaot/docs/containers.md
@@ -14,7 +14,7 @@ The [`releasesapi`](https://github.com/dotnet/dotnet-docker/blob/main/samples/re
 
 For cloud native apps, build and runtime OS typically match, at least if you use multi-stage build. Once you step out of containers (for app delivery), it is more likely that you are delivering binaries that you want to work in more places (like on older Linux distros).
 
-The .NET build has this exact same need. We produce several [container images to enable cross-building](https://github.com/dotnet/runtime/blob/main/docs/workflow/building/coreclr/linux-instructions.md).
+The .NET build has this exact same need. We produce several [container images to enable cross-building](https://github.com/dotnet/runtime/blob/main/docs/workflow/building/coreclr/linux-instructions.md#docker-images).
 
 You can use these images to build native AOT apps which work on distros as old as Ubuntu 16.04. These build images are not supported, but are expected to work (since we use them to build .NET on daily basis).