feat: Windows Container Support. Fixes #1507 and #1383

Dockerfile.windows

@@ -0,0 +1,72 @@
+####################################################################################################
+# Builder image
+# Initial stage which pulls prepares build dependencies and CLI tooling we need for our final image
+# Also used as the image in CI jobs so needs all dependencies
+####################################################################################################
+# had issues with official golange image for windows so I'm using plain servercore 
+FROM mcr.microsoft.com/windows/servercore:ltsc2019 as builder
+ENV GOLANG_VERSION=1.13.4
+SHELL ["powershell", "-Command"]
+
+ARG IMAGE_OS=windows
+ARG IMAGE_ARCH=amd64
+
+# install chocolatey package manager
+ENV chocolateyUseWindowsCompression=false
+RUN iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \
+    choco feature disable --name showDownloadProgress ; \
+    choco feature enable -n allowGlobalConfirmation
+
+# install golang, dep and other tools
+RUN choco install golang --version=$env:GOLANG_VERSION ; \
+    choco install make dep docker-cli git.portable
+
+####################################################################################################
+# argoexec-base
+# Used as the base for both the release and development version of argoexec
+####################################################################################################
+FROM mcr.microsoft.com/windows/nanoserver:1809 as argoexec-base
+COPY --from=builder /windows/system32/netapi32.dll /windows/system32/netapi32.dll
+
+ARG IMAGE_OS=windows
+ARG IMAGE_ARCH=amd64
+
+# NOTE: keep the version synced with https://storage.googleapis.com/kubernetes-release/release/stable.txt
+ENV KUBECTL_VERSION=1.15.1
+ENV JQ_VERSION=1.6
+
+RUN mkdir C:\app && \
+    curl -L -o C:\app\kubectl.exe "https://storage.googleapis.com/kubernetes-release/release/v%KUBECTL_VERSION%/bin/windows/amd64/kubectl.exe" && \
+    curl -L -o C:\app\jq.exe "https://github.com/stedolan/jq/releases/download/jq-%JQ_VERSION%/jq-win64.exe"
+
+COPY --from=builder C:/ProgramData/chocolatey/lib/docker-cli/tools/docker.exe C:/app/docker.exe
+COPY --from=builder C:/tools/git C:/app/git
+
+# add binaries to path
+USER Administrator
+RUN SETX /m path C:\app;C:\app\git\bin;%path%
+
+####################################################################################################
+# Argo Build stage which performs the actual build of Argo binaries
+####################################################################################################
+FROM builder as argo-build
+
+ARG IMAGE_OS=windows
+ARG IMAGE_ARCH=amd64
+
+# Perform the build
+WORKDIR C:/Users/ContainerAdministrator/go/src/github.com/argoproj/argo
+COPY . .
+# check we can use Git
+RUN git rev-parse HEAD
+# fail the build if we are "dirty"
+RUN git diff --exit-code
+# run in git bash for all the shell commands in Makefile to work
+RUN bash -c 'make dist/argoexec-windows-amd64'
+
+####################################################################################################
+# argoexec
+####################################################################################################
+FROM argoexec-base as argoexec
+COPY --from=argo-build C:/Users/ContainerAdministrator/go/src/github.com/argoproj/argo/dist/argoexec-windows-amd64 C:/app/argoexec.exe
+ENTRYPOINT [ "argoexec" ]

Makefile

@@ -206,6 +206,7 @@ $(CONTROLLER_IMAGE_FILE):
 # argoexec
 
 dist/argoexec-linux-amd64: GOARGS = GOOS=linux GOARCH=amd64
+dist/argoexec-windows-amd64: GOARGS = GOOS=windows GOARCH=amd64
 dist/argoexec-linux-arm64: GOARGS = GOOS=linux GOARCH=arm64
 
 dist/argoexec-%: $(ARGOEXEC_PKGS)

docs/README.md

@@ -41,6 +41,7 @@ Some use-case specific documentation is available:
 * [Transport Layer Security](tls.md)
 * [Workflow Variables](variables.md)
 * [Versioning](versioning.md)
+* [Windows Container Support](windows.md)
 * [Workflow Archive](workflow-archive.md)
 * [Workflow Controller Configmap](workflow-controller-configmap.md)
 * [Workflow Creator](workflow-creator.md)

docs/fields.md

@@ -82,6 +82,10 @@ Workflow is the definition of a workflow resource
 
 - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml)
 
+- [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml)
+
+- [`hello-windows.yaml`](../examples/hello-windows.yaml)
+
 - [`hello-world.yaml`](../examples/hello-world.yaml)
 
 - [`image-pull-secrets.yaml`](../examples/image-pull-secrets.yaml)
@@ -357,6 +361,10 @@ WorkflowSpec is the specification of a Workflow.
 
 - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml)
 
+- [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml)
+
+- [`hello-windows.yaml`](../examples/hello-windows.yaml)
+
 - [`hello-world.yaml`](../examples/hello-world.yaml)
 
 - [`image-pull-secrets.yaml`](../examples/image-pull-secrets.yaml)
@@ -645,6 +653,10 @@ CronWorkflowSpec is the specification of a CronWorkflow
 
 - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml)
 
+- [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml)
+
+- [`hello-windows.yaml`](../examples/hello-windows.yaml)
+
 - [`hello-world.yaml`](../examples/hello-world.yaml)
 
 - [`image-pull-secrets.yaml`](../examples/image-pull-secrets.yaml)
@@ -899,6 +911,10 @@ WorkflowTemplateSpec is a spec of WorkflowTemplate.
 
 - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml)
 
+- [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml)
+
+- [`hello-windows.yaml`](../examples/hello-windows.yaml)
+
 - [`hello-world.yaml`](../examples/hello-world.yaml)
 
 - [`image-pull-secrets.yaml`](../examples/image-pull-secrets.yaml)
@@ -1347,6 +1363,10 @@ Template is a reusable and composable unit of execution in a workflow
 
 - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml)
 
+- [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml)
+
+- [`hello-windows.yaml`](../examples/hello-windows.yaml)
+
 - [`hello-world.yaml`](../examples/hello-world.yaml)
 
 - [`image-pull-secrets.yaml`](../examples/image-pull-secrets.yaml)
@@ -2185,6 +2205,10 @@ Pod metdata
 
 - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml)
 
+- [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml)
+
+- [`hello-windows.yaml`](../examples/hello-windows.yaml)
+
 - [`hello-world.yaml`](../examples/hello-world.yaml)
 
 - [`image-pull-secrets.yaml`](../examples/image-pull-secrets.yaml)
@@ -2520,6 +2544,8 @@ WorkflowStep is a reference to a template to execute in a series of step
 
 - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml)
 
+- [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml)
+
 - [`influxdb-ci.yaml`](../examples/influxdb-ci.yaml)
 
 - [`k8s-orchestration.yaml`](../examples/k8s-orchestration.yaml)
@@ -3248,6 +3274,10 @@ ObjectMeta is metadata that all persisted resources must have, which includes al
 
 - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml)
 
+- [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml)
+
+- [`hello-windows.yaml`](../examples/hello-windows.yaml)
+
 - [`hello-world.yaml`](../examples/hello-world.yaml)
 
 - [`image-pull-secrets.yaml`](../examples/image-pull-secrets.yaml)
@@ -3676,6 +3706,10 @@ A single application container that you want to run within a pod.
 
 - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml)
 
+- [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml)
+
+- [`hello-windows.yaml`](../examples/hello-windows.yaml)
+
 - [`hello-world.yaml`](../examples/hello-world.yaml)
 
 - [`image-pull-secrets.yaml`](../examples/image-pull-secrets.yaml)
@@ -4250,6 +4284,10 @@ PersistentVolumeClaimSpec describes the common attributes of storage devicesand
 
 - [`hdfs-artifact.yaml`](../examples/hdfs-artifact.yaml)
 
+- [`hello-hybrid.yaml`](../examples/hello-hybrid.yaml)
+
+- [`hello-windows.yaml`](../examples/hello-windows.yaml)
+
 - [`hello-world.yaml`](../examples/hello-world.yaml)
 
 - [`image-pull-secrets.yaml`](../examples/image-pull-secrets.yaml)

docs/windows.md

@@ -0,0 +1,103 @@
+# Windows Container Support
+
+The Argo server and the workflow controller currently only run on Linux. The workflow executor however also runs on Windows nodes, meaning you can use Windows containers inside your workflows! Here are the steps to get started.
+
+## 0. Requirements
+* Kubernetes 1.14 or later, supporting Windows nodes
+* Hybrid cluster containing Linux and Windows nodes like described in the [Kubernetes docs](https://kubernetes.io/docs/setup/production-environment/windows/user-guide-windows-containers/)
+* Argo configured and running like described [here](getting-started.md) 
+
+## 1. Setting up the workflow executor
+
+Currently the worflow controller configuration doesn't support different configurations for the `dockerSockPath` based on the host OS. This means that the workflow executor, running in a Windows container can't use Docker for now.
+
+You therefore need to use `kubelet` or `k8sapi` instead in your workflow controller configmap:
+```yaml
+containerRuntimeExecutor: kubelet
+kubeletInsecure: true  # you can disable TLS verification of the kubelet executor for testing
+```
+
+## 2. Schedule workflows with Windows containers
+
+If you're running workflows in your hybrid Kubernetes cluster, always make sure to include a `nodeSelector` to run the steps on the correct host OS:
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: hello-windows-
+spec:
+  entrypoint: hello-win
+  templates:
+    - name: hello-win
+      nodeSelector:
+        kubernetes.io/os: windows    # specify the OS your step should run on
+      container:
+        image: mcr.microsoft.com/windows/nanoserver:1809
+        command: ["cmd", "/c"]
+        args: ["echo", "Hello from Windows Container!"]
+```
+
+You can run this example and get the logs:
+```
+$ argo submit --watch https://raw.githubusercontent.com/argoproj/argo/master/examples/hello-windows.yaml
+$ argo logs hello-windows-s9kk5
+hello-windows-s9kk5: "Hello from Windows Container!"
+```
+
+## Bonus: Hybrid workflows
+
+You can also run different steps on different host OSs. This can for example be very helpful when you need to compile your application on Windows and Linux.
+
+An example workflow can look like the following:
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: hello-hybrid-
+spec:
+  entrypoint: mytemplate
+  templates:
+    - name: mytemplate
+      steps:
+        - - name: step1
+            template: hello-win
+        - - name: step2
+            template: hello-linux
+
+    - name: hello-win
+      nodeSelector:
+        kubernetes.io/os: windows
+      container:
+        image: mcr.microsoft.com/windows/nanoserver:1809
+        command: ["cmd", "/c"]
+        args: ["echo", "Hello from Windows Container!"]
+    - name: hello-linux
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      container:
+        image: alpine
+        command: [echo]
+        args: ["Hello from Linux Container!"]
+
+```
+
+Again, you can run this example and get the logs:
+```
+$ argo submit --watch https://raw.githubusercontent.com/argoproj/argo/master/examples/hello-hybrid.yaml
+$ argo logs hello-hybrid-plqpp
+hello-hybrid-plqpp-1977432187: "Hello from Windows Container!"
+hello-hybrid-plqpp-764774907: Hello from Linux Container!
+```
+
+## Building the workflow executor image for Windows
+
+To build the workflow executor image for Windows you need a Windows machine running Windows Server 2019 with Docker installed like described [in the docs](https://docs.docker.com/ee/docker-ee/windows/docker-ee/#install-docker-engine---enterprise).
+
+You then clone the project and run the Docker build with the Dockerfile for Windows and `argoexec` as a target:
+
+```
+git clone https://github.com/argoproj/argo.git
+cd argo
+docker build -t myargoexec -f .\Dockerfile.windows --target argoexec .
+```

docs/workflow-controller-configmap.md

@@ -37,6 +37,8 @@ spec:
         image: argoproj/workflow-controller:latest
         name: workflow-controller
       serviceAccountName: argo
+      nodeSelector:
+              kubernetes.io/os: linux
 ```
 
 ## Alternate Structure

examples/hello-hybrid.yaml

@@ -0,0 +1,28 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: hello-hybrid-
+spec:
+  entrypoint: mytemplate
+  templates:
+    - name: mytemplate
+      steps:
+        - - name: step1
+            template: hello-win
+        - - name: step2
+            template: hello-linux
+
+    - name: hello-win
+      nodeSelector:
+        kubernetes.io/os: windows
+      container:
+        image: mcr.microsoft.com/windows/nanoserver:1809
+        command: ["cmd", "/c"]
+        args: ["echo", "Hello from Windows Container!"]
+    - name: hello-linux
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      container:
+        image: alpine
+        command: [echo]
+        args: ["Hello from Linux Container!"]

examples/hello-windows.yaml

@@ -0,0 +1,14 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: hello-windows-
+spec:
+  entrypoint: hello-win
+  templates:
+    - name: hello-win
+      nodeSelector:
+        kubernetes.io/os: windows
+      container:
+        image: mcr.microsoft.com/windows/nanoserver:1809
+        command: ["cmd", "/c"]
+        args: ["echo", "Hello from Windows Container!"]

manifests/base/argo-server/argo-server-deployment.yaml

@@ -26,3 +26,5 @@ spec:
               path: /
             initialDelaySeconds: 10
             periodSeconds: 20
+      nodeSelector:
+        kubernetes.io/os: linux

manifests/base/workflow-controller/workflow-controller-deployment.yaml

@@ -22,3 +22,5 @@ spec:
         - workflow-controller-configmap
         - --executor-image
         - argoproj/argoexec:latest
+      nodeSelector:
+        kubernetes.io/os: linux

manifests/install.yaml

@@ -407,6 +407,8 @@ spec:
             scheme: HTTP
           initialDelaySeconds: 10
           periodSeconds: 20
+      nodeSelector:
+        kubernetes.io/os: linux
       serviceAccountName: argo-server
 ---
 apiVersion: apps/v1
@@ -432,4 +434,6 @@ spec:
         - workflow-controller
         image: argoproj/workflow-controller:latest
         name: workflow-controller
+      nodeSelector:
+        kubernetes.io/os: linux
       serviceAccountName: argo