Test the policy controller admission webhook (#8008)

The policy controller has an validating webhook for `Server` resources,
but this functionality is not really tested.

Before adding more policy resources that need validation, let's add an
integration test that exercises resource validation. The initial test is
pretty simplistic, but this is just setup.

These test also help expose two issues:

1. The change in 8760c5f--to solely use the index for validation--is
   problematic, especially in CI where quick updates can pass validation
   when they should not. This is fixed by going back to making API calls
   when validating `Server` resources.
2. Our pod selector overlap detection is overly simplistic. This change
   updates it to at least detect when a server selects _all_ pods.
   There's probably more we can do here in followup changes.

Tests are added in a new `policy-test` crate that only includes these
tests and the utiltities they need. This crate is excluded when running
unit tests and is only executed when it has a Kubernetes cluster it can
execute against. A temporary namespace is created before each test is
run and deleted as the test completes.

The policy controller's CI workflow is updated to build the core control
plane, run a k3d cluster, and exercise tests. This workflow has minimal
dependencies on the existing script/CI tooling so that the dependencies
are explicit and we can avoid some of the complexity of the existing
test infrastructure.

Signed-off-by: Oliver Gould <ver@buoyant.io>

.github/workflows/policy_controller.yml

@@ -3,12 +3,15 @@ name: Policy Controller
 on:
   pull_request:
     paths:
-      - 'Cargo.lock'
-      - 'Cargo.toml'
-      - 'deny.toml'
-      - 'rust-toolchain'
-      - 'policy-controller/**'
-      - '.github/workflows/policy_controller.yml'
+      - .github/workflows/policy_controller.yml
+      - Cargo.lock
+      - Cargo.toml
+      - charts/linkerd-control-plane/templates/destination-rbac.yaml
+      - charts/linkerd-crds/templates/policy/**
+      - deny.toml
+      - policy-controller/**
+      - policy-test/**
+      - rust-toolchain
 
 permissions:
   contents: read
@@ -17,6 +20,8 @@ env:
   CARGO_ACTION_FMT_VERSION: v0.1.3
   CARGO_INCREMENTAL: 0
   CARGO_NET_RETRY: 10
+  K3D_VERSION: v5.3.0
+  PROXY_INIT_VERSION: v1.5.3
   RUST_BACKTRACE: short
   RUSTUP_MAX_RETRIES: 10
 
@@ -58,7 +63,7 @@ jobs:
       - run: |
           bin/scurl -o /usr/local/bin/cargo-action-fmt "https://github.com/olix0r/cargo-action-fmt/releases/download/release%2F${CARGO_ACTION_FMT_VERSION}/cargo-action-fmt-x86_64-unknown-linux-gnu"
           chmod 755 /usr/local/bin/cargo-action-fmt
-      - run: cargo fetch
+      - run: cargo fetch --locked
       - run: cargo clippy --frozen --all --no-deps --message-format=json | cargo-action-fmt
 
   check:
@@ -71,7 +76,7 @@ jobs:
       - run: |
           bin/scurl -o /usr/local/bin/cargo-action-fmt "https://github.com/olix0r/cargo-action-fmt/releases/download/release%2F${CARGO_ACTION_FMT_VERSION}/cargo-action-fmt-x86_64-unknown-linux-gnu"
           chmod 755 /usr/local/bin/cargo-action-fmt
-      - run: cargo fetch
+      - run: cargo fetch --locked
       # Check each crate independently to ensure its Cargo.toml is sufficient.
       - run: |
           for toml in $(find . -mindepth 2 -name Cargo.toml | sort -r)
@@ -89,9 +94,9 @@ jobs:
       image: docker://rust:1.59.0
     steps:
       - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846
-      - run: cargo fetch
-      - run: cargo test --workspace --frozen --no-run
-      - run: cargo test --workspace --frozen
+      - run: cargo fetch --locked
+      - run: cargo test --workspace --exclude=linkerd-policy-test --frozen --no-run
+      - run: cargo test --workspace --exclude=linkerd-policy-test --frozen
 
   rust-toolchain:
     name: rust toolchain
@@ -120,3 +125,96 @@ jobs:
           done
 
           exit $ex
+
+  docker_build:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        component:
+          - controller
+          - policy-controller
+          - proxy
+    name: Docker build (${{ matrix.component }})
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846
+      - uses: ./.github/actions/docker-build
+        with:
+          docker-registry: ghcr.io/linkerd
+          docker-target: linux-amd64
+          component: ${{ matrix.component }}
+      # TAG is set by docker-build
+      - run: echo $TAG
+      - name: Create artifact with CLI and image archives
+        run: |
+          mkdir -p /home/runner/archives
+          docker save "ghcr.io/linkerd/${{ matrix.component }}:$TAG" \
+            >/home/runner/archives/${{ matrix.component }}.tar
+      - name: Upload artifact
+        uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535
+        with:
+          name: image-archives
+          path: /home/runner/archives
+
+  integration:
+    needs: [docker_build]
+    name: Policy controller integration
+    runs-on: ubuntu-20.04
+    timeout-minutes: 20
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846
+
+      - uses: actions/setup-go@f6164bd8c8acb4a71fb2791a8b6c4024ff038dab
+        with:
+          go-version: '1.17'
+      - name: Build the Linkerd CLI
+        run: bin/linkerd version --short --client
+
+      - run: bin/scurl -v https://raw.githubusercontent.com/k3d-io/k3d/${K3D_VERSION}/install.sh | bash
+      - run: k3d --version
+      - run: k3d cluster create --no-lb --k3s-arg "--no-deploy=local-storage,traefik,servicelb,metrics-server@server:*"
+      - run: kubectl version
+
+      - name: Download image archives
+        uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741
+        with:
+          name: image-archives
+          path: image-archives
+      - name: Load images
+        run: |
+          docker load <image-archives/controller.tar
+          docker load <image-archives/policy-controller.tar
+          docker load <image-archives/proxy.tar
+          docker pull ghcr.io/linkerd/proxy-init:$PROXY_INIT_VERSION
+          docker image ls | grep ghcr.io/linkerd
+          tag="$(CI_FORCE_CLEAN=1 bin/root-tag)"
+          k3d image import \
+            ghcr.io/linkerd/controller:$tag \
+            ghcr.io/linkerd/policy-controller:$tag \
+            ghcr.io/linkerd/proxy:$tag \
+            ghcr.io/linkerd/proxy-init:$PROXY_INIT_VERSION
+
+      - run: bin/linkerd check --pre --wait=1m
+      - run: bin/linkerd install | kubectl apply -f -
+        env:
+          LINKERD_DOCKER_REGISTRY: ghcr.io/linkerd
+      - run: bin/linkerd check --wait=1m
+
+      - name: Install rust
+        run: |
+          rm -rf $HOME/.cargo
+          bin/scurl -v https://sh.rustup.rs | sh -s -- -y --default-toolchain $(cat rust-toolchain)
+          source $HOME/.cargo/env
+          echo "PATH=$PATH" >> $GITHUB_ENV
+          cargo version
+
+      - name: Install cargo-action-fmt
+        run: |
+          bin/scurl -o /usr/local/bin/cargo-action-fmt "https://github.com/olix0r/cargo-action-fmt/releases/download/release%2F${CARGO_ACTION_FMT_VERSION}/cargo-action-fmt-x86_64-unknown-linux-gnu"
+          chmod 755 /usr/local/bin/cargo-action-fmt
+
+      - run: cargo fetch --locked
+      - run: cargo test -p linkerd-policy-test --frozen --no-run | cargo-action-fmt
+      - run: cargo test -p linkerd-policy-test --frozen

Cargo.lock

@@ -22,6 +22,15 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "ansi_term"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.55"
@@ -921,6 +930,22 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "linkerd-policy-test"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "k8s-openapi",
+ "kube",
+ "linkerd-policy-controller-k8s-api",
+ "rand",
+ "serde",
+ "tokio",
+ "tokio-test",
+ "tracing",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "linkerd2-proxy-api"
 version = "0.3.1"
@@ -1822,6 +1847,19 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-test"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53474327ae5e166530d17f2d956afcb4f8a004de581b3cae10f12006bc8163e3"
+dependencies = [
+ "async-stream",
+ "bytes",
+ "futures-core",
+ "tokio",
+ "tokio-stream",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.6.9"
@@ -2017,6 +2055,7 @@ version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e0ab7bdc962035a87fba73f3acca9b8a8d0034c2e6f60b84aeaaddddc155dce"
 dependencies = [
+ "ansi_term",
  "lazy_static",
  "matchers",
  "regex",

Cargo.toml

@@ -1,11 +1,11 @@
 [workspace]
-resolver = "2"
 members = [
     "policy-controller",
     "policy-controller/core",
     "policy-controller/grpc",
     "policy-controller/k8s/api",
     "policy-controller/k8s/index",
+    "policy-test",
 ]
 
 [profile.release]

policy-controller/amd64.dockerfile

@@ -7,9 +7,10 @@ ARG TARGETARCH
 WORKDIR /build
 COPY Cargo.toml Cargo.lock .
 COPY policy-controller policy-controller
+RUN cargo new policy-test --lib
 RUN --mount=type=cache,target=target \
     --mount=type=cache,from=rust:1.59.0,source=/usr/local/cargo,target=/usr/local/cargo \
-    cargo fetch --locked
+    cargo fetch
 RUN --mount=type=cache,target=target \
     --mount=type=cache,from=rust:1.59.0,source=/usr/local/cargo,target=/usr/local/cargo \
     cargo build --frozen --target=x86_64-unknown-linux-gnu --release --package=linkerd-policy-controller && \

policy-controller/arm.dockerfile

@@ -10,9 +10,10 @@ ENV CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc
 WORKDIR /build
 COPY Cargo.toml Cargo.lock .
 COPY policy-controller policy-controller
+RUN cargo new policy-test --lib
 RUN --mount=type=cache,target=target \
     --mount=type=cache,from=rust:1.59.0,source=/usr/local/cargo,target=/usr/local/cargo \
-    cargo fetch --locked
+    cargo fetch
 # XXX(ver) we can't easily cross-compile against openssl, so use rustls on arm.
 RUN --mount=type=cache,target=target \
     --mount=type=cache,from=rust:1.59.0,source=/usr/local/cargo,target=/usr/local/cargo \

policy-controller/arm64.dockerfile

@@ -10,9 +10,10 @@ ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc
 WORKDIR /build
 COPY Cargo.toml Cargo.lock .
 COPY policy-controller policy-controller
+RUN cargo new policy-test --lib
 RUN --mount=type=cache,target=target \
     --mount=type=cache,from=rust:1.59.0,source=/usr/local/cargo,target=/usr/local/cargo \
-    cargo fetch --locked
+    cargo fetch
 # XXX(ver) we can't easily cross-compile against openssl, so use rustls on arm.
 RUN --mount=type=cache,target=target \
     --mount=type=cache,from=rust:1.59.0,source=/usr/local/cargo,target=/usr/local/cargo \

policy-controller/k8s/api/src/labels.rs

@@ -61,6 +61,16 @@ impl Selector {
         }
     }
 
+    /// Indicates whether this label selector matches all pods
+    pub fn selects_all(&self) -> bool {
+        match (self.match_labels.as_ref(), self.match_expressions.as_ref()) {
+            (None, None) => true,
+            (Some(l), None) => l.is_empty(),
+            (None, Some(e)) => e.is_empty(),
+            (Some(l), Some(e)) => l.is_empty() && e.is_empty(),
+        }
+    }
+
     pub fn matches(&self, labels: &Labels) -> bool {
         for expr in self.match_expressions.iter().flatten() {
             if !expr.matches(labels.as_ref()) {