New GCP Provider: Using kata's main branch

.gitignore

@@ -36,3 +36,6 @@ src/cloud-api-adaptor/*.qcow2
 .git-commit
 .git-commit.tmp
 *.tar.gz
+
+# Secrets
+install/overlays/gcp/GCP_CREDENTIALS

src/cloud-api-adaptor/Makefile

@@ -26,9 +26,9 @@ RESOURCE_CTRL ?= true
 YQ_CHECKSUM_${ARCH} ?= $(YQ_CHECKSUM)
 # BUILTIN_CLOUD_PROVIDERS is used for binary build -- what providers are built in the binaries.
 ifeq ($(RELEASE_BUILD),true)
-	BUILTIN_CLOUD_PROVIDERS ?= aws azure ibmcloud vsphere
+	BUILTIN_CLOUD_PROVIDERS ?= aws azure gcp ibmcloud vsphere
 else
-	BUILTIN_CLOUD_PROVIDERS ?= aws azure ibmcloud vsphere libvirt docker
+	BUILTIN_CLOUD_PROVIDERS ?= aws azure gcp ibmcloud vsphere libvirt docker
 endif
 
 all: build

src/cloud-api-adaptor/cmd/cloud-api-adaptor/gcp.go

@@ -0,0 +1,10 @@
+//go:build gcp
+
+// (C) Copyright Confidential Containers Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	_ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/gcp"
+)

src/cloud-api-adaptor/entrypoint.sh

@@ -86,6 +86,22 @@ azure() {
         ${optionals}
 }
 
+gcp() {
+test_vars GCP_CREDENTIALS
+
+[[ "${PODVM_IMAGE_NAME}" ]] && optionals+="-gcp-image-name ${PODVM_IMAGE_NAME} "
+[[ "${GCP_PROJECT_ID}" ]] && optionals+="-gcp-project-id ${GCP_PROJECT_ID} "
+[[ "${GCP_ZONE}" ]] && optionals+="-gcp-zone ${GCP_ZONE} " # if not set retrieved from IMDS
+[[ "${GCP_MACHINE_TYPE}" ]] && optionals+="-gcp-machine-type ${GCP_MACHINE_TYPE} " # default e2-medium
+[[ "${GCP_NETWORK}" ]] && optionals+="-gcp-network ${GCP_NETWORK} " # defaults to 'default'
+
+set -x
+exec cloud-api-adaptor gcp \
+	-pods-dir /run/peerpod/pods \
+	${optionals} \
+	-socket /run/peerpod/hypervisor.sock
+}
+
 ibmcloud() {
     one_of IBMCLOUD_API_KEY IBMCLOUD_IAM_PROFILE_ID
 
@@ -178,9 +194,10 @@ docker() {
 help_msg() {
     cat <<EOF
 Usage:
-	CLOUD_PROVIDER=aws|azure|ibmcloud|ibmcloud-powervs|libvirt|vsphere|docker $0
+	CLOUD_PROVIDER=aws|azure|gcp|ibmcloud|ibmcloud-powervs|libvirt|vsphere|docker $0
 or
-	$0 aws|azure|ibmcloud|ibmcloud-powervs|libvirt|vsphere|docker
+	$0 aws|azure|gcp|ibmcloud|ibmcloud-powervs|libvirt|vsphere|docker
+
 in addition all cloud provider specific env variables must be set and valid
 (CLOUD_PROVIDER is currently set to "$CLOUD_PROVIDER")
 EOF
@@ -190,6 +207,8 @@ if [[ "$CLOUD_PROVIDER" == "aws" ]]; then
     aws
 elif [[ "$CLOUD_PROVIDER" == "azure" ]]; then
     azure
+elif [[ "$CLOUD_PROVIDER" == "gcp" ]]; then
+    gcp
 elif [[ "$CLOUD_PROVIDER" == "ibmcloud" ]]; then
     ibmcloud
 elif [[ "$CLOUD_PROVIDER" == "ibmcloud-powervs" ]]; then

src/cloud-api-adaptor/gcp/README.md

@@ -0,0 +1,196 @@
+# Cloud API Adaptor on GCP
+
+This documentation will walk you through setting up Cloud API Adaptor (CAA) on
+Google Compute Engine (GCE). We will build the pod vm image, CCA dev image and
+experiment on a local libvirt cluster.
+
+## Build Pod VM Image
+
+### Modifying existing marketplace image
+
+Install packer by following [these
+instructions](https://learn.hashicorp.com/tutorials/packer/get-started-install-cli).
+
+Create 'packer' service account with compute instance admin role:
+
+```bash
+export GCP_PROJECT_ID="REPLACE_ME"
+gcloud iam service-accounts create packer \
+ --project ${GCP_PROJECT_ID} \
+  --description="Packer Service Account" \
+  --display-name="Packer Service Account"
+
+gcloud projects add-iam-policy-binding ${GCP_PROJECT_ID} \
+  --member=serviceAccount:packer@${GCP_PROJECT_ID}.iam.gserviceaccount.com \
+  --role=roles/compute.instanceAdmin.v1
+
+gcloud projects add-iam-policy-binding ${GCP_PROJECT_ID} \
+  --member=serviceAccount:packer@${GCP_PROJECT_ID}.iam.gserviceaccount.com \
+  --role=roles/iam.serviceAccountUser
+```
+
+Create application access token:
+
+```bash
+gcloud iam service-accounts keys create ${HOME}/.config/gcloud/packer_application_key.json \
+  --iam-account=packer@${GCP_PROJECT_ID}.iam.gserviceaccount.com
+
+export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/.config/gcloud/packer_application_key.json
+```
+
+Create a custom GCP VM image based on Ubuntu 20.04 having kata-agent,
+agent-protocol-forwarder and other dependencies.
+
+```bash
+cd image
+export GCP_ZONE="REPLACE_ME" # e.g. "us-west1-a"
+export GCP_MACHINE_TYPE="REPLACE_ME" # default is "e2-medium"
+export GCP_NETWORK="REPLACE_ME" # default is "default"
+export CLOUD_PROVIDER=gcp
+PODVM_DISTRO=ubuntu make image && cd -
+```
+
+You can also build the image using docker:
+
+```bash
+cd image
+DOCKER_BUILDKIT=1 docker build -t gcp \
+  --secret id=APPLICATION_KEY,src=${GOOGLE_APPLICATION_CREDENTIALS} \
+  --build-arg GCP_PROJECT_ID=${GCP_PROJECT_ID} \
+  --build-arg GCP_ZONE=${GCP_ZONE} \
+  -f Dockerfile .
+```
+
+## Local experimentation
+
+### Setup a local cluster
+
+For local development we'll use libvirt setup and a local Docker registry.
+Run local Docker registry:
+
+```bash
+docker stop registry
+docker rm registry
+docker run -d -p 5000:5000 --restart=always --name registry registry:2.7.0
+```
+
+Follow instructions in [libvirt/README.md](../libvirt/README.md) and deploy a
+Kubernetes cluster in a Kubeadm setup.
+
+```
+$ kcli list kube
++-----------+---------+-----------+-----------------------------------------+
+|  Cluster  |   Type  |    Plan   |                  Vms                    |
++-----------+---------+-----------+-----------------------------------------+
+| peer-pods | generic | peer-pods | peer-pods-ctlplane-0,peer-pods-worker-0 |
++-----------+---------+-----------+-----------------------------------------+
+
+$ kubectl get nodes
+NAME                 STATUS   ROLES                  AGE     VERSION
+peer-pods-ctlplane-0 Ready    control-plane,master   6m8s    v1.25.3
+peer-pods-worker-0   Ready    worker                 2m47s   v1.25.3
+```
+
+### Deploy the CoCo operator
+
+Deploy operator (copied from `deploy` target in [Makefile](../Makefile)):
+
+```bash
+kubectl apply -k "github.com/confidential-containers/operator/config/default"
+kubectl apply -k "github.com/confidential-containers/operator/config/samples/ccruntime/peer-pods"
+```
+
+Register "insecure" local Registry in worker node:
+
+```bash
+$ kcli ssh
+...
+ubuntu@peer-pods-worker-0:~$ sudo vim /etc/containerd/config.toml
+```
+
+Add the following:
+
+```
+[plugins."io.containerd.grpc.v1.cri".registry]
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."192.168.122.1:5000"]
+      endpoint = ["http://192.168.122.1:5000"]
+  [plugins."io.containerd.grpc.v1.cri".registry.configs]
+    [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.122.1:5000".tls]
+      insecure_skip_verify = true
+```
+
+Restart `containerd`:
+
+```bash
+ubuntu@peer-pods-worker-0:~$ sudo systemctl restart containerd
+```
+
+Now the worker node can download local dev CCA images from the local registry running on the host.
+
+### Build CCA dev image
+
+Build and push docker image to local registry:
+
+```bash
+export registry=192.168.122.1:5000
+export RELEASE_BUILD=true
+make image
+```
+
+### Configure VPC network
+
+Go to your GCP console, under "VPC networks" update the `default` (global)
+network and add a firewall rule that allows incoming TCP connections over port
+15150 from your workstation external IP address.
+
+## Deploy CCA
+
+Update [install/overlays/gcp/kustomization.yaml](../install/overlays/gcp/kustomization.yaml) with the required fields:
+
+```
+images:
+- name: cloud-api-adaptor
+  newName: 192.168.122.1:5000/cloud-api-adaptor # change image if needed
+  newTag: 47dcc2822b6c2489a02db83c520cf9fdcc833b3f-dirty # change if needed
+  ...
+configMapGenerator:
+- name: peer-pods-cm
+  namespace: confidential-containers-system
+  literals:
+  - CLOUD_PROVIDER="gcp" # leave as is.
+  - PODVM_IMAGE_NAME="" # set from step 1 above.
+  - GCP_PROJECT_ID="" # set
+  - GCP_ZONE="" # set
+  - GCP_MACHINE_TYPE="e2-medium" # defaults to e2-medium
+  - GCP_NETWORK="global/networks/default" # leave as is.
+  ...
+secretGenerator:
+  ...
+- name: peer-pods-secret
+  namespace: confidential-containers-system
+  files:
+  - GCP_CREDENTIALS # Make sure this file has the application creadentials. You can reuse the Packer creds: copy the file from ${HOME}/.config/gcloud/packer_application_key.json
+```
+
+```bash
+$ kubectl apply -k install/overlays/gcp/
+$ kubectl get pods -n confidential-containers-system
+NAME                                              READY   STATUS    RESTARTS   AGE
+cc-operator-controller-manager-546574cf87-b69pb   2/2     Running   0          7d10h
+cc-operator-daemon-install-mfjbj                  1/1     Running   0          7d10h
+cc-operator-pre-install-daemon-g2jsj              1/1     Running   0          7d10h
+cloud-api-adaptor-daemonset-5w8nw                 1/1     Running   0          7s
+```
+
+## Test with a simple workflow
+
+Deploy the `sample_busybox.yaml` (see [libvirt/README.md](../libvirt/README.md)):
+
+```
+$ kubectl apply -f sample_busybox.yaml
+pod/busybox created
+```
+
+Examine your GCP console, under "Compute Engine", "VM instances" you should see the new POD instance running.
+Examine `kubectl logs` and verify the tunnel to the podvm was established successfully.

src/cloud-api-adaptor/gcp/image/Dockerfile

@@ -0,0 +1,63 @@
+# Copyright Confidential Containers Contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Builds pod vm image inside container
+#
+# syntax=docker/dockerfile:1.3
+ARG BUILDER_IMG="quay.io/confidential-containers/podvm-builder-ubuntu"
+ARG BINARIES_IMG="quay.io/confidential-containers/podvm-binaries-ubuntu-amd64"
+
+FROM ${BINARIES_IMG} AS podvm_binaries
+FROM ${BUILDER_IMG} AS podvm_builder
+
+ARG CLOUD_PROVIDER=gcp
+ARG PODVM_DISTRO=ubuntu
+# If not provided, uses system architecture
+ARG ARCH=x86_64
+ARG CAA_SRC=""
+ARG CAA_SRC_REF=""
+
+ENV CLOUD_PROVIDER ${CLOUD_PROVIDER}
+ENV PODVM_DISTRO ${PODVM_DISTRO}
+
+RUN if [ -n "${CAA_SRC}" ]; then \
+      rm -rf cloud-api-adaptor && \
+      git clone ${CAA_SRC} cloud-api-adaptor;\
+    fi && \
+    if [ -n "${CAA_SRC_REF}" ]; then \
+      cd cloud-api-adaptor && \
+      git fetch origin ${CAA_SRC_REF} && \
+      git checkout FETCH_HEAD -b ${CAA_SRC_REF} ;\
+    fi
+
+RUN mkdir -p /src/cloud-api-adaptor/podvm/files
+
+# Copy the binaries to podvm/files folder
+COPY --from=podvm_binaries /podvm-binaries.tar.gz /src/cloud-api-adaptor/podvm/files
+RUN tar xvf /src/cloud-api-adaptor/podvm/files/podvm-binaries.tar.gz -C /src/cloud-api-adaptor/podvm/files
+
+# Copy the pause_bundle to podvm/files folder
+COPY --from=podvm_binaries /pause-bundle.tar.gz /src/cloud-api-adaptor/podvm/files
+RUN tar xvf /src/cloud-api-adaptor/podvm/files/pause-bundle.tar.gz -C /src/cloud-api-adaptor/podvm/files
+
+ARG GCP_PROJECT_ID
+ARG GCP_ZONE
+ARG GCP_MACHINE_TYPE=e2-medium
+ARG GCP_NETWORK=default
+ARG IMAGE_NAME=peer-pod-vmimage
+
+ENV GCP_PROJECT_ID ${GCP_PROJECT_ID}
+ENV GCP_ZONE ${GCP_ZONE}
+ENV GCP_MACHINE_TYPE ${GCP_MACHINE_TYPE}
+ENV GCP_NETWORK ${GCP_NETWORK}
+ENV IMAGE_NAME ${IMAGE_NAME}
+
+# Uncomment for development.
+COPY . cloud-api-adaptor/gcp/image
+
+RUN --mount=type=secret,id=APPLICATION_KEY \
+    export GOOGLE_APPLICATION_CREDENTIALS="/run/secrets/APPLICATION_KEY" && \
+    cd cloud-api-adaptor/gcp/image && \
+    packer init ${PODVM_DISTRO}/ && \
+    BINARIES= PAUSE_BUNDLE= CLOUD_PROVIDER=gcp PODVM_DISTRO=$PODVM_DISTRO make image

src/cloud-api-adaptor/gcp/image/Makefile

@@ -0,0 +1,59 @@
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+#include ../../podvm/Makefile.inc
+
+
+.PHONY: image clean
+
+GCP_MACHINE_TYPE ?= e2-medium
+GCP_NETWORK ?= default
+
+PODVM_DISTRO ?= ubuntu
+
+PODVM_BUILDER_IMAGE ?= $(REGISTRY)/podvm-builder-$(PODVM_DISTRO):$(PODVM_TAG)
+PODVM_BINARIES_IMAGE ?= $(REGISTRY)/podvm-binaries-$(PODVM_DISTRO)-$(ARCH):$(PODVM_TAG)
+
+IMAGE_FILE ?= podvm-$(or $(CLOUD_PROVIDER),gcp)-$(PODVM_DISTRO)-$(ARCH)
+PODVM_IMAGE ?= $(REGISTRY)/$(IMAGE_FILE):$(PODVM_TAG)
+
+image: $(IMAGE_FILE)
+
+$(IMAGE_FILE): setopts $(BINARIES) $(FILES)
+	mkdir -p toupload
+	packer build ${OPTS} \
+		-var zone=${GCP_ZONE} \
+		-var machine_type=${GCP_MACHINE_TYPE} \
+		-var network=${GCP_NETWORK} \
+		-var gce_image_name=${IMAGE_NAME} ./${PODVM_DISTRO}/
+	rm -fr toupload
+
+setopts:
+ifeq ($(PODVM_DISTRO),)
+	$(error PODVM_DISTRO is not defined)
+endif
+ifeq ($(GCP_ZONE),)
+	$(error GCP_ZONE is not defined)
+endif
+
+
+podvm-image:
+	docker buildx build --progress=plain --no-cache -t $(PODVM_IMAGE) -f Dockerfile \
+	--build-arg BUILDER_IMG=$(PODVM_BUILDER_IMAGE) \
+	--build-arg BINARIES_IMG=$(PODVM_BINARIES_IMAGE) \
+	--build-arg PODVM_DISTRO=$(PODVM_DISTRO) \
+	--build-arg ARCH=$(ARCH) \
+	--build-arg GCP_ZONE=$(GCP_ZONE) \
+	--build-arg GCP_PROJECT_ID=$(GCP_PROJECT_ID) \
+	--build-arg CLOUD_PROVIDER=$(or $(CLOUD_PROVIDER),gcp) \
+	--build-arg IMAGE_URL=$(IMAGE_URL) \
+	--build-arg IMAGE_CHECKSUM=$(IMAGE_CHECKSUM) \
+	 --secret id=APPLICATION_KEY,src=${GOOGLE_APPLICATION_CREDENTIALS} \
+	$(DOCKER_OPTS) .
+	rm -rf .git
+clean:
+	rm -f "$(IMAGE_FILE)" "$(UBUNTU_IMAGE_FILE)" $(BINARIES)
+	rm -fr "$(SKOPEO_SRC)" "$(UMOCI_SRC)" "$(PAUSE_SRC)" "$(FILES_DIR)/$(PAUSE_BUNDLE)"
+
+.PHONY: force
+force: