diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md new file mode 100644 index 000000000000..81c339a4c5ba --- /dev/null +++ b/scripts/e2e/README.md @@ -0,0 +1,91 @@ +Test cluster-api-provider-vsphere + +***Integration with Prow*** +apply hack/secret.yml to Prow cluster/local cluster +apply hack/job.yml at Prow cluster/local cluster +note: the actual Prow job definition file will be at k8s.io/test-infra + +``` + +-----------------------------------------------------+ + | | + | | + | container running on Prow cluster: | + | | + | create bootstrap cluster (on VMC) | + | transfer secret from Prow to bootstrap | + | launch a ci job at bootstrap | + | monitor job status | + | | + | | + | +---------------------+ | + | | secret | | + | +---------------------+ | + +-----------------------------------------------------+ + + +-------------------------------------------------------+ + | +--------------------------------------------+ | + | | secret: target VM SSH, bootstrap cluster | | + | | kubeconfig, vsphere info | | + | | | | + | +--------------------------------------------+ | + | | + | +-----------------------+ | + | | | | + | | CI job: | | + | | create target cluster | | + | | on VMC | | + | +-----------------------+ | + | | + | BOOTSTRAP CLUSTER (on VMC) | + | | + +-------------------------------------------------------+ +``` + + +***Launch CI from travis-ci*** +``` +docker run \ + --rm \ + -v $HOME/.ssh:/root/ssh \ + -e GOVC_URL=$GOVC_URL \ + -e GOVC_USERNAME=$GOVC_USERNAME \ + -e GOVC_PASSWORD=$GOVC_PASSWORD \ + -e JUMPHOST=$JUMPHOST \ + -e GOVC_INSECURE="true" \ + -e VSPHERE_MACHINE_CONTROLLER_REGISTRY=$VSPHERE_MACHINE_CONTROLLER_REGISTRY \ + -ti luoh/cluster-api-provider-vsphere-travis-ci:latest +``` +note: set `$VSPHER_MACHINE_CONTROLLER_REGISTRY` if you want to test your local build controller + + +***Architecture*** +``` + + +-----------------------------------+ + +----------------------+ | VMC Infra | + | travis-ci env | |-----------------------------------| + |----------------------| |+----+ +--------------------------+| + | | || | | bootstrap cluster || + | | || | | || + | cluster-api-vsphere- | ||JUMP| | cluster-api-vsphere-ci || + | travis-ci | SSH + HTTP ||HOST| | (a k8s job) || + | | +-----------> || | | || + | | <-----------+ || | | || + | | || | +--------------------------+| + | | || | | + | | || | +--------------------------+| + | | || | | target cluster || + | | || | | || + | | || | | || + | | |+----+ +--------------------------+| + +----------------------+ +-----------------------------------+ + +``` + +***Containers*** +the vsphere-machine-controller containers for CI purpose are hosted at +`gcr.io/cnx-cluster-api/vsphere-cluster-api-provider` +the cluster-api-provider-vsphere-travis-ci hosted at +`luoh/cluster-api-provider-vsphere-travis-ci` +the cluster-api-provider-vsphere-ci hosted at +`gcr/cnx-cluster-api/cluster-api-provider-vsphere-ci` diff --git a/scripts/e2e/bootstrap_job.yml b/scripts/e2e/bootstrap_job.yml new file mode 100644 index 000000000000..4b78c6a02680 --- /dev/null +++ b/scripts/e2e/bootstrap_job.yml @@ -0,0 +1,65 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: cluster-api-provider-vsphere-ci +spec: + backoffLimit: 4 + template: + spec: + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + containers: + - name: cluster-api-provider-vsphere-ci + image: gcr.io/cnx-cluster-api/cluster-api-provider-vsphere-ci:latest + env: + - name: TARGET_VM_SSH + valueFrom: + secretKeyRef: + name: clusterapi-provider-vsphere-ci-prow + key: target-vm-ssh + - name: TARGET_VM_SSH_PUB + valueFrom: + secretKeyRef: + name: clusterapi-provider-vsphere-ci-prow + key: target-vm-ssh-pub + - name: VSPHERE_CONTROLLER_VERSION + valueFrom: + secretKeyRef: + name: clusterapi-provider-vsphere-ci-prow + key: vsphere-controller-version + - name: PROVIDER_COMPONENT_SPEC + valueFrom: + secretKeyRef: + name: clusterapi-provider-vsphere-ci-prow + key: provider-component-spec + - name: VSPHERE_SERVER + valueFrom: + secretKeyRef: + name: clusterapi-provider-vsphere-ci-prow + key: vsphere-server + - name: VSPHERE_USERNAME + valueFrom: + secretKeyRef: + name: clusterapi-provider-vsphere-ci-prow + key: vsphere-username + - name: VSPHERE_PASSWORD + valueFrom: + secretKeyRef: + name: clusterapi-provider-vsphere-ci-prow + key: vsphere-password + volumeMounts: + - name: kube + mountPath: /root/.kube + - name: sshkeys + mountPath: /root/.ssh + command: + - "./clusterctl.sh" + volumes: + - name: kube + hostPath: + path: /home/vmware/.kube + - name: sshkeys + hostPath: + path: /home/vmware/.ssh + restartPolicy: Never diff --git a/scripts/e2e/bootstrap_job/Dockerfile b/scripts/e2e/bootstrap_job/Dockerfile new file mode 100644 index 000000000000..ce8ce3b4a656 --- /dev/null +++ b/scripts/e2e/bootstrap_job/Dockerfile @@ -0,0 +1,12 @@ +FROM photon:2.0 +LABEL maintainer="Hui Luo " + +RUN tdnf install -y iputils wget openssh + +COPY *.sh /clusterapi/ +COPY bin /clusterapi/bin +COPY spec /clusterapi/spec + +WORKDIR /clusterapi/ +CMD ["shell"] +ENTRYPOINT ["/clusterapi/clusterctl.sh"] diff --git a/scripts/e2e/bootstrap_job/Makefile b/scripts/e2e/bootstrap_job/Makefile new file mode 100644 index 000000000000..0f9182fcd93c --- /dev/null +++ b/scripts/e2e/bootstrap_job/Makefile @@ -0,0 +1,24 @@ +# Makefile + +VERSION ?= $(shell git describe --exact-match 2> /dev/null || \ + git describe --match=$(git rev-parse --short=8 HEAD) --always --dirty --abbrev=8) +REGISTRY ?=gcr.io/cnx-cluster-api/cluster-api-provider-vsphere-ci + +all: build upload clean +.PHONY : all + +.PHONY : build +build: + docker build . --tag $(REGISTRY):$(VERSION) + docker tag $(REGISTRY):$(VERSION) $(REGISTRY):latest + +upload: + @echo "logging into gcr.io registry with key file" + @echo $$GCR_KEY_FILE | docker login -u _json_key --password-stdin gcr.io + docker push $(REGISTRY):$(VERSION) + docker push $(REGISTRY):latest + @echo docker logout gcr.io + +clean: + docker image rm -f $(REGISTRY):$(VERSION) + docker image rm -f $(REGISTRY):latest diff --git a/scripts/e2e/bootstrap_job/clusterctl.sh b/scripts/e2e/bootstrap_job/clusterctl.sh new file mode 100755 index 000000000000..45cca0b87ab1 --- /dev/null +++ b/scripts/e2e/bootstrap_job/clusterctl.sh @@ -0,0 +1,56 @@ +#!/bin/sh + +# this script takes care of everything after bootstrap cluster created, it means +# bootstrap need be created beforehand. + +# specs requires following enviroments variables: +# VSPHERE_SERVER +# VSPHERE_USERNAME +# VSPHERE_PASSWORD +# VSPHERE_CONTROLLER_VERSION +# TARGET_VM_SSH (base64 encoded) +# TARGET_VM_SSH_PUB (base64 encoded) + + +# base64 encode SSH keys (k8s secret automatically decode it) +export TARGET_VM_SSH=$(echo -n "${TARGET_VM_SSH}" | base64 -w 0) +export TARGET_VM_SSH_PUB=$(echo -n "${TARGET_VM_SSH_PUB}" | base64 -w 0) + +for filename in spec/*.template; do + newfilename="$(echo "$filename" | sed 's/template/yml/g')" + rm -f "$newfilename" temp.sh + ( echo "cat <$newfilename"; + cat "$filename"; + echo "EOF"; + ) >temp.sh + chmod +x temp.sh + ./temp.sh +done +rm temp.sh + +# download kubectl binary +retry=20 +until [ "$(ping www.google.com -c 1)" ] +do + sleep 6 + retry=$((retry - 1)) + if [ $retry -lt 0 ] + then + echo "can not access internet" + exit 1 + fi +done +wget https://storage.googleapis.com/kubernetes-release/release/v1.10.2/bin/linux/amd64/kubectl \ + -O /usr/local/bin/kubectl +chmod +x /usr/local/bin/kubectl + +# run clusterctl +echo "test ${PROVIDER_COMPONENT_SPEC}" +./bin/clusterctl create cluster --existing-bootstrap-cluster-kubeconfig ~/.kube/config -c ./spec/cluster.yml \ + -m ./spec/machines.yml \ + -p ./spec/${PROVIDER_COMPONENT_SPEC} \ + --provider vsphere \ + -v 6 + +# cleanup the cluster +# TODO (clusterctl delete is not working) diff --git a/scripts/e2e/bootstrap_job/spec/cluster.template b/scripts/e2e/bootstrap_job/spec/cluster.template new file mode 100644 index 000000000000..5465539e9779 --- /dev/null +++ b/scripts/e2e/bootstrap_job/spec/cluster.template @@ -0,0 +1,18 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: Cluster +metadata: + name: test1 +spec: + clusterNetwork: + services: + cidrBlocks: ["10.96.0.0/12"] + pods: + cidrBlocks: ["10.244.0.0/16"] + serviceDomain: "cluster.local" + providerConfig: + value: + apiVersion: "vsphereproviderconfig/v1alpha1" + kind: "VsphereClusterProviderConfig" + vsphereUser: "$VSPHERE_USERNAME" + vspherePassword: "$VSPHERE_PASSWORD" + vsphereServer: "$VSPHERE_SERVER" diff --git a/scripts/e2e/bootstrap_job/spec/machines.template b/scripts/e2e/bootstrap_job/spec/machines.template new file mode 100644 index 000000000000..c94647926c56 --- /dev/null +++ b/scripts/e2e/bootstrap_job/spec/machines.template @@ -0,0 +1,32 @@ +items: +- apiVersion: "cluster.k8s.io/v1alpha1" + kind: Machine + metadata: + generateName: clusterapi-prow- + labels: + set: master + spec: + providerConfig: + value: + apiVersion: "vsphereproviderconfig/v1alpha1" + kind: "VsphereMachineProviderConfig" + machineSpec: + datacenter: "SDDC-Datacenter" + datastore: "WorkloadDatastore" + resourcePool: "clusterapi" + vmFolder: "clusterapi" + networks: + - networkName: "sddc-cgw-network-3" + ipConfig: + networkType: dhcp + numCPUs: 2 + memoryMB: 2048 + template: "ubuntu-16.04-server-cloudimg-amd64" + disks: + - diskLabel: "Hard disk 1" + diskSizeGB: 15 + versions: + kubelet: 1.11.2 + controlPlane: 1.11.2 + roles: + - Master diff --git a/scripts/e2e/bootstrap_job/spec/machineset.template b/scripts/e2e/bootstrap_job/spec/machineset.template new file mode 100644 index 000000000000..a650aca2243d --- /dev/null +++ b/scripts/e2e/bootstrap_job/spec/machineset.template @@ -0,0 +1,37 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineSet +metadata: + name: clusterapi-machineset-1 +spec: + replicas: 2 + selector: + matchLabels: + node-type: worker-node + template: + metadata: + labels: + node-type: worker-node + spec: + providerConfig: + value: + apiVersion: "vsphereproviderconfig/v1alpha1" + kind: "VsphereMachineProviderConfig" + machineSpec: + datacenter: "SDDC-Datacenter" + datastore: "WorkloadDatastore" + resourcePool: "clusterapi" + vmFolder: "clusterapi" + networks: + - networkName: "sddc-cgw-network-3" + ipConfig: + networkType: dhcp + numCPUs: 2 + memoryMB: 2048 + template: "ubuntu-16.04-server-cloudimg-amd64" + disks: + - diskLabel: "Hard disk 1" + diskSizeGB: 15 + versions: + kubelet: 1.11.2 + roles: + - Node diff --git a/scripts/e2e/bootstrap_job/spec/provider-components-v1.0.template b/scripts/e2e/bootstrap_job/spec/provider-components-v1.0.template new file mode 100644 index 000000000000..ede2e62c010b --- /dev/null +++ b/scripts/e2e/bootstrap_job/spec/provider-components-v1.0.template @@ -0,0 +1,105 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: clusterapi-controllers + namespace: default + labels: + api: clusterapi +spec: + replicas: 1 + template: + metadata: + labels: + api: clusterapi + spec: + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + key: node.alpha.kubernetes.io/notReady + operator: Exists + - effect: NoExecute + key: node.alpha.kubernetes.io/unreachable + operator: Exists + containers: + - name: controller-manager + image: gcr.io/k8s-cluster-api/controller-manager:0.0.7 + volumeMounts: + - name: config + mountPath: /etc/kubernetes + - name: certs + mountPath: /etc/ssl/certs + command: + - "./controller-manager" + args: + - --kubeconfig=/etc/kubernetes/admin.conf + resources: + requests: + cpu: 100m + memory: 20Mi + limits: + cpu: 100m + memory: 30Mi + - name: vsphere-machine-controller + image: $VSPHERE_CONTROLLER_VERSION + volumeMounts: + - name: config + mountPath: /etc/kubernetes + - name: certs + mountPath: /etc/ssl/certs + - name: machines-stage + mountPath: /tmp/cluster-api/machines + - name: sshkeys + mountPath: /root/.ssh/vsphere_tmp + subPath: vsphere_tmp + - name: sshkeys + mountPath: /root/.ssh/vsphere_tmp.pub + subPath: vsphere_tmp.pub + - name: kubeadm + mountPath: /usr/bin/kubeadm + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - "./vsphere-machine-controller" + args: + - --kubeconfig=/etc/kubernetes/admin.conf + resources: + requests: + cpu: 200m + memory: 200Mi + limits: + cpu: 400m + memory: 500Mi + volumes: + - name: config + hostPath: + path: /etc/kubernetes + - name: certs + hostPath: + path: /etc/ssl/certs + - name: machines-stage + emptyDir: {} + - name: sshkeys + secret: + defaultMode: 0600 + secretName: sshkeys + - name: kubeadm + hostPath: + path: /usr/bin/kubeadm +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: sshkeys + namespace: default +data: + vsphere_tmp: ${TARGET_VM_SSH} + vsphere_tmp.pub: ${TARGET_VM_SSH_PUB} diff --git a/scripts/e2e/bootstrap_job/spec/provider-components-v2.0.template b/scripts/e2e/bootstrap_job/spec/provider-components-v2.0.template new file mode 100644 index 000000000000..6f57056f6e06 --- /dev/null +++ b/scripts/e2e/bootstrap_job/spec/provider-components-v2.0.template @@ -0,0 +1,961 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + controller-tools.k8s.io: "1.0" + name: vsphere-provider-system +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + labels: + controller-tools.k8s.io: "1.0" + name: vsphereclusterproviderconfigs.vsphereproviderconfig.sigs.k8s.io +spec: + group: vsphereproviderconfig.sigs.k8s.io + names: + kind: VsphereClusterProviderConfig + plural: vsphereclusterproviderconfigs + scope: Namespaced + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + vspherePassword: + type: string + vsphereServer: + type: string + vsphereUser: + type: string + required: + - vsphereUser + - vspherePassword + - vsphereServer + version: v1alpha1 +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + labels: + controller-tools.k8s.io: "1.0" + name: vspheremachineproviderconfigs.vsphereproviderconfig.sigs.k8s.io +spec: + group: vsphereproviderconfig.sigs.k8s.io + names: + kind: VsphereMachineProviderConfig + plural: vspheremachineproviderconfigs + scope: Namespaced + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + machineRef: + type: string + machineSpec: + properties: + datacenter: + type: string + datastore: + type: string + disks: + items: + properties: + diskLabel: + type: string + diskSizeGB: + format: int64 + type: integer + type: object + type: array + memoryMB: + format: int64 + type: integer + networks: + items: + properties: + ipConfig: + properties: + dns: + items: + type: string + type: array + gateway: + type: string + ip: + type: string + netmask: + type: string + networkType: + type: string + required: + - networkType + type: object + networkName: + type: string + required: + - networkName + type: object + type: array + numCPUs: + format: int32 + type: integer + preloaded: + type: boolean + resourcePool: + type: string + template: + type: string + vmFolder: + type: string + vsphereCloudInit: + type: boolean + required: + - datacenter + - datastore + - networks + - template + - disks + type: object + machineVariables: + type: object + metadata: + type: object + vsphereMachine: + type: string + required: + - vsphereMachine + - machineVariables + version: v1alpha1 +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: vsphere-provider-manager-role +rules: +- apiGroups: + - apps + resources: + - deployments + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - vsphereproviderconfig.sigs.k8s.io + resources: + - vsphereclusterproviderconfigs + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - apps + resources: + - deployments + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - vsphereproviderconfig.sigs.k8s.io + resources: + - vspheremachineproviderconfigs + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - cluster.k8s.io + resources: + - clusters + - clusters/status + - machines + - machines/status + - machinesets + - machinedeployments + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - secrets + - events + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + name: vsphere-provider-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-provider-manager-role +subjects: +- kind: ServiceAccount + name: default + namespace: vsphere-provider-system +--- +apiVersion: v1 +data: + vsphere_tmp: ${TARGET_VM_SSH} + vsphere_tmp.pub: ${TARGET_VM_SSH_PUB} +kind: Secret +metadata: + name: vsphere-provider-machine-sshkeys-546f6942dk + namespace: vsphere-provider-system +type: Opaque +--- +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + controller-tools.k8s.io: "1.0" + name: vsphere-provider-controller-manager-service + namespace: vsphere-provider-system +spec: + ports: + - port: 443 + selector: + control-plane: controller-manager + controller-tools.k8s.io: "1.0" +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + control-plane: controller-manager + controller-tools.k8s.io: "1.0" + name: vsphere-provider-controller-manager + namespace: vsphere-provider-system +spec: + selector: + matchLabels: + control-plane: controller-manager + controller-tools.k8s.io: "1.0" + serviceName: vsphere-provider-controller-manager-service + template: + metadata: + labels: + control-plane: controller-manager + controller-tools.k8s.io: "1.0" + spec: + containers: + - args: + - --logtostderr + - --v=6 + command: + - /root/manager + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: $VSPHERE_CONTROLLER_VERSION + name: manager + resources: + limits: + cpu: 400m + memory: 500Mi + requests: + cpu: 200m + memory: 200Mi + volumeMounts: + - mountPath: /etc/kubernetes + name: config + - mountPath: /etc/ssl/certs + name: certs + - mountPath: /tmp/cluster-api/machines + name: machines-stage + - mountPath: /root/.ssh/vsphere_tmp + name: sshkeys + subPath: vsphere_tmp + - mountPath: /root/.ssh/vsphere_tmp.pub + name: sshkeys + subPath: vsphere_tmp.pub + - mountPath: /usr/bin/kubeadm + name: kubeadm + terminationGracePeriodSeconds: 10 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + key: node.alpha.kubernetes.io/notReady + operator: Exists + - effect: NoExecute + key: node.alpha.kubernetes.io/unreachable + operator: Exists + volumes: + - hostPath: + path: /etc/kubernetes + name: config + - hostPath: + path: /etc/ssl/certs + name: certs + - emptyDir: {} + name: machines-stage + - name: sshkeys + secret: + defaultMode: 384 + secretName: vsphere-provider-machine-sshkeys-546f6942dk + - hostPath: + path: /usr/bin/kubeadm + name: kubeadm +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + controller-tools.k8s.io: "1.0" + name: cluster-api-system +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + labels: + controller-tools.k8s.io: "1.0" + name: clusters.cluster.k8s.io +spec: + group: cluster.k8s.io + names: + kind: Cluster + plural: clusters + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + clusterNetwork: + properties: + pods: + properties: + cidrBlocks: + items: + type: string + type: array + required: + - cidrBlocks + type: object + serviceDomain: + type: string + services: + properties: + cidrBlocks: + items: + type: string + type: array + required: + - cidrBlocks + type: object + required: + - services + - pods + - serviceDomain + type: object + providerConfig: + properties: + value: + type: object + valueFrom: + properties: + machineClass: + properties: + provider: + type: string + type: object + type: object + type: object + required: + - clusterNetwork + type: object + status: + properties: + apiEndpoints: + items: + properties: + host: + type: string + port: + format: int64 + type: integer + required: + - host + - port + type: object + type: array + errorMessage: + type: string + errorReason: + type: string + providerStatus: + type: object + type: object + version: v1alpha1 +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + labels: + controller-tools.k8s.io: "1.0" + name: machinedeployments.cluster.k8s.io +spec: + group: cluster.k8s.io + names: + kind: MachineDeployment + plural: machinedeployments + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + minReadySeconds: + format: int32 + type: integer + paused: + type: boolean + progressDeadlineSeconds: + format: int32 + type: integer + replicas: + format: int32 + type: integer + revisionHistoryLimit: + format: int32 + type: integer + selector: + type: object + strategy: + properties: + rollingUpdate: + properties: + maxSurge: {} + maxUnavailable: {} + type: object + type: + type: string + type: object + template: + properties: + metadata: + type: object + spec: + properties: + configSource: + type: object + metadata: + type: object + providerConfig: + properties: + value: + type: object + valueFrom: + properties: + machineClass: + properties: + provider: + type: string + type: object + type: object + type: object + taints: + items: + type: object + type: array + versions: + properties: + controlPlane: + type: string + kubelet: + type: string + required: + - kubelet + type: object + required: + - providerConfig + type: object + type: object + required: + - selector + - template + type: object + status: + properties: + availableReplicas: + format: int32 + type: integer + observedGeneration: + format: int64 + type: integer + readyReplicas: + format: int32 + type: integer + replicas: + format: int32 + type: integer + unavailableReplicas: + format: int32 + type: integer + updatedReplicas: + format: int32 + type: integer + type: object + version: v1alpha1 +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + labels: + controller-tools.k8s.io: "1.0" + name: machines.cluster.k8s.io +spec: + group: cluster.k8s.io + names: + kind: Machine + plural: machines + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + configSource: + type: object + metadata: + type: object + providerConfig: + properties: + value: + type: object + valueFrom: + properties: + machineClass: + properties: + provider: + type: string + type: object + type: object + type: object + taints: + items: + type: object + type: array + versions: + properties: + controlPlane: + type: string + kubelet: + type: string + required: + - kubelet + type: object + required: + - providerConfig + type: object + status: + properties: + addresses: + items: + type: object + type: array + conditions: + items: + type: object + type: array + errorMessage: + type: string + errorReason: + type: string + lastUpdated: + format: date-time + type: string + nodeRef: + type: object + providerStatus: + type: object + versions: + properties: + controlPlane: + type: string + kubelet: + type: string + required: + - kubelet + type: object + type: object + version: v1alpha1 +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + labels: + controller-tools.k8s.io: "1.0" + name: machinesets.cluster.k8s.io +spec: + group: cluster.k8s.io + names: + kind: MachineSet + plural: machinesets + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + minReadySeconds: + format: int32 + type: integer + replicas: + format: int32 + type: integer + selector: + type: object + template: + properties: + metadata: + type: object + spec: + properties: + configSource: + type: object + metadata: + type: object + providerConfig: + properties: + value: + type: object + valueFrom: + properties: + machineClass: + properties: + provider: + type: string + type: object + type: object + type: object + taints: + items: + type: object + type: array + versions: + properties: + controlPlane: + type: string + kubelet: + type: string + required: + - kubelet + type: object + required: + - providerConfig + type: object + type: object + required: + - selector + type: object + status: + properties: + availableReplicas: + format: int32 + type: integer + errorMessage: + type: string + errorReason: + type: string + fullyLabeledReplicas: + format: int32 + type: integer + observedGeneration: + format: int64 + type: integer + readyReplicas: + format: int32 + type: integer + replicas: + format: int32 + type: integer + required: + - replicas + type: object + version: v1alpha1 +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: cluster-api-manager-role +rules: +- apiGroups: + - cluster.k8s.io + resources: + - clusters + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - cluster.k8s.io + resources: + - machines + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - cluster.k8s.io + resources: + - machinedeployments + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - cluster.k8s.io + resources: + - machinesets + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - cluster.k8s.io + resources: + - machines + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - cluster.k8s.io + resources: + - machines + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + name: cluster-api-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-api-manager-role +subjects: +- kind: ServiceAccount + name: default + namespace: cluster-api-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + controller-tools.k8s.io: "1.0" + name: cluster-api-controller-manager-service + namespace: cluster-api-system +spec: + ports: + - port: 443 + selector: + control-plane: controller-manager + controller-tools.k8s.io: "1.0" +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + control-plane: controller-manager + controller-tools.k8s.io: "1.0" + name: cluster-api-controller-manager + namespace: cluster-api-system +spec: + selector: + matchLabels: + control-plane: controller-manager + controller-tools.k8s.io: "1.0" + serviceName: cluster-api-controller-manager-service + template: + metadata: + labels: + control-plane: controller-manager + controller-tools.k8s.io: "1.0" + spec: + containers: + - command: + - /root/manager + image: gcr.io/k8s-cluster-api/cluster-api-controller:latest + name: manager + resources: + limits: + cpu: 100m + memory: 30Mi + requests: + cpu: 100m + memory: 20Mi + terminationGracePeriodSeconds: 10 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + key: node.alpha.kubernetes.io/notReady + operator: Exists + - effect: NoExecute + key: node.alpha.kubernetes.io/unreachable + operator: Exists diff --git a/scripts/e2e/bootstrap_secret.template b/scripts/e2e/bootstrap_secret.template new file mode 100644 index 000000000000..58b1b2ec9b11 --- /dev/null +++ b/scripts/e2e/bootstrap_secret.template @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: clusterapi-provider-vsphere-ci-prow +type: Opaque +data: + vsphere-controller-version: ${VSPHERE_CONTROLLER_VERSION} + provider-component-spec: ${PROVIDER_COMPONENT_SPEC} + target-vm-ssh: ${TARGET_VM_SSH} + target-vm-ssh-pub: ${TARGET_VM_SSH_PUB} + vsphere-username: ${VSPHERE_USERNAME} + vsphere-password: ${VSPHERE_PASSWORD} + vsphere-server: ${VSPHERE_SERVER} diff --git a/scripts/e2e/e2e.sh b/scripts/e2e/e2e.sh new file mode 100755 index 000000000000..ab96fb44cdbe --- /dev/null +++ b/scripts/e2e/e2e.sh @@ -0,0 +1,180 @@ +#!/bin/sh + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# it requires the following enviroment variables: +# JUMPHOST +# GOVC_URL +# GOVC_USERNAME +# GOVC_PASSWORD +# VSPHERE_CONTROLLER_VERSION + +# and it requires container has volumes +# /root/ssh/.jumphost/jumphost-key +# /root/ssh/.bootstrapper/bootstrapper-key + +# the first argument should be the vsphere controller version + +fill_file_with_value() { + newfilename="$(echo "$1" | sed 's/template/yml/g')" + rm -f "$newfilename" temp.sh + ( echo "cat <$newfilename"; + cat "$1"; + echo "EOF"; + ) >temp.sh + chmod +x temp.sh + ./temp.sh +} + +revert_bootstrap_vm() { + bootstrap_vm=$(govc find / -type m -name clusterapi-bootstrap-prow) + snapshot_name="cluster-api-provider-vsphere-ci-0.0.1" + govc snapshot.revert -vm "${bootstrap_vm}" "${snapshot_name}" + bootstrap_vm_ip=$(govc vm.ip "${bootstrap_vm}") +} + +run_cmd_on_bootstrap() { + ssh -o ProxyCommand="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/ssh/.jumphost/jumphost-key -W %h:%p luoh@$JUMPHOST" vmware@"$1" \ + -i "/root/ssh/.bootstrapper/bootstrapper-key" \ + -o "StrictHostKeyChecking=no" \ + -o "UserKnownHostsFile=/dev/null" "$2" +} + +delete_vm() { + vm=$(govc find / -type m -name "$1""-*") + govc vm.power -off "${vm}" + govc vm.destroy "${vm}" +} + +get_bootstrap_vm() { + export GOVC_INSECURE=1 + retry=10 + bootstrap_vm_ip="" + until [ $bootstrap_vm_ip ] + do + sleep 6 + revert_bootstrap_vm + retry=$((retry - 1)) + if [ $retry -lt 0 ] + then + break + fi + done + + if [ -z "$bootstrap_vm_ip" ] ; then + echo "bootstrap vm ip is empty" + exit 1 + fi + echo "bootstrapper VM ip: ${bootstrap_vm_ip}" +} + +export_base64_value() { + base64_value=$(echo -n "$2" | base64 -w 0) + export "$1"="$base64_value" +} + +apply_secret_to_bootstrap() { + provider_component=${PROVIDER_COMPONENT_SPEC:=provider-components-v2.0.yml} + export_base64_value "PROVIDER_COMPONENT_SPEC" "${provider_component}" + echo "test ${provider_component}" + + echo "test controller version $1" + vsphere_controller_version="gcr.io/cnx-cluster-api/vsphere-cluster-api-provider:$1" + export_base64_value "VSPHERE_CONTROLLER_VERSION" "${vsphere_controller_version}" + echo "test ${vsphere_controller_version}" + + export_base64_value "VSPHERE_SERVER" "${GOVC_URL}" + export_base64_value "VSPHERE_USERNAME" "${GOVC_USERNAME}" + export_base64_value "VSPHERE_PASSWORD" "${GOVC_PASSWORD}" + export_base64_value "TARGET_VM_SSH" "${TARGET_VM_SSH}" + export_base64_value "TARGET_VM_SSH_PUB" "${TARGET_VM_SSH_PUB}" + + fill_file_with_value "bootstrap_secret.template" + run_cmd_on_bootstrap "${bootstrap_vm_ip}" "cat > /tmp/bootstrap_secret.yml" < bootstrap_secret.yml + run_cmd_on_bootstrap "${bootstrap_vm_ip}" "kubectl create -f /tmp/bootstrap_secret.yml" +} + +start_docker() { + service docker start + # the service can be started but the docker socket not ready, wait for ready + WAIT_N=0 + MAX_WAIT=5 + while true; do + # docker ps -q should only work if the daemon is ready + docker ps -q > /dev/null 2>&1 && break + if [ ${WAIT_N} -lt ${MAX_WAIT} ]; then + WAIT_N=$((WAIT_N+1)) + echo "Waiting for docker to be ready, sleeping for ${WAIT_N} seconds." + sleep ${WAIT_N} + else + echo "Reached maximum attempts, not waiting any longer..." + break + fi + done +} + +clone_clusterapi_vsphere_repo() { + mkdir -p /go/src/sigs.k8s.io/cluster-api-provider-vsphere + git clone https://github.com/kubernetes-sigs/cluster-api-provider-vsphere.git \ + /go/src/sigs.k8s.io/cluster-api-provider-vsphere/ +} + +install_govc() { + govc_bin="/tmp/govc/bin" + mkdir -p "${govc_bin}" + curl -sL https://github.com/vmware/govmomi/releases/download/v0.19.0/govc_linux_amd64.gz -o "${govc_bin}"/govc.gz + gunzip "${govc_bin}"/govc.gz + chmod +x "${govc_bin}"/govc + export PATH=${govc_bin}:$PATH +} + +vsphere_controller_version="" +# the main loop +if [ -z "${PROW_JOB_ID}" ] ; then + vsphere_controller_version="$1" + start_docker + clone_clusterapi_vsphere_repo + current=$(pwd) + cd /go/src/sigs.k8s.io/cluster-api-provider-vsphere || exit 1 + export VERSION="${vsphere_controller_version}" && make ci-push + cd "${current}" || exit 1 +else + # in Prow context, clusterapi-vsphere already been checked out + vsphere_controller_version="${PULL_PULL_SHA}" + export VERSION="${vsphere_controller_version}" && make ci-push + cd ./scripts/e2e || exit 1 +fi +echo "build vSphere controller version: ${vsphere_controller_version}" + +# get bootstrap VM +install_govc +get_bootstrap_vm + +# apply secret at bootstrap cluster +apply_secret_to_bootstrap "${vsphere_controller_version}" + +# launch the job at bootstrap cluster +run_cmd_on_bootstrap "${bootstrap_vm_ip}" "cat > /tmp/bootstrap_job.yml" < bootstrap_job.yml +run_cmd_on_bootstrap "${bootstrap_vm_ip}" "kubectl create -f /tmp/bootstrap_job.yml" + +# wait for job to finish +run_cmd_on_bootstrap "${bootstrap_vm_ip}" 'bash -s' < wait_for_job.sh +ret="$?" + +# cleanup +delete_vm "clusterapi-prow" +get_bootstrap_vm + +exit "${ret}" diff --git a/scripts/e2e/hack/Dockerfile b/scripts/e2e/hack/Dockerfile new file mode 100644 index 000000000000..c90e7b1f3dbc --- /dev/null +++ b/scripts/e2e/hack/Dockerfile @@ -0,0 +1,11 @@ +# build from test-infra prow image +FROM gcr.io/k8s-testimages/kubekins-e2e:v20181120-dea0825e3-master + +COPY *.sh /ci/ +COPY *.template /ci/ +COPY *.yml /ci/ + +RUN chmod +x /ci/e2e.sh +WORKDIR /ci/ +CMD ["shell"] +ENTRYPOINT ["/ci/e2e.sh"] diff --git a/scripts/e2e/hack/Makefile b/scripts/e2e/hack/Makefile new file mode 100644 index 000000000000..dfd6f5caedf5 --- /dev/null +++ b/scripts/e2e/hack/Makefile @@ -0,0 +1,22 @@ +# Makefile + +VERSION ?= $(shell git describe --exact-match 2> /dev/null || \ + git describe --match=$(git rev-parse --short=8 HEAD) --always --dirty --abbrev=8) +REGISTRY ?=luoh/cluster-api-provider-vsphere-prow-ci + +all: build upload clean +.PHONY : all + +.PHONY : build +build: + docker build . --tag $(REGISTRY):$(VERSION) + docker tag $(REGISTRY):$(VERSION) $(REGISTRY):debug + +upload: + docker login -u="$(DOCKER_USERNAME)" -p="$(DOCKER_PASSWORD)"; + docker push $(REGISTRY):$(VERSION) + docker push $(REGISTRY):debug + +clean: + docker image rm -f $(REGISTRY):$(VERSION) + docker image rm -f $(REGISTRY):debug diff --git a/scripts/e2e/hack/job.yml b/scripts/e2e/hack/job.yml new file mode 100644 index 000000000000..6fec785c5238 --- /dev/null +++ b/scripts/e2e/hack/job.yml @@ -0,0 +1,80 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: cluster-api-provider-vsphere-ci +spec: + backoffLimit: 4 + template: + spec: + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + containers: + - name: cluster-api-provider-vsphere-ci + image: luoh/cluster-api-provider-vsphere-prow-ci:latest + env: + - name: JUMPHOST + valueFrom: + secretKeyRef: + name: clusterapi-provider-vsphere-ci-prow + key: jumphost + - name: GOVC_URL + valueFrom: + secretKeyRef: + name: clusterapi-provider-vsphere-ci-prow + key: vsphere-server + - name: GOVC_USERNAME + valueFrom: + secretKeyRef: + name: clusterapi-provider-vsphere-ci-prow + key: vsphere-username + - name: GOVC_PASSWORD + valueFrom: + secretKeyRef: + name: clusterapi-provider-vsphere-ci-prow + key: vsphere-password + - name: TARGET_VM_SSH + valueFrom: + secretKeyRef: + name: clusterapi-provider-vsphere-ci-prow + key: target-vm-ssh + - name: TARGET_VM_SSH_PUB + valueFrom: + secretKeyRef: + name: clusterapi-provider-vsphere-ci-prow + key: target-vm-ssh-pub + - name: GCR_KEY_FILE + valueFrom: + secretKeyRef: + name: clusterapi-provider-vsphere-ci-prow + key: cluster-api-vsphere-gcr-service-account + volumeMounts: + - name: jumphost-key + mountPath: /root/ssh/.jumphost + - name: bootstrapper-key + mountPath: /root/ssh/.bootstrapper + - name: docker-socket + mountPath: /var/run/docker.sock + command: + - "./e2e.sh" + args: + - ${VSPHERE_CONTROLLER_VERSION} + volumes: + - name: docker-socket + hostPath: + path: /var/run/docker.sock + - name: jumphost-key + secret: + secretName: clusterapi-provider-vsphere-ci-prow + defaultMode: 256 + items: + - key: jumphost-key + path: jumphost-key + - name: bootstrapper-key + secret: + secretName: clusterapi-provider-vsphere-ci-prow + defaultMode: 256 + items: + - key: bootstrapper-key + path: bootstrapper-key + restartPolicy: Never diff --git a/scripts/e2e/hack/secret.yml b/scripts/e2e/hack/secret.yml new file mode 100644 index 000000000000..0405d61c56f2 --- /dev/null +++ b/scripts/e2e/hack/secret.yml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Secret +metadata: + name: clusterapi-provider-vsphere-ci-prow +type: Opaque +data: + jumphost: {JUMPHOST} + jumphost-key: {JUMPHOST-KEY} + bootstrapper-key: {BOOTSTRAPPER-KEY} + target-vm-ssh: {TARGET-VM-SSH} + target-vm-ssh-pub: {TARGET-VM-SSH-PUB} + vsphere-username: {VSPHERE-USERNAME} + vsphere-password: {VSPHERE-PASSWORD} + vsphere-server: {VSPHERE-SERVER} + cluster-api-vsphere-gcr-service-account: {CLUSTER-API-VSPHERE-GCR-SERVICE-ACCOUNT} diff --git a/scripts/e2e/wait_for_job.sh b/scripts/e2e/wait_for_job.sh new file mode 100644 index 000000000000..357d11ea226b --- /dev/null +++ b/scripts/e2e/wait_for_job.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TOTAL=600 +INTERVAL=6 +retry=$((${TOTAL}/ INTERVAL)) +ret=0 +until kubectl get jobs --no-headers | awk -F" " '{print $2}' | awk -F"/" '{s+=($1!=$2)} END {exit s}'; +do + sleep ${INTERVAL}; + retry=$((retry - 1)) + if [ $retry -lt 0 ]; + then + ret=1 + echo "job timeout" + break + fi; + kubectl get jobs --no-headers; +done; +echo "all jobs finished"; + +echo "--------- vsphere manager log begin ----------" +manager_pod_name=$(kubectl get pods -a --no-headers -n vsphere-provider-system | grep vsphere | awk -F" " '{print $1}') +kubectl logs "${manager_pod_name}" -n vsphere-provider-system +echo "--------- vsphere manager log end ----------" + +job_pod_names=$(kubectl get pods -a --no-headers | grep cluster-api-provider-vsphere-ci | awk -F" " '{print $1}' | tr "\\\n" "\n") +for job_pod_name in ${job_pod_names} +do + echo "--------- ci job log begin ----------" + kubectl logs "${job_pod_name}" + echo "--------- ci job log end ----------" +done + +exit ${ret}