Merge 9235741 into 20b679b

.github/workflows/acceptance.yaml

@@ -0,0 +1,18 @@
+name: Acceptance Tests
+on: push
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - name: Run acceptance
+        run: docker-compose run -v $HOME/go/pkg/mod:/go/pkg/mod test

.gitignore

@@ -15,3 +15,6 @@ ginkgo.report
 test/library/wego-library-test
 tilt_modules
 .envrc
+
+# generate by k3s when running acceptance tests
+kubeconfig.yaml

docker-compose.yaml

@@ -0,0 +1,57 @@
+# to run define K3S_TOKEN, K3S_VERSION is optional, eg:
+#   K3S_TOKEN=${RANDOM}${RANDOM}${RANDOM} docker-compose up
+
+version: '3'
+services:
+  k3s:
+    image: "rancher/k3s:${K3S_VERSION:-latest}"
+    command: server --no-deploy traefik --tls-san k3s
+    networks:
+      cluster:
+        ipv4_address: 172.10.0.150
+    tmpfs:
+    - /run
+    - /var/run
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 65535
+        hard: 65535
+    privileged: true
+    restart: always
+    environment:
+    - K3S_TOKEN=foo
+    - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml
+    - K3S_KUBECONFIG_MODE=666
+    volumes:
+    - k3s-server:/var/lib/rancher/k3s
+    # This is just so that we get the kubeconfig file out
+    - .:/output
+    ports:
+    - 6443:6443  # Kubernetes API Server
+    - 80:80    # Ingress controller port 80
+    - 443:443   # Ingress controller port 443
+
+  test:
+    build:
+      context: .
+      dockerfile: test.dockerfile
+    command: go test github.com/weaveworks/weave-gitops/test/acceptance/vcluster
+    environment:
+    - KUBECONFIG=/app/kubeconfig.yaml
+    volumes:
+    - .:/app
+    - ~/go/pkg/mod:/go/pkg/mod
+    working_dir: /app
+    depends_on:
+    - k3s
+    networks:
+      - cluster
+volumes:
+  k3s-server: {}
+
+networks:
+  cluster:
+    ipam:
+      config:
+        - subnet: 172.10.0.0/24

test.dockerfile

@@ -0,0 +1,15 @@
+# Go build
+FROM golang:1.17 AS go-build
+# Add a kubectl
+RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
+    && install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl \
+    && curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash \
+    && curl -s -L "https://github.com/loft-sh/vcluster/releases/latest" | sed -nE 's!.*"([^"]*vcluster-linux-amd64)".*!https://github.com\1!p' | xargs -n 1 curl -L -o vcluster \
+    && chmod +x vcluster \
+    && mv vcluster /usr/local/bin
+
+RUN mkdir /app
+WORKDIR /app
+# COPY go.mod .
+# COPY go.sum .
+# RUN go mod download

test/acceptance/vcluster/acceptance.go

@@ -0,0 +1,39 @@
+package vcluster
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/weaveworks/weave-gitops/test/vcluster"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/rand"
+)
+
+var _ = Describe("Acceptance PoC", func() {
+	var (
+		clusterFactory vcluster.Factory
+		clusterName    string
+	)
+
+	BeforeEach(func() {
+		var err error
+		clusterName = "test-" + rand.String(10)
+		clusterFactory, err = vcluster.NewFactory()
+		Expect(err).To(BeNil(), "creating new factory")
+		client, err := clusterFactory.Create(context.TODO(), clusterName)
+		Expect(err).To(BeNil(), "creating new cluster")
+
+		namespaceObj := &corev1.Namespace{}
+		namespaceObj.Name = "test"
+		Expect(client.Create(context.TODO(), namespaceObj)).To(Succeed())
+	})
+
+	AfterEach(func() {
+		Expect(clusterFactory.Delete(context.TODO(), clusterName)).To(Succeed())
+	})
+
+	It("Testing creation and deletion of a vcluster", func() {
+
+	})
+})

test/acceptance/vcluster/vcluster_suite_test.go

@@ -0,0 +1,29 @@
+package vcluster_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/weaveworks/weave-gitops/test/vcluster"
+)
+
+func TestVcluster(t *testing.T) {
+	if err := vcluster.UpdateHostKubeconfig(); err != nil {
+		t.Errorf("failed updating host kubeconfig: %w", err)
+		t.FailNow()
+	}
+
+	if err := vcluster.WaitClusterConnectivity(); err != nil {
+		t.Errorf("failed waiting cluster to be ready: %w", err)
+		t.FailNow()
+	}
+
+	if err := vcluster.InstallNginxIngressController(); err != nil {
+		t.Errorf("failed installing ingress controller: %w", err)
+		t.FailNow()
+	}
+
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Vcluster Suite")
+}

test/vcluster/manifests/manifests.yaml.tpl

@@ -0,0 +1,214 @@
+---
+# Source: vcluster/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: vc-{{.Name}}
+  namespace: {{.Name}}
+  labels:
+    app: vcluster
+    chart: "vcluster-0.5.3"
+    release: "{{.Name}}"
+    heritage: "Helm"
+---
+# Source: vcluster/templates/rbac/role.yaml
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{.Name}}
+  namespace: {{.Name}}
+  labels:
+    app: vcluster
+    chart: "vcluster-0.5.3"
+    release: "{{.Name}}"
+    heritage: "Helm"
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps", "secrets", "services", "pods", "pods/attach", "pods/portforward", "pods/exec", "endpoints", "persistentvolumeclaims"]
+    verbs: ["create", "delete", "patch", "update", "get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events", "pods/log"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["ingresses"]
+    verbs: ["create", "delete", "patch", "update", "get", "list", "watch"]
+  - apiGroups: ["apps"]
+    resources: ["statefulsets", "replicasets", "deployments"]
+    verbs: ["get", "list", "watch"]
+---
+# Source: vcluster/templates/rbac/rolebinding.yaml
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{.Name}}
+  namespace: {{.Name}}
+  labels:
+    app: vcluster
+    chart: "vcluster-0.5.3"
+    release: "{{.Name}}"
+    heritage: "Helm"
+subjects:
+  - kind: ServiceAccount
+    name: vc-{{.Name}}
+    namespace: {{.Name}}
+roleRef:
+  kind: Role
+  name: {{.Name}}
+  apiGroup: rbac.authorization.k8s.io
+---
+# Source: vcluster/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{.Name}}
+  namespace: {{.Name}}
+  labels:
+    app: vcluster
+    chart: "vcluster-0.5.3"
+    release: "{{.Name}}"
+    heritage: "Helm"
+spec:
+  type: ClusterIP
+  ports:
+    - name: https
+      port: 443
+      targetPort: 8443
+      protocol: TCP
+  selector:
+    app: vcluster
+    release: {{.Name}}
+---
+# Source: vcluster/templates/statefulset-service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{.Name}}-headless
+  namespace: {{.Name}}
+  labels:
+    app: {{.Name}}
+    chart: "vcluster-0.5.3"
+    release: "{{.Name}}"
+    heritage: "Helm"
+spec:
+  ports:
+    - name: https
+      port: 443
+      targetPort: 8443
+      protocol: TCP
+  clusterIP: None
+  selector:
+    app: vcluster
+    release: "{{.Name}}"
+---
+# Source: vcluster/templates/statefulset.yaml
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{.Name}}
+  namespace: {{.Name}}
+  labels:
+    app: vcluster
+    chart: "vcluster-0.5.3"
+    release: "{{.Name}}"
+    heritage: "Helm"
+spec:
+  serviceName: {{.Name}}-headless
+  replicas: 1
+  selector:
+    matchLabels:
+      app: vcluster
+      release: {{.Name}}
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        storageClassName:
+        resources:
+          requests:
+            storage: 5Gi
+  template:
+    metadata:
+      labels:
+        app: vcluster
+        release: {{.Name}}
+    spec:
+      terminationGracePeriodSeconds: 10
+      nodeSelector:
+        {}
+      affinity:
+        {}
+      tolerations:
+        []
+      serviceAccountName: vc-{{.Name}}
+      volumes:
+      containers:
+      - image: rancher/k3s:v1.21.4-k3s1
+        name: vcluster
+        # k3s has a problem running as pid 1 and disabled agents on cgroupv2
+        # nodes as it will try to evacuate the cgroups there. Starting k3s
+        # through a shell makes it non pid 1 and prevents this from happening
+        command:
+          - /bin/sh
+        args:
+          - -c
+          - /bin/k3s
+            server
+            --write-kubeconfig=/data/k3s-config/kube-config.yaml
+            --data-dir=/data
+            --disable=traefik,servicelb,metrics-server,local-storage,coredns
+            --disable-network-policy
+            --disable-agent
+            --disable-scheduler
+            --disable-cloud-controller
+            --flannel-backend=none
+            --kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle
+            --service-cidr=10.43.0.0/12
+            && true
+        env:
+          []
+        securityContext:
+          allowPrivilegeEscalation: false
+        volumeMounts:
+          - mountPath: /data
+            name: data
+        resources:
+          limits:
+            memory: 2Gi
+          requests:
+            cpu: 200m
+            memory: 256Mi
+      - name: syncer
+        image: "loftsh/vcluster:0.5.3"
+        args:
+          - --name={{.Name}}
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8443
+            scheme: HTTPS
+          failureThreshold: 10
+          initialDelaySeconds: 60
+          periodSeconds: 2
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 8443
+            scheme: HTTPS
+          failureThreshold: 30
+          periodSeconds: 2
+        securityContext:
+          allowPrivilegeEscalation: false
+        env:
+          - name: DEFAULT_IMAGE_REGISTRY
+            value:
+        volumeMounts:
+          - mountPath: /data
+            name: data
+            readOnly: true
+        resources:
+          limits:
+            memory: 1Gi
+          requests:
+            cpu: 100m
+            memory: 128Mi