try adding very basic of ci/tests (#1)

* try adding very basic of ci/tests
* hyperqueue not working yet - next item of action

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.github/workflows/build-deploy.yaml

@@ -0,0 +1,38 @@
+name: build hyperqueue-operator
+
+on:
+  pull_request: []
+  push:
+    branches:
+    - main
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # catalog and bundle were removed, nobody using
+        command: [docker]
+
+    name: make and build ${{ matrix.command }}
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v3
+    - uses: actions/setup-go@v3
+      with:
+        go-version: ^1.18.1
+    - name: GHCR Login
+      if: (github.event_name != 'pull_request')
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Build Container
+      run: make ${{ matrix.command }}-build
+
+    - name: Deploy Container
+      if: (github.event_name != 'pull_request')
+      run: make ${{ matrix.command }}-push

.github/workflows/main.yaml

@@ -0,0 +1,90 @@
+name: test hyperqueue-operator
+
+on:
+  pull_request: []
+
+jobs:
+  formatting:
+    name: Formatting
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Setup black linter
+      run: conda create --quiet --name black pyflakes
+
+    - name: Check Spelling
+      uses: crate-ci/typos@7ad296c72fa8265059cc03d1eda562fbdfcd6df2 # v1.9.0
+      with:
+        files: ./README.md ./config/samples
+
+  unit-tests:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Setup Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: ^1.18
+    - name: fmt check
+      run: make fmt
+
+    # These aren't written yet
+    - name: Run Unit tests
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: make test
+
+    # Ensure build-config is the same as the one we have
+    - name: Check Updated hyperqueue-operator.yaml
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        cp examples/dist/hyperqueue-operator.yaml /tmp/hyperqueue-operator.yaml
+        make build-config
+        diff examples/dist/hyperqueue-operator.yaml /tmp/hyperqueue-operator.yaml
+
+  test-jobs:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [["hello-world", "ubuntu", 20]]
+
+    steps:
+    - name: Clone the code
+      uses: actions/checkout@v3
+
+    - name: Setup Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: ^1.18
+
+    - name: Start minikube
+      uses: medyagh/setup-minikube@697f2b7aaed5f70bf2a94ee21a4ec3dde7b12f92 # v0.0.9
+
+    - name: Create the namespace
+      run: kubectl create namespace hyperqueue-operator
+
+    - name: Pull Docker Containers to MiniKube
+      env:
+        container: ${{ matrix.test[1] }}
+        test: ${{ matrix[0] }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        export SHELL=/bin/bash
+        eval $(minikube -p minikube docker-env)
+        minikube ssh docker pull ${container}
+        make deploy-local
+        minikube image load ghcr.io/converged-computing/hyperqueue-operator:test
+        kubectl apply -f examples/dist/hyperqueue-operator-local.yaml
+
+    - name: Test ${{ matrix.test[0] }}
+      env:
+        name: ${{ matrix.test[0] }}
+        jobtime: ${{ matrix.test[2] }}
+      run: |
+        echo "TODO test ${name}"
+        # /bin/bash ./script/test.sh ${name} ${jobtime}

Makefile

@@ -130,6 +130,15 @@ docker-build: test ## Build docker image with the manager.
 docker-push: ## Push docker image with the manager.
 	docker push ${IMG}
 
+# Build a local test image, load into minikube or kind and apply the build-config
+.PHONY: deploy-local
+deploy-local: manifests kustomize build
+	kubectl delete -f examples/dist/hyperqueue-operator-local.yaml || true
+	docker build -t ${DEVIMG} .
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${DEVIMG}
+	$(KUSTOMIZE) build config/default > examples/dist/hyperqueue-operator-local.yaml
+	sed -i 's/        imagePullPolicy: Always/        imagePullPolicy: Never/' examples/dist/hyperqueue-operator-local.yaml
+
 # PLATFORMS defines the target platforms for  the manager image be build to provide support to multiple
 # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
 # - able to use docker buildx . More info: https://docs.docker.com/build/buildx/
@@ -289,3 +298,7 @@ catalog-build: opm ## Build a catalog image.
 .PHONY: catalog-push
 catalog-push: ## Push a catalog image.
 	$(MAKE) docker-push IMG=$(CATALOG_IMG)
+
+.PHONY: pre-push
+pre-push: generate build-config
+	git status

README.md

@@ -13,7 +13,7 @@ This isn't tested or working yet (and I'm new to the tool so please don't expect
 mkdir hyperqueue-operator
 cd hyperqueue-operator/
 operator-sdk init --domain flux-framework.org --repo github.com/converged-computing/hyperqueue-operator
-operator-sdk create api --group hyperqueues --version v1alpha1 --kind Hyperqueue --resource --controller
+operator-sdk create api --version v1alpha1 --kind Hyperqueue --resource --controller
 ```
 
 ## Getting Started
@@ -41,6 +41,9 @@ Generate the custom resource definition
 ```bash
 # Build and push the image, and generate the examples/dist/hyperqueue-operator-dev.yaml
 $ make test-deploy DEVIMG=<some-registry>/hyperqueue-operator:tag
+
+# As an example
+$ make test-deploy DEVIMG=vanessa/hyperqueue-operator:test
 ```
 
 Apply the new config!
@@ -62,8 +65,6 @@ When you are done, cleanup.
 $ kind delete cluster
 ```
 
-
-
 ### How it works
 
 This project aims to follow the Kubernetes [Operator pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/).

api/v1alpha1/hyperqueue_types.go

@@ -43,6 +43,10 @@ type HyperqueueSpec struct {
 	// Size of the hyperqueue (1 server + (N-1) nodes)
 	Size int32 `json:"size"`
 
+	// Interactive mode keeps the cluster running
+	// +optional
+	Interactive bool `json:"interactive"`
+
 	// Time limit for the job
 	// Approximately one year. This cannot be zero or job won't start
 	// +kubebuilder:default=31500000

config/crd/bases/flux-framework.org_hyperqueues.yaml

@@ -46,6 +46,9 @@ spec:
                 description: Release of Hyperqueue to installed (if hq binary not
                   found in PATH)
                 type: string
+              interactive:
+                description: Interactive mode keeps the cluster running
+                type: boolean
               resources:
                 additionalProperties:
                   anyOf:

config/manager/kustomization.yaml

@@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: vanessa/hyperqueue-operator
-  newTag: test
+  newName: ghcr.io/converged-computing/hyperqueue-operator
+  newTag: latest

controllers/hyperqueue/hyperqueue.go

@@ -95,7 +95,8 @@ func (r *HyperqueueReconciler) getCluster(
 					"Namespace:", job.Namespace,
 					"Name:", job.Name,
 				)
-				return job, ctrl.Result{}, err
+				// If there is an error, return the existing (empty)
+				return existing, ctrl.Result{}, err
 			}
 
 			r.Log.Info(
@@ -112,7 +113,7 @@ func (r *HyperqueueReconciler) getCluster(
 					"Namespace:", job.Namespace,
 					"Name:", job.Name,
 				)
-				return job, ctrl.Result{}, err
+				return existing, ctrl.Result{}, err
 			}
 			// Successful - return and requeue
 			return job, ctrl.Result{Requeue: true}, nil
@@ -159,6 +160,7 @@ func (r *HyperqueueReconciler) getConfigMap(
 		data["start-server"] = serverStart
 		data["start-worker"] = workerStart
 	}
+	fmt.Println(data)
 
 	// Create the config map with respective data!
 	cm = &corev1.ConfigMap{

controllers/hyperqueue/jobset.go

@@ -28,6 +28,9 @@ func (r *HyperqueueReconciler) newJobSet(
 	cluster *api.Hyperqueue,
 ) (*jobset.JobSet, error) {
 
+	// When we have a success policy
+	// serverName := cluster.Name + "-server"
+
 	// When suspend is true we have a hard time debugging jobs, so keep false
 	suspend := false
 	jobs := jobset.JobSet{
@@ -37,6 +40,12 @@ func (r *HyperqueueReconciler) newJobSet(
 		},
 		Spec: jobset.JobSetSpec{
 
+			// The job is successful when the broker job finishes with completed (0)
+			//SuccessPolicy: &jobset.SuccessPolicy{
+			//	Operator:             jobset.OperatorAny,
+			//	TargetReplicatedJobs: []string{serverName},
+			//},
+
 			// This might be the control for child jobs (worker)
 			// But I don't think we need this anymore.
 			Suspend: &suspend,
@@ -48,6 +57,7 @@ func (r *HyperqueueReconciler) newJobSet(
 	// Get leader server job, the parent in the JobSet
 	serverJob, err := r.getJob(cluster, cluster.Spec.Server, 1, "server", true)
 	if err != nil {
+		r.Log.Error(err, "There was an error getting the server ReplicatedJob")
 		return &jobs, err
 	}
 
@@ -63,6 +73,7 @@ func (r *HyperqueueReconciler) newJobSet(
 
 		workerJob, err := r.getJob(cluster, workerNode, workerNodes, "worker", true)
 		if err != nil {
+			r.Log.Error(err, "There was an error getting the worker ReplicatedJob")
 			return &jobs, err
 		}
 		jobs.Spec.ReplicatedJobs = []jobset.ReplicatedJob{serverJob, workerJob}
@@ -113,14 +124,6 @@ func (r *HyperqueueReconciler) getJob(
 		Replicas: 1,
 	}
 
-	// Do we have a pull secret for the image?
-	pullSecret := corev1.LocalObjectReference{}
-	if node.PullSecret != "" {
-		pullSecret = corev1.LocalObjectReference{
-			Name: node.PullSecret,
-		}
-	}
-
 	// Create the JobSpec for the job -> Template -> Spec
 	jobspec := batchv1.JobSpec{
 		BackoffLimit:          &backoffLimit,
@@ -138,18 +141,25 @@ func (r *HyperqueueReconciler) getJob(
 			},
 			Spec: corev1.PodSpec{
 				// matches the service
-				Subdomain:        serviceName,
-				Volumes:          getVolumes(cluster),
-				RestartPolicy:    corev1.RestartPolicyOnFailure,
-				ImagePullSecrets: []corev1.LocalObjectReference{pullSecret},
+				Subdomain:     serviceName,
+				Volumes:       getVolumes(cluster),
+				RestartPolicy: corev1.RestartPolicyOnFailure,
 			},
 		},
 	}
+
+	// Do we have a pull secret for the image?
+	if node.PullSecret != "" {
+		jobspec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{
+			{Name: node.PullSecret},
+		}
+	}
+
 	// Get resources for the node (server or worker)
 	resources, err := r.getNodeResources(cluster, node)
-	r.Log.Info("🌀 Hyperqueue", "Pod.Resources", resources)
+	r.Log.Info("👑️ Hyperqueue", "Pod.Resources", resources)
 	if err != nil {
-		r.Log.Info("🌀 Hyperqueue", "Pod.Resources", resources)
+		r.Log.Error(err, "❌ Hyperqueue", "Pod.Resources", resources)
 		return job, err
 	}
 	jobspec.Template.Spec.Overhead = resources
@@ -161,6 +171,11 @@ func (r *HyperqueueReconciler) getJob(
 		mounts,
 		entrypoint,
 	)
+	// Error creating containers
+	if err != nil {
+		r.Log.Error(err, "❌ Hyperqueue", "Pod.Resources", resources)
+		return job, err
+	}
 	jobspec.Template.Spec.Containers = containers
 	job.Template.Spec = jobspec
 	return job, err

controllers/hyperqueue/templates/components.sh

@@ -20,4 +20,8 @@ workdir=${PWD}
 mkdir -p ${workdir}
 
 # End init logic
-{{end}}
+{{end}}
+
+{{define "exit"}}
+{{ if .Spec.Interactive }}sleep infinity{{ end }}
+{{ end }}

controllers/hyperqueue/templates/server.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-# This script handles start logic for the broker
+# This script handles shared start logic
 {{template "init" .}}
 
 # Start the server
@@ -12,3 +12,5 @@ else
     hq server start &
     hq server submit $@
 fi
+
+{{template "exit" .}}

controllers/hyperqueue/templates/worker.sh

@@ -2,4 +2,5 @@
 
 # Shared logic to install hq
 {{template "init" .}}
-hq worker start
+hq worker start
+{{template "exit" .}}

examples/dist/hyperqueue-operator.yaml

@@ -52,6 +52,9 @@ spec:
                 default: 0.15.0
                 description: Release of Hyperqueue to installed (if hq binary not found in PATH)
                 type: string
+              interactive:
+                description: Interactive mode keeps the cluster running
+                type: boolean
               resources:
                 additionalProperties:
                   anyOf:
@@ -694,7 +697,8 @@ spec:
         - --leader-elect
         command:
         - /manager
-        image: vanessa/hyperqueue-operator:test
+        image: ghcr.io/converged-computing/hyperqueue-operator:latest
+        imagePullPolicy: Always
         livenessProbe:
           httpGet:
             path: /healthz