Merge pull request #1834 from barney-s/grpc-integrate3

Integrate GRPC based expander into the composition engine

experiments/compositions/composition/Dockerfile

@@ -29,6 +29,7 @@ RUN go mod download
 COPY cmd/main.go cmd/main.go
 COPY api/ api/
 COPY internal/controller/ internal/controller/
+COPY proto/ proto/
 COPY pkg/ pkg/
 
 # Build

experiments/compositions/composition/api/v1alpha1/plan_types.go

@@ -35,7 +35,7 @@ type PlanSpec struct {
 
 // StageStatus captures the status of a stage
 type StageStatus struct {
-	ResourceCount int `json:"resourceCount,omitempty"`
+	ResourceCount int `json:"resourceCount"`
 }
 
 // PlanStatus defines the observed state of Plan

experiments/compositions/composition/config/crd/bases/composition.google.com_plans.yaml

@@ -141,6 +141,8 @@ spec:
                   properties:
                     resourceCount:
                       type: integer
+                  required:
+                  - resourceCount
                   type: object
                 type: object
             type: object

experiments/compositions/composition/config/expanders/jinja2-v0.0.1.yaml

@@ -101,7 +101,7 @@ metadata:
     app.kubernetes.io/part-of: composition
     app.kubernetes.io/instance: jinja2-v0.0.1
     app.kubernetes.io/component: expanders
-  name: jinja2-v0-0-1
+  name: gjinja2-v0-0-1
   namespace: system
 spec:
   # type: LoadBalancer

experiments/compositions/composition/expanders.mk

@@ -59,8 +59,8 @@ docker-run-expander-jinja2: docker-build-expander-jinja2
 
 .PHONY: unit-test-expander-jinja2
 unit-test-expander-jinja2: deploy-kind
-	kubectl patch service -n composition-system composition-jinja2-v0-0-1 -p '{"spec":{"type":"LoadBalancer"}}'
+	kubectl patch service -n composition-system composition-gjinja2-v0-0-1 -p '{"spec":{"type":"LoadBalancer"}}'
 	nodeip=$$(kubectl get nodes -o json  | jq '.items[0].status.addresses[0].address' | xargs echo );\
-	nodeport=$$(kubectl get service -n composition-system composition-jinja2-v0-0-1 -o json | jq ".spec.ports[0].nodePort");\
+	nodeport=$$(kubectl get service -n composition-system composition-gjinja2-v0-0-1 -o json | jq ".spec.ports[0].nodePort");\
 	echo $$nodeip:$$nodeport; \
 	cd expanders/jinja2 && go test -v --addr=$$nodeip:$$nodeport

experiments/compositions/composition/internal/controller/expander_reconciler.go

@@ -16,12 +16,17 @@ package controller
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/go-logr/logr"
 	compositionv1alpha1 "google.com/composition/api/v1alpha1"
 	"google.com/composition/pkg/containerexecutor/jobcontainerexecutor"
+	pb "google.com/composition/proto"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -58,6 +63,12 @@ var planGVK schema.GroupVersionKind = schema.GroupVersionKind{
 	Kind:    "Plan",
 }
 
+var contextGVK schema.GroupVersionKind = schema.GroupVersionKind{
+	Group:   "composition.google.com",
+	Version: "v1alpha1",
+	Kind:    "Context",
+}
+
 func (r *ExpanderReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	logger := log.FromContext(ctx)
 	logger = logger.WithName(r.Composition.Name).WithName(r.InputGVK.Group)
@@ -198,7 +209,7 @@ func (r *ExpanderReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 				return ctrl.Result{}, err
 			}
 		} else {
-			reason, err := r.evaluateAndSavePlan(ctx, logger, &inputcr, expander.Name, planNN.Name, value, ev.Spec.ImageRegistry)
+			reason, err := r.evaluateAndSavePlan(ctx, logger, &inputcr, expander, planNN, value)
 			if err != nil {
 				newStatus.AppendErrorCondition(expander.Name, err.Error(), reason)
 				return ctrl.Result{}, err
@@ -370,6 +381,99 @@ func (r *ExpanderReconciler) runJob(ctx context.Context, logger logr.Logger,
 }
 
 func (r *ExpanderReconciler) evaluateAndSavePlan(ctx context.Context, logger logr.Logger,
-	cr *unstructured.Unstructured, expanderName, planName, value, registry string) (string, error) {
-	return "NotImplemented", fmt.Errorf("GRPC Caller not implemented")
+	cr *unstructured.Unstructured, expander compositionv1alpha1.Expander,
+	planNN types.NamespacedName, grpcService string) (string, error) {
+	// Set up a connection to the server.
+	conn, err := grpc.Dial(grpcService, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		logger.Error(err, "grpc dial failed: "+grpcService)
+		return "GRPCConnError", err
+	}
+
+	// read context in cr.namespace
+	contextcr := unstructured.Unstructured{}
+	contextcr.SetGroupVersionKind(contextGVK)
+	contextNN := types.NamespacedName{Namespace: cr.GetNamespace(), Name: "context"}
+	if err := r.Get(ctx, contextNN, &contextcr); err != nil {
+		logger.Error(err, "unable to fetch Context CR", "context", contextNN)
+		return "GetContextFailed", err
+	}
+	contextBytes, err := json.Marshal(contextcr.Object)
+	if err != nil {
+		logger.Error(err, "failed to marshal Context Object")
+		return "MarshallContextFailed", err
+	}
+
+	// marshall facade cr
+	facadeBytes, err := json.Marshal(cr.Object)
+	if err != nil {
+		logger.Error(err, "failed to marshall Facade Object")
+		return "MarshallFacadeFailed", err
+	}
+
+	// marshall expande config
+	configBytes, err := json.Marshal(expander.Template)
+	if err != nil {
+		logger.Error(err, "failed to marshall Expander Config")
+		return "MarshallExpanderConfigFailed", err
+	}
+
+	expanderClient := pb.NewExpanderClient(conn)
+	result, err := expanderClient.Evaluate(ctx,
+		// TODO (barney-s) value (pass as parameter)
+		&pb.EvaluateRequest{
+			Config:   configBytes,
+			Context:  contextBytes,
+			Facade:   facadeBytes,
+			Resource: r.InputGVR.Resource,
+			Value:    []byte{},
+		})
+	if err != nil {
+		logger.Error(err, "expander.Evaluate() Failed", "expander", expander.Name)
+		return "EvaluateError", err
+	}
+	if result.Status != pb.Status_SUCCESS {
+		logger.Error(nil, "expander.Evaluate() Status is not Success", "expander", expander.Name, "status", result.Status)
+		err = fmt.Errorf("Evaluate Failed: %s", result.Error.Message)
+		return "EvaluateStatusFailed", err
+	}
+
+	// Write to Plan object
+	// Re-read the Plan CR to load the expanded manifests
+	plancr := compositionv1alpha1.Plan{}
+	if err := r.Client.Get(ctx, planNN, &plancr); err != nil {
+		logger.Error(err, "unable to read Plan CR", "plan", planNN)
+		return "GetPlanFailed", err
+	}
+
+	if plancr.Spec.Stages == nil {
+		plancr.Spec.Stages = map[string]compositionv1alpha1.Stage{}
+	}
+	if result.Type == pb.ResultType_MANIFESTS {
+		s, err := strconv.Unquote(string(result.Manifests))
+		if err != nil {
+			logger.Error(err, "unable to unquote grpc response")
+			return "UnquoteResponseFailed", err
+		}
+		plancr.Spec.Stages[expander.Name] = compositionv1alpha1.Stage{
+			Manifest: s,
+		}
+	} else {
+		s, err := strconv.Unquote(string(result.Values))
+		if err != nil {
+			logger.Error(err, "unable to unquote grpc response")
+			return "UnquoteResponseFailed", err
+		}
+		plancr.Spec.Stages[expander.Name] = compositionv1alpha1.Stage{
+			Manifest: s,
+		}
+	}
+
+	err = r.Client.Update(ctx, &plancr)
+	if err != nil {
+		logger.Error(err, "error updating plan", "plan", planNN)
+		return "UpdatePlanFailed", err
+	}
+
+	return "", nil
 }

experiments/compositions/composition/internal/controller/expanderversion_controller.go

@@ -132,7 +132,7 @@ func (r *ExpanderVersionReconciler) processExpanderVersion(
 			value = fmt.Sprintf("%s/expander-%s:%s", ev.Spec.ImageRegistry, expander, key)
 		} else {
 			svcVersion := strings.Replace(key, ".", "-", -1)
-			value = fmt.Sprintf("composition-%s-%s", expander, svcVersion)
+			value = fmt.Sprintf("composition-%s-%s:8443", expander, svcVersion)
 		}
 		ev.Status.VersionMap[key] = value
 	}

experiments/compositions/composition/tests/data/TestSimpleCompositionExpansionGrpc/input.yaml

@@ -0,0 +1,144 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: pconfigs.facade.foocorp.com
+spec:
+  group: facade.foocorp.com
+  names:
+    kind: PConfig
+    listKind: PConfigList
+    plural: pconfigs
+    singular: pconfig
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: Schema for the pconfig
+        properties:
+          apiVersion:
+            description: api-version of api
+            type: string
+          kind:
+            description: gvk Kind
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: PConfig spec
+            properties:
+              project:
+                type: string
+            required:
+            - project
+            type: object
+          status:
+            description: PConfig status
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: composition-facade-foocorp
+rules:
+- apiGroups:
+  - facade.foocorp.com
+  resources:
+  - '*'
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - facade.foocorp.com
+  resources:
+  - "*/status"
+  verbs:
+  - get
+  - update
+- apiGroups: ["*"]
+  resources: ["*"]
+  verbs: ["*"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: composition-facade-foocorp
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: composition-facade-foocorp
+subjects:
+- kind: ServiceAccount
+  name: composition-controller-manager
+  namespace: composition-system
+---
+apiVersion: composition.google.com/v1alpha1
+kind: Composition
+metadata:
+  name: projectconfigmap
+  namespace: default
+spec:
+  inputAPIGroup: pconfigs.facade.foocorp.com
+  expanders:
+  - type: gjinja2
+    name: project
+    template: |
+      {% set hostProject = 'compositions-foobar' %}
+      {% set managedProject = pconfigs.spec.project %}
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: {{ managedProject }}
+        namespace: {{ pconfigs.metadata.namespace }}
+        labels:
+          createdby: "composition-namespaceconfigmap"
+      data:
+        name: {{ managedProject }}
+        billingAccountRef: "010101-ABABCD-BCAB11"
+        folderRef: "000000111100"
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: team-a
+---
+apiVersion: composition.google.com/v1alpha1
+kind: Context
+metadata:
+  name: context
+  namespace: team-a
+spec:
+  project: proj-a
+---
+apiVersion: facade.foocorp.com/v1alpha1
+kind: PConfig
+metadata:
+  name: team-a-config
+  namespace: team-a
+spec:
+  project: proj-a