Needed changes operator sdk upgrade

.circleci/config.yml

@@ -42,7 +42,7 @@ commands:
       - run:
           name: Install operator-sdk
           command: |
-            export OPERATOR_SDK_RELEASE_VERSION=v0.8.0
+            export OPERATOR_SDK_RELEASE_VERSION=v0.15.2
             curl -OJL https://github.com/operator-framework/operator-sdk/releases/download/${OPERATOR_SDK_RELEASE_VERSION}/operator-sdk-${OPERATOR_SDK_RELEASE_VERSION}-x86_64-linux-gnu
             chmod +x operator-sdk-${OPERATOR_SDK_RELEASE_VERSION}-x86_64-linux-gnu && sudo cp operator-sdk-${OPERATOR_SDK_RELEASE_VERSION}-x86_64-linux-gnu /usr/local/bin/operator-sdk && rm operator-sdk-${OPERATOR_SDK_RELEASE_VERSION}-x86_64-linux-gnu
 

Makefile

@@ -52,7 +52,7 @@ tag:
 
 ## local: push operator docker image to remote repo
 local:
-	OPERATOR_NAME=$(OPERATOR_NAME) $(OPERATOR_SDK) up local --namespace $(NAMESPACE)
+	OPERATOR_NAME=$(OPERATOR_NAME) $(OPERATOR_SDK) run --local --namespace $(NAMESPACE)
 
 ## e2e-setup: create OCP project for the operator
 e2e-setup:

build/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi7/ubi-minimal:latest
+FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
 
 ENV OPERATOR=/usr/local/bin/3scale-operator \
     USER_UID=1001 \

build/bin/entrypoint

@@ -1,12 +1,3 @@
 #!/bin/sh -e
 
-# This is documented here:
-# https://docs.openshift.com/container-platform/3.11/creating_images/guidelines.html#openshift-specific-guidelines
-
-if ! whoami &>/dev/null; then
-  if [ -w /etc/passwd ]; then
-    echo "${USER_NAME:-3scale-operator}:x:$(id -u):$(id -g):${USER_NAME:-3scale-operator} user:${HOME}:/sbin/nologin" >> /etc/passwd
-  fi
-fi
-
 exec ${OPERATOR} $@

build/bin/user_setup

@@ -2,12 +2,10 @@
 set -x
 
 # ensure $HOME exists and is accessible by group 0 (we don't know what the runtime UID will be)
+echo "${USER_NAME}:x:${USER_UID}:0:${USER_NAME} user:${HOME}:/sbin/nologin" >> /etc/passwd
 mkdir -p ${HOME}
 chown ${USER_UID}:0 ${HOME}
 chmod ug+rwx ${HOME}
 
-# runtime user will need to be able to self-insert in /etc/passwd
-chmod g+rw /etc/passwd
-
 # no need for this script to remain in the image after running
 rm $0

cmd/manager/main.go

@@ -2,13 +2,15 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"os"
 	"runtime"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/rest"
 
 	"github.com/3scale/3scale-operator/pkg/3scale/amp/product"
 	"github.com/3scale/3scale-operator/pkg/apis"
@@ -17,27 +19,32 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
+	kubemetrics "github.com/operator-framework/operator-sdk/pkg/kube-metrics"
 	"github.com/operator-framework/operator-sdk/pkg/leader"
 	"github.com/operator-framework/operator-sdk/pkg/log/zap"
 	"github.com/operator-framework/operator-sdk/pkg/metrics"
-	"github.com/operator-framework/operator-sdk/pkg/restmapper"
 	sdkVersion "github.com/operator-framework/operator-sdk/version"
 	"github.com/spf13/pflag"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 	crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
-	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
-	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
 )
 
 // Change below variables to serve metrics on different host or port.
 var (
-	metricsHost       = "0.0.0.0"
-	metricsPort int32 = 8383
+	metricsHost               = "0.0.0.0"
+	metricsPort         int32 = 8383
+	operatorMetricsPort int32 = 8686
 )
 var log = logf.Log.WithName("cmd")
 
 func printVersion() {
+	log.Info(fmt.Sprintf("Operator Version: %s", version.Version))
 	log.Info(fmt.Sprintf("Go Version: %s", runtime.Version()))
 	log.Info(fmt.Sprintf("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH))
 	log.Info(fmt.Sprintf("Version of operator-sdk: %v", sdkVersion.Version))
@@ -80,7 +87,6 @@ func main() {
 	}
 
 	ctx := context.TODO()
-
 	// Become the leader before proceeding
 	err = leader.Become(ctx, "3scale-operator-lock")
 	if err != nil {
@@ -91,7 +97,6 @@ func main() {
 	// Create a new Cmd to provide shared dependencies and start components
 	mgr, err := manager.New(cfg, manager.Options{
 		Namespace:          namespace,
-		MapperProvider:     restmapper.NewDynamicRESTMapper,
 		MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort),
 	})
 	if err != nil {
@@ -115,11 +120,8 @@ func main() {
 
 	register3scaleVersionInfoMetric()
 
-	// Create Service object to expose the metrics port.
-	_, err = metrics.ExposeMetricsPort(ctx, metricsPort)
-	if err != nil {
-		log.Info(err.Error())
-	}
+	// Add the Metrics Service
+	addMetrics(ctx, cfg, namespace)
 
 	log.Info("Starting the Cmd.")
 
@@ -144,3 +146,134 @@ func register3scaleVersionInfoMetric() {
 	// Register custom metrics with the global prometheus registry
 	crmetrics.Registry.MustRegister(threeScaleVersionInfo)
 }
+
+// addMetrics will create the Services and Service Monitors to allow the operator export the metrics by using
+// the Prometheus operator
+func addMetrics(ctx context.Context, cfg *rest.Config, namespace string) {
+	if err := serveCRMetrics(cfg); err != nil {
+		if errors.Is(err, k8sutil.ErrRunLocal) {
+			log.Info("Skipping CR metrics server creation; not running in a cluster.")
+			return
+		}
+		log.Info("Could not generate and serve custom resource metrics", "error", err.Error())
+	}
+
+	// Add to the below struct any other metrics ports you want to expose.
+	servicePorts := []v1.ServicePort{
+		{Port: metricsPort, Name: metrics.OperatorPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: metricsPort}},
+		{Port: operatorMetricsPort, Name: metrics.CRPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: operatorMetricsPort}},
+	}
+
+	// Create Service object to expose the metrics port(s).
+	service, err := metrics.CreateMetricsService(ctx, cfg, servicePorts)
+	if err != nil {
+		log.Info("Could not create metrics Service", "error", err.Error())
+	}
+
+	// CreateServiceMonitors will automatically create the prometheus-operator ServiceMonitor resources
+	// necessary to configure Prometheus to scrape metrics from this operator.
+	services := []*v1.Service{service}
+	_, err = metrics.CreateServiceMonitors(cfg, namespace, services)
+	if err != nil {
+		log.Info("Could not create ServiceMonitor object", "error", err.Error())
+		// If this operator is deployed to a cluster without the prometheus-operator running, it will return
+		// ErrServiceMonitorNotPresent, which can be used to safely skip ServiceMonitor creation.
+		if err == metrics.ErrServiceMonitorNotPresent {
+			log.Info("Install prometheus-operator in your cluster to create ServiceMonitor objects", "error", err.Error())
+		}
+	}
+}
+
+// serveCRMetrics gets the Operator/CustomResource GVKs and generates metrics based on those types.
+// It serves those metrics on "http://metricsHost:operatorMetricsPort".
+func serveCRMetrics(cfg *rest.Config) error {
+	// Below function returns filtered operator/CustomResource specific GVKs.
+	// For more control override the below GVK list with your own custom logic.
+	gvks, err := k8sutil.GetGVKsFromAddToScheme(apis.AddToScheme)
+	if err != nil {
+		return err
+	}
+
+	// We perform our custom GKV filtering on top of the one performed
+	// by operator-sdk code
+	filteredGVK := filterGKVsFromAddToScheme(gvks)
+	if err != nil {
+		return err
+	}
+
+	// Get the namespace the operator is currently deployed in.
+	operatorNs, err := k8sutil.GetOperatorNamespace()
+	if err != nil {
+		return err
+	}
+	// To generate metrics in other namespaces, add the values below.
+	ns := []string{operatorNs}
+	// Generate and serve custom resource specific metrics.
+	err = kubemetrics.GenerateAndServeCRMetrics(cfg, ns, filteredGVK, metricsHost, operatorMetricsPort)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func filterGKVsFromAddToScheme(gvks []schema.GroupVersionKind) []schema.GroupVersionKind {
+	// We use gkvFilters to filter from the existing GKVs defined in the used
+	// runtime.Schema for the operator. The reason for that is that
+	// kube-metrics tries to list all of the defined Kinds in the schemas
+	// that are passed, including Kinds that the operator doesn't use and
+	// thus the role used the operator doesn't have them set and we don't want
+	// to set as they are not used by the operator.
+	// For the fields that the filters have we have defined the value '*' to
+	// specify any will be a match (accepted)
+	matchAnyValue := "*"
+	gvkFilters := []schema.GroupVersionKind{
+		// Kubernetes types
+		schema.GroupVersionKind{Kind: "PersistentVolumeClaim", Version: matchAnyValue},
+		schema.GroupVersionKind{Kind: "ServiceAccount", Version: matchAnyValue},
+		schema.GroupVersionKind{Kind: "Secret", Version: matchAnyValue},
+		schema.GroupVersionKind{Kind: "Pod", Version: matchAnyValue},
+		schema.GroupVersionKind{Kind: "ConfigMap", Version: matchAnyValue},
+		schema.GroupVersionKind{Kind: "Service", Version: matchAnyValue},
+
+		// OpenShift types
+		schema.GroupVersionKind{Group: "route.openshift.io", Kind: "Route", Version: matchAnyValue},
+		schema.GroupVersionKind{Group: "image.openshift.io", Kind: "ImageStream", Version: matchAnyValue},
+		schema.GroupVersionKind{Group: "apps.openshift.io", Kind: "DeploymentConfig", Version: matchAnyValue},
+
+		// Custom resource types
+		schema.GroupVersionKind{Group: "capabilities.3scale.net", Kind: "Plan", Version: matchAnyValue},
+		schema.GroupVersionKind{Group: "capabilities.3scale.net", Kind: "API", Version: matchAnyValue},
+		schema.GroupVersionKind{Group: "capabilities.3scale.net", Kind: "Limit", Version: matchAnyValue},
+		schema.GroupVersionKind{Group: "capabilities.3scale.net", Kind: "MappingRule", Version: matchAnyValue},
+		schema.GroupVersionKind{Group: "capabilities.3scale.net", Kind: "Tenant", Version: matchAnyValue},
+		schema.GroupVersionKind{Group: "capabilities.3scale.net", Kind: "Metric", Version: matchAnyValue},
+		schema.GroupVersionKind{Group: "capabilities.3scale.net", Kind: "Binding", Version: matchAnyValue},
+		schema.GroupVersionKind{Group: "apps.3scale.net", Kind: "APIManager", Version: matchAnyValue},
+	}
+
+	ownGVKs := []schema.GroupVersionKind{}
+	for _, gvk := range gvks {
+		for _, gvkFilter := range gvkFilters {
+			match := true
+			if gvkFilter.Kind == matchAnyValue && gvkFilter.Group == matchAnyValue && gvkFilter.Version == matchAnyValue {
+				log.V(1).Info("gvkFilter should at least have one of its fields defined. Skipping...")
+				match = false
+			} else {
+				if gvkFilter.Kind != matchAnyValue && gvkFilter.Kind != gvk.Kind {
+					match = false
+				}
+				if gvkFilter.Group != matchAnyValue && gvkFilter.Group != gvk.Group {
+					match = false
+				}
+				if gvkFilter.Version != matchAnyValue && gvkFilter.Version != gvk.Version {
+					match = false
+				}
+			}
+			if match {
+				ownGVKs = append(ownGVKs, gvk)
+			}
+		}
+	}
+
+	return ownGVKs
+}

deploy/crds/apps_v1alpha1_apimanager_crd.yaml → deploy/crds/apps.3scale.net_apimanagers_crd.yaml

@@ -14,20 +14,22 @@ spec:
     status: {}
   validation:
     openAPIV3Schema:
+      description: APIManager is the Schema for the apimanagers API
       properties:
         apiVersion:
           description: 'APIVersion defines the versioned schema of this representation
             of an object. Servers should convert recognized schemas to the latest
-            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
           type: string
         kind:
           description: 'Kind is a string value representing the REST resource this
             object represents. Servers may infer this from the endpoint the client
-            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
           type: string
         metadata:
           type: object
         spec:
+          description: APIManagerSpec defines the desired state of APIManager
           properties:
             apicast:
               properties:
@@ -126,18 +128,20 @@ spec:
                           description: Deprecated
                           type: string
                         awsCredentialsSecret:
+                          description: Deprecated
                           properties:
                             name:
+                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                TODO: Add other useful fields. apiVersion, kind, uid?'
                               type: string
-                          description: Deprecated
                           type: object
                         awsRegion:
                           description: Deprecated
                           type: string
                       required:
                       - awsBucket
-                      - awsRegion
                       - awsCredentialsSecret
+                      - awsRegion
                       type: object
                     persistentVolumeClaim:
                       description: Union type. Only one of the fields can be set.
@@ -148,10 +152,15 @@ spec:
                     simpleStorageService:
                       properties:
                         configurationSecretRef:
-                          type: object
+                          description: LocalObjectReference contains enough information
+                            to let you locate the referenced object inside the same
+                            namespace.
                           properties:
                             name:
+                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                TODO: Add other useful fields. apiVersion, kind, uid?'
                               type: string
+                          type: object
                       required:
                       - configurationSecretRef
                       type: object
@@ -172,6 +181,7 @@ spec:
             tenantName:
               type: string
             wildcardDomain:
+              description: Wildcard domain as configured in the API Manager object
               type: string
             zync:
               properties:
@@ -196,6 +206,7 @@ spec:
           - wildcardDomain
           type: object
         status:
+          description: APIManagerStatus defines the observed state of APIManager
           properties:
             conditions:
               items:
@@ -205,11 +216,12 @@ spec:
                   type:
                     type: string
                 required:
-                - type
                 - status
+                - type
                 type: object
               type: array
             deployments:
+              description: APIManager Deployment Configs
               properties:
                 ready:
                   description: Deployments are ready to serve requests
@@ -231,6 +243,7 @@ spec:
           required:
           - deployments
           type: object
+      type: object
   version: v1alpha1
   versions:
   - name: v1alpha1