Rework reconcile loop for workspace controller

This commit represents an almost-complete overhaul of the main
reconcile loop in the workspace controller.

A high level overview of changes:

- All controllers do full reconciling of all objects they are
  responsible for; this means deleting a route or the workspace
  deployment means it will be recreated
  - The main workspace controller now watches all resources it creates
    to trigger reconciles
- The main reconcile loop is split into phases with subcontrollers; it
  only progresses based on status of earlier steps (i.e. if components
  aren't ready, we don't try to create routing)
- All service/ingress/route creation is delegated to WorkspaceRouting
- The openshift-oauth routingClass results in the openshift oauth-proxy
  container running in the main workspace deployment
- There's a cleaner separation between elements in `pkg/controller` --
  no imports across controllers (i.e. WorkspaceRouting imports nothing
  from Workspace)
  - All shared structs are extracted to `apis` folder
- One service is created for all workspace endpoints (except
  discoverable endpoints)
- Add Component subcontroller that converts devfile components into
  k8s objects

A design doc and more detailed history for these changes is found at
https://github.com/amisevsk/che-workspace-operator-rework

Signed-off-by: Angel Misevski <amisevsk@redhat.com>

cmd/manager/main.go

@@ -1,44 +1,59 @@
+//
+// Copyright (c) 2019-2020 Red Hat, Inc.
+// This program and the accompanying materials are made
+// available under the terms of the Eclipse Public License 2.0
+// which is available at https://www.eclipse.org/legal/epl-2.0/
+//
+// SPDX-License-Identifier: EPL-2.0
+//
+// Contributors:
+//   Red Hat, Inc. - initial API and implementation
+//
+
 package main
 
 import (
 	"context"
 	"flag"
 	"fmt"
-	"github.com/che-incubator/che-workspace-operator/pkg/controller/registry"
-	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/util/intstr"
-	"k8s.io/client-go/rest"
 	"os"
 	"runtime"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/rest"
 
 	"github.com/che-incubator/che-workspace-operator/pkg/apis"
 	"github.com/che-incubator/che-workspace-operator/pkg/controller"
 	"github.com/che-incubator/che-workspace-operator/pkg/webhook"
+	"github.com/che-incubator/che-workspace-operator/version"
 
 	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
+	kubemetrics "github.com/operator-framework/operator-sdk/pkg/kube-metrics"
 	"github.com/operator-framework/operator-sdk/pkg/leader"
 	"github.com/operator-framework/operator-sdk/pkg/log/zap"
 	"github.com/operator-framework/operator-sdk/pkg/metrics"
+	"github.com/operator-framework/operator-sdk/pkg/restmapper"
 	sdkVersion "github.com/operator-framework/operator-sdk/version"
 	"github.com/spf13/pflag"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
-	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
-
-	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
+	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 )
 
 // Change below variables to serve metrics on different host or port.
 var (
-	metricsHost       = "0.0.0.0"
-	metricsPort int32 = 8383
+	metricsHost               = "0.0.0.0"
+	metricsPort         int32 = 8383
+	operatorMetricsPort int32 = 8686
 )
 var log = logf.Log.WithName("cmd")
 
 func printVersion() {
+	log.Info(fmt.Sprintf("Operator Version: %s", version.Version))
 	log.Info(fmt.Sprintf("Go Version: %s", runtime.Version()))
 	log.Info(fmt.Sprintf("Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH))
 	log.Info(fmt.Sprintf("Version of operator-sdk: %v", sdkVersion.Version))
@@ -81,9 +96,8 @@ func main() {
 	}
 
 	ctx := context.TODO()
-
 	// Become the leader before proceeding
-	err = leader.Become(ctx, "che-workspace-controller-lock")
+	err = leader.Become(ctx, "che-workspace-operator-lock")
 	if err != nil {
 		log.Error(err, "")
 		os.Exit(1)
@@ -92,6 +106,7 @@ func main() {
 	// Create a new Cmd to provide shared dependencies and start components
 	mgr, err := manager.New(cfg, manager.Options{
 		Namespace:          namespace,
+		MapperProvider:     restmapper.NewDynamicRESTMapper,
 		MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort),
 	})
 	if err != nil {
@@ -107,14 +122,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	log.Info("Expose Plugin Registry Port.")
-
-	// Create Service object to expose the embedded plugin registry.
-	_, err = registry.ExposeRegistryPort(ctx, 8080)
-	if err != nil {
-		log.Info(err.Error())
-	}
-
 	log.Info("Setting up Controllers.")
 
 	// Setup all Controllers
@@ -123,20 +130,43 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Setup all webhooks
 	log.Info("Setting up webhooks")
 	if err := webhook.SetUpWebhooks(mgr, ctx); err != nil {
 		log.Error(err, "unable to register webhooks to the manager")
 		os.Exit(1)
 	}
 
+	// TODO: Required to filter GVK for metrics, since we add routes and templates to the scheme.
+	// TODO: see: https://github.com/operator-framework/operator-sdk/pull/2606
+	//if err = serveCRMetrics(cfg); err != nil {
+	//	log.Info("Could not generate and serve custom resource metrics", "error", err.Error())
+	//}
+
 	log.Info("Expose Metrics Port.")
+	// Add to the below struct any other metrics ports you want to expose.
 	servicePorts := []v1.ServicePort{
 		{Port: metricsPort, Name: metrics.OperatorPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: metricsPort}},
+		// TODO: See above re: CR metrics
+		//{Port: operatorMetricsPort, Name: metrics.CRPortName, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: operatorMetricsPort}},
+	}
+	// Create Service object to expose the metrics port(s).
+	service, err := metrics.CreateMetricsService(ctx, cfg, servicePorts)
+	if err != nil {
+		log.Info("Could not create metrics Service", "error", err.Error())
 	}
-	// Create Service object to expose the metrics port.
-	_, err = metrics.CreateMetricsService(ctx, &rest.Config{}, servicePorts)
+
+	// CreateServiceMonitors will automatically create the prometheus-operator ServiceMonitor resources
+	// necessary to configure Prometheus to scrape metrics from this operator.
+	services := []*v1.Service{service}
+	_, err = metrics.CreateServiceMonitors(cfg, namespace, services)
 	if err != nil {
-		log.Info(err.Error())
+		log.Info("Could not create ServiceMonitor object", "error", err.Error())
+		// If this operator is deployed to a cluster without the prometheus-operator running, it will return
+		// ErrServiceMonitorNotPresent, which can be used to safely skip ServiceMonitor creation.
+		if err == metrics.ErrServiceMonitorNotPresent {
+			log.Info("Install prometheus-operator in your cluster to create ServiceMonitor objects", "error", err.Error())
+		}
 	}
 
 	log.Info("Starting the Cmd.")
@@ -147,3 +177,27 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+// serveCRMetrics gets the Operator/CustomResource GVKs and generates metrics based on those types.
+// It serves those metrics on "http://metricsHost:operatorMetricsPort".
+func serveCRMetrics(cfg *rest.Config) error {
+	// Below function returns filtered operator/CustomResource specific GVKs.
+	// For more control override the below GVK list with your own custom logic.
+	filteredGVK, err := k8sutil.GetGVKsFromAddToScheme(apis.AddToScheme)
+	if err != nil {
+		return err
+	}
+	// Get the namespace the operator is currently deployed in.
+	operatorNs, err := k8sutil.GetOperatorNamespace()
+	if err != nil {
+		return err
+	}
+	// To generate metrics in other namespaces, add the values below.
+	ns := []string{operatorNs}
+	// Generate and serve custom resource specific metrics.
+	err = kubemetrics.GenerateAndServeCRMetrics(cfg, ns, filteredGVK, metricsHost, operatorMetricsPort)
+	if err != nil {
+		return err
+	}
+	return nil
+}

deploy/controller_config.yaml

@@ -8,6 +8,6 @@ data:
   ingress.global.domain: 192.168.99.100.nip.io
   plugin.registry.url: http://che-plugin-registry.192.168.99.100.nip.io/v3
   che.workspace.plugin_broker.artifacts.image: quay.io/eclipse/che-plugin-artifacts-broker:v3.1.0
-  cherestapis.image.name: quay.io/dfestal/che-workspace-crd-rest-apis:newone
+  cherestapis.image.name: amisevsk/che-rest-apis:latest
   che.webhooks.enabled: "false"
   che.default_routing_class: "basic"