THREESCALE-9001 / Zync Routes Resync Function with multitenant

go.mod

@@ -30,6 +30,8 @@ require (
 	sigs.k8s.io/controller-runtime v0.12.2
 )
 
+require github.com/moby/spdystream v0.2.0 // indirect
+
 require (
 	cloud.google.com/go/compute v1.7.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect

go.sum

@@ -104,6 +104,7 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
@@ -185,6 +186,7 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -623,6 +625,7 @@ github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
 github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=

pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go

@@ -1,7 +1,13 @@
 package operator
 
 import (
+	"context"
+	"encoding/xml"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"time"
 
 	appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1"
 	"github.com/3scale/3scale-operator/pkg/common"
@@ -19,6 +25,7 @@ import (
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 type BaseAPIManagerLogicReconciler struct {
@@ -35,6 +42,16 @@ type baseAPIManagerLogicReconcilerCRDAvailabilityCache struct {
 	serviceMonitorCRDAvailable   *bool
 }
 
+type Accounts struct {
+    XMLName xml.Name `xml:"accounts"`
+    Account []Account `xml:"account"`
+}
+
+type Account struct {
+    AdminBaseURL string `xml:"admin_base_url"`
+    BaseURL      string `xml:"base_url"`
+}
+
 func NewBaseAPIManagerLogicReconciler(b *reconcilers.BaseReconciler, apiManager *appsv1alpha1.APIManager) *BaseAPIManagerLogicReconciler {
 	return &BaseAPIManagerLogicReconciler{
 		BaseReconciler:       b,
@@ -116,6 +133,269 @@ func (r *BaseAPIManagerLogicReconciler) ReconcileGrafanaDashboard(desired *grafa
 	return r.ReconcileResource(&grafanav1alpha1.GrafanaDashboard{}, desired, mutateFn)
 }
 
+func (r *BaseAPIManagerLogicReconciler) findSystemSidekiqPod(apimanager *appsv1alpha1.APIManager) (string, error) {
+	namespace := apimanager.GetNamespace()
+	podName := ""
+	podList := &v1.PodList{}
+
+    // system-sidekiq pod needs to be up & running
+    err := r.waitForSystemSidekiq(apimanager)
+    if err != nil {
+        return "", fmt.Errorf("failed to wait for system-sidekiq: %w", err)
+    }
+
+	listOps := []client.ListOption{
+		client.InNamespace(namespace),
+		client.MatchingLabels(map[string]string{"deploymentConfig": "system-sidekiq"}),
+	}
+
+	err = r.Client().List(context.TODO(), podList, listOps...)
+	if err != nil {
+		return "", fmt.Errorf("failed to list pods: %w", err)
+	}
+
+	for _, pod := range podList.Items {
+		if pod.Status.Phase == "Running" {
+			podName = pod.ObjectMeta.Name
+			break
+		}
+	}
+
+	if podName == "" {
+		return "",fmt.Errorf("no matching pod found")
+	}
+
+	return podName, nil
+}
+
+func (r *BaseAPIManagerLogicReconciler) getAccessToken() (string, error) {
+
+    namespace := r.apiManager.GetNamespace()
+
+	secretName := types.NamespacedName{
+        Namespace: namespace,
+        Name:      "system-seed",
+    }
+
+    secret := &v1.Secret{}
+    err := r.Client().Get(context.TODO(), secretName, secret)
+    if err != nil {
+        return "", fmt.Errorf("failed to get secret: %w", err)
+    }
+
+    // Retrieve the access token from the secret data
+    accessToken, ok := secret.Data["MASTER_ACCESS_TOKEN"]
+    if !ok {
+        return "", fmt.Errorf("access token not found in secret")
+    }
+
+    return string(accessToken), nil
+}
+
+func (r *BaseAPIManagerLogicReconciler) getMasterRoute() (string, error) {
+
+	masterPrefix := "master"
+
+	opts := []client.ListOption{
+        client.InNamespace(r.apiManager.GetNamespace()),
+    }
+	foundRoute := ""
+	routes := routev1.RouteList{}
+    err := r.Client().List(context.TODO(), &routes, opts ...)
+    if err != nil {
+        return "", err
+    }
+
+	for _, route := range routes.Items {
+		if strings.HasPrefix(route.Spec.Host, masterPrefix) {
+			foundRoute = "https://" + route.Spec.Host
+			return foundRoute, nil
+		}
+	}
+
+	return "", fmt.Errorf("route not found")
+}
+
+func (r *BaseAPIManagerLogicReconciler) baseRoutesExist() (bool, error) {
+	expectedRoutes := 2
+    serviceNames := []string{"system-master", "backend-listener"}
+    opts := []client.ListOption{
+        client.InNamespace(r.apiManager.GetNamespace()),
+    }
+
+    routes := routev1.RouteList{}
+    err := r.Client().List(context.TODO(), &routes, opts ...)
+    if err != nil {
+        return false, err
+    }
+
+    serviceCount := 0
+    for _, service := range serviceNames {
+        found := false
+        for _, route := range routes.Items {
+            if route.Spec.To.Name == service {
+                found = true
+                break
+            }
+        }
+        if found {
+            serviceCount++
+        }
+    }
+
+    if serviceCount >= expectedRoutes {
+        return true, nil
+    }
+
+    return false, fmt.Errorf("base routes not found")
+}
+
+func (r *BaseAPIManagerLogicReconciler) getAccountUrls() ([]string, []string, error){
+
+	state := "approved"
+
+	masterRoute, err := r.getMasterRoute()
+	if err != nil {
+		r.logger.Error(err, "Error getting Master Route")
+		return nil, nil, err
+	}
+
+	url := masterRoute + "/admin/api/accounts.xml"
+
+    accessToken, err := r.getAccessToken()
+	if err != nil {
+		fmt.Println("Error getting Access Token:", err)
+		return nil, nil, err
+	}
+
+    // Create a new HTTP GET request
+    req, err := http.NewRequest("GET", url, nil)
+    if err != nil {
+        fmt.Println("Error Creating HTTP Request:", err)
+        return nil, nil, err
+    }
+
+    // Add query parameters to the request
+    q := req.URL.Query()
+    q.Add("access_token", accessToken)
+    q.Add("state", state)
+    req.URL.RawQuery = q.Encode()
+
+    // Send the HTTP request
+    client := &http.Client{}
+    resp, err := client.Do(req)
+    if err != nil {
+        fmt.Println("Error sending HTTP request:", err)
+        return nil, nil, err
+    }
+    defer resp.Body.Close()
+
+    // Read and parse the XML response
+    body, err := ioutil.ReadAll(resp.Body)
+    if err != nil {
+        fmt.Println("Error reading HTTP response:", err)
+        return nil, nil, err
+    }
+
+    var accounts Accounts
+	err = xml.Unmarshal([]byte(body), &accounts)
+	if err != nil {
+		fmt.Println("Error unmarshalling HTTP response:", err)
+		return nil, nil, err
+	}
+
+    var adminBaseURLs []string
+    var baseURLs []string
+    for _, account := range accounts.Account {
+        adminBaseURLs = append(adminBaseURLs, account.AdminBaseURL)
+        baseURLs = append(baseURLs, account.BaseURL)
+    }
+
+    return adminBaseURLs, baseURLs, nil
+}
+
+func (r *BaseAPIManagerLogicReconciler) routesExist() (bool, error) {
+	_, err:= r.baseRoutesExist()
+	if err != nil {
+		return false, nil
+	}
+
+	adminBaseURLs, baseURLs, err := r.getAccountUrls()
+    if err != nil {
+        return false, err
+    }
+
+	opts := []client.ListOption{
+        client.InNamespace(r.apiManager.GetNamespace()),
+    }
+
+    routes := routev1.RouteList{}
+    err = r.Client().List(context.TODO(), &routes, opts ...)
+    if err != nil {
+        return false, err
+    }
+
+    // Create a map to track the presence of adminBaseURLs and baseURLs
+    urlMap := make(map[string]bool)
+    for _, url := range append(adminBaseURLs, baseURLs...) {
+        urlMap[url] = false
+    }
+
+    // Check if all adminBaseURLs and baseURLs are present in the route location URLs
+    for _, route := range routes.Items {
+		routeHost := "https://" + route.Spec.Host
+        for url := range urlMap {
+            if routeHost == url {
+                urlMap[url] = true
+            }
+        }
+    }
+
+    missingURLs := []string{}
+    for url, found := range urlMap {
+        if !found {
+            missingURLs = append(missingURLs, url)
+        }
+    }
+
+    if len(missingURLs) == 0 {
+		return true, nil
+    }
+
+	return false, nil
+}
+
+
+func (r *BaseAPIManagerLogicReconciler) executeCommandOnPod(containerName string, namespace string, podName string, command []string) (string, string, error) {
+	podExecutor := helper.NewPodExecutor(r.logger) 
+
+	stdout, stderr, err := podExecutor.ExecuteRemoteContainerCommand(namespace, podName, containerName, command)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to execute command on pod: %w, stderr: %s", err, stderr)
+	}
+
+	fmt.Println("Command output (stdout):", stdout)
+	fmt.Println("Command output (stderr):", stderr)
+    return stdout, stderr, err
+}
+
+func (r *BaseAPIManagerLogicReconciler) waitForSystemSidekiq(apimanager *appsv1alpha1.APIManager) error {
+
+    // Wait until system-sidekiq deployments are ready
+    for !helper.ArrayContains(apimanager.Status.Deployments.Ready, "system-sidekiq") {
+        r.Logger().Info("system-sidekiq deployments not ready. Waiting", "APIManager", apimanager.Name)
+        time.Sleep(5 * time.Second)
+
+        // Refresh APIManager status
+        err := r.GetResource(types.NamespacedName{Name: r.apiManager.Name, Namespace: r.apiManager.Namespace}, apimanager)
+        if err != nil {
+            return fmt.Errorf("failed to get APIManager: %w", err)
+        }
+    }
+
+    return nil
+}
+
 func (r *BaseAPIManagerLogicReconciler) ReconcilePrometheusRules(desired *monitoringv1.PrometheusRule, mutateFn reconcilers.MutateFn) error {
 	kindExists, err := r.HasPrometheusRules()
 	if err != nil {