diff --git a/go.mod b/go.mod index 56315ec51..1da6fbe6d 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,8 @@ require ( sigs.k8s.io/controller-runtime v0.12.2 ) +require github.com/moby/spdystream v0.2.0 // indirect + require ( cloud.google.com/go/compute v1.7.0 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect diff --git a/go.sum b/go.sum index d0ae4ba89..c99e91374 100644 --- a/go.sum +++ b/go.sum @@ -104,6 +104,7 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -185,6 +186,7 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -623,6 +625,7 @@ github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= diff --git a/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go b/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go index 1a7488478..d9a8a8428 100644 --- a/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go +++ b/pkg/3scale/amp/operator/base_apimanager_logic_reconciler.go @@ -1,7 +1,13 @@ package operator import ( + "context" + "encoding/xml" "fmt" + "io/ioutil" + "net/http" + "strings" + "time" appsv1alpha1 "github.com/3scale/3scale-operator/apis/apps/v1alpha1" "github.com/3scale/3scale-operator/pkg/common" @@ -19,6 +25,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) type BaseAPIManagerLogicReconciler struct { @@ -35,6 +42,16 @@ type baseAPIManagerLogicReconcilerCRDAvailabilityCache struct { serviceMonitorCRDAvailable *bool } +type Accounts struct { + XMLName xml.Name `xml:"accounts"` + Account []Account `xml:"account"` +} + +type Account struct { + AdminBaseURL string `xml:"admin_base_url"` + BaseURL string `xml:"base_url"` +} + func NewBaseAPIManagerLogicReconciler(b *reconcilers.BaseReconciler, apiManager *appsv1alpha1.APIManager) *BaseAPIManagerLogicReconciler { return &BaseAPIManagerLogicReconciler{ BaseReconciler: b, @@ -116,6 +133,269 @@ func (r *BaseAPIManagerLogicReconciler) ReconcileGrafanaDashboard(desired *grafa return r.ReconcileResource(&grafanav1alpha1.GrafanaDashboard{}, desired, mutateFn) } +func (r *BaseAPIManagerLogicReconciler) findSystemSidekiqPod(apimanager *appsv1alpha1.APIManager) (string, error) { + namespace := apimanager.GetNamespace() + podName := "" + podList := &v1.PodList{} + + // system-sidekiq pod needs to be up & running + err := r.waitForSystemSidekiq(apimanager) + if err != nil { + return "", fmt.Errorf("failed to wait for system-sidekiq: %w", err) + } + + listOps := []client.ListOption{ + client.InNamespace(namespace), + client.MatchingLabels(map[string]string{"deploymentConfig": "system-sidekiq"}), + } + + err = r.Client().List(context.TODO(), podList, listOps...) + if err != nil { + return "", fmt.Errorf("failed to list pods: %w", err) + } + + for _, pod := range podList.Items { + if pod.Status.Phase == "Running" { + podName = pod.ObjectMeta.Name + break + } + } + + if podName == "" { + return "",fmt.Errorf("no matching pod found") + } + + return podName, nil +} + +func (r *BaseAPIManagerLogicReconciler) getAccessToken() (string, error) { + + namespace := r.apiManager.GetNamespace() + + secretName := types.NamespacedName{ + Namespace: namespace, + Name: "system-seed", + } + + secret := &v1.Secret{} + err := r.Client().Get(context.TODO(), secretName, secret) + if err != nil { + return "", fmt.Errorf("failed to get secret: %w", err) + } + + // Retrieve the access token from the secret data + accessToken, ok := secret.Data["MASTER_ACCESS_TOKEN"] + if !ok { + return "", fmt.Errorf("access token not found in secret") + } + + return string(accessToken), nil +} + +func (r *BaseAPIManagerLogicReconciler) getMasterRoute() (string, error) { + + masterPrefix := "master" + + opts := []client.ListOption{ + client.InNamespace(r.apiManager.GetNamespace()), + } + foundRoute := "" + routes := routev1.RouteList{} + err := r.Client().List(context.TODO(), &routes, opts ...) + if err != nil { + return "", err + } + + for _, route := range routes.Items { + if strings.HasPrefix(route.Spec.Host, masterPrefix) { + foundRoute = "https://" + route.Spec.Host + return foundRoute, nil + } + } + + return "", fmt.Errorf("route not found") +} + +func (r *BaseAPIManagerLogicReconciler) baseRoutesExist() (bool, error) { + expectedRoutes := 2 + serviceNames := []string{"system-master", "backend-listener"} + opts := []client.ListOption{ + client.InNamespace(r.apiManager.GetNamespace()), + } + + routes := routev1.RouteList{} + err := r.Client().List(context.TODO(), &routes, opts ...) + if err != nil { + return false, err + } + + serviceCount := 0 + for _, service := range serviceNames { + found := false + for _, route := range routes.Items { + if route.Spec.To.Name == service { + found = true + break + } + } + if found { + serviceCount++ + } + } + + if serviceCount >= expectedRoutes { + return true, nil + } + + return false, fmt.Errorf("base routes not found") +} + +func (r *BaseAPIManagerLogicReconciler) getAccountUrls() ([]string, []string, error){ + + state := "approved" + + masterRoute, err := r.getMasterRoute() + if err != nil { + r.logger.Error(err, "Error getting Master Route") + return nil, nil, err + } + + url := masterRoute + "/admin/api/accounts.xml" + + accessToken, err := r.getAccessToken() + if err != nil { + fmt.Println("Error getting Access Token:", err) + return nil, nil, err + } + + // Create a new HTTP GET request + req, err := http.NewRequest("GET", url, nil) + if err != nil { + fmt.Println("Error Creating HTTP Request:", err) + return nil, nil, err + } + + // Add query parameters to the request + q := req.URL.Query() + q.Add("access_token", accessToken) + q.Add("state", state) + req.URL.RawQuery = q.Encode() + + // Send the HTTP request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + fmt.Println("Error sending HTTP request:", err) + return nil, nil, err + } + defer resp.Body.Close() + + // Read and parse the XML response + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading HTTP response:", err) + return nil, nil, err + } + + var accounts Accounts + err = xml.Unmarshal([]byte(body), &accounts) + if err != nil { + fmt.Println("Error unmarshalling HTTP response:", err) + return nil, nil, err + } + + var adminBaseURLs []string + var baseURLs []string + for _, account := range accounts.Account { + adminBaseURLs = append(adminBaseURLs, account.AdminBaseURL) + baseURLs = append(baseURLs, account.BaseURL) + } + + return adminBaseURLs, baseURLs, nil +} + +func (r *BaseAPIManagerLogicReconciler) routesExist() (bool, error) { + _, err:= r.baseRoutesExist() + if err != nil { + return false, nil + } + + adminBaseURLs, baseURLs, err := r.getAccountUrls() + if err != nil { + return false, err + } + + opts := []client.ListOption{ + client.InNamespace(r.apiManager.GetNamespace()), + } + + routes := routev1.RouteList{} + err = r.Client().List(context.TODO(), &routes, opts ...) + if err != nil { + return false, err + } + + // Create a map to track the presence of adminBaseURLs and baseURLs + urlMap := make(map[string]bool) + for _, url := range append(adminBaseURLs, baseURLs...) { + urlMap[url] = false + } + + // Check if all adminBaseURLs and baseURLs are present in the route location URLs + for _, route := range routes.Items { + routeHost := "https://" + route.Spec.Host + for url := range urlMap { + if routeHost == url { + urlMap[url] = true + } + } + } + + missingURLs := []string{} + for url, found := range urlMap { + if !found { + missingURLs = append(missingURLs, url) + } + } + + if len(missingURLs) == 0 { + return true, nil + } + + return false, nil +} + + +func (r *BaseAPIManagerLogicReconciler) executeCommandOnPod(containerName string, namespace string, podName string, command []string) (string, string, error) { + podExecutor := helper.NewPodExecutor(r.logger) + + stdout, stderr, err := podExecutor.ExecuteRemoteContainerCommand(namespace, podName, containerName, command) + if err != nil { + return "", "", fmt.Errorf("failed to execute command on pod: %w, stderr: %s", err, stderr) + } + + fmt.Println("Command output (stdout):", stdout) + fmt.Println("Command output (stderr):", stderr) + return stdout, stderr, err +} + +func (r *BaseAPIManagerLogicReconciler) waitForSystemSidekiq(apimanager *appsv1alpha1.APIManager) error { + + // Wait until system-sidekiq deployments are ready + for !helper.ArrayContains(apimanager.Status.Deployments.Ready, "system-sidekiq") { + r.Logger().Info("system-sidekiq deployments not ready. Waiting", "APIManager", apimanager.Name) + time.Sleep(5 * time.Second) + + // Refresh APIManager status + err := r.GetResource(types.NamespacedName{Name: r.apiManager.Name, Namespace: r.apiManager.Namespace}, apimanager) + if err != nil { + return fmt.Errorf("failed to get APIManager: %w", err) + } + } + + return nil +} + func (r *BaseAPIManagerLogicReconciler) ReconcilePrometheusRules(desired *monitoringv1.PrometheusRule, mutateFn reconcilers.MutateFn) error { kindExists, err := r.HasPrometheusRules() if err != nil { diff --git a/pkg/3scale/amp/operator/base_apimanager_logic_reconciler_test.go b/pkg/3scale/amp/operator/base_apimanager_logic_reconciler_test.go index 375a65c26..76ac9875c 100644 --- a/pkg/3scale/amp/operator/base_apimanager_logic_reconciler_test.go +++ b/pkg/3scale/amp/operator/base_apimanager_logic_reconciler_test.go @@ -19,6 +19,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" logf "sigs.k8s.io/controller-runtime/pkg/log" + routev1 "github.com/openshift/api/route/v1" ) func TestBaseAPIManagerLogicReconcilerUpdateOwnerRef(t *testing.T) { @@ -502,3 +503,124 @@ func TestBaseAPIManagerLogicReconcilerHasServiceMonitors(t *testing.T) { t.Fatalf("Unexpected exists value received. Expected: %t, got: %t", false, exists) } } + +func TestBaseAPIManagerLogicReconciler_FindSystemSidekiqPod(t *testing.T) { + apimanagerName := "example-apimanager" + namespace := "operator-unittest" + log := logf.Log.WithName("operator_test") + + ctx := context.TODO() + + apimanager := &appsv1alpha1.APIManager{ + ObjectMeta: metav1.ObjectMeta{ + Name: apimanagerName, + Namespace: namespace, + }, + Spec: appsv1alpha1.APIManagerSpec{}, + } + + // Register operator types with the runtime scheme. + s := scheme.Scheme + s.AddKnownTypes(appsv1alpha1.GroupVersion, apimanager) + s.AddKnownTypes(routev1.SchemeGroupVersion, &routev1.RouteList{}) + err := appsv1.AddToScheme(s) + if err != nil { + t.Fatal(err) + } + + // Objects to track in the fake client. + objs := []runtime.Object{apimanager} + + // Create a fake client to mock API calls. + cl := fake.NewFakeClient(objs...) + clientAPIReader := fake.NewFakeClient(objs...) + clientset := fakeclientset.NewSimpleClientset() + recorder := record.NewFakeRecorder(10000) + + baseReconciler := reconcilers.NewBaseReconciler(ctx, cl, s, clientAPIReader, log, clientset.Discovery(), recorder) + r := NewBaseAPIManagerLogicReconciler(baseReconciler, apimanager) + + // Mock the APIManager status with the necessary deployment + apimanager.Status.Deployments.Ready = []string{"system-sidekiq"} + + pod := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-pod", + Namespace: namespace, + Labels: map[string]string{ + "deploymentConfig": "system-sidekiq", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "example-container", + Image: "example-image", + }, + }, + }, + Status: v1.PodStatus{ + Phase: v1.PodRunning, + }, + } + + err = cl.Create(ctx, &pod) + if err != nil { + t.Fatalf("Failed to create pod: %s", err) + } + + // Mock the routes for the APIManager + routes := routev1.RouteList{ + Items: []routev1.Route{ + { + Spec: routev1.RouteSpec{ + To: routev1.RouteTargetReference{ + Name: "system-provider", + }, + }, + }, + { + Spec: routev1.RouteSpec{ + To: routev1.RouteTargetReference{ + Name: "system-master", + }, + }, + }, + { + Spec: routev1.RouteSpec{ + To: routev1.RouteTargetReference{ + Name: "system-developer", + }, + }, + }, + { + Spec: routev1.RouteSpec{ + To: routev1.RouteTargetReference{ + Name: "backend-listener", + }, + }, + }, + }, + } + + // Create a pointer to client.ListOptions + opts := &client.ListOptions{} + + // Call the real List method with the pointer to client.ListOptions + err = cl.List(ctx, &routes, opts) + + if err != nil { + t.Fatalf("Unexpected error while listing routes: %s", err) + } + + foundPodName, err := r.findSystemSidekiqPod(apimanager) + + if err != nil { + t.Errorf("failed to execute command on pod: %s, stderr: %s", "", "") + } + if foundPodName != "example-pod" { + t.Errorf("expected: %s, got: %s", "example-pod", foundPodName) + } +} + + diff --git a/pkg/3scale/amp/operator/zync_reconciler.go b/pkg/3scale/amp/operator/zync_reconciler.go index c650c2aa5..2b52a2004 100644 --- a/pkg/3scale/amp/operator/zync_reconciler.go +++ b/pkg/3scale/amp/operator/zync_reconciler.go @@ -160,6 +160,32 @@ func (r *ZyncReconciler) Reconcile() (reconcile.Result, error) { return reconcile.Result{}, err } + if len(r.apiManager.Status.Deployments.Starting) == 0 && len(r.apiManager.Status.Deployments.Stopped) == 0 && len(r.apiManager.Status.Deployments.Ready) > 0 { + exist, err := r.routesExist() + if err != nil { + return reconcile.Result{}, err + } + if exist { + return reconcile.Result{}, nil + } else { + // If the system-provider route does not exist at this point (i.e. when Deployments are ready) + // we can force a resync of routes. see below for more details on why this is required: + // https://access.redhat.com/documentation/en-us/red_hat_3scale_api_management/2.7/html/operating_3scale/backup-restore#creating_equivalent_zync_routes + // This scenario will manifest during a backup and restore and also if the product ns was accidentally deleted. + podName, err := r.findSystemSidekiqPod(r.apiManager) + if err != nil { + return reconcile.Result{}, err + } + if podName != "" { + // Execute a resync of routes + _, _, err := r.executeCommandOnPod("system-sidekiq", r.apiManager.Namespace, podName, []string{"bundle", "exec", "rake", "zync:resync:domains"}) + if err != nil { + return reconcile.Result{}, err + } + } + } + } + return reconcile.Result{}, nil } diff --git a/pkg/helper/podHelper.go b/pkg/helper/podHelper.go new file mode 100644 index 000000000..f7c03d1c3 --- /dev/null +++ b/pkg/helper/podHelper.go @@ -0,0 +1,131 @@ +package helper + +import ( + "bytes" + l "github.com/go-logr/logr" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + kube "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" +) + +//go:generate moq -out pod_executor_moq.go . PodExecutorInterface +type PodExecutorInterface interface { + ExecuteRemoteCommand(ns string, podName string, command []string) (string, string, error) + ExecuteRemoteContainerCommand(ns string, podName string, container string, command []string) (string, string, error) +} + +type PodExecutor struct { + Log l.Logger +} + +var _ PodExecutorInterface = &PodExecutor{} + +func NewPodExecutor(log l.Logger) *PodExecutor { + return &PodExecutor{ + Log: log, + } +} + +func (p PodExecutor) ExecuteRemoteCommand(ns string, podName string, command []string) (string, string, error) { + + kubeClient, restConfig, err := getClient() + if err != nil { + return "", "", errors.Wrapf(err, "Failed to get client") + } + + req := kubeClient.CoreV1().RESTClient().Post().Resource("pods").Name(podName). + Namespace(ns).SubResource("exec") + option := &v1.PodExecOptions{ + Command: command, + Stdin: false, + Stdout: true, + Stderr: true, + TTY: true, + //Container: container, + } + req.VersionedParams( + option, + scheme.ParameterCodec, + ) + exec, err := remotecommand.NewSPDYExecutor(restConfig, "POST", req.URL()) + if err != nil { + return "", "", errors.Wrapf(err, "Failed executing command %s on %s/%s", command, ns, podName) + } + + buf := &bytes.Buffer{} + errBuf := &bytes.Buffer{} + + err = exec.Stream(remotecommand.StreamOptions{ + Stdout: buf, + Stderr: errBuf, + }) + if err != nil { + return "", "", errors.Wrapf(err, "Failed executing command %s on %s/%s", command, ns, podName) + } + + return buf.String(), errBuf.String(), nil +} + +func (p PodExecutor) ExecuteRemoteContainerCommand(ns string, podName string, container string, command []string) (string, string, error) { + kubeClient, restConfig, err := getClient() + if err != nil { + return "", "", errors.Wrapf(err, "Failed to get client") + } + + req := kubeClient.CoreV1().RESTClient().Post().Resource("pods").Name(podName). + Namespace(ns).SubResource("exec") + option := &v1.PodExecOptions{ + Command: command, + Stdin: false, + Stdout: true, + Stderr: true, + TTY: true, + Container: container, + } + req.VersionedParams( + option, + scheme.ParameterCodec, + ) + exec, err := remotecommand.NewSPDYExecutor(restConfig, "POST", req.URL()) + if err != nil { + return "", "", errors.Wrapf(err, "Failed executing command %s on %s/%s", command, ns, podName) + } + + buf := &bytes.Buffer{} + errBuf := &bytes.Buffer{} + + streamOptions := remotecommand.StreamOptions{ + Stdout: buf, + Stderr: errBuf, + Tty: option.TTY, + } + + err = exec.Stream(streamOptions) + if err != nil { + return "", "", errors.Wrapf(err, "Failed executing command %s on %s/%s", command, ns, podName) + } + + return buf.String(), errBuf.String(), nil +} + +func getClient() (*kube.Clientset, *restclient.Config, error) { + + kubeCfg := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), + &clientcmd.ConfigOverrides{}, + ) + restCfg, err := kubeCfg.ClientConfig() + if err != nil { + return nil, nil, err + } + + kubeClient, err := kube.NewForConfig(restCfg) + if err != nil { + return nil, nil, errors.Wrapf(err, "Failed to generate new client") + } + return kubeClient, restCfg, nil +}