Add reparent field to global service controller

Dockerfile

@@ -70,7 +70,7 @@ ENV HELM_VERSION=v3.10.3
 ENV TERRAFORM_VERSION=v1.2.9
 
 # renovate: datasource=github-releases depName=pluralsh/plural-cli
-ENV CLI_VERSION=v0.9.3
+ENV CLI_VERSION=v0.9.4
 
 # renovate: datasource=github-tags depName=kubernetes/kubernetes
 ENV KUBECTL_VERSION=v1.25.5

assets/src/generated/graphql.ts

@@ -519,6 +519,21 @@ export type CanaryStatus = {
   phase?: Maybe<Scalars['String']['output']>;
 };
 
+/** A spec for specifying cascade behavior on an owning resource */
+export type Cascade = {
+  __typename?: 'Cascade';
+  /** whether to perform a drain-delete for all owned resources */
+  delete?: Maybe<Scalars['Boolean']['output']>;
+  /** whether to perform a detach-delete for all owned resources */
+  detach?: Maybe<Scalars['Boolean']['output']>;
+};
+
+/** Whether you want to delete or detach owned resources */
+export type CascadeAttributes = {
+  delete?: InputMaybe<Scalars['Boolean']['input']>;
+  detach?: InputMaybe<Scalars['Boolean']['input']>;
+};
+
 export type Certificate = {
   __typename?: 'Certificate';
   events?: Maybe<Array<Maybe<Event>>>;
@@ -1645,13 +1660,16 @@ export enum GitHealth {
 /** a representation of where to pull manifests from git */
 export type GitRef = {
   __typename?: 'GitRef';
+  /** a list of individual files to include as well */
+  files?: Maybe<Array<Scalars['String']['output']>>;
   /** the folder manifests live under */
   folder: Scalars['String']['output'];
   /** a general git ref, either a branch name or commit sha understandable by `git checkout <ref>` */
   ref: Scalars['String']['output'];
 };
 
 export type GitRefAttributes = {
+  files?: InputMaybe<Array<Scalars['String']['input']>>;
   folder: Scalars['String']['input'];
   ref: Scalars['String']['input'];
 };
@@ -1706,6 +1724,8 @@ export type GitStatus = {
 /** a rules based mechanism to redeploy a service across a fleet of clusters */
 export type GlobalService = {
   __typename?: 'GlobalService';
+  /** behavior for all owned resources when this global service is deleted */
+  cascade?: Maybe<Cascade>;
   /** the kubernetes distribution to target with this global service */
   distro?: Maybe<ClusterDistro>;
   /** internal id of this global service */
@@ -1739,6 +1759,8 @@ export type GlobalServiceServicesArgs = {
 
 /** A reference for a globalized service, which targets clusters based on the configured criteria */
 export type GlobalServiceAttributes = {
+  /** behavior for all owned resources when this global service is deleted */
+  cascade?: InputMaybe<CascadeAttributes>;
   /** kubernetes distribution to target */
   distro?: InputMaybe<ClusterDistro>;
   /** name for this global service */
@@ -2272,6 +2294,8 @@ export type ManagedNamespace = {
   __typename?: 'ManagedNamespace';
   /** annotations for this namespace */
   annotations?: Maybe<Scalars['Map']['output']>;
+  /** behavior for all owned resources when this global service is deleted */
+  cascade?: Maybe<Cascade>;
   /** the timestamp this namespace was deleted at, indicating it's currently draining */
   deletedAt?: Maybe<Scalars['DateTime']['output']>;
   /** A short description of the purpose of this namespace */
@@ -2306,6 +2330,8 @@ export type ManagedNamespaceServicesArgs = {
 export type ManagedNamespaceAttributes = {
   /** annotations for this namespace */
   annotations?: InputMaybe<Scalars['Json']['input']>;
+  /** behavior for all owned resources when this global service is deleted */
+  cascade?: InputMaybe<CascadeAttributes>;
   /** A short description of the purpose of this namespace */
   description?: InputMaybe<Scalars['String']['input']>;
   /** labels for this namespace */

charts/controller/crds/deployments.plural.sh_globalservices.yaml

@@ -98,6 +98,10 @@ spec:
                     type: string
                 type: object
                 x-kubernetes-map-type: atomic
+              reparent:
+                description: Whether you'd want this global service to take ownership
+                  of existing Plural services
+                type: boolean
               serviceRef:
                 description: ServiceRef to replicate across clusters
                 properties:

controller/api/v1alpha1/globalservice_types.go

@@ -33,6 +33,10 @@ type GlobalServiceSpec struct {
 	// +kubebuilder:validation:Optional
 	Tags map[string]string `json:"tags,omitempty"`
 
+	// Whether you'd want this global service to take ownership of existing Plural services
+	// +kubebuilder:validation:Optional
+	Reparent *bool `json:"reparent,omitempty"`
+
 	// Distro of kubernetes this cluster is running
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:validation:Enum=GENERIC;EKS;AKS;GKE;RKE;K3S

controller/config/crd/bases/deployments.plural.sh_globalservices.yaml

@@ -98,6 +98,10 @@ spec:
                     type: string
                 type: object
                 x-kubernetes-map-type: atomic
+              reparent:
+                description: Whether you'd want this global service to take ownership
+                  of existing Plural services
+                type: boolean
               serviceRef:
                 description: ServiceRef to replicate across clusters
                 properties:

controller/go.mod

@@ -7,7 +7,7 @@ toolchain go1.21.1
 // Dependencies
 require (
 	github.com/Yamashou/gqlgenc v0.18.1
-	github.com/pluralsh/console-client-go v0.5.1
+	github.com/pluralsh/console-client-go v0.5.4
 	github.com/pluralsh/polly v0.1.7
 	github.com/samber/lo v1.39.0
 	github.com/spf13/pflag v1.0.5

controller/go.sum

@@ -623,6 +623,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pluralsh/console-client-go v0.5.1 h1:q0jSEJh3atHX7khEBoPJXWH79VW+W4OnMP7JNg6N3nk=
 github.com/pluralsh/console-client-go v0.5.1/go.mod h1:eyCiLA44YbXiYyJh8303jk5JdPkt9McgCo5kBjk4lKo=
+github.com/pluralsh/console-client-go v0.5.4 h1:R0S5MNrVQGFfcjjiM3fAcXf1pm1DDgWl8IQj0tuqf5A=
+github.com/pluralsh/console-client-go v0.5.4/go.mod h1:eyCiLA44YbXiYyJh8303jk5JdPkt9McgCo5kBjk4lKo=
 github.com/pluralsh/polly v0.1.7 h1:MUuTb6rCUV1doisaFGC+iz+33ZPU4FZHOb/kFwSDkjw=
 github.com/pluralsh/polly v0.1.7/go.mod h1:Yo1/jcW+4xwhWG+ZJikZy4J4HJkMNPZ7sq5auL2c/tY=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

controller/internal/controller/globalservice_controller.go

@@ -131,6 +131,7 @@ func (r *GlobalServiceReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		Name:       globalService.Name,
 		Distro:     globalService.Spec.Distro,
 		ProviderID: provider.Status.ID,
+		Reparent:   globalService.Spec.Reparent,
 	}
 	if globalService.Spec.Template != nil {
 		namespace := globalService.GetNamespace()
@@ -156,27 +157,18 @@ func (r *GlobalServiceReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 
 	if !globalService.Status.HasID() {
 		controllerutil.AddFinalizer(globalService, GlobalServiceFinalizer)
-		var err error
-		var createGlobalService *console.GlobalServiceFragment
-		if service == nil {
-			createGlobalService, err = r.ConsoleClient.CreateGlobalServiceFromTemplate(attr)
-		} else {
-			createGlobalService, err = r.ConsoleClient.CreateGlobalService(*service.Status.ID, attr)
-		}
-		if err != nil {
-			utils.MarkCondition(globalService.SetCondition, v1alpha1.SynchronizedConditionType, v1.ConditionFalse, v1alpha1.SynchronizedConditionReasonError, err.Error())
-			return ctrl.Result{}, err
-		}
-		globalService.Status.ID = &createGlobalService.ID
-		globalService.Status.SHA = &sha
-		utils.MarkCondition(globalService.SetCondition, v1alpha1.SynchronizedConditionType, v1.ConditionTrue, v1alpha1.SynchronizedConditionReason, "")
-		return ctrl.Result{}, nil
+		return ctrl.Result{}, r.handleCreate(sha, globalService, service, attr)
 	}
 
 	existingGlobalService, err := r.ConsoleClient.GetGlobalService(globalService.Status.GetID())
+	if errors.IsNotFound(err) {
+		globalService.Status.ID = nil
+		return ctrl.Result{}, r.handleCreate(sha, globalService, service, attr)
+	}
+
 	if err != nil {
 		utils.MarkCondition(globalService.SetCondition, v1alpha1.SynchronizedConditionType, v1.ConditionFalse, v1alpha1.SynchronizedConditionReasonError, err.Error())
-		return ctrl.Result{}, err
+		return requeue, err
 	}
 
 	if !globalService.Status.IsSHAEqual(sha) {
@@ -199,6 +191,24 @@ func (r *GlobalServiceReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	return ctrl.Result{}, nil
 }
 
+func (r *GlobalServiceReconciler) handleCreate(sha string, global *v1alpha1.GlobalService, svc *v1alpha1.ServiceDeployment, attrs console.GlobalServiceAttributes) error {
+	var err error
+	var createGlobalService *console.GlobalServiceFragment
+	if svc == nil {
+		createGlobalService, err = r.ConsoleClient.CreateGlobalServiceFromTemplate(attrs)
+	} else {
+		createGlobalService, err = r.ConsoleClient.CreateGlobalService(*svc.Status.ID, attrs)
+	}
+	if err != nil {
+		utils.MarkCondition(global.SetCondition, v1alpha1.SynchronizedConditionType, v1.ConditionFalse, v1alpha1.SynchronizedConditionReasonError, err.Error())
+		return err
+	}
+	global.Status.ID = &createGlobalService.ID
+	global.Status.SHA = &sha
+	utils.MarkCondition(global.SetCondition, v1alpha1.SynchronizedConditionType, v1.ConditionTrue, v1alpha1.SynchronizedConditionReason, "")
+	return nil
+}
+
 func (r *GlobalServiceReconciler) handleDelete(ctx context.Context, service *v1alpha1.GlobalService) error {
 	logger := log.FromContext(ctx)
 	if controllerutil.ContainsFinalizer(service, GlobalServiceFinalizer) {

lib/console/deployments/git.ex

@@ -104,6 +104,7 @@ defmodule Console.Deployments.Git do
   def delete_repository(id, %User{} = user) do
     try do
       get_repository!(id)
+      |> GitRepository.changeset()
       |> allow(user, :git)
       |> when_ok(:delete)
       |> notify(:delete, user)

lib/console/deployments/git/agent.ex

@@ -83,16 +83,16 @@ defmodule Console.Deployments.Git.Agent do
     {:reply, Cache.commit(cache, ref), state}
   end
 
-  def handle_call({:fetch, %Service.Git{ref: ref, folder: path}}, _, %State{cache: cache} = state) do
-    case Cache.fetch(cache, ref, path) do
+  def handle_call({:fetch, %Service.Git{} = ref}, _, %State{cache: cache} = state) do
+    case Cache.fetch(cache, ref) do
       {:ok, %Cache.Line{file: f}, cache} -> {:reply, File.open(f), %{state | cache: cache}}
       err -> {:reply, err, state}
     end
   end
 
-  def handle_call({:fetch, %Service{git: %{ref: ref, folder: path}} = svc}, _, %State{cache: cache} = state) do
+  def handle_call({:fetch, %Service{git: %Service.Git{} = ref} = svc}, _, %State{cache: cache} = state) do
     svc = Console.Repo.preload(svc, [:revision])
-    with {:ok, %Cache.Line{file: f, sha: sha, message: msg}, cache} <- Cache.fetch(cache, ref, path),
+    with {:ok, %Cache.Line{file: f, sha: sha, message: msg}, cache} <- Cache.fetch(cache, ref),
          {:ok, _} <- Services.update_sha(svc, sha, msg) do
       {:reply, File.open(f), %{state | cache: cache}}
     else

lib/console/deployments/git/cache.ex

@@ -8,7 +8,7 @@ defmodule Console.Deployments.Git.Cache do
   This should be relatively disk-efficient, and refresh cycles will sync with polling the git repo
   """
   import Console.Deployments.Git.Cmd
-  alias Console.Schema.GitRepository
+  alias Console.Schema.{GitRepository, Service.Git}
 
   defstruct [:git, :dir, :heads, cache: %{}]
 
@@ -34,10 +34,14 @@ defmodule Console.Deployments.Git.Cache do
     %__MODULE__{git: git, dir: dir}
   end
 
+  def fetch(%__MODULE__{} = cache, %Git{ref: ref, folder: folder, files: [_ | _] = files}),
+    do: fetch(cache, ref, folder, files)
+  def fetch(%__MODULE__{} = cache, %Git{ref: ref, folder: folder}), do: fetch(cache, ref, folder)
+
   def fetch(%__MODULE__{} = cache, ref, path, filter \\ fn _ -> true end) do
     with {:ok, sha} <- commit(cache, ref),
          {:ok, line} <- find_commit(cache, sha, path, filter),
-      do: {:ok, line, put_in(cache.cache[{sha, path}], line)}
+      do: {:ok, line, put_in(cache.cache[cache_key(sha, path, filter)], line)}
   end
 
   def refresh(%__MODULE__{git: git, cache: cache} = c) do
@@ -47,7 +51,8 @@ defmodule Console.Deployments.Git.Cache do
   end
 
   defp find_commit(%__MODULE__{git: g, cache: cache} = c, sha, path, filter) do
-    case Map.get(cache, {sha, path}) do
+    cache_key = cache_key(sha, path, filter)
+    case Map.get(cache, cache_key) do
       %Line{} = l -> {:ok, Line.touch(l)}
       _ -> new_line(c, g, sha, path, filter)
     end
@@ -82,7 +87,21 @@ defmodule Console.Deployments.Git.Cache do
     end
   end
 
-  defp tarball(%GitRepository{dir: dir}, path, tgz_path, filter) do
+  defp tarball(%GitRepository{dir: dir}, path, tgz_path, [_ | _] = files) do
+    subpath = Path.join(dir, path)
+    additional_files = Enum.map(files, &Path.join(dir, &1))
+                       |> Enum.map(& {to_charlist(Path.basename(&1)), to_charlist(&1)})
+
+    Console.ls_r(subpath)
+    |> Enum.map(&tar_path(&1, subpath))
+    |> Enum.concat(additional_files)
+    |> case do
+      [_ | _] = files -> tar_files(files, tgz_path)
+      [] -> {:error, "folder is empty"}
+    end
+  end
+
+  defp tarball(%GitRepository{dir: dir}, path, tgz_path, filter) when is_function(filter) do
     subpath = Path.join(dir, path)
 
     Console.ls_r(subpath)
@@ -108,6 +127,9 @@ defmodule Console.Deployments.Git.Cache do
     {relative_path, to_charlist(filename)}
   end
 
+  defp cache_key(sha, path, [_ | _] = filter), do: {sha, path, filter}
+  defp cache_key(sha, path, _), do: {sha, path}
+
   defp sha?(ref), do: String.match?(ref, ~r/^[0-9A-Fa-f]{40}$/)
 
   defp cache_path(dir, sha, path), do: Path.join([dir, "#{sha}-#{Base.encode32(path)}.tgz"])

lib/console/deployments/global.ex

@@ -79,16 +79,25 @@ defmodule Console.Deployments.Global do
   @spec delete(binary, User.t) :: global_resp
   def delete(global_id, %User{} = user) do
     start_transaction()
-    |> add_operation(:owned, fn _ ->
-      Service.for_owner(global_id)
-      |> Repo.update_all(set: [owner_id: nil])
-      |> ok()
-    end)
     |> add_operation(:global, fn _ ->
       get!(global_id)
       |> allow(user, :write)
-      |> when_ok(:delete)
     end)
+    |> add_operation(:cascade, fn
+      %{global: %GlobalService{cascade: %GlobalService.Cascade{delete: true}}} ->
+        Service.for_owner(global_id)
+        |> Repo.update_all(set: [deleted_at: Timex.now()])
+        |> ok()
+      %{global: %GlobalService{cascade: %GlobalService.Cascade{detach: true}}} ->
+        Service.for_owner(global_id)
+        |> Repo.delete_all()
+        |> ok()
+      %{global: _} ->
+        Service.for_owner(global_id)
+        |> Repo.update_all(set: [owner_id: nil])
+        |> ok()
+    end)
+    |> add_operation(:delete, fn %{global: global} -> Repo.delete(global) end)
     |> execute(extract: :global)
     |> notify(:delete, user)
   end