apis/tenancy/workspaces: add region label column

cmd/apigen/main.go

@@ -32,6 +32,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
 
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
@@ -41,7 +42,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/kubernetes/pkg/apis/admissionregistration"
 	"sigs.k8s.io/yaml"
 
 	"github.com/kcp-dev/kcp/pkg/apis/apis"
@@ -173,7 +173,7 @@ func loadCustomResourceDefinitions(logger logr.Logger, baseDir string) (map[meta
 			Group:    parts[0],
 			Resource: parts[1],
 		}
-		if gr.Group == apis.GroupName || gr.Group == rbacv1.GroupName || gr.Group == admissionregistration.GroupName {
+		if gr.Group == apis.GroupName || gr.Group == rbacv1.GroupName || gr.Group == admissionregistrationv1.GroupName {
 			logger.Info(fmt.Sprintf("Skipping CustomResourceDefinition %s from %s", gr.String(), path))
 			return nil
 		}

cmd/sharded-test-server/main.go

@@ -25,14 +25,20 @@ import (
 	"path/filepath"
 	"strings"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	machineryutilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/sets"
 	kuser "k8s.io/apiserver/pkg/authentication/user"
 	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/retry"
 
 	"github.com/kcp-dev/kcp/cmd/sharded-test-server/third_party/library-go/crypto"
 	shard "github.com/kcp-dev/kcp/cmd/test-server/kcp"
+	"github.com/kcp-dev/kcp/pkg/apis/core"
 	"github.com/kcp-dev/kcp/pkg/authorization/bootstrap"
+	kcpclientset "github.com/kcp-dev/kcp/pkg/client/clientset/versioned/cluster"
 )
 
 func main() {
@@ -262,6 +268,37 @@ func start(proxyFlags, shardFlags []string, logDirPath, workDirPath string, numb
 		return err
 	}
 
+	// Label region of shards
+	clientConfig, err := loadKubeConfig(filepath.Join(workDirPath, ".kcp/admin.kubeconfig"), "base")
+	if err != nil {
+		return err
+	}
+	config, err := clientConfig.ClientConfig()
+	if err != nil {
+		return err
+	}
+	client, err := kcpclientset.NewForConfig(config)
+	if err != nil {
+		return err
+	}
+	for i := range shards {
+		name := fmt.Sprintf("shard-%d", i)
+		if i == 0 {
+			name = "root"
+		}
+
+		if i >= len(regions) {
+			break
+		}
+		patch := fmt.Sprintf(`{"metadata":{"labels":{"region":%q}}}`, regions[i])
+		if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+			_, err := client.Cluster(core.RootCluster.Path()).CoreV1alpha1().Shards().Patch(ctx, name, types.MergePatchType, []byte(patch), metav1.PatchOptions{})
+			return err
+		}); err != nil {
+			return err
+		}
+	}
+
 	select {
 	case shardIndexErr := <-shardsErrCh:
 		return fmt.Errorf("shard %d exited: %w", shardIndexErr.index, shardIndexErr.error)
@@ -278,3 +315,49 @@ type indexErrTuple struct {
 	index int
 	error error
 }
+
+func loadKubeConfig(kubeconfigPath, contextName string) (clientcmd.ClientConfig, error) {
+	fs, err := os.Stat(kubeconfigPath)
+	if err != nil {
+		return nil, err
+	}
+	if fs.Size() == 0 {
+		return nil, fmt.Errorf("%s points to an empty file", kubeconfigPath)
+	}
+
+	rawConfig, err := clientcmd.LoadFromFile(kubeconfigPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load admin kubeconfig: %w", err)
+	}
+
+	return clientcmd.NewNonInteractiveClientConfig(*rawConfig, contextName, nil, nil), nil
+}
+
+var regions = []string{
+	"us-east-2",
+	"us-east-1",
+	"us-west-1",
+	"us-west-2",
+	"af-south-1",
+	"ap-east-1",
+	"ap-south-2",
+	"ap-southeast-3",
+	"ap-south-1",
+	"ap-northeast-3",
+	"ap-northeast-2",
+	"ap-southeast-1",
+	"ap-southeast-2",
+	"ap-northeast-1",
+	"ca-central-1",
+	"eu-central-1",
+	"eu-west-1",
+	"eu-west-2",
+	"eu-south-1",
+	"eu-west-3",
+	"eu-south-2",
+	"eu-north-1",
+	"eu-central-2",
+	"me-south-1",
+	"me-central-1",
+	"sa-east-1",
+}

config/crds/core.kcp.io_shards.yaml

@@ -18,6 +18,10 @@ spec:
   scope: Cluster
   versions:
   - additionalPrinterColumns:
+    - description: The region this workspace is in
+      jsonPath: .metadata.labels['region']
+      name: Region
+      type: string
     - description: Type URL to directly connect to the shard
       jsonPath: .spec.baseURL
       name: URL

config/crds/tenancy.kcp.io_workspaces.yaml

@@ -23,8 +23,12 @@ spec:
       jsonPath: .spec.type.name
       name: Type
       type: string
+    - description: The region this workspace is in
+      jsonPath: .metadata.labels['region']
+      name: Region
+      type: string
     - description: The current phase (e.g. Scheduling, Initializing, Ready, Deleting)
-      jsonPath: .metadata.labels['tenancy\.kcp\.dev/phase']
+      jsonPath: .metadata.labels['tenancy\.kcp\.io/phase']
       name: Phase
       type: string
     - description: URL to access the workspace

config/root-phase0/apiexport-shards.core.kcp.io.yaml

@@ -5,5 +5,5 @@ metadata:
   name: shards.core.kcp.io
 spec:
   latestResourceSchemas:
-  - v221219-c92ed8152.shards.core.kcp.io
+  - v230116-943e458f6.shards.core.kcp.io
 status: {}

config/root-phase0/apiexport-tenancy.kcp.io.yaml

@@ -7,7 +7,7 @@ spec:
   latestResourceSchemas:
   - v221219-c92ed8152.clusterworkspaces.tenancy.kcp.io
   - v230110-89146c99.workspacetypes.tenancy.kcp.io
-  - v230112-eb0f15cec.workspaces.tenancy.kcp.io
+  - v230116-832a4a55d.workspaces.tenancy.kcp.io
   maximalPermissionPolicy:
     local: {}
 status: {}

config/root-phase0/apiresourceschema-logicalclusters.core.kcp.io.yaml

@@ -2,7 +2,7 @@ apiVersion: apis.kcp.io/v1alpha1
 kind: APIResourceSchema
 metadata:
   creationTimestamp: null
-  name: v221219-dab0c5392.logicalclusters.core.kcp.io
+  name: v230116-943e458f6.logicalclusters.core.kcp.io
 spec:
   group: core.kcp.io
   names:

config/root-phase0/apiresourceschema-shards.core.kcp.io.yaml

@@ -2,7 +2,7 @@ apiVersion: apis.kcp.io/v1alpha1
 kind: APIResourceSchema
 metadata:
   creationTimestamp: null
-  name: v221219-c92ed8152.shards.core.kcp.io
+  name: v230116-943e458f6.shards.core.kcp.io
 spec:
   group: core.kcp.io
   names:
@@ -15,6 +15,10 @@ spec:
   scope: Cluster
   versions:
   - additionalPrinterColumns:
+    - description: The region this workspace is in
+      jsonPath: .metadata.labels['region']
+      name: Region
+      type: string
     - description: Type URL to directly connect to the shard
       jsonPath: .spec.baseURL
       name: URL

config/root-phase0/apiresourceschema-workspaces.tenancy.kcp.io.yaml

@@ -2,7 +2,7 @@ apiVersion: apis.kcp.io/v1alpha1
 kind: APIResourceSchema
 metadata:
   creationTimestamp: null
-  name: v230112-eb0f15cec.workspaces.tenancy.kcp.io
+  name: v230116-832a4a55d.workspaces.tenancy.kcp.io
 spec:
   group: tenancy.kcp.io
   names:
@@ -21,8 +21,12 @@ spec:
       jsonPath: .spec.type.name
       name: Type
       type: string
+    - description: The region this workspace is in
+      jsonPath: .metadata.labels['region']
+      name: Region
+      type: string
     - description: The current phase (e.g. Scheduling, Initializing, Ready, Deleting)
-      jsonPath: .metadata.labels['tenancy\.kcp\.dev/phase']
+      jsonPath: .metadata.labels['tenancy\.kcp\.io/phase']
       name: Phase
       type: string
     - description: URL to access the workspace

pkg/apis/core/v1alpha1/shard_types.go

@@ -35,6 +35,7 @@ var RootShard = "root"
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +kubebuilder:subresource:status
 // +kubebuilder:resource:scope=Cluster,categories=kcp
+// +kubebuilder:printcolumn:name="Region",type=string,JSONPath=`.metadata.labels['region']`,description="The region this workspace is in"
 // +kubebuilder:printcolumn:name="URL",type=string,JSONPath=`.spec.baseURL`,description="Type URL to directly connect to the shard"
 // +kubebuilder:printcolumn:name="External URL",type=string,JSONPath=`.spec.externalURL`,description="The URL exposed in logical clusters created on that shard"
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"

pkg/apis/tenancy/v1alpha1/types_workspace.go

@@ -111,7 +111,8 @@ const LogicalClusterTypeAnnotationKey = "internal.tenancy.kcp.io/type"
 // +kubebuilder:subresource:status
 // +kubebuilder:resource:scope=Cluster,categories=kcp,shortName=ws
 // +kubebuilder:printcolumn:name="Type",type=string,JSONPath=`.spec.type.name`,description="Type of the workspace"
-// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`,description="The current phase (e.g. Scheduling, Initializing, Ready, Deleting)"
+// +kubebuilder:printcolumn:name="Region",type=string,JSONPath=`.metadata.labels['region']`,description="The region this workspace is in"
+// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.metadata.labels['tenancy\.kcp\.io/phase']`,description="The current phase (e.g. Scheduling, Initializing, Ready, Deleting)"
 // +kubebuilder:printcolumn:name="URL",type=string,JSONPath=`.spec.URL`,description="URL to access the workspace"
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
 type Workspace struct {

pkg/reconciler/tenancy/workspace/workspace_reconcile_scheduling.go

@@ -97,20 +97,23 @@ func (r *schedulingReconciler) reconcile(ctx context.Context, workspace *tenancy
 		}
 
 		if !hasShard {
-			shardName, reason, err := r.chooseShardAndMarkCondition(logger, workspace) // call first with status side-effect, before any annotation aka spec change
+			shard, reason, err := r.chooseShardAndMarkCondition(logger, workspace) // call first with status side-effect, before any annotation aka spec change
 			if err != nil {
 				return reconcileStatusStopAndRequeue, err
 			}
-			if len(shardName) == 0 {
+			if shard == nil {
 				conditions.MarkFalse(workspace, tenancyv1alpha1.WorkspaceScheduled, tenancyv1alpha1.WorkspaceReasonUnschedulable, conditionsv1alpha1.ConditionSeverityError, reason)
 				return reconcileStatusContinue, nil // retry is automatic when new shards show up
 			}
-			logger.V(1).Info("Chose shard", "shard", shardName)
-			shardNameHash = ByBase36Sha224NameValue(shardName)
+			logger.V(1).Info("Chose shard", "shard", shard.Name)
+			shardNameHash = ByBase36Sha224NameValue(shard.Name)
 			if workspace.Annotations == nil {
 				workspace.Annotations = map[string]string{}
 			}
 			workspace.Annotations[WorkspaceShardHashAnnotationKey] = shardNameHash
+			if region, found := shard.Labels["region"]; found {
+				workspace.Labels["region"] = region
+			}
 		}
 		if !hasCluster {
 			cluster := r.generateClusterName(logicalcluster.From(workspace).Path().Join(workspace.Name))
@@ -169,21 +172,21 @@ func (r *schedulingReconciler) reconcile(ctx context.Context, workspace *tenancy
 	return reconcileStatusContinue, nil
 }
 
-func (r *schedulingReconciler) chooseShardAndMarkCondition(logger klog.Logger, workspace *tenancyv1alpha1.Workspace) (shard string, reason string, err error) {
+func (r *schedulingReconciler) chooseShardAndMarkCondition(logger klog.Logger, workspace *tenancyv1alpha1.Workspace) (shard *corev1alpha1.Shard, reason string, err error) {
 	selector := labels.Everything()
 	if workspace.Spec.Location != nil {
 		if workspace.Spec.Location.Selector != nil {
 			var err error
 			selector, err = metav1.LabelSelectorAsSelector(workspace.Spec.Location.Selector)
 			if err != nil {
-				return "", fmt.Sprintf("spec.location.selector is invalid: %v", err), nil // don't retry, cannot do anything useful
+				return nil, fmt.Sprintf("spec.location.selector is invalid: %v", err), nil // don't retry, cannot do anything useful
 			}
 		}
 	}
 
 	shards, err := r.listShards(selector)
 	if err != nil {
-		return "", "", err
+		return nil, "", err
 	}
 
 	validShards := make([]*corev1alpha1.Shard, 0, len(shards))
@@ -209,10 +212,10 @@ func (r *schedulingReconciler) chooseShardAndMarkCondition(logger klog.Logger, w
 			failures = append(failures, fmt.Errorf("  %s: reason %q, message %q", name, x.reason, x.message))
 		}
 		logger.Error(utilerrors.NewAggregate(failures), "no valid shards found for workspace, skipping")
-		return "", "No available shards to schedule the workspace", nil // retry is automatic when new shards show up
+		return nil, "No available shards to schedule the workspace", nil // retry is automatic when new shards show up
 	}
 	targetShard := validShards[rand.Intn(len(validShards))]
-	return targetShard.Name, "", nil
+	return targetShard, "", nil
 }
 
 func (r *schedulingReconciler) createLogicalCluster(ctx context.Context, shard *corev1alpha1.Shard, cluster logicalcluster.Path, parent *corev1alpha1.LogicalCluster, workspace *tenancyv1alpha1.Workspace) error {