Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions internal/tools/sigmigrate/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ d8 tools sig-migrate [flags]
| `--log-level` | Set the log level (INFO, DEBUG, TRACE) | `DEBUG` |
| `--kubeconfig` | Path to the kubeconfig file to use for CLI requests | `$HOME/.kube/config` or `$KUBECONFIG` |
| `--context` | The name of the kubeconfig context to use | `kubernetes-admin@kubernetes` |
| `--object` | Process only one specific object in `namespace/name/kind` format (`clusterwide/name/kind` for cluster) | `""` |
| `--object` | Process object(s) by `namespace/name/resource` (`clusterwide/name/resource` for cluster-scoped) | `""` |

## Usage Examples

Expand Down Expand Up @@ -91,7 +91,7 @@ d8 tools sig-migrate \

1. **Resource Collection**: The command uses Kubernetes API discovery to automatically discover all available resources (both namespaced and cluster-wide).

2. **Optional Object Filter**: If `--object` is specified, the command skips full resource fetching/listing and processes only the matching object (`namespace/name/kind`).
2. **Optional Object Filter**: If `--object` is specified, the command skips full resource fetching/listing and processes matching object(s) by `namespace/name/resource`. If multiple API groups contain the same `namespace/name/resource`, all matching entries are selected.

3. **Adding Annotation**: For each selected resource, an annotation `d8-migration=<timestamp>` is added, where `timestamp` is the current time in Unix timestamp format.
Comment on lines 92 to 96

Expand All @@ -108,11 +108,11 @@ d8 tools sig-migrate \

The command creates run-scoped files to track failed operations:

- `/tmp/failed_annotations_<timestamp>.txt` - list of objects in `namespace|name|kind` format that failed to be processed
- `/tmp/failed_annotations_<timestamp>.txt` - list of objects in `namespace|name|kind|group|version` format that failed to be processed
- `/tmp/failed_errors_<timestamp>.txt` - detailed error information in `namespace|name|kind|error_message` format
- `/tmp/skipped_objects_<timestamp>.txt` - skipped objects with reason/details

For retry compatibility, failed annotations are also mirrored into legacy `/tmp/failed_annotations.txt` and `--retry` reads from that legacy file.
For retry compatibility, failed annotations are also mirrored into legacy `/tmp/failed_annotations.txt` and `--retry` reads from that legacy file (supports both old `namespace|name|kind` and new `namespace|name|kind|group|version` lines).

### Automatic Failure Detection

Expand Down
2 changes: 1 addition & 1 deletion internal/tools/sigmigrate/cmd/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,6 @@ func addFlags(flags *pflag.FlagSet) {
flags.String(
"object",
"",
"Scan only a specific object in format <namespace>/<name>/<resource_name>. Use 'clusterwide' namespace for cluster-scoped resources. To get the resource name, use the 'kubectl api-resources' command.",
"Process objects by identifier in format <namespace>/<name>/<resource_name>. Use 'clusterwide' namespace for cluster-scoped resources. Resource name must match kubectl api-resources output.",
)
}
81 changes: 62 additions & 19 deletions internal/tools/sigmigrate/sigmigrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,60 @@ type ObjectRef struct {
GVR schema.GroupVersionResource
}

func preferredVersionByGroup(apiGroupList *metav1.APIGroupList) map[string]string {
preferred := make(map[string]string)
if apiGroupList == nil {
return preferred
}

for _, group := range apiGroupList.Groups {
if group.Name == "" {
continue
}
if group.PreferredVersion.Version != "" {
preferred[group.Name] = group.PreferredVersion.Version
}
}

return preferred
}

func objectCollectionKey(namespace, name string, gvr schema.GroupVersionResource) string {
return fmt.Sprintf("%s|%s|%s|%s", namespace, name, gvr.Group, gvr.Resource)
}

func upsertCollectedObject(
Comment thread
Suselz marked this conversation as resolved.
objects map[string]ObjectRef,
namespace string,
name string,
gvr schema.GroupVersionResource,
preferredByGroup map[string]string,
) {
key := objectCollectionKey(namespace, name, gvr)
candidate := ObjectRef{
Namespace: namespace,
Name: name,
Kind: gvr.Resource,
GVR: gvr,
}

existing, exists := objects[key]
if !exists {
objects[key] = candidate
return
}

preferredVersion := preferredByGroup[gvr.Group]
if preferredVersion == "" {
// Keep first discovered version if preferred version is unknown.
return
}

if existing.GVR.Version != preferredVersion && gvr.Version == preferredVersion {
objects[key] = candidate
}
}

type SigMigrateConfig struct {
RetryFailed bool
KubectlAs string
Expand Down Expand Up @@ -349,6 +403,7 @@ func collectAllObjects(discoveryClient discovery.DiscoveryInterface, dynamicClie
tracef("failed to discover API groups: %v", err)
return nil, fmt.Errorf("failed to discover API groups: %w", err)
}
preferredByGroup := preferredVersionByGroup(apiGroupList)

namespacedResources := []schema.GroupVersionResource{}
clusterResources := []schema.GroupVersionResource{}
Expand Down Expand Up @@ -459,13 +514,7 @@ func collectAllObjects(discoveryClient discovery.DiscoveryInterface, dynamicClie
namespace = "clusterwide"
}
name := item.GetName()
key := fmt.Sprintf("%s|%s|%s", namespace, name, gvr.Resource)
objects[key] = ObjectRef{
Namespace: namespace,
Name: name,
Kind: gvr.Resource,
GVR: gvr,
}
upsertCollectedObject(objects, namespace, name, gvr, preferredByGroup)
}
}

Expand All @@ -492,13 +541,7 @@ func collectAllObjects(discoveryClient discovery.DiscoveryInterface, dynamicClie

for _, item := range list.Items {
name := item.GetName()
key := fmt.Sprintf("clusterwide|%s|%s", name, gvr.Resource)
objects[key] = ObjectRef{
Namespace: "clusterwide",
Name: name,
Kind: gvr.Resource,
GVR: gvr,
}
upsertCollectedObject(objects, "clusterwide", name, gvr, preferredByGroup)
}
}

Expand Down Expand Up @@ -789,7 +832,7 @@ func loadFailedObjects() (map[string]ObjectRef, error) {
gvr.Version = strings.TrimSpace(parts[4])
}

key := fmt.Sprintf("%s|%s|%s", namespace, name, kind)
key := objectCollectionKey(namespace, name, gvr)
objects[key] = ObjectRef{
Namespace: namespace,
Name: name,
Expand Down Expand Up @@ -1032,11 +1075,11 @@ func filterObjectsByIdentifier(objects map[string]ObjectRef, objectIdentifier st
return map[string]ObjectRef{}
}

key := fmt.Sprintf("%s|%s|%s", namespace, name, kind)

filtered := make(map[string]ObjectRef)
if object, exists := objects[key]; exists {
filtered[key] = object
for key, object := range objects {
if object.Namespace == namespace && object.Name == name && object.Kind == kind {
filtered[key] = object
}
Comment on lines 1078 to +1082
}

return filtered
Expand Down
123 changes: 111 additions & 12 deletions internal/tools/sigmigrate/sigmigrate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,9 +113,108 @@ func TestParseObjectIdentifier_InvalidFormat(t *testing.T) {
require.Error(t, err)
}

func TestPreferredVersionByGroup(t *testing.T) {
groups := &metav1.APIGroupList{
Groups: []metav1.APIGroup{
{
Name: "deckhouse.io",
PreferredVersion: metav1.GroupVersionForDiscovery{
GroupVersion: "deckhouse.io/v1",
Version: "v1",
},
},
{
Name: "apps",
PreferredVersion: metav1.GroupVersionForDiscovery{
GroupVersion: "apps/v1",
Version: "v1",
},
},
},
}

preferred := preferredVersionByGroup(groups)
require.Equal(t, "v1", preferred["deckhouse.io"])
require.Equal(t, "v1", preferred["apps"])
}

func TestUpsertCollectedObject_PrefersPreferredVersion(t *testing.T) {
objects := make(map[string]ObjectRef)
preferred := map[string]string{"deckhouse.io": "v1"}

upsertCollectedObject(
objects,
"clusterwide",
"worker",
schema.GroupVersionResource{Group: "deckhouse.io", Version: "v1alpha1", Resource: "nodegroups"},
preferred,
)
upsertCollectedObject(
objects,
"clusterwide",
"worker",
schema.GroupVersionResource{Group: "deckhouse.io", Version: "v1", Resource: "nodegroups"},
preferred,
)

obj := objects["clusterwide|worker|deckhouse.io|nodegroups"]
require.Equal(t, "v1", obj.GVR.Version)
require.Equal(t, "deckhouse.io", obj.GVR.Group)
require.Equal(t, "nodegroups", obj.GVR.Resource)
}

func TestLoadFailedObjects_DoesNotOverwriteSameNamespaceNameKindAcrossGroups(t *testing.T) {
tmpDir := t.TempDir()
legacyRetryFile := filepath.Join(tmpDir, "failed_annotations_legacy.txt")
setCurrentRunState(&sigMigrateRunState{LegacyFailedRetryFile: legacyRetryFile})
defer setCurrentRunState(nil)

testData := strings.Join([]string{
"clusterwide|worker|nodegroups|deckhouse.io|v1",
"clusterwide|worker|nodegroups|example.io|v1",
}, "\n") + "\n"

err := os.WriteFile(legacyRetryFile, []byte(testData), 0644)
require.NoError(t, err)

objects, err := loadFailedObjects()
require.NoError(t, err)
require.Len(t, objects, 2)

objDeckhouse := objects["clusterwide|worker|deckhouse.io|nodegroups"]
require.Equal(t, "deckhouse.io", objDeckhouse.GVR.Group)
require.Equal(t, "v1", objDeckhouse.GVR.Version)

objExample := objects["clusterwide|worker|example.io|nodegroups"]
require.Equal(t, "example.io", objExample.GVR.Group)
require.Equal(t, "v1", objExample.GVR.Version)
}

func TestFilterObjectsByIdentifier_ReturnsAllMatchingGroups(t *testing.T) {
objects := map[string]ObjectRef{
"clusterwide|worker|deckhouse.io|nodegroups": {
Namespace: "clusterwide",
Name: "worker",
Kind: "nodegroups",
GVR: schema.GroupVersionResource{Group: "deckhouse.io", Version: "v1", Resource: "nodegroups"},
},
"clusterwide|worker|example.io|nodegroups": {
Namespace: "clusterwide",
Name: "worker",
Kind: "nodegroups",
GVR: schema.GroupVersionResource{Group: "example.io", Version: "v1", Resource: "nodegroups"},
},
}

filtered := filterObjectsByIdentifier(objects, "clusterwide/worker/nodegroups")
require.Len(t, filtered, 2)
require.Contains(t, filtered, "clusterwide|worker|deckhouse.io|nodegroups")
require.Contains(t, filtered, "clusterwide|worker|example.io|nodegroups")
}

func TestFilterObjectsByIdentifier_SpecificObject(t *testing.T) {
objects := map[string]ObjectRef{
"default|cm-one|configmaps": {
"default|cm-one||configmaps": {
Namespace: "default",
Name: "cm-one",
Kind: "configmaps",
Expand All @@ -125,7 +224,7 @@ func TestFilterObjectsByIdentifier_SpecificObject(t *testing.T) {
Resource: "configmaps",
},
},
"kube-system|cm-two|configmaps": {
"kube-system|cm-two||configmaps": {
Namespace: "kube-system",
Name: "cm-two",
Kind: "configmaps",
Expand All @@ -139,13 +238,13 @@ func TestFilterObjectsByIdentifier_SpecificObject(t *testing.T) {

filtered := filterObjectsByIdentifier(objects, "kube-system/cm-two/configmaps")
require.Len(t, filtered, 1)
require.Contains(t, filtered, "kube-system|cm-two|configmaps")
require.Equal(t, "cm-two", filtered["kube-system|cm-two|configmaps"].Name)
require.Contains(t, filtered, "kube-system|cm-two||configmaps")
require.Equal(t, "cm-two", filtered["kube-system|cm-two||configmaps"].Name)
}

func TestFilterObjectsByIdentifier_NotFound(t *testing.T) {
objects := map[string]ObjectRef{
"default|cm-one|configmaps": {
"default|cm-one||configmaps": {
Namespace: "default",
Name: "cm-one",
Kind: "configmaps",
Expand All @@ -163,7 +262,7 @@ func TestFilterObjectsByIdentifier_NotFound(t *testing.T) {

func TestFilterObjectsByIdentifier_InvalidFormat(t *testing.T) {
objects := map[string]ObjectRef{
"default|cm-one|configmaps": {
"default|cm-one||configmaps": {
Namespace: "default",
Name: "cm-one",
Kind: "configmaps",
Expand Down Expand Up @@ -274,19 +373,19 @@ func TestLoadFailedObjects(t *testing.T) {
require.NoError(t, err)
require.Len(t, objects, 3)

first := objects["default|test-pod|pods"]
first := objects["default|test-pod||pods"]
require.Equal(t, "default", first.Namespace)
require.Equal(t, "test-pod", first.Name)
require.Equal(t, "pods", first.Kind)
require.Equal(t, "pods", first.GVR.Resource)

second := objects["kube-system|test-cm|configmaps"]
second := objects["kube-system|test-cm||configmaps"]
require.Equal(t, "kube-system", second.Namespace)
require.Equal(t, "test-cm", second.Name)
require.Equal(t, "configmaps", second.Kind)
require.Equal(t, "configmaps", second.GVR.Resource)

nodeGroup := objects["clusterwide|worker|nodegroups"]
nodeGroup := objects["clusterwide|worker||nodegroups"]
require.Equal(t, "clusterwide", nodeGroup.Namespace)
require.Equal(t, "worker", nodeGroup.Name)
require.Equal(t, "nodegroups", nodeGroup.Kind)
Expand All @@ -308,7 +407,7 @@ func TestLoadFailedObjects_ExtendedRetryFormatIncludesGVR(t *testing.T) {
require.NoError(t, err)
require.Len(t, objects, 1)

obj := objects["clusterwide|worker|nodegroups"]
obj := objects["clusterwide|worker|deckhouse.io|nodegroups"]
require.Equal(t, "clusterwide", obj.Namespace)
require.Equal(t, "worker", obj.Name)
require.Equal(t, "nodegroups", obj.Kind)
Expand All @@ -335,15 +434,15 @@ func TestLoadFailedObjects_ExtendedRetryFormat_NamespacedObjects(t *testing.T) {
require.NoError(t, err)
require.Len(t, objects, 2)

deployment := objects["default|web-app|deployments"]
deployment := objects["default|web-app|apps|deployments"]
require.Equal(t, "default", deployment.Namespace)
require.Equal(t, "web-app", deployment.Name)
require.Equal(t, "deployments", deployment.Kind)
require.Equal(t, "deployments", deployment.GVR.Resource)
require.Equal(t, "apps", deployment.GVR.Group)
require.Equal(t, "v1", deployment.GVR.Version)

configMap := objects["kube-system|coredns|configmaps"]
configMap := objects["kube-system|coredns||configmaps"]
require.Equal(t, "kube-system", configMap.Namespace)
require.Equal(t, "coredns", configMap.Name)
require.Equal(t, "configmaps", configMap.Kind)
Expand Down
Loading