fix: pre-release validation fixes for kubeadm, networking, and CTK

cmd/action/ci/entrypoint.go

@@ -96,7 +96,7 @@ func entrypoint(log *logger.FunLogger) error {
 	}
 
 	if cfg.Spec.Kubernetes.Install {
-		err = utils.GetKubeConfig(log, &cfg, hostUrl, kubeconfig)
+		err = utils.GetKubeConfig(log, &cfg, hostUrl, kubeconfig, "")
 		if err != nil {
 			return fmt.Errorf("failed to get kubeconfig: %w", err)
 		}

cmd/cli/common/host.go

@@ -27,6 +27,7 @@ import (
 	"github.com/NVIDIA/holodeck/internal/logger"
 	"github.com/NVIDIA/holodeck/pkg/provider/aws"
 	"github.com/NVIDIA/holodeck/pkg/sshutil"
+	"github.com/NVIDIA/holodeck/pkg/utils"
 )
 
 // GetHostURL resolves the SSH-reachable host URL for an environment.
@@ -82,6 +83,10 @@ const (
 // ConnectSSH establishes an SSH connection with retries.
 // Host key verification uses Trust-On-First-Use (TOFU).
 func ConnectSSH(log *logger.FunLogger, keyPath, userName, hostUrl string) (*ssh.Client, error) {
+	keyPath, err := utils.ExpandPath(keyPath)
+	if err != nil {
+		return nil, fmt.Errorf("expanding key path: %w", err)
+	}
 	key, err := os.ReadFile(keyPath) //nolint:gosec // keyPath is from trusted env config
 	if err != nil {
 		return nil, fmt.Errorf("failed to read key file %s: %w", keyPath, err)

cmd/cli/create/create.go

@@ -485,7 +485,7 @@ func runSingleNodeProvision(log *logger.FunLogger, opts *options) error {
 				break
 			}
 		}
-		if err = utils.GetKubeConfig(log, &opts.cache, hostUrl, opts.kubeconfig); err != nil {
+		if err = utils.GetKubeConfig(log, &opts.cache, hostUrl, opts.kubeconfig, ""); err != nil {
 			return fmt.Errorf("failed to get kubeconfig: %w", err)
 		}
 	}
@@ -563,7 +563,7 @@ func runMultinodeProvision(log *logger.FunLogger, opts *options) error {
 			}
 		}
 		if hostUrl != "" {
-			if err := utils.GetKubeConfig(log, &opts.cache, hostUrl, opts.kubeconfig); err != nil {
+			if err := utils.GetKubeConfig(log, &opts.cache, hostUrl, opts.kubeconfig, ""); err != nil {
 				return fmt.Errorf("failed to get kubeconfig: %w", err)
 			}
 		}

cmd/cli/delete/delete_test.go

@@ -131,8 +131,8 @@ var _ = Describe("Delete Command", func() {
 
 			It("should delete single SSH instance successfully", func() {
 				// Create a valid SSH cache file
-				yaml := sshCacheYAML("sshdelete1", "ssh-delete-test")
-				cacheFile := filepath.Join(tempDir, "sshdelete1.yaml")
+				yaml := sshCacheYAML("a1b2c3d4", "ssh-delete-test")
+				cacheFile := filepath.Join(tempDir, "a1b2c3d4.yaml")
 				err := os.WriteFile(cacheFile, []byte(yaml), 0600)
 				Expect(err).NotTo(HaveOccurred())
 
@@ -141,7 +141,7 @@ var _ = Describe("Delete Command", func() {
 					Commands: []*cli.Command{cmd},
 				}
 
-				err = app.Run([]string{"holodeck", "delete", "--cachepath", tempDir, "sshdelete1"})
+				err = app.Run([]string{"holodeck", "delete", "--cachepath", tempDir, "a1b2c3d4"})
 				Expect(err).NotTo(HaveOccurred())
 
 				// Verify cache file was removed
@@ -150,15 +150,15 @@ var _ = Describe("Delete Command", func() {
 
 				// Verify success message
 				Expect(buf.String()).To(ContainSubstring("Successfully deleted"))
-				Expect(buf.String()).To(ContainSubstring("sshdelete1"))
+				Expect(buf.String()).To(ContainSubstring("a1b2c3d4"))
 			})
 
 			It("should delete multiple SSH instances successfully", func() {
 				// Create two cache files
-				yaml1 := sshCacheYAML("sshmulti1", "ssh-multi-1")
-				yaml2 := sshCacheYAML("sshmulti2", "ssh-multi-2")
-				cacheFile1 := filepath.Join(tempDir, "sshmulti1.yaml")
-				cacheFile2 := filepath.Join(tempDir, "sshmulti2.yaml")
+				yaml1 := sshCacheYAML("e5f6a7b8", "ssh-multi-1")
+				yaml2 := sshCacheYAML("c9d0e1f2", "ssh-multi-2")
+				cacheFile1 := filepath.Join(tempDir, "e5f6a7b8.yaml")
+				cacheFile2 := filepath.Join(tempDir, "c9d0e1f2.yaml")
 				err := os.WriteFile(cacheFile1, []byte(yaml1), 0600)
 				Expect(err).NotTo(HaveOccurred())
 				err = os.WriteFile(cacheFile2, []byte(yaml2), 0600)
@@ -169,7 +169,7 @@ var _ = Describe("Delete Command", func() {
 					Commands: []*cli.Command{cmd},
 				}
 
-				err = app.Run([]string{"holodeck", "delete", "--cachepath", tempDir, "sshmulti1", "sshmulti2"})
+				err = app.Run([]string{"holodeck", "delete", "--cachepath", tempDir, "e5f6a7b8", "c9d0e1f2"})
 				Expect(err).NotTo(HaveOccurred())
 
 				// Verify both cache files were removed
@@ -179,14 +179,14 @@ var _ = Describe("Delete Command", func() {
 				Expect(os.IsNotExist(err)).To(BeTrue())
 
 				// Verify success messages for both
-				Expect(buf.String()).To(ContainSubstring("sshmulti1"))
-				Expect(buf.String()).To(ContainSubstring("sshmulti2"))
+				Expect(buf.String()).To(ContainSubstring("e5f6a7b8"))
+				Expect(buf.String()).To(ContainSubstring("c9d0e1f2"))
 			})
 
 			It("should stop on first error with multiple instances", func() {
 				// Create only one valid cache file
-				yaml := sshCacheYAML("sshvalid1", "ssh-valid")
-				cacheFile := filepath.Join(tempDir, "sshvalid1.yaml")
+				yaml := sshCacheYAML("a3b4c5d6", "ssh-valid")
+				cacheFile := filepath.Join(tempDir, "a3b4c5d6.yaml")
 				err := os.WriteFile(cacheFile, []byte(yaml), 0600)
 				Expect(err).NotTo(HaveOccurred())
 
@@ -196,7 +196,7 @@ var _ = Describe("Delete Command", func() {
 				}
 
 				// First instance doesn't exist, should fail before processing second
-				err = app.Run([]string{"holodeck", "delete", "--cachepath", tempDir, "nonexistent", "sshvalid1"})
+				err = app.Run([]string{"holodeck", "delete", "--cachepath", tempDir, "nonexistent", "a3b4c5d6"})
 				Expect(err).To(HaveOccurred())
 				Expect(err.Error()).To(ContainSubstring("failed to get instance nonexistent"))
 
@@ -207,8 +207,8 @@ var _ = Describe("Delete Command", func() {
 
 			It("should fail if second instance doesn't exist", func() {
 				// Create only one valid cache file
-				yaml := sshCacheYAML("sshfirst1", "ssh-first")
-				cacheFile := filepath.Join(tempDir, "sshfirst1.yaml")
+				yaml := sshCacheYAML("e7f8a9b0", "ssh-first")
+				cacheFile := filepath.Join(tempDir, "e7f8a9b0.yaml")
 				err := os.WriteFile(cacheFile, []byte(yaml), 0600)
 				Expect(err).NotTo(HaveOccurred())
 
@@ -218,7 +218,7 @@ var _ = Describe("Delete Command", func() {
 				}
 
 				// First succeeds, second fails
-				err = app.Run([]string{"holodeck", "delete", "--cachepath", tempDir, "sshfirst1", "nonexistent"})
+				err = app.Run([]string{"holodeck", "delete", "--cachepath", tempDir, "e7f8a9b0", "nonexistent"})
 				Expect(err).To(HaveOccurred())
 				Expect(err.Error()).To(ContainSubstring("failed to get instance nonexistent"))
 
@@ -235,8 +235,8 @@ var _ = Describe("Delete Command", func() {
 				DeferCleanup(os.RemoveAll, tempDir)
 
 				// Create a cache file in custom path
-				yaml := sshCacheYAML("customdel", "custom-delete")
-				cacheFile := filepath.Join(tempDir, "customdel.yaml")
+				yaml := sshCacheYAML("f1e2d3c4", "custom-delete")
+				cacheFile := filepath.Join(tempDir, "f1e2d3c4.yaml")
 				err = os.WriteFile(cacheFile, []byte(yaml), 0600)
 				Expect(err).NotTo(HaveOccurred())
 
@@ -246,7 +246,7 @@ var _ = Describe("Delete Command", func() {
 				}
 
 				// Use -c alias for cachepath
-				err = app.Run([]string{"holodeck", "delete", "-c", tempDir, "customdel"})
+				err = app.Run([]string{"holodeck", "delete", "-c", tempDir, "f1e2d3c4"})
 				Expect(err).NotTo(HaveOccurred())
 
 				// Verify cache file was removed

cmd/cli/dryrun/dryrun.go

@@ -27,6 +27,7 @@ import (
 	"github.com/NVIDIA/holodeck/pkg/provider/aws"
 	"github.com/NVIDIA/holodeck/pkg/provisioner"
 	"github.com/NVIDIA/holodeck/pkg/sshutil"
+	"github.com/NVIDIA/holodeck/pkg/utils"
 
 	cli "github.com/urfave/cli/v2"
 	"golang.org/x/crypto/ssh"
@@ -131,6 +132,10 @@ func validateAWS(log *logger.FunLogger, opts *options) error {
 // createSshClient creates a ssh client, and retries if it fails to connect
 func connectOrDie(keyPath, userName, hostUrl string) error {
 	var err error
+	keyPath, err = utils.ExpandPath(keyPath)
+	if err != nil {
+		return fmt.Errorf("expanding key path: %w", err)
+	}
 	key, err := os.ReadFile(keyPath) // nolint:gosec
 	if err != nil {
 		return fmt.Errorf("failed to read key file: %w", err)

cmd/cli/get/get.go

@@ -181,7 +181,7 @@ func (m command) runKubeconfig(instanceID string) error {
 	}
 
 	// Download kubeconfig
-	if err := utils.GetKubeConfig(m.log, &env, hostUrl, outputPath); err != nil {
+	if err := utils.GetKubeConfig(m.log, &env, hostUrl, outputPath, ""); err != nil {
 		return fmt.Errorf("failed to download kubeconfig: %w", err)
 	}
 

cmd/cli/status/status_test.go

@@ -130,14 +130,14 @@ var _ = Describe("Status Command", func() {
 			Expect(err).NotTo(HaveOccurred())
 			DeferCleanup(os.RemoveAll, tempDir)
 
-			instanceID := "test12345678"
+			instanceID := "ab12cd34"
 			cacheFile := filepath.Join(tempDir, instanceID+".yaml")
 			validYAML := `apiVersion: holodeck.nvidia.com/v1alpha1
 kind: Environment
 metadata:
   name: test-environment
   labels:
-    holodeck-instance-id: test12345678
+    holodeck-instance-id: ab12cd34
 spec:
   provider: ssh
   username: testuser

docs/examples/README.md

@@ -129,9 +129,12 @@ Install the NVIDIA driver using the official `.run` installer.
 
 **Files:**
 
-- [`examples/ctk_package_pinned.yaml`](../../examples/ctk_package_pinned.yaml) — CTK pinned to a specific version
-- [`examples/ctk_git_source.yaml`](../../examples/ctk_git_source.yaml) — CTK built from git
-- [`examples/ctk_latest_source.yaml`](../../examples/ctk_latest_source.yaml) — CTK tracking latest branch
+- [`ctk_package_pinned.yaml`](../../examples/ctk_package_pinned.yaml)
+  — CTK pinned to a specific version
+- [`ctk_git_source.yaml`](../../examples/ctk_git_source.yaml)
+  — CTK built from git
+- [`ctk_latest_source.yaml`](../../examples/ctk_latest_source.yaml)
+  — CTK tracking latest branch
 
 See the [CTK Installation Sources Guide](../guides/ctk-sources.md) for
 detailed configuration options.
@@ -140,8 +143,10 @@ detailed configuration options.
 
 **Files:**
 
-- [`examples/runtime_containerd_git.yaml`](../../examples/runtime_containerd_git.yaml) — Containerd built from git
-- [`examples/runtime_containerd_latest.yaml`](../../examples/runtime_containerd_latest.yaml) — Containerd tracking latest
+- [`runtime_containerd_git.yaml`](../../examples/runtime_containerd_git.yaml)
+  — Containerd built from git
+- [`runtime_containerd_latest.yaml`](../../examples/runtime_containerd_latest.yaml)
+  — Containerd tracking latest
 
 See the [Container Runtime Sources Guide](../guides/runtime-sources.md)
 for all runtime options.

docs/guides/custom-templates.md

@@ -178,15 +178,15 @@ customTemplates:
 
 1. **Use `pre-install` for system prerequisites** like package
    repos, kernel parameters, or certificates.
-2. **Use `post-kubernetes` for workload deployment** since the
+1. **Use `post-kubernetes` for workload deployment** since the
    cluster is ready at that point.
-3. **Use `post-install` for validation scripts** that verify
+1. **Use `post-install` for validation scripts** that verify
    the full stack.
-4. **Set `continueOnError: true` for non-critical scripts**
+1. **Set `continueOnError: true` for non-critical scripts**
    like monitoring or logging.
-5. **Add checksums for URL sources** to ensure script integrity.
-6. **Keep scripts idempotent** so re-runs produce the same result.
-7. **Test with `holodeck dryrun`** to validate configuration
+1. **Add checksums for URL sources** to ensure script integrity.
+1. **Keep scripts idempotent** so re-runs produce the same result.
+1. **Test with `holodeck dryrun`** to validate configuration
    before provisioning.
 
 ## Related

pkg/provider/aws/cluster.go

@@ -323,6 +323,32 @@ func (p *Provider) createClusterSecurityGroup(cache *ClusterCache) error {
 		},
 	}
 
+	// Self-referencing rules: allow all traffic between instances in this SG.
+	// Covers webhooks (dynamic ports), NodePort (30000-32767), IPIP (Calico),
+	// and any future K8s inter-node communication.
+	// Uses explicit TCP+UDP+ICMP (not protocol -1) for stricter compliance.
+	sgRef := []types.UserIdGroupPair{{GroupId: sgOutput.GroupId}}
+	permissions = append(permissions,
+		types.IpPermission{
+			FromPort:         aws.Int32(0),
+			ToPort:           aws.Int32(65535),
+			IpProtocol:       aws.String("tcp"),
+			UserIdGroupPairs: sgRef,
+		},
+		types.IpPermission{
+			FromPort:         aws.Int32(0),
+			ToPort:           aws.Int32(65535),
+			IpProtocol:       aws.String("udp"),
+			UserIdGroupPairs: sgRef,
+		},
+		types.IpPermission{
+			FromPort:         aws.Int32(-1),
+			ToPort:           aws.Int32(-1),
+			IpProtocol:       aws.String("icmp"),
+			UserIdGroupPairs: sgRef,
+		},
+	)
+
 	irInput := &ec2.AuthorizeSecurityGroupIngressInput{
 		GroupId:       sgOutput.GroupId,
 		IpPermissions: permissions,

pkg/provider/aws/create.go

@@ -404,6 +404,25 @@ func (p *Provider) createSecurityGroup(cache *AWS) error {
 				IpProtocol: &tcp,
 				IpRanges:   ipRanges,
 			},
+			// Self-referencing: allow all TCP/UDP/ICMP between SG members
+			{
+				FromPort:         aws.Int32(0),
+				ToPort:           aws.Int32(65535),
+				IpProtocol:       aws.String("tcp"),
+				UserIdGroupPairs: []types.UserIdGroupPair{{GroupId: sgOutput.GroupId}},
+			},
+			{
+				FromPort:         aws.Int32(0),
+				ToPort:           aws.Int32(65535),
+				IpProtocol:       aws.String("udp"),
+				UserIdGroupPairs: []types.UserIdGroupPair{{GroupId: sgOutput.GroupId}},
+			},
+			{
+				FromPort:         aws.Int32(-1),
+				ToPort:           aws.Int32(-1),
+				IpProtocol:       aws.String("icmp"),
+				UserIdGroupPairs: []types.UserIdGroupPair{{GroupId: sgOutput.GroupId}},
+			},
 		},
 	}
 

pkg/provider/aws/nlb.go

@@ -45,7 +45,14 @@ func (p *Provider) createNLB(cache *ClusterCache) error {
 	cancelLoading := p.log.Loading("Creating Network Load Balancer")
 
 	lbType := elbv2types.LoadBalancerTypeEnumNetwork
-	lbName := fmt.Sprintf("%s-nlb", p.ObjectMeta.Name)
+	// AWS load balancer names are limited to 32 characters.
+	const nlbSuffix = "-nlb"
+	maxNLBNameLen := 32 - len(nlbSuffix)
+	nlbBaseName := p.ObjectMeta.Name
+	if len(nlbBaseName) > maxNLBNameLen {
+		nlbBaseName = nlbBaseName[:maxNLBNameLen]
+	}
+	lbName := nlbBaseName + nlbSuffix
 
 	// Determine subnet IDs (use the same subnet for NLB)
 	subnetIDs := []string{cache.Subnetid}
@@ -91,7 +98,15 @@ func (p *Provider) createTargetGroup(cache *ClusterCache) error {
 
 	cancelLoading := p.log.Loading("Creating target group for Kubernetes API")
 
-	tgName := fmt.Sprintf("%s-k8s-api-tg", p.ObjectMeta.Name)
+	// AWS target group names are limited to 32 characters.
+	// Truncate the environment name to fit within the limit.
+	const tgSuffix = "-k8s-tg"
+	maxNameLen := 32 - len(tgSuffix)
+	name := p.ObjectMeta.Name
+	if len(name) > maxNameLen {
+		name = name[:maxNameLen]
+	}
+	tgName := name + tgSuffix
 
 	// Create target group for Kubernetes API (port 6443)
 	createTGInput := &elasticloadbalancingv2.CreateTargetGroupInput{

pkg/provisioner/cluster.go

@@ -149,13 +149,17 @@ func (cp *ClusterProvisioner) ProvisionCluster(nodes []NodeInfo) error {
 	return nil
 }
 
-// determineControlPlaneEndpoint returns the control plane endpoint
+// determineControlPlaneEndpoint returns the control plane endpoint for cluster-internal
+// communication (kubeadm init, join, API server binding). For HA with NLB, returns the
+// NLB DNS. For non-HA, returns the first CP's private IP since all nodes are in the
+// same VPC and the private IP is always routable. External access (kubeconfig) is
+// handled separately by RewriteKubeConfigServer.
 func (cp *ClusterProvisioner) determineControlPlaneEndpoint(firstCP NodeInfo) string {
 	// Check if HA is enabled and we have a load balancer DNS
 	if cp.Environment.Status.Cluster != nil && cp.Environment.Status.Cluster.LoadBalancerDNS != "" {
 		return cp.Environment.Status.Cluster.LoadBalancerDNS
 	}
-	// Fall back to first control-plane private IP
+	// Use private IP for intra-VPC communication (init + join)
 	return firstCP.PrivateIP
 }
 

pkg/provisioner/cluster_test.go

@@ -310,6 +310,7 @@ func TestClusterProvisioner_determineControlPlaneEndpoint(t *testing.T) {
 				Status: v1alpha1.EnvironmentStatus{},
 			},
 			firstCP: NodeInfo{
+				PublicIP:  "54.1.2.3",
 				PrivateIP: "10.0.0.1",
 			},
 			expected: "10.0.0.1",
@@ -322,6 +323,7 @@ func TestClusterProvisioner_determineControlPlaneEndpoint(t *testing.T) {
 				},
 			},
 			firstCP: NodeInfo{
+				PublicIP:  "54.1.2.4",
 				PrivateIP: "10.0.0.2",
 			},
 			expected: "10.0.0.2",