Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 110 additions & 11 deletions cmd/launch.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import (
"github.com/spore-host/spawn/pkg/sweep"
"github.com/spore-host/spawn/pkg/userdata"
"github.com/spore-host/spawn/pkg/wizard"
truffleaws "github.com/spore-host/truffle/pkg/aws"
"gopkg.in/yaml.v3"
)

Expand Down Expand Up @@ -1218,6 +1219,14 @@ func launchWithProgress(ctx context.Context, awsClient *aws.Client, config *aws.
}
}

// Pre-flight: validate instance-type feature constraints (MPI placement group,
// EFA, hibernation) BEFORE creating any AWS resources (IAM role, security
// group), so an unsupported combination fails fast with an actionable message
// instead of cryptically after several API calls (#110).
if err := preflightInstanceConstraints(ctx, awsClient, config, mpiEnabled, efaEnabled, hibernate || hibernateOnIdle); err != nil {
return err
}

// Step 2: Setup SSH key
prog.Start("Setting up SSH key")
if config.KeyName == "" {
Expand Down Expand Up @@ -1517,19 +1526,41 @@ func launchWithProgress(ctx context.Context, awsClient *aws.Client, config *aws.
return fmt.Errorf("--mpi requires --job-array-name")
}

// Validate instance type supports placement groups if enabled
if mpiAutoPlacementGroup || mpiPlacementGroup != "" {
if err := awsClient.ValidateInstanceTypeForPlacementGroup(ctx, config.InstanceType); err != nil {
return fmt.Errorf("placement group validation: %w", err)
// Decide on a cluster placement group. HPC instance types (hpc6a/hpc7a/
// hpc7g) don't support cluster placement groups — they get low-latency
// networking from AWS HPC infrastructure — so --auto-placement-group
// (on by default) must SKIP them gracefully rather than hard-fail (#104).
// An explicitly requested --placement-group still errors if unsupported.
tc := truffleaws.NewClientFromConfig(awsClient.Config())
clusterPG := func() (bool, error) {
caps, err := tc.GetCapabilities(ctx, config.InstanceType, config.Region)
if err != nil {
return false, err
}
return caps.ClusterPlacement, nil
}

// Create auto placement group if needed
if mpiAutoPlacementGroup && mpiPlacementGroup == "" {
mpiPlacementGroup = fmt.Sprintf("spawn-mpi-%s", jobArrayName)
fmt.Fprintf(os.Stderr, "Creating placement group: %s\n", mpiPlacementGroup)
if err := awsClient.CreatePlacementGroup(ctx, mpiPlacementGroup, config.Region); err != nil {
return fmt.Errorf("create placement group: %w", err)
if mpiPlacementGroup != "" {
// Explicit request: must be supported (authoritative capability check).
supported, err := clusterPG()
if err != nil {
return fmt.Errorf("placement group validation: %w", err)
}
if !supported {
return fmt.Errorf("instance type %s does not support cluster placement groups (required for --placement-group)", config.InstanceType)
}
} else if mpiAutoPlacementGroup {
supported, err := clusterPG()
if err != nil {
return fmt.Errorf("placement group validation: %w", err)
}
if supported {
mpiPlacementGroup = fmt.Sprintf("spawn-mpi-%s", jobArrayName)
fmt.Fprintf(os.Stderr, "Creating placement group: %s\n", mpiPlacementGroup)
if err := awsClient.CreatePlacementGroup(ctx, mpiPlacementGroup, config.Region); err != nil {
return fmt.Errorf("create placement group: %w", err)
}
} else {
fmt.Fprintf(os.Stderr, "ℹ️ %s doesn't support cluster placement groups (HPC instance types use AWS HPC networking); skipping placement group.\n", config.InstanceType)
}
}

Expand Down Expand Up @@ -1994,6 +2025,74 @@ func guardWindowsInstanceType(targetOS, instanceType string) error {
return nil
}

// preflightInstanceConstraints validates that the requested instance type
// supports the requested features (MPI cluster placement group, EFA,
// hibernation) BEFORE any AWS resources are created, with actionable errors
// (#110). One DescribeInstanceTypes call backs all checks. HPC types are exempt
// from the MPI/placement-group requirement — they use AWS HPC networking and
// spawn skips the placement group for them (#104), so --mpi alone is fine.
func preflightInstanceConstraints(ctx context.Context, awsClient *aws.Client, config *aws.LaunchConfig, wantMPI, wantEFA, wantHibernate bool) error {
if !wantMPI && !wantEFA && !wantHibernate {
return nil // nothing feature-specific to check
}
// truffle is the instance-type capability authority — consume it rather than
// re-querying EC2 from spawn. Build a truffle client from spawn's AWS config
// so creds/region match.
tc := truffleaws.NewClientFromConfig(awsClient.Config())
caps, err := tc.GetCapabilities(ctx, config.InstanceType, config.Region)
if err != nil {
return fmt.Errorf("pre-flight instance-type check: %w", err)
}
if !caps.Found {
return fmt.Errorf("instance type %q not found in region %s", config.InstanceType, config.Region)
}

// --efa: must support EFA.
if wantEFA && !caps.EFA {
return fmt.Errorf("instance type %q does not support EFA (required for --efa).\n Find EFA-capable types: truffle find \"%s\" efa (e.g. c5n.18xlarge, hpc6a.48xlarge)",
config.InstanceType, instanceFamilyHint(config.InstanceType))
}

// --hibernate / --hibernate-on-idle: must support hibernation.
if wantHibernate && !caps.Hibernation {
return fmt.Errorf("instance type %q does not support hibernation (required for --hibernate/--hibernate-on-idle).\n Choose a hibernation-capable type, or drop the hibernation flag.", config.InstanceType)
}

// --mpi: needs a cluster placement group UNLESS it's an HPC type (which spawn
// skips the placement group for). So only block --mpi when neither holds.
if wantMPI && !caps.ClusterPlacement && !isHPCInstanceType(config.InstanceType) {
return fmt.Errorf("instance type %q does not support cluster placement groups (needed for --mpi).\n Use an MPI-capable type (e.g. c5n.18xlarge, c6i.32xlarge) or an HPC type (hpc6a/hpc7a/hpc7g), or run: truffle find \"%s\" efa",
config.InstanceType, instanceFamilyHint(config.InstanceType))
}
return nil
}

// instanceFamilyHint returns a glob hint for the instance's family for use in
// suggested truffle commands, e.g. "c5n.18xlarge" -> "c5n*".
func instanceFamilyHint(instanceType string) string {
if i := strings.IndexByte(instanceType, '.'); i > 0 {
return instanceType[:i] + "*"
}
return instanceType
}

// isHPCInstanceType reports whether the type is in the AWS HPC family, which
// gets low-latency networking from HPC infrastructure rather than placement
// groups (so --mpi is valid without a cluster placement group). Detected by the
// "hpc" family prefix rather than a hardcoded list, so new HPC families
// (hpc6a/hpc6id/hpc7a/hpc7g/hpc8a/… as of June 2026, and future ones) are
// covered automatically — the EC2 naming convention is the contract. A real
// family is "hpc" followed by a generation digit (hpc6a, hpc7g…), so we require
// the digit to avoid matching a stray "hpc.weird".
func isHPCInstanceType(instanceType string) bool {
const p = "hpc"
if !strings.HasPrefix(instanceType, p) || len(instanceType) <= len(p) {
return false
}
c := instanceType[len(p)]
return c >= '0' && c <= '9'
}

// windowsLifecycleGuard enforces cost safety for Windows launches. Windows has
// no in-instance spored yet (#77), so idle-timeout cannot work and the only
// thing that will stop the instance is its TTL plus the server-side reaper
Expand Down
32 changes: 32 additions & 0 deletions cmd/preflight_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package cmd

import "testing"

func TestInstanceFamilyHint(t *testing.T) {
cases := map[string]string{
"c5n.18xlarge": "c5n*",
"hpc6a.48xlarge": "hpc6a*",
"m7i.xlarge": "m7i*",
"weird": "weird", // no dot
}
for in, want := range cases {
if got := instanceFamilyHint(in); got != want {
t.Errorf("instanceFamilyHint(%q) = %q, want %q", in, got, want)
}
}
}

func TestIsHPCInstanceType(t *testing.T) {
hpc := []string{"hpc6a.48xlarge", "hpc7a.96xlarge", "hpc7g.16xlarge"}
notHPC := []string{"c5n.18xlarge", "m7i.xlarge", "hpc.weird", "c6i.32xlarge"}
for _, it := range hpc {
if !isHPCInstanceType(it) {
t.Errorf("%s should be HPC", it)
}
}
for _, it := range notHPC {
if isHPCInstanceType(it) {
t.Errorf("%s should NOT be HPC", it)
}
}
}
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/imagebuilder v1.55.6
github.com/aws/aws-sdk-go-v2/service/kms v1.51.1
github.com/aws/aws-sdk-go-v2/service/lambda v1.90.1
github.com/aws/aws-sdk-go-v2/service/pricing v1.41.2
github.com/aws/aws-sdk-go-v2/service/pricing v1.42.2
github.com/aws/aws-sdk-go-v2/service/route53 v1.62.7
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
github.com/aws/aws-sdk-go-v2/service/scheduler v1.17.24
Expand All @@ -38,6 +38,7 @@ require (
github.com/scttfrdmn/substrate v0.70.0
github.com/spf13/cobra v1.10.2
github.com/spore-host/libs v0.36.0
github.com/spore-host/truffle v0.38.1
go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws v0.68.0
go.opentelemetry.io/otel v1.43.0
go.opentelemetry.io/otel/sdk v1.43.0
Expand Down
8 changes: 6 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -54,14 +54,16 @@ github.com/aws/aws-sdk-go-v2/service/kms v1.51.1 h1:zuSf4olLKZW8cF/W9Y5wvGT+/0ra
github.com/aws/aws-sdk-go-v2/service/kms v1.51.1/go.mod h1:Y0+uxvxz6ib4KktRdK0V4X45Vcs/JyYoz8H71pO8xeI=
github.com/aws/aws-sdk-go-v2/service/lambda v1.90.1 h1:odCeJgHXfQoXEWQUIzPkKvsJTWcLMsaOWowNpovPFFw=
github.com/aws/aws-sdk-go-v2/service/lambda v1.90.1/go.mod h1:NbtJVztitG7JkuoI4GSrDUlsB32zeXqKBvXj6bUxcMo=
github.com/aws/aws-sdk-go-v2/service/pricing v1.41.2 h1:Ujj2QuBZCrQRek/VmnPwjz6LRGdKoxldpp8fNXwLUUg=
github.com/aws/aws-sdk-go-v2/service/pricing v1.41.2/go.mod h1:zXv2YjVkSugNoBHG8WrHHNqCyTFplsE8B8hCAV9riRA=
github.com/aws/aws-sdk-go-v2/service/pricing v1.42.2 h1:qLe0KpIqzUuBQk6iV7oiOGW/EEWLs87uTP/xNKpfe88=
github.com/aws/aws-sdk-go-v2/service/pricing v1.42.2/go.mod h1:aciuNKM3vUImiRzhEquRAAfetzdIKAdbEIL3cTm1XE4=
github.com/aws/aws-sdk-go-v2/service/route53 v1.62.7 h1:twRRMmtSITnt/rrp+D7UDLzE5pKMZe759aalkUdN+OY=
github.com/aws/aws-sdk-go-v2/service/route53 v1.62.7/go.mod h1:ztM1lr+sRoCAI8336ZUvlRPbToue0d3gE/wd6jomSJ8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
github.com/aws/aws-sdk-go-v2/service/scheduler v1.17.24 h1:F4gvh9TJcEZVqirKpX/FBWEMK6tvnGSVf4FWDvFtSQw=
github.com/aws/aws-sdk-go-v2/service/scheduler v1.17.24/go.mod h1:0/mvOL++cUfpS4KgHigHDo+x8KLYG/grOTyIOw3KCPM=
github.com/aws/aws-sdk-go-v2/service/servicequotas v1.34.7 h1:LRcX5C4jwSmkbmkamPVLU3/VALAo0Fuy77a2TgMKYx0=
github.com/aws/aws-sdk-go-v2/service/servicequotas v1.34.7/go.mod h1:52QJsp2N27Em8o5H/cgkBwjTY4I/TYpTBHMlqhuCHMQ=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
github.com/aws/aws-sdk-go-v2/service/sns v1.39.17 h1:synXIPC/L4Cc489P0XDcrVJzHSLj7krKRpFLalbGM2k=
Expand Down Expand Up @@ -174,6 +176,8 @@ github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/spore-host/libs v0.36.0 h1:zKsxYuZZlq9wmpFOh5RPP7pu4M8irj90L8pKL8rHLLU=
github.com/spore-host/libs v0.36.0/go.mod h1:1NdCOXkDuw6T0MAcWOyO630e0OPaGaYiMbLcRBIHeyM=
github.com/spore-host/truffle v0.38.1 h1:IPj0GvrCVhPfp/zDdgcnRspKZzL3G8y9pIS4v+Htv6A=
github.com/spore-host/truffle v0.38.1/go.mod h1:20ZI1wEvNcqi/xefx0akXn5ZfgvquD2p27sQy88bvi0=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
Expand Down
43 changes: 42 additions & 1 deletion pkg/aws/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ type LaunchConfig struct {
SecurityGroupIDs []string // Security group IDs; a default spawn SG is created if empty
SubnetID string // VPC subnet ID; leave empty to use default subnet
UserData string // User-data script (plain text or base64); spored is injected automatically
ClientToken string // Optional RunInstances idempotency token; deterministic in (cluster,entity,generation) for callers like cohort (#108). Empty = today's behavior.
Spot bool // If true, launch as a Spot instance
SpotMaxPrice string // Optional Spot max price in $/hr, e.g. "0.50"; empty = on-demand cap
ReservationID string // On-Demand Capacity Reservation ID to target
Expand Down Expand Up @@ -213,6 +214,38 @@ type LaunchResult struct {
KeyName string // EC2 key pair name used for SSH access
}

// LaunchError wraps a RunInstances failure with the verbatim AWS error code
// extracted as a Go value, so callers can classify failures (capacity vs quota
// vs config) on an explicit code rather than string-matching a wrapped message
// (#108). Code is the AWS API error code (e.g. "InsufficientInstanceCapacity",
// "RequestLimitExceeded", "Unsupported", "MaxSpotInstanceCountExceeded"), or ""
// if the underlying error wasn't an AWS API error. The original error is
// preserved via Unwrap, so errors.As(err, &smithyAPIErr) still works too.
type LaunchError struct {
Code string
err error
}

func (e *LaunchError) Error() string {
if e.Code != "" {
return fmt.Sprintf("failed to launch instance: %s: %v", e.Code, e.err)
}
return fmt.Sprintf("failed to launch instance: %v", e.err)
}

func (e *LaunchError) Unwrap() error { return e.err }

// newLaunchError builds a LaunchError, extracting the verbatim AWS error code
// from the smithy.APIError in the chain if present.
func newLaunchError(err error) error {
le := &LaunchError{err: err}
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
le.Code = apiErr.ErrorCode()
}
return le
}

// Launch starts a new EC2 instance as described by launchConfig and returns its
// ID, IP addresses, and initial state. All spawn: lifecycle tags (TTL, idle
// timeout, DNS name, cost limit, etc.) are applied at launch time so spored
Expand Down Expand Up @@ -278,6 +311,14 @@ func (c *Client) Launch(ctx context.Context, launchConfig LaunchConfig) (*Launch
BlockDeviceMappings: blockDevices,
}

// Idempotency token (optional). With it, a retry after a network timeout
// won't double-launch, and the caller can resolve the Ambiguous fault class
// ("did RunInstances succeed before the response was lost?"). Empty = today's
// behavior (no token). (#108)
if launchConfig.ClientToken != "" {
input.ClientToken = aws.String(launchConfig.ClientToken)
}

// Add IAM instance profile if specified
if launchConfig.IamInstanceProfile != "" {
input.IamInstanceProfile = &types.IamInstanceProfileSpecification{
Expand Down Expand Up @@ -379,7 +420,7 @@ func (c *Client) Launch(ctx context.Context, launchConfig LaunchConfig) (*Launch
// Launch instance
result, err := ec2Client.RunInstances(ctx, input)
if err != nil {
return nil, fmt.Errorf("failed to launch instance: %w", err)
return nil, newLaunchError(err)
}

if len(result.Instances) == 0 {
Expand Down
49 changes: 49 additions & 0 deletions pkg/aws/launcherror_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package aws

import (
"errors"
"fmt"
"testing"
)

// TestNewLaunchError_ExtractsCode verifies that an AWS API error in the chain
// surfaces its verbatim code on LaunchError.Code, so callers can classify the
// failure on a code rather than string-matching (#108).
func TestNewLaunchError_ExtractsCode(t *testing.T) {
apiErr := &fakeAPIError{code: "InsufficientInstanceCapacity"}
err := newLaunchError(apiErr)

var le *LaunchError
if !errors.As(err, &le) {
t.Fatalf("newLaunchError did not produce a *LaunchError: %T", err)
}
if le.Code != "InsufficientInstanceCapacity" {
t.Errorf("Code = %q, want InsufficientInstanceCapacity", le.Code)
}
// The original error must remain reachable via Unwrap so callers can still
// errors.As(&smithyAPIError).
if !errors.Is(err, apiErr) {
t.Errorf("Unwrap chain lost the original API error")
}
if !contains(err.Error(), "InsufficientInstanceCapacity") {
t.Errorf("Error() = %q, want it to mention the code", err.Error())
}
}

// TestNewLaunchError_NonAPIError verifies that a plain error yields an empty
// Code (not a panic) and is still wrapped + unwrappable.
func TestNewLaunchError_NonAPIError(t *testing.T) {
base := fmt.Errorf("dial tcp: connection refused")
err := newLaunchError(base)

var le *LaunchError
if !errors.As(err, &le) {
t.Fatalf("newLaunchError did not produce a *LaunchError: %T", err)
}
if le.Code != "" {
t.Errorf("Code = %q, want empty for a non-API error", le.Code)
}
if !errors.Is(err, base) {
t.Errorf("Unwrap chain lost the original error")
}
}
Loading
Loading