diff --git a/cmd/launch.go b/cmd/launch.go index 3bdd85a..ea89aa6 100644 --- a/cmd/launch.go +++ b/cmd/launch.go @@ -42,6 +42,7 @@ import ( "github.com/spore-host/spawn/pkg/sweep" "github.com/spore-host/spawn/pkg/userdata" "github.com/spore-host/spawn/pkg/wizard" + truffleaws "github.com/spore-host/truffle/pkg/aws" "gopkg.in/yaml.v3" ) @@ -1218,6 +1219,14 @@ func launchWithProgress(ctx context.Context, awsClient *aws.Client, config *aws. } } + // Pre-flight: validate instance-type feature constraints (MPI placement group, + // EFA, hibernation) BEFORE creating any AWS resources (IAM role, security + // group), so an unsupported combination fails fast with an actionable message + // instead of cryptically after several API calls (#110). + if err := preflightInstanceConstraints(ctx, awsClient, config, mpiEnabled, efaEnabled, hibernate || hibernateOnIdle); err != nil { + return err + } + // Step 2: Setup SSH key prog.Start("Setting up SSH key") if config.KeyName == "" { @@ -1517,19 +1526,41 @@ func launchWithProgress(ctx context.Context, awsClient *aws.Client, config *aws. return fmt.Errorf("--mpi requires --job-array-name") } - // Validate instance type supports placement groups if enabled - if mpiAutoPlacementGroup || mpiPlacementGroup != "" { - if err := awsClient.ValidateInstanceTypeForPlacementGroup(ctx, config.InstanceType); err != nil { - return fmt.Errorf("placement group validation: %w", err) + // Decide on a cluster placement group. HPC instance types (hpc6a/hpc7a/ + // hpc7g) don't support cluster placement groups — they get low-latency + // networking from AWS HPC infrastructure — so --auto-placement-group + // (on by default) must SKIP them gracefully rather than hard-fail (#104). + // An explicitly requested --placement-group still errors if unsupported. + tc := truffleaws.NewClientFromConfig(awsClient.Config()) + clusterPG := func() (bool, error) { + caps, err := tc.GetCapabilities(ctx, config.InstanceType, config.Region) + if err != nil { + return false, err } + return caps.ClusterPlacement, nil } - - // Create auto placement group if needed - if mpiAutoPlacementGroup && mpiPlacementGroup == "" { - mpiPlacementGroup = fmt.Sprintf("spawn-mpi-%s", jobArrayName) - fmt.Fprintf(os.Stderr, "Creating placement group: %s\n", mpiPlacementGroup) - if err := awsClient.CreatePlacementGroup(ctx, mpiPlacementGroup, config.Region); err != nil { - return fmt.Errorf("create placement group: %w", err) + if mpiPlacementGroup != "" { + // Explicit request: must be supported (authoritative capability check). + supported, err := clusterPG() + if err != nil { + return fmt.Errorf("placement group validation: %w", err) + } + if !supported { + return fmt.Errorf("instance type %s does not support cluster placement groups (required for --placement-group)", config.InstanceType) + } + } else if mpiAutoPlacementGroup { + supported, err := clusterPG() + if err != nil { + return fmt.Errorf("placement group validation: %w", err) + } + if supported { + mpiPlacementGroup = fmt.Sprintf("spawn-mpi-%s", jobArrayName) + fmt.Fprintf(os.Stderr, "Creating placement group: %s\n", mpiPlacementGroup) + if err := awsClient.CreatePlacementGroup(ctx, mpiPlacementGroup, config.Region); err != nil { + return fmt.Errorf("create placement group: %w", err) + } + } else { + fmt.Fprintf(os.Stderr, "ℹ️ %s doesn't support cluster placement groups (HPC instance types use AWS HPC networking); skipping placement group.\n", config.InstanceType) } } @@ -1994,6 +2025,74 @@ func guardWindowsInstanceType(targetOS, instanceType string) error { return nil } +// preflightInstanceConstraints validates that the requested instance type +// supports the requested features (MPI cluster placement group, EFA, +// hibernation) BEFORE any AWS resources are created, with actionable errors +// (#110). One DescribeInstanceTypes call backs all checks. HPC types are exempt +// from the MPI/placement-group requirement — they use AWS HPC networking and +// spawn skips the placement group for them (#104), so --mpi alone is fine. +func preflightInstanceConstraints(ctx context.Context, awsClient *aws.Client, config *aws.LaunchConfig, wantMPI, wantEFA, wantHibernate bool) error { + if !wantMPI && !wantEFA && !wantHibernate { + return nil // nothing feature-specific to check + } + // truffle is the instance-type capability authority — consume it rather than + // re-querying EC2 from spawn. Build a truffle client from spawn's AWS config + // so creds/region match. + tc := truffleaws.NewClientFromConfig(awsClient.Config()) + caps, err := tc.GetCapabilities(ctx, config.InstanceType, config.Region) + if err != nil { + return fmt.Errorf("pre-flight instance-type check: %w", err) + } + if !caps.Found { + return fmt.Errorf("instance type %q not found in region %s", config.InstanceType, config.Region) + } + + // --efa: must support EFA. + if wantEFA && !caps.EFA { + return fmt.Errorf("instance type %q does not support EFA (required for --efa).\n Find EFA-capable types: truffle find \"%s\" efa (e.g. c5n.18xlarge, hpc6a.48xlarge)", + config.InstanceType, instanceFamilyHint(config.InstanceType)) + } + + // --hibernate / --hibernate-on-idle: must support hibernation. + if wantHibernate && !caps.Hibernation { + return fmt.Errorf("instance type %q does not support hibernation (required for --hibernate/--hibernate-on-idle).\n Choose a hibernation-capable type, or drop the hibernation flag.", config.InstanceType) + } + + // --mpi: needs a cluster placement group UNLESS it's an HPC type (which spawn + // skips the placement group for). So only block --mpi when neither holds. + if wantMPI && !caps.ClusterPlacement && !isHPCInstanceType(config.InstanceType) { + return fmt.Errorf("instance type %q does not support cluster placement groups (needed for --mpi).\n Use an MPI-capable type (e.g. c5n.18xlarge, c6i.32xlarge) or an HPC type (hpc6a/hpc7a/hpc7g), or run: truffle find \"%s\" efa", + config.InstanceType, instanceFamilyHint(config.InstanceType)) + } + return nil +} + +// instanceFamilyHint returns a glob hint for the instance's family for use in +// suggested truffle commands, e.g. "c5n.18xlarge" -> "c5n*". +func instanceFamilyHint(instanceType string) string { + if i := strings.IndexByte(instanceType, '.'); i > 0 { + return instanceType[:i] + "*" + } + return instanceType +} + +// isHPCInstanceType reports whether the type is in the AWS HPC family, which +// gets low-latency networking from HPC infrastructure rather than placement +// groups (so --mpi is valid without a cluster placement group). Detected by the +// "hpc" family prefix rather than a hardcoded list, so new HPC families +// (hpc6a/hpc6id/hpc7a/hpc7g/hpc8a/… as of June 2026, and future ones) are +// covered automatically — the EC2 naming convention is the contract. A real +// family is "hpc" followed by a generation digit (hpc6a, hpc7g…), so we require +// the digit to avoid matching a stray "hpc.weird". +func isHPCInstanceType(instanceType string) bool { + const p = "hpc" + if !strings.HasPrefix(instanceType, p) || len(instanceType) <= len(p) { + return false + } + c := instanceType[len(p)] + return c >= '0' && c <= '9' +} + // windowsLifecycleGuard enforces cost safety for Windows launches. Windows has // no in-instance spored yet (#77), so idle-timeout cannot work and the only // thing that will stop the instance is its TTL plus the server-side reaper diff --git a/cmd/preflight_test.go b/cmd/preflight_test.go new file mode 100644 index 0000000..3c649d5 --- /dev/null +++ b/cmd/preflight_test.go @@ -0,0 +1,32 @@ +package cmd + +import "testing" + +func TestInstanceFamilyHint(t *testing.T) { + cases := map[string]string{ + "c5n.18xlarge": "c5n*", + "hpc6a.48xlarge": "hpc6a*", + "m7i.xlarge": "m7i*", + "weird": "weird", // no dot + } + for in, want := range cases { + if got := instanceFamilyHint(in); got != want { + t.Errorf("instanceFamilyHint(%q) = %q, want %q", in, got, want) + } + } +} + +func TestIsHPCInstanceType(t *testing.T) { + hpc := []string{"hpc6a.48xlarge", "hpc7a.96xlarge", "hpc7g.16xlarge"} + notHPC := []string{"c5n.18xlarge", "m7i.xlarge", "hpc.weird", "c6i.32xlarge"} + for _, it := range hpc { + if !isHPCInstanceType(it) { + t.Errorf("%s should be HPC", it) + } + } + for _, it := range notHPC { + if isHPCInstanceType(it) { + t.Errorf("%s should NOT be HPC", it) + } + } +} diff --git a/go.mod b/go.mod index 9a92321..78014b1 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/imagebuilder v1.55.6 github.com/aws/aws-sdk-go-v2/service/kms v1.51.1 github.com/aws/aws-sdk-go-v2/service/lambda v1.90.1 - github.com/aws/aws-sdk-go-v2/service/pricing v1.41.2 + github.com/aws/aws-sdk-go-v2/service/pricing v1.42.2 github.com/aws/aws-sdk-go-v2/service/route53 v1.62.7 github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 github.com/aws/aws-sdk-go-v2/service/scheduler v1.17.24 @@ -38,6 +38,7 @@ require ( github.com/scttfrdmn/substrate v0.70.0 github.com/spf13/cobra v1.10.2 github.com/spore-host/libs v0.36.0 + github.com/spore-host/truffle v0.38.1 go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws v0.68.0 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/sdk v1.43.0 diff --git a/go.sum b/go.sum index a62c24d..757a0b6 100644 --- a/go.sum +++ b/go.sum @@ -54,14 +54,16 @@ github.com/aws/aws-sdk-go-v2/service/kms v1.51.1 h1:zuSf4olLKZW8cF/W9Y5wvGT+/0ra github.com/aws/aws-sdk-go-v2/service/kms v1.51.1/go.mod h1:Y0+uxvxz6ib4KktRdK0V4X45Vcs/JyYoz8H71pO8xeI= github.com/aws/aws-sdk-go-v2/service/lambda v1.90.1 h1:odCeJgHXfQoXEWQUIzPkKvsJTWcLMsaOWowNpovPFFw= github.com/aws/aws-sdk-go-v2/service/lambda v1.90.1/go.mod h1:NbtJVztitG7JkuoI4GSrDUlsB32zeXqKBvXj6bUxcMo= -github.com/aws/aws-sdk-go-v2/service/pricing v1.41.2 h1:Ujj2QuBZCrQRek/VmnPwjz6LRGdKoxldpp8fNXwLUUg= -github.com/aws/aws-sdk-go-v2/service/pricing v1.41.2/go.mod h1:zXv2YjVkSugNoBHG8WrHHNqCyTFplsE8B8hCAV9riRA= +github.com/aws/aws-sdk-go-v2/service/pricing v1.42.2 h1:qLe0KpIqzUuBQk6iV7oiOGW/EEWLs87uTP/xNKpfe88= +github.com/aws/aws-sdk-go-v2/service/pricing v1.42.2/go.mod h1:aciuNKM3vUImiRzhEquRAAfetzdIKAdbEIL3cTm1XE4= github.com/aws/aws-sdk-go-v2/service/route53 v1.62.7 h1:twRRMmtSITnt/rrp+D7UDLzE5pKMZe759aalkUdN+OY= github.com/aws/aws-sdk-go-v2/service/route53 v1.62.7/go.mod h1:ztM1lr+sRoCAI8336ZUvlRPbToue0d3gE/wd6jomSJ8= github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU= github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= github.com/aws/aws-sdk-go-v2/service/scheduler v1.17.24 h1:F4gvh9TJcEZVqirKpX/FBWEMK6tvnGSVf4FWDvFtSQw= github.com/aws/aws-sdk-go-v2/service/scheduler v1.17.24/go.mod h1:0/mvOL++cUfpS4KgHigHDo+x8KLYG/grOTyIOw3KCPM= +github.com/aws/aws-sdk-go-v2/service/servicequotas v1.34.7 h1:LRcX5C4jwSmkbmkamPVLU3/VALAo0Fuy77a2TgMKYx0= +github.com/aws/aws-sdk-go-v2/service/servicequotas v1.34.7/go.mod h1:52QJsp2N27Em8o5H/cgkBwjTY4I/TYpTBHMlqhuCHMQ= github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= github.com/aws/aws-sdk-go-v2/service/sns v1.39.17 h1:synXIPC/L4Cc489P0XDcrVJzHSLj7krKRpFLalbGM2k= @@ -174,6 +176,8 @@ github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/spore-host/libs v0.36.0 h1:zKsxYuZZlq9wmpFOh5RPP7pu4M8irj90L8pKL8rHLLU= github.com/spore-host/libs v0.36.0/go.mod h1:1NdCOXkDuw6T0MAcWOyO630e0OPaGaYiMbLcRBIHeyM= +github.com/spore-host/truffle v0.38.1 h1:IPj0GvrCVhPfp/zDdgcnRspKZzL3G8y9pIS4v+Htv6A= +github.com/spore-host/truffle v0.38.1/go.mod h1:20ZI1wEvNcqi/xefx0akXn5ZfgvquD2p27sQy88bvi0= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= diff --git a/pkg/aws/client.go b/pkg/aws/client.go index bdce5ae..08a72fc 100644 --- a/pkg/aws/client.go +++ b/pkg/aws/client.go @@ -109,6 +109,7 @@ type LaunchConfig struct { SecurityGroupIDs []string // Security group IDs; a default spawn SG is created if empty SubnetID string // VPC subnet ID; leave empty to use default subnet UserData string // User-data script (plain text or base64); spored is injected automatically + ClientToken string // Optional RunInstances idempotency token; deterministic in (cluster,entity,generation) for callers like cohort (#108). Empty = today's behavior. Spot bool // If true, launch as a Spot instance SpotMaxPrice string // Optional Spot max price in $/hr, e.g. "0.50"; empty = on-demand cap ReservationID string // On-Demand Capacity Reservation ID to target @@ -213,6 +214,38 @@ type LaunchResult struct { KeyName string // EC2 key pair name used for SSH access } +// LaunchError wraps a RunInstances failure with the verbatim AWS error code +// extracted as a Go value, so callers can classify failures (capacity vs quota +// vs config) on an explicit code rather than string-matching a wrapped message +// (#108). Code is the AWS API error code (e.g. "InsufficientInstanceCapacity", +// "RequestLimitExceeded", "Unsupported", "MaxSpotInstanceCountExceeded"), or "" +// if the underlying error wasn't an AWS API error. The original error is +// preserved via Unwrap, so errors.As(err, &smithyAPIErr) still works too. +type LaunchError struct { + Code string + err error +} + +func (e *LaunchError) Error() string { + if e.Code != "" { + return fmt.Sprintf("failed to launch instance: %s: %v", e.Code, e.err) + } + return fmt.Sprintf("failed to launch instance: %v", e.err) +} + +func (e *LaunchError) Unwrap() error { return e.err } + +// newLaunchError builds a LaunchError, extracting the verbatim AWS error code +// from the smithy.APIError in the chain if present. +func newLaunchError(err error) error { + le := &LaunchError{err: err} + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + le.Code = apiErr.ErrorCode() + } + return le +} + // Launch starts a new EC2 instance as described by launchConfig and returns its // ID, IP addresses, and initial state. All spawn: lifecycle tags (TTL, idle // timeout, DNS name, cost limit, etc.) are applied at launch time so spored @@ -278,6 +311,14 @@ func (c *Client) Launch(ctx context.Context, launchConfig LaunchConfig) (*Launch BlockDeviceMappings: blockDevices, } + // Idempotency token (optional). With it, a retry after a network timeout + // won't double-launch, and the caller can resolve the Ambiguous fault class + // ("did RunInstances succeed before the response was lost?"). Empty = today's + // behavior (no token). (#108) + if launchConfig.ClientToken != "" { + input.ClientToken = aws.String(launchConfig.ClientToken) + } + // Add IAM instance profile if specified if launchConfig.IamInstanceProfile != "" { input.IamInstanceProfile = &types.IamInstanceProfileSpecification{ @@ -379,7 +420,7 @@ func (c *Client) Launch(ctx context.Context, launchConfig LaunchConfig) (*Launch // Launch instance result, err := ec2Client.RunInstances(ctx, input) if err != nil { - return nil, fmt.Errorf("failed to launch instance: %w", err) + return nil, newLaunchError(err) } if len(result.Instances) == 0 { diff --git a/pkg/aws/launcherror_test.go b/pkg/aws/launcherror_test.go new file mode 100644 index 0000000..e107b94 --- /dev/null +++ b/pkg/aws/launcherror_test.go @@ -0,0 +1,49 @@ +package aws + +import ( + "errors" + "fmt" + "testing" +) + +// TestNewLaunchError_ExtractsCode verifies that an AWS API error in the chain +// surfaces its verbatim code on LaunchError.Code, so callers can classify the +// failure on a code rather than string-matching (#108). +func TestNewLaunchError_ExtractsCode(t *testing.T) { + apiErr := &fakeAPIError{code: "InsufficientInstanceCapacity"} + err := newLaunchError(apiErr) + + var le *LaunchError + if !errors.As(err, &le) { + t.Fatalf("newLaunchError did not produce a *LaunchError: %T", err) + } + if le.Code != "InsufficientInstanceCapacity" { + t.Errorf("Code = %q, want InsufficientInstanceCapacity", le.Code) + } + // The original error must remain reachable via Unwrap so callers can still + // errors.As(&smithyAPIError). + if !errors.Is(err, apiErr) { + t.Errorf("Unwrap chain lost the original API error") + } + if !contains(err.Error(), "InsufficientInstanceCapacity") { + t.Errorf("Error() = %q, want it to mention the code", err.Error()) + } +} + +// TestNewLaunchError_NonAPIError verifies that a plain error yields an empty +// Code (not a panic) and is still wrapped + unwrappable. +func TestNewLaunchError_NonAPIError(t *testing.T) { + base := fmt.Errorf("dial tcp: connection refused") + err := newLaunchError(base) + + var le *LaunchError + if !errors.As(err, &le) { + t.Fatalf("newLaunchError did not produce a *LaunchError: %T", err) + } + if le.Code != "" { + t.Errorf("Code = %q, want empty for a non-API error", le.Code) + } + if !errors.Is(err, base) { + t.Errorf("Unwrap chain lost the original error") + } +} diff --git a/pkg/aws/s3.go b/pkg/aws/s3.go index 40d2259..87d43a4 100644 --- a/pkg/aws/s3.go +++ b/pkg/aws/s3.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" @@ -47,6 +48,15 @@ func (c *Client) CreateS3BucketWithTags(ctx context.Context, config S3BucketConf return nil } + // A bucket that exists in a DIFFERENT region answers HeadBucket with 301 + // MovedPermanently / PermanentRedirect (not NotFound). The bucket exists — + // it's just elsewhere — so for an existence check, treat that as "exists, + // don't create" (#103). FSx itself validates cross-region S3 usability and + // returns a meaningful error if the bucket truly can't back the filesystem. + if isCrossRegionRedirect(err) { + return nil + } + // Check if error is "not found" - if so, create bucket var apiErr smithy.APIError if errors.As(err, &apiErr) { @@ -125,6 +135,29 @@ func (c *Client) CreateS3BucketWithTags(ctx context.Context, config S3BucketConf return fmt.Errorf("failed to check if bucket exists: %w", err) } +// isCrossRegionRedirect reports whether a HeadBucket error is the 301 redirect +// AWS returns when the bucket exists but lives in another region. The SDK +// surfaces this two ways depending on the code path: a smithy.APIError with a +// PermanentRedirect/MovedPermanently/301 code, or a bare HTTP-301 +// awshttp.ResponseError with no error code. Detect both (#103). +func isCrossRegionRedirect(err error) bool { + if err == nil { + return false + } + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch apiErr.ErrorCode() { + case "PermanentRedirect", "MovedPermanently", "301": + return true + } + } + var respErr *awshttp.ResponseError + if errors.As(err, &respErr) && respErr.HTTPStatusCode() == 301 { + return true + } + return false +} + // GetFSxConfigFromS3Bucket retrieves FSx configuration from S3 bucket tags func (c *Client) GetFSxConfigFromS3Bucket(ctx context.Context, stackName, region string) (*FSxConfig, error) { cfg := c.cfg.Copy() diff --git a/pkg/aws/s3_test.go b/pkg/aws/s3_test.go index ec761ce..c680128 100644 --- a/pkg/aws/s3_test.go +++ b/pkg/aws/s3_test.go @@ -2,8 +2,13 @@ package aws import ( "context" + "errors" + "fmt" + "net/http" "testing" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + smithyhttp "github.com/aws/smithy-go/transport/http" "github.com/spore-host/spawn/pkg/testutil" ) @@ -74,3 +79,37 @@ func TestCreateS3BucketWithTags(t *testing.T) { }) } } + +// TestIsCrossRegionRedirect covers the two shapes a cross-region HeadBucket +// 301 takes (a coded smithy.APIError vs a bare HTTP-301 ResponseError), plus +// the negatives, so a bucket that merely lives in another region is treated as +// "exists, don't create" rather than a hard error (#103). +func TestIsCrossRegionRedirect(t *testing.T) { + respErr301 := &awshttp.ResponseError{ + ResponseError: &smithyhttp.ResponseError{ + Response: &smithyhttp.Response{Response: &http.Response{StatusCode: 301}}, + Err: errors.New("moved"), + }, + } + cases := []struct { + name string + err error + want bool + }{ + {"nil", nil, false}, + {"PermanentRedirect code", &fakeAPIError{code: "PermanentRedirect"}, true}, + {"MovedPermanently code", &fakeAPIError{code: "MovedPermanently"}, true}, + {"301 code", &fakeAPIError{code: "301"}, true}, + {"bare http 301", respErr301, true}, + {"wrapped http 301", fmt.Errorf("head bucket: %w", respErr301), true}, + {"NotFound is not a redirect", &fakeAPIError{code: "NotFound"}, false}, + {"plain error", errors.New("connection refused"), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := isCrossRegionRedirect(tc.err); got != tc.want { + t.Errorf("isCrossRegionRedirect(%v) = %v, want %v", tc.err, got, tc.want) + } + }) + } +}