feat: Add add_services instruction to start services in bulk (#912)

Closes #802

api/golang/core/lib/services/service_config_builder.go

@@ -44,33 +44,51 @@ func NewServiceConfigBuilder(containerImageName string) *ServiceConfigBuilder {
 	}
 }
 
+// NewServiceConfigBuilderFromServiceConfig returns a builder from the already built serviceConfig object
+// This is useful to create a variant of a serviceConfig without having to copy all values manually
+func NewServiceConfigBuilderFromServiceConfig(serviceConfig *kurtosis_core_rpc_api_bindings.ServiceConfig) *ServiceConfigBuilder {
+	return &ServiceConfigBuilder{
+		containerImageName:         serviceConfig.ContainerImageName,
+		privatePorts:               copyPortsMap(serviceConfig.PrivatePorts),
+		publicPorts:                copyPortsMap(serviceConfig.PublicPorts),
+		entrypointArgs:             copySlice(serviceConfig.EntrypointArgs),
+		cmdArgs:                    copySlice(serviceConfig.CmdArgs),
+		envVars:                    copyMap(serviceConfig.EnvVars),
+		filesArtifactMountDirpaths: copyMap(serviceConfig.FilesArtifactMountpoints),
+		cpuAllocationMillicpus:     serviceConfig.CpuAllocationMillicpus,
+		memoryAllocationMegabytes:  serviceConfig.MemoryAllocationMegabytes,
+		privateIPAddrPlaceholder:   serviceConfig.PrivateIpAddrPlaceholder,
+		subnetwork:                 *serviceConfig.Subnetwork,
+	}
+}
+
 func (builder *ServiceConfigBuilder) WithPrivatePorts(privatePorts map[string]*kurtosis_core_rpc_api_bindings.Port) *ServiceConfigBuilder {
-	builder.privatePorts = privatePorts
+	builder.privatePorts = copyPortsMap(privatePorts)
 	return builder
 }
 
 func (builder *ServiceConfigBuilder) WithPublicPorts(publicPorts map[string]*kurtosis_core_rpc_api_bindings.Port) *ServiceConfigBuilder {
-	builder.publicPorts = publicPorts
+	builder.publicPorts = copyPortsMap(publicPorts)
 	return builder
 }
 
 func (builder *ServiceConfigBuilder) WithEntryPointArgs(entryPointArgs []string) *ServiceConfigBuilder {
-	builder.entrypointArgs = entryPointArgs
+	builder.entrypointArgs = copySlice(entryPointArgs)
 	return builder
 }
 
 func (builder *ServiceConfigBuilder) WithCmdArgs(cmdArgs []string) *ServiceConfigBuilder {
-	builder.cmdArgs = cmdArgs
+	builder.cmdArgs = copySlice(cmdArgs)
 	return builder
 }
 
 func (builder *ServiceConfigBuilder) WithEnvVars(envVars map[string]string) *ServiceConfigBuilder {
-	builder.envVars = envVars
+	builder.envVars = copyMap(envVars)
 	return builder
 }
 
 func (builder *ServiceConfigBuilder) WithFilesArtifactMountDirpaths(filesArtifactMountDirpaths map[string]string) *ServiceConfigBuilder {
-	builder.filesArtifactMountDirpaths = filesArtifactMountDirpaths
+	builder.filesArtifactMountDirpaths = copyMap(filesArtifactMountDirpaths)
 	return builder
 }
 
@@ -115,6 +133,28 @@ func (builder *ServiceConfigBuilder) Build() *kurtosis_core_rpc_api_bindings.Ser
 	)
 }
 
+func copyPortsMap(ports map[string]*kurtosis_core_rpc_api_bindings.Port) map[string]*kurtosis_core_rpc_api_bindings.Port {
+	newPorts := make(map[string]*kurtosis_core_rpc_api_bindings.Port, len(ports))
+	for name, port := range ports {
+		newPorts[name] = binding_constructors.NewPort(port.Number, port.TransportProtocol, port.MaybeApplicationProtocol)
+	}
+	return newPorts
+}
+
+func copySlice(value []string) []string {
+	newSlice := make([]string, len(value))
+	copy(newSlice, value)
+	return newSlice
+}
+
+func copyMap(keyValue map[string]string) map[string]string {
+	newMap := make(map[string]string, len(keyValue))
+	for key, value := range keyValue {
+		newMap[key] = value
+	}
+	return newMap
+}
+
 func portToStarlark(port *kurtosis_core_rpc_api_bindings.Port) string {
 	starlarkFields := []string{}
 	starlarkFields = append(starlarkFields, fmt.Sprintf(`number=%d`, port.GetNumber()))

api/golang/core/lib/services/service_config_builder_test.go

@@ -0,0 +1,84 @@
+package services
+
+import (
+	"github.com/kurtosis-tech/kurtosis/api/golang/core/kurtosis_core_rpc_api_bindings"
+	"github.com/kurtosis-tech/kurtosis/api/golang/core/lib/binding_constructors"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestServiceConfigBuilderFrom_Invariant(t *testing.T) {
+	initialServiceConfig := NewServiceConfigBuilder(
+		"test-image",
+	).WithPrivatePorts(
+		map[string]*kurtosis_core_rpc_api_bindings.Port{
+			"grpc": binding_constructors.NewPort(1323, kurtosis_core_rpc_api_bindings.Port_TCP, "https"),
+		},
+	).WithPublicPorts(
+		map[string]*kurtosis_core_rpc_api_bindings.Port{
+			"grpc": binding_constructors.NewPort(1323, kurtosis_core_rpc_api_bindings.Port_TCP, "https"),
+		},
+	).WithEntryPointArgs(
+		[]string{"echo", "'Hello World!'"},
+	).WithCmdArgs(
+		[]string{"sleep", "999999"},
+	).WithEnvVars(
+		map[string]string{
+			"VAR_1": "VALUE",
+		},
+	).WithFilesArtifactMountDirpaths(
+		map[string]string{
+			"/path/to/file": "artifact",
+		},
+	).WithCpuAllocationMillicpus(
+		1000,
+	).WithMemoryAllocationMegabytes(
+		1024,
+	).WithPrivateIPAddressPlaceholder(
+		"<IP_ADDRESS>",
+	).WithSubnetwork(
+		"subnetwork_1",
+	).Build()
+
+	newServiceConfigBuilder := NewServiceConfigBuilderFromServiceConfig(initialServiceConfig)
+	newServiceConfig := newServiceConfigBuilder.Build()
+
+	require.Equal(t, initialServiceConfig, newServiceConfig)
+
+	// modify random values
+	newServiceConfig.ContainerImageName = "new-test-image"
+	newServiceConfig.PrivatePorts["grpc"].Number = 9876
+	newServiceConfig.PublicPorts["grpc"].MaybeApplicationProtocol = "ftp"
+	newServiceConfig.EntrypointArgs[0] = "new-echo"
+	newServiceConfig.CmdArgs[1] = "1234"
+	newServiceConfig.EnvVars["VAR_1"] = "NEW_VALUE"
+	newServiceConfig.FilesArtifactMountpoints["new/path/to/file"] = "new-artifact"
+	newServiceConfig.CpuAllocationMillicpus = 500
+	newServiceConfig.MemoryAllocationMegabytes = 512
+	newSubnetwork := "new-subnetwork"
+	newServiceConfig.Subnetwork = &newSubnetwork
+
+	// test that initial value has not changed
+	require.Equal(t, "test-image", initialServiceConfig.ContainerImageName)
+	require.Equal(t, binding_constructors.NewPort(1323, kurtosis_core_rpc_api_bindings.Port_TCP, "https"), initialServiceConfig.PrivatePorts["grpc"])
+	require.Equal(t, binding_constructors.NewPort(1323, kurtosis_core_rpc_api_bindings.Port_TCP, "https"), initialServiceConfig.PublicPorts["grpc"])
+	require.Equal(t, []string{"echo", "'Hello World!'"}, initialServiceConfig.EntrypointArgs)
+	require.Equal(t, []string{"sleep", "999999"}, initialServiceConfig.CmdArgs)
+	require.Equal(t, map[string]string{"VAR_1": "VALUE"}, initialServiceConfig.EnvVars)
+	require.Equal(t, map[string]string{"/path/to/file": "artifact"}, initialServiceConfig.FilesArtifactMountpoints)
+	require.Equal(t, uint64(1000), initialServiceConfig.CpuAllocationMillicpus)
+	require.Equal(t, uint64(1024), initialServiceConfig.MemoryAllocationMegabytes)
+	require.Equal(t, "subnetwork_1", *initialServiceConfig.Subnetwork)
+
+	// test that new value are as expected
+	require.Equal(t, "new-test-image", newServiceConfig.ContainerImageName)
+	require.Equal(t, binding_constructors.NewPort(9876, kurtosis_core_rpc_api_bindings.Port_TCP, "https"), newServiceConfig.PrivatePorts["grpc"])
+	require.Equal(t, binding_constructors.NewPort(1323, kurtosis_core_rpc_api_bindings.Port_TCP, "ftp"), newServiceConfig.PublicPorts["grpc"])
+	require.Equal(t, []string{"new-echo", "'Hello World!'"}, newServiceConfig.EntrypointArgs)
+	require.Equal(t, []string{"sleep", "1234"}, newServiceConfig.CmdArgs)
+	require.Equal(t, map[string]string{"VAR_1": "NEW_VALUE"}, newServiceConfig.EnvVars)
+	require.Equal(t, map[string]string{"/path/to/file": "artifact", "new/path/to/file": "new-artifact"}, newServiceConfig.FilesArtifactMountpoints)
+	require.Equal(t, uint64(500), newServiceConfig.CpuAllocationMillicpus)
+	require.Equal(t, uint64(512), newServiceConfig.MemoryAllocationMegabytes)
+	require.Equal(t, "new-subnetwork", *newServiceConfig.Subnetwork)
+}

core/server/api_container/server/startosis_engine/kurtosis_builtins.go

@@ -37,6 +37,7 @@ import (
 func KurtosisPlanInstructions(serviceNetwork service_network.ServiceNetwork, runtimeValueStore *runtime_value_store.RuntimeValueStore) []*kurtosis_plan_instruction.KurtosisPlanInstruction {
 	return []*kurtosis_plan_instruction.KurtosisPlanInstruction{
 		render_templates.NewRenderTemplatesInstruction(serviceNetwork),
+		add_service.NewAddServices(serviceNetwork, runtimeValueStore),
 	}
 }
 

core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_instruction.go

@@ -7,10 +7,8 @@ import (
 	"github.com/kurtosis-tech/kurtosis/api/golang/core/lib/binding_constructors"
 	kurtosis_backend_service "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network"
-	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network/partition_topology"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers"
-	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers/magic_string_helper"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/runtime_value_store"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors"
@@ -35,7 +33,7 @@ func GenerateAddServiceBuiltin(instructionsQueue *[]kurtosis_instruction.Kurtosi
 			return nil, interpretationError
 		}
 		*instructionsQueue = append(*instructionsQueue, addServiceInstruction)
-		returnValue, interpretationError := addServiceInstruction.makeAddServiceInterpretationReturnValue()
+		returnValue, interpretationError := makeAddServiceInterpretationReturnValue(addServiceInstruction.serviceName, addServiceInstruction.serviceConfig)
 		if interpretationError != nil {
 			return nil, interpretationError
 		}
@@ -89,111 +87,30 @@ func (instruction *AddServiceInstruction) GetCanonicalInstruction() *kurtosis_co
 }
 
 func (instruction *AddServiceInstruction) Execute(ctx context.Context) (*string, error) {
-	serviceNameStr, err := magic_string_helper.ReplaceRuntimeValueInString(string(instruction.serviceName), instruction.runtimeValueStore)
+	replacedServiceName, replacedServiceConfig, err := replaceMagicStrings(instruction.serviceNetwork, instruction.runtimeValueStore, instruction.serviceName, instruction.serviceConfig)
 	if err != nil {
-		return nil, stacktrace.Propagate(err, "Error occurred while replacing facts in service id for '%v'", instruction.serviceName)
-	}
-	instruction.serviceName = kurtosis_backend_service.ServiceName(serviceNameStr)
-	err = instruction.replaceMagicStrings()
-	if err != nil {
-		return nil, stacktrace.Propagate(err, "An error occurred replacing IP Address with actual values in add service instruction for service '%v'", instruction.serviceName)
+		return nil, stacktrace.Propagate(err, "An error occurred replacing a magic string in '%s' instruction arguments. Execution cannot proceed", AddServiceBuiltinName)
 	}
 
-	startedService, err := instruction.serviceNetwork.StartService(ctx, instruction.serviceName, instruction.serviceConfig)
+	startedService, err := instruction.serviceNetwork.StartService(ctx, replacedServiceName, replacedServiceConfig)
 	if err != nil {
-		return nil, stacktrace.Propagate(err, "Failed adding service '%s' to enclave with an unexpected error", instruction.serviceName)
+		return nil, stacktrace.Propagate(err, "Failed adding service '%s' to enclave with an unexpected error", replacedServiceName)
 	}
-	instructionResult := fmt.Sprintf("Service '%s' added with service UUID '%s'", instruction.serviceName, startedService.GetRegistration().GetUUID())
+	instructionResult := fmt.Sprintf("Service '%s' added with service UUID '%s'", replacedServiceName, startedService.GetRegistration().GetUUID())
 	return &instructionResult, nil
 }
 
 func (instruction *AddServiceInstruction) ValidateAndUpdateEnvironment(environment *startosis_validator.ValidatorEnvironment) error {
-	if partition_topology.ParsePartitionId(instruction.serviceConfig.Subnetwork) != partition_topology.DefaultPartitionId {
-		if !environment.IsNetworkPartitioningEnabled() {
-			return startosis_errors.NewValidationError("Service was about to be started inside subnetwork '%s' but the Kurtosis enclave was started with subnetwork capabilities disabled. Make sure to run the Starlark script with subnetwork enabled.", *instruction.serviceConfig.Subnetwork)
-		}
-	}
-	if environment.DoesServiceNameExist(instruction.serviceName) {
-		return startosis_errors.NewValidationError("There was an error validating '%v' as service name '%v' already exists", AddServiceBuiltinName, instruction.serviceName)
+	if err := validateSingleService(environment, instruction.serviceName, instruction.serviceConfig); err != nil {
+		return err
 	}
-	for _, artifactName := range instruction.serviceConfig.FilesArtifactMountpoints {
-		if !environment.DoesArtifactNameExist(artifactName) {
-			return startosis_errors.NewValidationError("There was an error validating '%v' as artifact name '%v' does not exist", AddServiceBuiltinName, artifactName)
-		}
-	}
-	environment.AddServiceName(instruction.serviceName)
-	environment.AppendRequiredContainerImage(instruction.serviceConfig.ContainerImageName)
 	return nil
 }
 
 func (instruction *AddServiceInstruction) String() string {
 	return shared_helpers.CanonicalizeInstruction(AddServiceBuiltinName, kurtosis_instruction.NoArgs, instruction.starlarkKwargs)
 }
 
-func (instruction *AddServiceInstruction) replaceMagicStrings() error {
-	serviceNameStr := string(instruction.serviceName)
-	entryPointArgs := instruction.serviceConfig.EntrypointArgs
-	for index, entryPointArg := range entryPointArgs {
-		entryPointArgWithIPAddressReplaced, err := magic_string_helper.ReplaceIPAddressInString(entryPointArg, instruction.serviceNetwork, serviceNameStr)
-		if err != nil {
-			return stacktrace.Propagate(err, "Error occurred while replacing IP address in entry point args for '%v'", entryPointArg)
-		}
-		entryPointArgWithIPAddressAndRuntimeValueReplaced, err := magic_string_helper.ReplaceRuntimeValueInString(entryPointArgWithIPAddressReplaced, instruction.runtimeValueStore)
-		if err != nil {
-			return stacktrace.Propagate(err, "Error occurred while replacing runtime value in entry point args for '%v'", entryPointArg)
-		}
-		entryPointArgs[index] = entryPointArgWithIPAddressAndRuntimeValueReplaced
-	}
-
-	cmdArgs := instruction.serviceConfig.CmdArgs
-	for index, cmdArg := range cmdArgs {
-		cmdArgWithIPAddressReplaced, err := magic_string_helper.ReplaceIPAddressInString(cmdArg, instruction.serviceNetwork, serviceNameStr)
-		if err != nil {
-			return stacktrace.Propagate(err, "Error occurred while replacing IP address in command args for '%v'", cmdArg)
-		}
-		cmdArgWithIPAddressAndRuntimeValueReplaced, err := magic_string_helper.ReplaceRuntimeValueInString(cmdArgWithIPAddressReplaced, instruction.runtimeValueStore)
-		if err != nil {
-			return stacktrace.Propagate(err, "Error occurred while replacing runtime value in command args for '%v'", cmdArg)
-		}
-		cmdArgs[index] = cmdArgWithIPAddressAndRuntimeValueReplaced
-	}
-
-	envVars := instruction.serviceConfig.EnvVars
-	for envVarName, envVarValue := range envVars {
-		envVarValueWithIPAddressReplaced, err := magic_string_helper.ReplaceIPAddressInString(envVarValue, instruction.serviceNetwork, serviceNameStr)
-		if err != nil {
-			return stacktrace.Propagate(err, "Error occurred while replacing IP address in env vars for '%v'", envVarValue)
-		}
-		envVarValueWithIPAddressAndRuntimeValueReplaced, err := magic_string_helper.ReplaceRuntimeValueInString(envVarValueWithIPAddressReplaced, instruction.runtimeValueStore)
-		if err != nil {
-			return stacktrace.Propagate(err, "Error occurred while replacing runtime value in command args for '%v'", envVars)
-		}
-		envVars[envVarName] = envVarValueWithIPAddressAndRuntimeValueReplaced
-	}
-
-	return nil
-}
-
-func (instruction *AddServiceInstruction) makeAddServiceInterpretationReturnValue() (*kurtosis_types.Service, *startosis_errors.InterpretationError) {
-	ports := instruction.serviceConfig.GetPrivatePorts()
-	portSpecsDict := starlark.NewDict(len(ports))
-	for portId, port := range ports {
-		number := port.GetNumber()
-		transportProtocol := port.GetTransportProtocol()
-		maybeApplicationProtocol := port.GetMaybeApplicationProtocol()
-
-		portSpec := kurtosis_types.NewPortSpec(number, transportProtocol, maybeApplicationProtocol)
-		if err := portSpecsDict.SetKey(starlark.String(portId), portSpec); err != nil {
-			return nil, startosis_errors.NewInterpretationError("An error occurred while creating a port spec for values "+
-				"(number: '%v', transport_protocol: '%v', application_protocol: '%v') the add instruction return value",
-				number, transportProtocol, maybeApplicationProtocol)
-		}
-	}
-	ipAddress := starlark.String(fmt.Sprintf(magic_string_helper.IpAddressReplacementPlaceholderFormat, instruction.serviceName))
-	returnValue := kurtosis_types.NewService(ipAddress, portSpecsDict)
-	return returnValue, nil
-}
-
 func (instruction *AddServiceInstruction) parseStartosisArgs(b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) *startosis_errors.InterpretationError {
 	var serviceNameArg starlark.String
 	var serviceConfigArg *kurtosis_types.ServiceConfig

core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_intruction_test.go

@@ -4,15 +4,12 @@ import (
 	"github.com/kurtosis-tech/kurtosis/api/golang/core/kurtosis_core_rpc_api_bindings"
 	"github.com/kurtosis-tech/kurtosis/api/golang/core/lib/binding_constructors"
 	"github.com/kurtosis-tech/kurtosis/api/golang/core/lib/services"
-	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
-	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_constants"
 	"github.com/stretchr/testify/require"
 	"go.starlark.net/starlark"
-	"net"
 	"testing"
 )
 
@@ -42,36 +39,6 @@ func TestAddServiceInstruction_GetCanonicalizedInstruction(t *testing.T) {
 	require.Equal(t, expectedOutput, addServiceInstruction.String())
 }
 
-func TestAddServiceInstruction_EntryPointArgsAreReplaced(t *testing.T) {
-	ipAddresses := map[service.ServiceName]net.IP{
-		"foo_service": net.ParseIP("172.17.3.13"),
-	}
-	serviceNetwork := service_network.NewMockServiceNetwork(ipAddresses)
-	addServiceInstruction := NewAddServiceInstruction(
-		serviceNetwork,
-		kurtosis_instruction.NewInstructionPosition(22, 26, "dummyFile"),
-		"example-datastore-server-2",
-		services.NewServiceConfigBuilder(
-			testContainerImageName,
-		).WithPrivatePorts(
-			map[string]*kurtosis_core_rpc_api_bindings.Port{
-				"grpc": {
-					Number:            1323,
-					TransportProtocol: kurtosis_core_rpc_api_bindings.Port_TCP,
-				},
-			},
-		).WithEntryPointArgs(
-			[]string{"-- {{kurtosis:foo_service.ip_address}}"},
-		).Build(),
-		starlark.StringDict{}, // Unused
-		nil,
-	)
-
-	err := addServiceInstruction.replaceMagicStrings()
-	require.Nil(t, err)
-	require.Equal(t, "-- 172.17.3.13", addServiceInstruction.serviceConfig.EntrypointArgs[0])
-}
-
 func TestAddServiceInstruction_SerializeAndParseAgain(t *testing.T) {
 	serviceConfigBuilder := services.NewServiceConfigBuilder(
 		testContainerImageName,