Skip to content

Commit

Permalink
feat: enable image build spec for tasks (#2374)
Browse files Browse the repository at this point in the history 
## Description
ex. 
```
def run(plan, args):
    result = plan.run_sh(
        run="echo ${SOME}",
        # build image containing openssl, gh cli, python, and bash, etc.
        image=ImageBuildSpec(
            build_context_dir="./",
            image_name="taskimg",
        ),
        env_vars={
            "SOME": "THING",
        }
    )
    plan.print(result)
```

## Is this change user facing?
YES

## References:
#2224
  • Loading branch information
@tedim52
tedim52 committed Apr 17, 2024
1 parent 4eaae9f commit 3d5e6ec
Show file tree
Hide file tree
Showing 6 changed files with 138 additions and 64 deletions.
4 changes: 2 additions & 2 deletions core/server/api_container/server/startosis_engine/kurtosis_builtins.go
View file
Expand Up @@ -81,8 +81,8 @@ func KurtosisPlanInstructions(
render_templates.NewRenderTemplatesInstruction(serviceNetwork, runtimeValueStore),
request.NewRequest(serviceNetwork, runtimeValueStore),
start_service.NewStartService(serviceNetwork),
tasks.NewRunPythonService(serviceNetwork, runtimeValueStore, nonBlockingMode),
tasks.NewRunShService(serviceNetwork, runtimeValueStore, nonBlockingMode),
tasks.NewRunPythonService(serviceNetwork, runtimeValueStore, nonBlockingMode, packageId, packageContentProvider, packageReplaceOptions),
tasks.NewRunShService(serviceNetwork, runtimeValueStore, nonBlockingMode, packageId, packageContentProvider, packageReplaceOptions),
stop_service.NewStopService(serviceNetwork),
store_service_files.NewStoreServiceFiles(serviceNetwork),
upload_files.NewUploadFiles(packageId, serviceNetwork, packageContentProvider, packageReplaceOptions),
Expand Down
3 changes: 2 additions & 1 deletion ...ver/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go
View file
Expand Up @@ -326,7 +326,8 @@ func validateAndConvertConfigAndReadyCondition(
locatorOfModuleInWhichThisBuiltInIsBeingCalled,
packageId,
packageContentProvider,
packageReplaceOptions, imageDownloadMode)
packageReplaceOptions,
imageDownloadMode)
if interpretationErr != nil {
return nil, nil, interpretationErr
}
Expand Down
83 changes: 56 additions & 27 deletions core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go
View file
Expand Up @@ -4,6 +4,9 @@ import (
"context"
"fmt"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/exec_result"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/store_spec"
Expand All @@ -19,6 +22,7 @@ import (
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/plan_yaml"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/runtime_value_store"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_validator"
"github.com/kurtosis-tech/stacktrace"
"github.com/xtgo/uuid"
Expand All @@ -44,7 +48,13 @@ const (
runPythonDefaultDescription = "Running Python script"
)

func NewRunPythonService(serviceNetwork service_network.ServiceNetwork, runtimeValueStore *runtime_value_store.RuntimeValueStore, nonBlockingMode bool) *kurtosis_plan_instruction.KurtosisPlanInstruction {
func NewRunPythonService(
serviceNetwork service_network.ServiceNetwork,
runtimeValueStore *runtime_value_store.RuntimeValueStore,
nonBlockingMode bool,
packageId string,
packageContentProvider startosis_packages.PackageContentProvider,
packageReplaceOptions map[string]string) *kurtosis_plan_instruction.KurtosisPlanInstruction {
return &kurtosis_plan_instruction.KurtosisPlanInstruction{
KurtosisBaseBuiltin: &kurtosis_starlark_framework.KurtosisBaseBuiltin{
Name: RunPythonBuiltinName,
Expand All @@ -69,10 +79,8 @@ func NewRunPythonService(serviceNetwork service_network.ServiceNetwork, runtimeV
{
Name: ImageNameArgName,
IsOptional: true,
ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String],
Validator: func(argumentValue starlark.Value) *startosis_errors.InterpretationError {
return builtin_argument.NonEmptyString(argumentValue, ImageNameArgName)
},
ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.Value],
Validator: nil,
},
{
Name: FilesArgName,
Expand All @@ -99,19 +107,22 @@ func NewRunPythonService(serviceNetwork service_network.ServiceNetwork, runtimeV

Capabilities: func() kurtosis_plan_instruction.KurtosisPlanInstructionCapabilities {
return &RunPythonCapabilities{
serviceNetwork: serviceNetwork,
runtimeValueStore: runtimeValueStore,
pythonArguments: nil,
packages: nil,
name: "",
nonBlockingMode: nonBlockingMode,
serviceConfig: nil, // populated at interpretation time
run: "", // populated at interpretation time
resultUuid: "", // populated at interpretation time
storeSpecList: nil,
wait: DefaultWaitTimeoutDurationStr,
description: "", // populated at interpretation time
returnValue: nil, // populated at interpretation time
serviceNetwork: serviceNetwork,
runtimeValueStore: runtimeValueStore,
pythonArguments: nil,
packages: nil,
name: "",
nonBlockingMode: nonBlockingMode,
packageId: packageId,
packageContentProvider: packageContentProvider,
packageReplaceOptions: packageReplaceOptions,
serviceConfig: nil, // populated at interpretation time
run: "", // populated at interpretation time
resultUuid: "", // populated at interpretation time
storeSpecList: nil,
wait: DefaultWaitTimeoutDurationStr,
description: "", // populated at interpretation time
returnValue: nil, // populated at interpretation time
}
},

Expand Down Expand Up @@ -139,14 +150,19 @@ type RunPythonCapabilities struct {
pythonArguments []string
packages []string

// fields required for image building
packageId string
packageContentProvider startosis_packages.PackageContentProvider
packageReplaceOptions map[string]string

returnValue *starlarkstruct.Struct
serviceConfig *service.ServiceConfig
storeSpecList []*store_spec.StoreSpec
wait string
description string
}

func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RunPythonCapabilities) Interpret(locatorOfModuleInWhichThisBuiltinIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
randomUuid := uuid.NewRandom()
builtin.name = fmt.Sprintf("task-%v", randomUuid.String())

Expand Down Expand Up @@ -180,15 +196,28 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg
builtin.packages = packagesList
}

var image string
var maybeImageName string
var maybeImageBuildSpec *image_build_spec.ImageBuildSpec
var maybeImageRegistrySpec *image_registry_spec.ImageRegistrySpec
var maybeNixBuildSpec *nix_build_spec.NixBuildSpec
var interpretationErr *startosis_errors.InterpretationError
if arguments.IsSet(ImageNameArgName) {
imageStarlark, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ImageNameArgName)
rawImageVal, err := builtin_argument.ExtractArgumentValue[starlark.Value](arguments, ImageNameArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ImageNameArgName)
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract raw image attribute.")
}

maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, interpretationErr = service_config.ConvertImage(
rawImageVal,
locatorOfModuleInWhichThisBuiltinIsBeingCalled,
builtin.packageId,
builtin.packageContentProvider,
builtin.packageReplaceOptions)
if interpretationErr != nil {
return nil, startosis_errors.WrapWithInterpretationError(interpretationErr, "An error occurred converting image for run sh.")
}
image = imageStarlark.GoString()
} else {
image = defaultRunPythonImageName
maybeImageName = defaultRunPythonImageName
}

var filesArtifactExpansion *service_directory.FilesArtifactsExpansion
Expand Down Expand Up @@ -219,9 +248,9 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg
}

// build a service config from image and files artifacts expansion.
builtin.serviceConfig, err = getServiceConfig(image, filesArtifactExpansion, envVars)
builtin.serviceConfig, err = getServiceConfig(maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, filesArtifactExpansion, envVars)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config using image '%s'", image)
return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config for run python.")
}

if arguments.IsSet(StoreFilesArgName) {
Expand Down Expand Up @@ -258,7 +287,7 @@ func (builtin *RunPythonCapabilities) Validate(_ *builtin_argument.ArgumentValue
if builtin.serviceConfig.GetFilesArtifactsExpansion() != nil {
serviceDirpathsToArtifactIdentifiers = builtin.serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers
}
return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig.GetContainerImageName())
return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig)
}

// Execute This is just v0 for run_python task - we can later improve on it.
Expand Down
79 changes: 54 additions & 25 deletions core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go
View file
Expand Up @@ -3,6 +3,9 @@ package tasks
import (
"context"
"fmt"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/store_spec"
Expand All @@ -18,6 +21,7 @@ import (
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/plan_yaml"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/runtime_value_store"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages"
"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_validator"
"github.com/kurtosis-tech/stacktrace"
"github.com/xtgo/uuid"
Expand All @@ -33,7 +37,13 @@ const (
runningShScriptPrefix = "Running sh script"
)

func NewRunShService(serviceNetwork service_network.ServiceNetwork, runtimeValueStore *runtime_value_store.RuntimeValueStore, nonBlockingMode bool) *kurtosis_plan_instruction.KurtosisPlanInstruction {
func NewRunShService(
serviceNetwork service_network.ServiceNetwork,
runtimeValueStore *runtime_value_store.RuntimeValueStore,
nonBlockingMode bool,
packageId string,
packageContentProvider startosis_packages.PackageContentProvider,
packageReplaceOptions map[string]string) *kurtosis_plan_instruction.KurtosisPlanInstruction {
return &kurtosis_plan_instruction.KurtosisPlanInstruction{
KurtosisBaseBuiltin: &kurtosis_starlark_framework.KurtosisBaseBuiltin{
Name: RunShBuiltinName,
Expand All @@ -47,10 +57,8 @@ func NewRunShService(serviceNetwork service_network.ServiceNetwork, runtimeValue
{
Name: ImageNameArgName,
IsOptional: true,
ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String],
Validator: func(argumentValue starlark.Value) *startosis_errors.InterpretationError {
return builtin_argument.NonEmptyString(argumentValue, ImageNameArgName)
},
ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.Value],
Validator: nil,
},
{
Name: FilesArgName,
Expand Down Expand Up @@ -83,17 +91,20 @@ func NewRunShService(serviceNetwork service_network.ServiceNetwork, runtimeValue

Capabilities: func() kurtosis_plan_instruction.KurtosisPlanInstructionCapabilities {
return &RunShCapabilities{
serviceNetwork: serviceNetwork,
runtimeValueStore: runtimeValueStore,
name: "",
nonBlockingMode: nonBlockingMode,
serviceConfig: nil, // populated at interpretation time
run: "", // populated at interpretation time
resultUuid: "", // populated at interpretation time
storeSpecList: nil,
wait: DefaultWaitTimeoutDurationStr,
description: "", // populated at interpretation time
returnValue: nil, // populated at interpretation time
serviceNetwork: serviceNetwork,
runtimeValueStore: runtimeValueStore,
packageId: packageId,
packageReplaceOptions: packageReplaceOptions,
packageContentProvider: packageContentProvider,
name: "",
nonBlockingMode: nonBlockingMode,
serviceConfig: nil, // populated at interpretation time
run: "", // populated at interpretation time
resultUuid: "", // populated at interpretation time
storeSpecList: nil,
wait: DefaultWaitTimeoutDurationStr,
description: "", // populated at interpretation time
returnValue: nil, // populated at interpretation time
}
},

Expand All @@ -117,29 +128,47 @@ type RunShCapabilities struct {
run string
nonBlockingMode bool

// fields required for image building
packageId string
packageContentProvider startosis_packages.PackageContentProvider
packageReplaceOptions map[string]string

serviceConfig *service.ServiceConfig
storeSpecList []*store_spec.StoreSpec
returnValue *starlarkstruct.Struct
wait string
description string
}

func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RunShCapabilities) Interpret(locatorOfModuleInWhichThisBuiltinIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
runCommand, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, RunArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", RunArgName)
}
builtin.run = runCommand.GoString()

var image string
var maybeImageName string
var maybeImageBuildSpec *image_build_spec.ImageBuildSpec
var maybeImageRegistrySpec *image_registry_spec.ImageRegistrySpec
var maybeNixBuildSpec *nix_build_spec.NixBuildSpec
var interpretationErr *startosis_errors.InterpretationError
if arguments.IsSet(ImageNameArgName) {
imageStarlark, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ImageNameArgName)
rawImageVal, err := builtin_argument.ExtractArgumentValue[starlark.Value](arguments, ImageNameArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ImageNameArgName)
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract raw image attribute.")
}

maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, interpretationErr = service_config.ConvertImage(
rawImageVal,
locatorOfModuleInWhichThisBuiltinIsBeingCalled,
builtin.packageId,
builtin.packageContentProvider,
builtin.packageReplaceOptions)
if interpretationErr != nil {
return nil, startosis_errors.WrapWithInterpretationError(interpretationErr, "An error occurred converting image for run sh.")
}
image = imageStarlark.GoString()
} else {
image = defaultRunShImageName
maybeImageName = defaultRunShImageName
}

var filesArtifactExpansion *service_directory.FilesArtifactsExpansion
Expand Down Expand Up @@ -170,9 +199,9 @@ func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argumen
}

// build a service config from image and files artifacts expansion.
builtin.serviceConfig, err = getServiceConfig(image, filesArtifactExpansion, envVars)
builtin.serviceConfig, err = getServiceConfig(maybeImageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, filesArtifactExpansion, envVars)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config using image '%s'", image)
return nil, startosis_errors.WrapWithInterpretationError(err, "An error occurred creating service config using for run sh task.")
}

if arguments.IsSet(StoreFilesArgName) {
Expand Down Expand Up @@ -215,7 +244,7 @@ func (builtin *RunShCapabilities) Validate(_ *builtin_argument.ArgumentValuesSet
if builtin.serviceConfig.GetFilesArtifactsExpansion() != nil {
serviceDirpathsToArtifactIdentifiers = builtin.serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers
}
return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig.GetContainerImageName())
return validateTasksCommon(validatorEnvironment, builtin.storeSpecList, serviceDirpathsToArtifactIdentifiers, builtin.serviceConfig)
}

// Execute This is just v0 for run_sh task - we can later improve on it.
Expand Down

0 comments on commit 3d5e6ec

Please sign in to comment.