Skip to content

Commit

Permalink
feat: upload files support relative locators (#930)
Browse files Browse the repository at this point in the history 
## Description:
had missed upload when i fixed read, import

## Is this change user facing?
YES
  • Loading branch information
@h4ck3rk3y
h4ck3rk3y committed Jul 18, 2023
1 parent 9386120 commit 8d60968
Show file tree
Hide file tree
Showing 26 changed files with 39 additions and 35 deletions.
2 changes: 1 addition & 1 deletion ...ver/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go
View file
Expand Up @@ -85,7 +85,7 @@ type AddServiceCapabilities struct {
resultUuid string
}

func (builtin *AddServiceCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *AddServiceCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
serviceName, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ServiceNameArgName)
Expand Down
2 changes: 1 addition & 1 deletion ...er/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_services.go
View file
Expand Up @@ -80,7 +80,7 @@ type AddServicesCapabilities struct {
resultUuids map[service.ServiceName]string
}

func (builtin *AddServicesCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *AddServicesCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
ServiceConfigsDict, err := builtin_argument.ExtractArgumentValue[*starlark.Dict](arguments, ConfigsArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ConfigsArgName)
Expand Down
2 changes: 1 addition & 1 deletion core/server/api_container/server/startosis_engine/kurtosis_instruction/assert/assert.go
View file
Expand Up @@ -92,7 +92,7 @@ type AssertCapabilities struct {
target starlark.Comparable
}

func (builtin *AssertCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *AssertCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
runtimeValue, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, RuntimeValueArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", RuntimeValueArgName)
Expand Down
2 changes: 1 addition & 1 deletion core/server/api_container/server/startosis_engine/kurtosis_instruction/exec/exec.go
View file
Expand Up @@ -98,7 +98,7 @@ type ExecCapabilities struct {
skipCodeCheck bool
}

func (builtin *ExecCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *ExecCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {

serviceNameArgumentValue, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion ...i_container/server/startosis_engine/kurtosis_instruction/kurtosis_print/kurtosis_print.go
View file
Expand Up @@ -57,7 +57,7 @@ type PrintCapabilities struct {
msg starlark.Value
}

func (builtin *PrintCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *PrintCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
msg, err := builtin_argument.ExtractArgumentValue[starlark.Value](arguments, PrintArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", PrintArgName)
Expand Down
2 changes: 1 addition & 1 deletion ...ainer/server/startosis_engine/kurtosis_instruction/remove_connection/remove_connection.go
View file
Expand Up @@ -59,7 +59,7 @@ type RemoveConnectionCapabilities struct {
subnetwork2 service_network_types.PartitionID
}

func (builtin *RemoveConnectionCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RemoveConnectionCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
subnetworks, err := builtin_argument.ExtractArgumentValue[starlark.Tuple](arguments, SubnetworksArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", SubnetworksArgName)
Expand Down
2 changes: 1 addition & 1 deletion ...i_container/server/startosis_engine/kurtosis_instruction/remove_service/remove_service.go
View file
Expand Up @@ -58,7 +58,7 @@ type RemoveServiceCapabilities struct {
serviceName service.ServiceName
}

func (builtin *RemoveServiceCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RemoveServiceCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
serviceName, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ServiceNameArgName)
Expand Down
2 changes: 1 addition & 1 deletion ...ntainer/server/startosis_engine/kurtosis_instruction/render_templates/render_templates.go
View file
Expand Up @@ -79,7 +79,7 @@ type RenderTemplatesCapabilities struct {
runtimeValueStore *runtime_value_store.RuntimeValueStore
}

func (builtin *RenderTemplatesCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RenderTemplatesCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
if !arguments.IsSet(ArtifactNameArgName) {
natureThemeName, err := builtin.serviceNetwork.GetUniqueNameForFileArtifact()
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion core/server/api_container/server/startosis_engine/kurtosis_instruction/request/request.go
View file
Expand Up @@ -108,7 +108,7 @@ type RequestCapabilities struct {
skipCodeCheck bool
}

func (builtin *RequestCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RequestCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {

serviceNameArgumentValue, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion ...i_container/server/startosis_engine/kurtosis_instruction/set_connection/set_connection.go
View file
Expand Up @@ -81,7 +81,7 @@ type SetConnectionCapabilities struct {
connectionConfig *partition_topology.PartitionConnection
}

func (builtin *SetConnectionCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *SetConnectionCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
connectionConfigStarlark, err := builtin_argument.ExtractArgumentValue[*connection_config.ConnectionConfig](arguments, ConnectionConfigArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ConnectionConfigArgName)
Expand Down
2 changes: 1 addition & 1 deletion ...api_container/server/startosis_engine/kurtosis_instruction/start_service/start_service.go
View file
Expand Up @@ -58,7 +58,7 @@ type StartServiceCapabilities struct {
serviceName service.ServiceName
}

func (builtin *StartServiceCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *StartServiceCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
serviceName, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ServiceNameArgName)
Expand Down
2 changes: 1 addition & 1 deletion ...r/api_container/server/startosis_engine/kurtosis_instruction/stop_service/stop_service.go
View file
Expand Up @@ -58,7 +58,7 @@ type StopServiceCapabilities struct {
serviceName service.ServiceName
}

func (builtin *StopServiceCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *StopServiceCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
serviceName, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ServiceNameArgName)
Expand Down
2 changes: 1 addition & 1 deletion ...r/server/startosis_engine/kurtosis_instruction/store_service_files/store_service_files.go
View file
Expand Up @@ -75,7 +75,7 @@ type StoreServiceFilesCapabilities struct {
artifactName string
}

func (builtin *StoreServiceFilesCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *StoreServiceFilesCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
if !arguments.IsSet(ArtifactNameArgName) {
natureThemeName, err := builtin.serviceNetwork.GetUniqueNameForFileArtifact()
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go
View file
Expand Up @@ -146,7 +146,7 @@ type RunPythonCapabilities struct {
wait string
}

func (builtin *RunPythonCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
pythonScript, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, RunArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", RunArgName)
Expand Down
2 changes: 1 addition & 1 deletion core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go
View file
Expand Up @@ -107,7 +107,7 @@ type RunShCapabilities struct {
wait string
}

func (builtin *RunShCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
runCommand, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, RunArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", RunArgName)
Expand Down
2 changes: 1 addition & 1 deletion ...i_container/server/startosis_engine/kurtosis_instruction/update_service/update_service.go
View file
Expand Up @@ -78,7 +78,7 @@ type UpdateServiceCapabilities struct {
updateServiceConfig *kurtosis_core_rpc_api_bindings.UpdateServiceConfig
}

func (builtin *UpdateServiceCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *UpdateServiceCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
serviceName, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ServiceNameArgName)
Expand Down
9 changes: 7 additions & 2 deletions ...r/api_container/server/startosis_engine/kurtosis_instruction/upload_files/upload_files.go
View file
Expand Up @@ -73,7 +73,7 @@ type UploadFilesCapabilities struct {
pathOnDisk string
}

func (builtin *UploadFilesCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *UploadFilesCapabilities) Interpret(locatorOfModuleInWhichThisBuiltInIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
if !arguments.IsSet(ArtifactNameArgName) {
natureThemeName, err := builtin.serviceNetwork.GetUniqueNameForFileArtifact()
if err != nil {
Expand All @@ -93,7 +93,12 @@ func (builtin *UploadFilesCapabilities) Interpret(arguments *builtin_argument.Ar
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", SrcArgName)
}

pathOnDisk, interpretationErr := builtin.packageContentProvider.GetOnDiskAbsoluteFilePath(src.GoString())
absoluteLocator, interpretationErr := builtin.packageContentProvider.GetAbsoluteLocatorForRelativeModuleLocator(locatorOfModuleInWhichThisBuiltInIsBeingCalled, src.GoString())
if interpretationErr != nil {
return nil, startosis_errors.WrapWithInterpretationError(interpretationErr, "Tried to convert locator '%v' into absolute locator but failed", src.GoString())
}

pathOnDisk, interpretationErr := builtin.packageContentProvider.GetOnDiskAbsoluteFilePath(absoluteLocator)
if interpretationErr != nil {
return nil, interpretationErr
}
Expand Down
2 changes: 1 addition & 1 deletion core/server/api_container/server/startosis_engine/kurtosis_instruction/wait/wait.go
View file
Expand Up @@ -129,7 +129,7 @@ type WaitCapabilities struct {
resultUuid string
}

func (builtin *WaitCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *WaitCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {

serviceNameArgumentValue, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
Expand Down
3 changes: 2 additions & 1 deletion ...engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction.go
View file
Expand Up @@ -44,7 +44,8 @@ func (builtin *KurtosisPlanInstructionWrapper) CreateBuiltin() func(thread *star
}

instructionWrapper := newKurtosisPlanInstructionInternal(wrappedBuiltin, builtin.Capabilities(), builtin.DefaultDisplayArguments)
returnedFutureValue, interpretationErr := instructionWrapper.interpret()
locatorOfModuleInWhichInstructionIsBeingInterpreted := thread.Name
returnedFutureValue, interpretationErr := instructionWrapper.interpret(locatorOfModuleInWhichInstructionIsBeingInterpreted)
if interpretationErr != nil {
return nil, interpretationErr
}
Expand Down
2 changes: 1 addition & 1 deletion ...is_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_capabilities.go
View file
Expand Up @@ -9,7 +9,7 @@ import (
)

type KurtosisPlanInstructionCapabilities interface {
Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError)
Interpret(locatorOfModuleInWhichThisBuiltInIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError)

Validate(arguments *builtin_argument.ArgumentValuesSet, validatorEnvironment *startosis_validator.ValidatorEnvironment) *startosis_errors.ValidationError

Expand Down
4 changes: 2 additions & 2 deletions ...rtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_internal.go
View file
Expand Up @@ -75,8 +75,8 @@ func (builtin *kurtosisPlanInstructionInternal) Execute(ctx context.Context) (*s
return &result, nil
}

func (builtin *kurtosisPlanInstructionInternal) interpret() (starlark.Value, *startosis_errors.InterpretationError) {
result, interpretationErr := builtin.capabilities.Interpret(builtin.GetArguments())
func (builtin *kurtosisPlanInstructionInternal) interpret(locatorOfModuleInWhichThisBuiltInIsBeingCalled string) (starlark.Value, *startosis_errors.InterpretationError) {
result, interpretationErr := builtin.capabilities.Interpret(locatorOfModuleInWhichThisBuiltInIsBeingCalled, builtin.GetArguments())
if interpretationErr != nil {
return nil, interpretationErr
}
Expand Down
4 changes: 2 additions & 2 deletions ...r/startosis_engine/kurtosis_starlark_framework/test_engine/upload_files_framework_test.go
View file
Expand Up @@ -29,7 +29,7 @@ func (t *uploadFilesTestCase) GetId() string {
func (t *uploadFilesTestCase) GetInstruction() *kurtosis_plan_instruction.KurtosisPlanInstruction {
serviceNetwork := service_network.NewMockServiceNetwork(t)
packageContentProvider := mock_package_content_provider.NewMockPackageContentProvider()
require.Nil(t, packageContentProvider.AddFileContent(TestSrcPath, "Hello World!"))
require.Nil(t, packageContentProvider.AddFileContent(TestModuleFileName, "Hello World!"))

serviceNetwork.EXPECT().UploadFilesArtifact(
mock.Anything, // data gets written to disk and compressed to it's a bit tricky to replicate here.
Expand All @@ -43,7 +43,7 @@ func (t *uploadFilesTestCase) GetInstruction() *kurtosis_plan_instruction.Kurtos
}

func (t uploadFilesTestCase) GetStarlarkCode() string {
return fmt.Sprintf("%s(%s=%q, %s=%q)", upload_files.UploadFilesBuiltinName, upload_files.SrcArgName, TestSrcPath, upload_files.ArtifactNameArgName, TestArtifactName)
return fmt.Sprintf("%s(%s=%q, %s=%q)", upload_files.UploadFilesBuiltinName, upload_files.SrcArgName, TestModuleFileName, upload_files.ArtifactNameArgName, TestArtifactName)
}

func (t *uploadFilesTestCase) GetStarlarkCodeForAssertion() string {
Expand Down
4 changes: 2 additions & 2 deletions ...ngine/kurtosis_starlark_framework/test_engine/upload_files_without_name_framework_test.go
View file
Expand Up @@ -29,7 +29,7 @@ func (t *uploadFilesWithoutNameTestCase) GetId() string {
func (t *uploadFilesWithoutNameTestCase) GetInstruction() *kurtosis_plan_instruction.KurtosisPlanInstruction {
serviceNetwork := service_network.NewMockServiceNetwork(t)
packageContentProvider := mock_package_content_provider.NewMockPackageContentProvider()
require.Nil(t, packageContentProvider.AddFileContent(TestSrcPath, "Hello World!"))
require.Nil(t, packageContentProvider.AddFileContent(TestModuleFileName, "Hello World!"))

serviceNetwork.EXPECT().GetUniqueNameForFileArtifact().Times(1).Return(
mockedFileArtifactName,
Expand All @@ -48,7 +48,7 @@ func (t *uploadFilesWithoutNameTestCase) GetInstruction() *kurtosis_plan_instruc
}

func (t uploadFilesWithoutNameTestCase) GetStarlarkCode() string {
return fmt.Sprintf("%s(%s=%q)", upload_files.UploadFilesBuiltinName, upload_files.SrcArgName, TestSrcPath)
return fmt.Sprintf("%s(%s=%q)", upload_files.UploadFilesBuiltinName, upload_files.SrcArgName, TestModuleFileName)
}

func (t *uploadFilesWithoutNameTestCase) GetStarlarkCodeForAssertion() string {
Expand Down
9 changes: 3 additions & 6 deletions core/server/api_container/server/startosis_engine/startosis_interpreter.go
View file
Expand Up @@ -28,18 +28,15 @@ import (
)

const (
starlarkGoThreadName = "Startosis interpreter thread"

multipleInterpretationErrorMsg = "Multiple errors caught interpreting the Starlark script. Listing each of them below."
evaluationErrorPrefix = "Evaluation error: "

skipImportInstructionInStacktraceValue = "import_module"

runFunctionName = "run"

paramsRequiredForArgs = 2
minimumParamsRequiredForPlan = 1
maximumParamsAllowedForRunFunction = 2
paramsRequiredForArgs = 2
minimumParamsRequiredForPlan = 1

planParamIndex = 0
planParamName = "plan"
Expand Down Expand Up @@ -268,7 +265,7 @@ func (interpreter *StartosisInterpreter) Interpret(
return startosis_constants.NoOutputObject, nil, missingMainFunctionError(packageId, mainFunctionName)
}

runFunctionExecutionThread := newStarlarkThread(starlarkGoThreadName)
runFunctionExecutionThread := newStarlarkThread(moduleLocator)

var argsTuple starlark.Tuple
var kwArgs []starlark.Tuple
Expand Down
4 changes: 2 additions & 2 deletions ...tsuites/golang/testsuite/startosis_package_test/startosis_package_relative_import_test.go
View file
Expand Up @@ -17,6 +17,6 @@ func (suite *StartosisPackageTestSuite) TestStartosisPackage_RelativeImports() {
require.Nil(t, runResult.InterpretationError)
require.Empty(t, runResult.ValidationErrors)
require.Nil(t, runResult.ExecutionError)
expectedResult := "John Doe\nOpen Sesame\n"
require.Equal(t, expectedResult, string(runResult.RunOutput))
expectedResult := "Files with artifact name 'upload' uploaded with artifact UUID '[a-f0-9]{32}'\nJohn Doe\nOpen Sesame\n"
require.Regexp(t, expectedResult, string(runResult.RunOutput))
}
1 change: 1 addition & 0 deletions internal_testsuites/starlark/valid-package-with-relative-imports/main.star
View file
Expand Up @@ -2,5 +2,6 @@ lib = import_module("./src/lib.star")
password = read_file("./static_files/password.txt")

def run(plan):
plan.upload_files("./static_files/password.txt", "upload")
plan.print(lib.NAME)
plan.print(password)

0 comments on commit 8d60968

Please sign in to comment.