Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: upload files support relative locators #930

Merged
merged 9 commits into from Jul 18, 2023
Expand Up @@ -85,7 +85,7 @@ type AddServiceCapabilities struct {
resultUuid string
}

func (builtin *AddServiceCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *AddServiceCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
serviceName, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ServiceNameArgName)
Expand Down
Expand Up @@ -80,7 +80,7 @@ type AddServicesCapabilities struct {
resultUuids map[service.ServiceName]string
}

func (builtin *AddServicesCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *AddServicesCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
ServiceConfigsDict, err := builtin_argument.ExtractArgumentValue[*starlark.Dict](arguments, ConfigsArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ConfigsArgName)
Expand Down
Expand Up @@ -92,7 +92,7 @@ type AssertCapabilities struct {
target starlark.Comparable
}

func (builtin *AssertCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *AssertCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
runtimeValue, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, RuntimeValueArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", RuntimeValueArgName)
Expand Down
Expand Up @@ -98,7 +98,7 @@ type ExecCapabilities struct {
skipCodeCheck bool
}

func (builtin *ExecCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *ExecCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {

serviceNameArgumentValue, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
Expand Down
Expand Up @@ -57,7 +57,7 @@ type PrintCapabilities struct {
msg starlark.Value
}

func (builtin *PrintCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *PrintCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
msg, err := builtin_argument.ExtractArgumentValue[starlark.Value](arguments, PrintArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", PrintArgName)
Expand Down
Expand Up @@ -59,7 +59,7 @@ type RemoveConnectionCapabilities struct {
subnetwork2 service_network_types.PartitionID
}

func (builtin *RemoveConnectionCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RemoveConnectionCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
subnetworks, err := builtin_argument.ExtractArgumentValue[starlark.Tuple](arguments, SubnetworksArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", SubnetworksArgName)
Expand Down
Expand Up @@ -58,7 +58,7 @@ type RemoveServiceCapabilities struct {
serviceName service.ServiceName
}

func (builtin *RemoveServiceCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RemoveServiceCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
serviceName, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ServiceNameArgName)
Expand Down
Expand Up @@ -79,7 +79,7 @@ type RenderTemplatesCapabilities struct {
runtimeValueStore *runtime_value_store.RuntimeValueStore
}

func (builtin *RenderTemplatesCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RenderTemplatesCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
if !arguments.IsSet(ArtifactNameArgName) {
natureThemeName, err := builtin.serviceNetwork.GetUniqueNameForFileArtifact()
if err != nil {
Expand Down
Expand Up @@ -108,7 +108,7 @@ type RequestCapabilities struct {
skipCodeCheck bool
}

func (builtin *RequestCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RequestCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {

serviceNameArgumentValue, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
Expand Down
Expand Up @@ -81,7 +81,7 @@ type SetConnectionCapabilities struct {
connectionConfig *partition_topology.PartitionConnection
}

func (builtin *SetConnectionCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *SetConnectionCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
connectionConfigStarlark, err := builtin_argument.ExtractArgumentValue[*connection_config.ConnectionConfig](arguments, ConnectionConfigArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ConnectionConfigArgName)
Expand Down
Expand Up @@ -58,7 +58,7 @@ type StartServiceCapabilities struct {
serviceName service.ServiceName
}

func (builtin *StartServiceCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *StartServiceCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
serviceName, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ServiceNameArgName)
Expand Down
Expand Up @@ -58,7 +58,7 @@ type StopServiceCapabilities struct {
serviceName service.ServiceName
}

func (builtin *StopServiceCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *StopServiceCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
serviceName, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ServiceNameArgName)
Expand Down
Expand Up @@ -75,7 +75,7 @@ type StoreServiceFilesCapabilities struct {
artifactName string
}

func (builtin *StoreServiceFilesCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *StoreServiceFilesCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
if !arguments.IsSet(ArtifactNameArgName) {
natureThemeName, err := builtin.serviceNetwork.GetUniqueNameForFileArtifact()
if err != nil {
Expand Down
Expand Up @@ -146,7 +146,7 @@ type RunPythonCapabilities struct {
wait string
}

func (builtin *RunPythonCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
pythonScript, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, RunArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", RunArgName)
Expand Down
Expand Up @@ -107,7 +107,7 @@ type RunShCapabilities struct {
wait string
}

func (builtin *RunShCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
runCommand, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, RunArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", RunArgName)
Expand Down
Expand Up @@ -78,7 +78,7 @@ type UpdateServiceCapabilities struct {
updateServiceConfig *kurtosis_core_rpc_api_bindings.UpdateServiceConfig
}

func (builtin *UpdateServiceCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *UpdateServiceCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
serviceName, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", ServiceNameArgName)
Expand Down
Expand Up @@ -73,7 +73,7 @@ type UploadFilesCapabilities struct {
pathOnDisk string
}

func (builtin *UploadFilesCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *UploadFilesCapabilities) Interpret(locatorOfModuleInWhichThisBuiltInIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
if !arguments.IsSet(ArtifactNameArgName) {
natureThemeName, err := builtin.serviceNetwork.GetUniqueNameForFileArtifact()
if err != nil {
Expand All @@ -93,7 +93,12 @@ func (builtin *UploadFilesCapabilities) Interpret(arguments *builtin_argument.Ar
return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", SrcArgName)
}

pathOnDisk, interpretationErr := builtin.packageContentProvider.GetOnDiskAbsoluteFilePath(src.GoString())
absoluteLocator, interpretationErr := builtin.packageContentProvider.GetAbsoluteLocatorForRelativeModuleLocator(locatorOfModuleInWhichThisBuiltInIsBeingCalled, src.GoString())
if interpretationErr != nil {
return nil, startosis_errors.WrapWithInterpretationError(interpretationErr, "Tried to convert locator '%v' into absolute locator but failed", src.GoString())
}

pathOnDisk, interpretationErr := builtin.packageContentProvider.GetOnDiskAbsoluteFilePath(absoluteLocator)
if interpretationErr != nil {
return nil, interpretationErr
}
Expand Down
Expand Up @@ -129,7 +129,7 @@ type WaitCapabilities struct {
resultUuid string
}

func (builtin *WaitCapabilities) Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
func (builtin *WaitCapabilities) Interpret(_ string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {

serviceNameArgumentValue, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ServiceNameArgName)
if err != nil {
Expand Down
Expand Up @@ -44,7 +44,8 @@ func (builtin *KurtosisPlanInstructionWrapper) CreateBuiltin() func(thread *star
}

instructionWrapper := newKurtosisPlanInstructionInternal(wrappedBuiltin, builtin.Capabilities(), builtin.DefaultDisplayArguments)
returnedFutureValue, interpretationErr := instructionWrapper.interpret()
locatorOfModuleInWhichInstructionIsBeingInterpreted := thread.Name
returnedFutureValue, interpretationErr := instructionWrapper.interpret(locatorOfModuleInWhichInstructionIsBeingInterpreted)
if interpretationErr != nil {
return nil, interpretationErr
}
Expand Down
Expand Up @@ -9,7 +9,7 @@ import (
)

type KurtosisPlanInstructionCapabilities interface {
Interpret(arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError)
Interpret(locatorOfModuleInWhichThisBuiltInIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError)

Validate(arguments *builtin_argument.ArgumentValuesSet, validatorEnvironment *startosis_validator.ValidatorEnvironment) *startosis_errors.ValidationError

Expand Down
Expand Up @@ -75,8 +75,8 @@ func (builtin *kurtosisPlanInstructionInternal) Execute(ctx context.Context) (*s
return &result, nil
}

func (builtin *kurtosisPlanInstructionInternal) interpret() (starlark.Value, *startosis_errors.InterpretationError) {
result, interpretationErr := builtin.capabilities.Interpret(builtin.GetArguments())
func (builtin *kurtosisPlanInstructionInternal) interpret(locatorOfModuleInWhichThisBuiltInIsBeingCalled string) (starlark.Value, *startosis_errors.InterpretationError) {
result, interpretationErr := builtin.capabilities.Interpret(locatorOfModuleInWhichThisBuiltInIsBeingCalled, builtin.GetArguments())
if interpretationErr != nil {
return nil, interpretationErr
}
Expand Down
Expand Up @@ -29,7 +29,7 @@ func (t *uploadFilesTestCase) GetId() string {
func (t *uploadFilesTestCase) GetInstruction() *kurtosis_plan_instruction.KurtosisPlanInstruction {
serviceNetwork := service_network.NewMockServiceNetwork(t)
packageContentProvider := mock_package_content_provider.NewMockPackageContentProvider()
require.Nil(t, packageContentProvider.AddFileContent(TestSrcPath, "Hello World!"))
require.Nil(t, packageContentProvider.AddFileContent(TestModuleFileName, "Hello World!"))

serviceNetwork.EXPECT().UploadFilesArtifact(
mock.Anything, // data gets written to disk and compressed to it's a bit tricky to replicate here.
Expand All @@ -43,7 +43,7 @@ func (t *uploadFilesTestCase) GetInstruction() *kurtosis_plan_instruction.Kurtos
}

func (t uploadFilesTestCase) GetStarlarkCode() string {
return fmt.Sprintf("%s(%s=%q, %s=%q)", upload_files.UploadFilesBuiltinName, upload_files.SrcArgName, TestSrcPath, upload_files.ArtifactNameArgName, TestArtifactName)
return fmt.Sprintf("%s(%s=%q, %s=%q)", upload_files.UploadFilesBuiltinName, upload_files.SrcArgName, TestModuleFileName, upload_files.ArtifactNameArgName, TestArtifactName)
}

func (t *uploadFilesTestCase) GetStarlarkCodeForAssertion() string {
Expand Down
Expand Up @@ -29,7 +29,7 @@ func (t *uploadFilesWithoutNameTestCase) GetId() string {
func (t *uploadFilesWithoutNameTestCase) GetInstruction() *kurtosis_plan_instruction.KurtosisPlanInstruction {
serviceNetwork := service_network.NewMockServiceNetwork(t)
packageContentProvider := mock_package_content_provider.NewMockPackageContentProvider()
require.Nil(t, packageContentProvider.AddFileContent(TestSrcPath, "Hello World!"))
require.Nil(t, packageContentProvider.AddFileContent(TestModuleFileName, "Hello World!"))

serviceNetwork.EXPECT().GetUniqueNameForFileArtifact().Times(1).Return(
mockedFileArtifactName,
Expand All @@ -48,7 +48,7 @@ func (t *uploadFilesWithoutNameTestCase) GetInstruction() *kurtosis_plan_instruc
}

func (t uploadFilesWithoutNameTestCase) GetStarlarkCode() string {
return fmt.Sprintf("%s(%s=%q)", upload_files.UploadFilesBuiltinName, upload_files.SrcArgName, TestSrcPath)
return fmt.Sprintf("%s(%s=%q)", upload_files.UploadFilesBuiltinName, upload_files.SrcArgName, TestModuleFileName)
}

func (t *uploadFilesWithoutNameTestCase) GetStarlarkCodeForAssertion() string {
Expand Down
Expand Up @@ -28,18 +28,15 @@ import (
)

const (
starlarkGoThreadName = "Startosis interpreter thread"

multipleInterpretationErrorMsg = "Multiple errors caught interpreting the Starlark script. Listing each of them below."
evaluationErrorPrefix = "Evaluation error: "

skipImportInstructionInStacktraceValue = "import_module"

runFunctionName = "run"

paramsRequiredForArgs = 2
minimumParamsRequiredForPlan = 1
maximumParamsAllowedForRunFunction = 2
paramsRequiredForArgs = 2
minimumParamsRequiredForPlan = 1

planParamIndex = 0
planParamName = "plan"
Expand Down Expand Up @@ -268,7 +265,7 @@ func (interpreter *StartosisInterpreter) Interpret(
return startosis_constants.NoOutputObject, nil, missingMainFunctionError(packageId, mainFunctionName)
}

runFunctionExecutionThread := newStarlarkThread(starlarkGoThreadName)
runFunctionExecutionThread := newStarlarkThread(moduleLocator)

var argsTuple starlark.Tuple
var kwArgs []starlark.Tuple
Expand Down
Expand Up @@ -17,6 +17,6 @@ func (suite *StartosisPackageTestSuite) TestStartosisPackage_RelativeImports() {
require.Nil(t, runResult.InterpretationError)
require.Empty(t, runResult.ValidationErrors)
require.Nil(t, runResult.ExecutionError)
expectedResult := "John Doe\nOpen Sesame\n"
require.Equal(t, expectedResult, string(runResult.RunOutput))
expectedResult := "Files with artifact name 'upload' uploaded with artifact UUID '[a-f0-9]{32}'\nJohn Doe\nOpen Sesame\n"
require.Regexp(t, expectedResult, string(runResult.RunOutput))
}
Expand Up @@ -2,5 +2,6 @@ lib = import_module("./src/lib.star")
password = read_file("./static_files/password.txt")

def run(plan):
plan.upload_files("./static_files/password.txt", "upload")
plan.print(lib.NAME)
plan.print(password)