Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add cli argument to control image download #1495

Merged
merged 27 commits into from Oct 28, 2023
Merged
Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
2b8a14e
Add image pulling mode to docker manager
lostbean Oct 5, 2023
045551b
swap inverted defaults
lostbean Oct 6, 2023
cf38e01
Add CLI argument to capture image download mode
lostbean Oct 6, 2023
66ab73e
Add image download mode to API
lostbean Oct 6, 2023
3f86ebf
Updating the correct abstraction - KurtosisBackend
lostbean Oct 7, 2023
556107f
Apply image download mode via validator
lostbean Oct 7, 2023
4f99e61
make API type safe
lostbean Oct 7, 2023
ba6e70c
remove duplicated enum definition
lostbean Oct 7, 2023
0c5105a
Move switch case to a common location
lostbean Oct 7, 2023
6434b1e
Correct flag message
lostbean Oct 7, 2023
8cd67a3
rename file
lostbean Oct 7, 2023
c59972a
Actually genarate code using enumer
lostbean Oct 7, 2023
d3d2887
pass image download mode to fetchImageFromBackend
lostbean Oct 19, 2023
7f36410
fix RPC translator
lostbean Oct 19, 2023
eee5e3d
Log mode when fetching image
lostbean Oct 19, 2023
24124f2
Remove unimplemented flag 'never'
lostbean Oct 19, 2023
6fef551
Fix wrong removal
lostbean Oct 19, 2023
c931780
Document new flag `--image-download`
lostbean Oct 19, 2023
563798c
remove `never` enum from protobuf
lostbean Oct 19, 2023
1971a1b
do not return value if fail to parse
lostbean Oct 20, 2023
e77cbd7
rename functions for clarity and consistency
lostbean Oct 20, 2023
198a4dd
remove intermediary data structure to simplify codebase
lostbean Oct 20, 2023
d7b08ea
moving conversion func to apic
lostbean Oct 20, 2023
c957ccf
rename args to match CamelCase format
lostbean Oct 20, 2023
a079a35
no magic and early ejection
lostbean Oct 25, 2023
82e18e0
Merge branch 'main' into edgar/image-download-flag
lostbean Oct 25, 2023
70d5ce7
Merge branch 'main' into edgar/image-download-flag
lostbean Oct 27, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1,372 changes: 728 additions & 644 deletions api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go

Large diffs are not rendered by default.

12 changes: 12 additions & 0 deletions api/golang/core/lib/binding_constructors/binding_constructors.go
Expand Up @@ -41,11 +41,14 @@ func NewRunStarlarkScriptArgs(
experimentalFeatures []kurtosis_core_rpc_api_bindings.KurtosisFeatureFlag,
cloudInstanceId string,
cloudUserId string,
imageDownloadMode kurtosis_core_rpc_api_bindings.ImageDownloadMode,
) *kurtosis_core_rpc_api_bindings.RunStarlarkScriptArgs {
cloudInstanceIdCopy := new(string)
*cloudInstanceIdCopy = cloudInstanceId
cloudUserIdCopy := new(string)
*cloudUserIdCopy = cloudUserId
imageDownloadModeCopy := new(kurtosis_core_rpc_api_bindings.ImageDownloadMode)
*imageDownloadModeCopy = imageDownloadMode
parallelismCopy := new(int32)
*parallelismCopy = parallelism
return &kurtosis_core_rpc_api_bindings.RunStarlarkScriptArgs{
Expand All @@ -57,6 +60,7 @@ func NewRunStarlarkScriptArgs(
ExperimentalFeatures: experimentalFeatures,
CloudInstanceId: cloudInstanceIdCopy,
CloudUserId: cloudUserIdCopy,
ImageDownloadMode: imageDownloadModeCopy,
}
}

Expand All @@ -70,13 +74,16 @@ func NewRunStarlarkPackageArgs(
experimentalFeatures []kurtosis_core_rpc_api_bindings.KurtosisFeatureFlag,
cloudInstanceId string,
cloudUserId string,
imageDownloadMode kurtosis_core_rpc_api_bindings.ImageDownloadMode,
) *kurtosis_core_rpc_api_bindings.RunStarlarkPackageArgs {
parallelismCopy := new(int32)
*parallelismCopy = parallelism
cloudInstanceIdCopy := new(string)
*cloudInstanceIdCopy = cloudInstanceId
cloudUserIdCopy := new(string)
*cloudUserIdCopy = cloudUserId
imageDownloadModeCopy := new(kurtosis_core_rpc_api_bindings.ImageDownloadMode)
*imageDownloadModeCopy = imageDownloadMode
clonePackage := false
return &kurtosis_core_rpc_api_bindings.RunStarlarkPackageArgs{
PackageId: packageId,
Expand All @@ -90,6 +97,7 @@ func NewRunStarlarkPackageArgs(
ExperimentalFeatures: experimentalFeatures,
CloudInstanceId: cloudInstanceIdCopy,
CloudUserId: cloudUserIdCopy,
ImageDownloadMode: imageDownloadModeCopy,
}
}

Expand All @@ -103,13 +111,16 @@ func NewRunStarlarkRemotePackageArgs(
experimentalFeatures []kurtosis_core_rpc_api_bindings.KurtosisFeatureFlag,
cloudInstanceId string,
cloudUserId string,
imageDownloadMode kurtosis_core_rpc_api_bindings.ImageDownloadMode,
) *kurtosis_core_rpc_api_bindings.RunStarlarkPackageArgs {
parallelismCopy := new(int32)
*parallelismCopy = parallelism
cloudInstanceIdCopy := new(string)
*cloudInstanceIdCopy = cloudInstanceId
cloudUserIdCopy := new(string)
*cloudUserIdCopy = cloudUserId
imageDownloadModeCopy := new(kurtosis_core_rpc_api_bindings.ImageDownloadMode)
*imageDownloadModeCopy = imageDownloadMode
clonePackage := true
return &kurtosis_core_rpc_api_bindings.RunStarlarkPackageArgs{
PackageId: packageId,
Expand All @@ -123,6 +134,7 @@ func NewRunStarlarkRemotePackageArgs(
ExperimentalFeatures: experimentalFeatures,
CloudInstanceId: cloudInstanceIdCopy,
CloudUserId: cloudUserIdCopy,
ImageDownloadMode: imageDownloadModeCopy,
}
}

Expand Down
18 changes: 10 additions & 8 deletions api/golang/core/lib/enclaves/enclave_context.go
Expand Up @@ -20,6 +20,11 @@ package enclaves
import (
"context"
"encoding/json"
"io"
"os"
"path"
"strings"

yaml_convert "github.com/ghodss/yaml"
"github.com/kurtosis-tech/kurtosis/api/golang/core/kurtosis_core_rpc_api_bindings"
"github.com/kurtosis-tech/kurtosis/api/golang/core/lib/binding_constructors"
Expand All @@ -31,10 +36,6 @@ import (
"github.com/sirupsen/logrus"
"google.golang.org/grpc"
"google.golang.org/protobuf/types/known/emptypb"
"io"
"os"
"path"
"strings"
)

type EnclaveUUID string
Expand Down Expand Up @@ -95,7 +96,7 @@ func (enclaveCtx *EnclaveContext) RunStarlarkScript(
return nil, nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for script '%v'", oldSerializedParams)
}
ctxWithCancel, cancelCtxFunc := context.WithCancel(ctx)
executeStartosisScriptArgs := binding_constructors.NewRunStarlarkScriptArgs(runConfig.MainFunctionName, serializedScript, serializedParams, runConfig.DryRun, runConfig.Parallelism, runConfig.ExperimentalFeatureFlags, runConfig.CloudInstanceId, runConfig.CloudUserId)
executeStartosisScriptArgs := binding_constructors.NewRunStarlarkScriptArgs(runConfig.MainFunctionName, serializedScript, serializedParams, runConfig.DryRun, runConfig.Parallelism, runConfig.ExperimentalFeatureFlags, runConfig.CloudInstanceId, runConfig.CloudUserId, runConfig.ImageDownload)
starlarkResponseLineChan := make(chan *kurtosis_core_rpc_api_bindings.StarlarkRunResponseLine)

stream, err := enclaveCtx.client.RunStarlarkScript(ctxWithCancel, executeStartosisScriptArgs)
Expand Down Expand Up @@ -147,7 +148,7 @@ func (enclaveCtx *EnclaveContext) RunStarlarkPackage(
return nil, nil, stacktrace.Propagate(err, "An error occurred getting Kurtosis yaml file from path '%s'", packageRootPath)
}

executeStartosisPackageArgs, err := enclaveCtx.assembleRunStartosisPackageArg(kurtosisYml, runConfig.RelativePathToMainFile, runConfig.MainFunctionName, serializedParams, runConfig.DryRun, runConfig.Parallelism, runConfig.ExperimentalFeatureFlags, runConfig.CloudInstanceId, runConfig.CloudUserId)
executeStartosisPackageArgs, err := enclaveCtx.assembleRunStartosisPackageArg(kurtosisYml, runConfig.RelativePathToMainFile, runConfig.MainFunctionName, serializedParams, runConfig.DryRun, runConfig.Parallelism, runConfig.ExperimentalFeatureFlags, runConfig.CloudInstanceId, runConfig.CloudUserId, runConfig.ImageDownload)
if err != nil {
return nil, nil, stacktrace.Propagate(err, "Error preparing package '%s' for execution", packageRootPath)
}
Expand Down Expand Up @@ -233,7 +234,7 @@ func (enclaveCtx *EnclaveContext) RunStarlarkRemotePackage(
}()

starlarkResponseLineChan := make(chan *kurtosis_core_rpc_api_bindings.StarlarkRunResponseLine)
executeStartosisScriptArgs := binding_constructors.NewRunStarlarkRemotePackageArgs(packageId, runConfig.RelativePathToMainFile, runConfig.MainFunctionName, serializedParams, runConfig.DryRun, runConfig.Parallelism, runConfig.ExperimentalFeatureFlags, runConfig.CloudInstanceId, runConfig.CloudUserId)
executeStartosisScriptArgs := binding_constructors.NewRunStarlarkRemotePackageArgs(packageId, runConfig.RelativePathToMainFile, runConfig.MainFunctionName, serializedParams, runConfig.DryRun, runConfig.Parallelism, runConfig.ExperimentalFeatureFlags, runConfig.CloudInstanceId, runConfig.CloudUserId, runConfig.ImageDownload)

stream, err := enclaveCtx.client.RunStarlarkPackage(ctxWithCancel, executeStartosisScriptArgs)
if err != nil {
Expand Down Expand Up @@ -531,9 +532,10 @@ func (enclaveCtx *EnclaveContext) assembleRunStartosisPackageArg(
experimentalFeatures []kurtosis_core_rpc_api_bindings.KurtosisFeatureFlag,
cloudInstanceId string,
cloudUserId string,
imageDownloadMode kurtosis_core_rpc_api_bindings.ImageDownloadMode,
) (*kurtosis_core_rpc_api_bindings.RunStarlarkPackageArgs, error) {

return binding_constructors.NewRunStarlarkPackageArgs(kurtosisYaml.PackageName, relativePathToMainFile, mainFunctionName, serializedParams, dryRun, parallelism, experimentalFeatures, cloudInstanceId, cloudUserId), nil
return binding_constructors.NewRunStarlarkPackageArgs(kurtosisYaml.PackageName, relativePathToMainFile, mainFunctionName, serializedParams, dryRun, parallelism, experimentalFeatures, cloudInstanceId, cloudUserId, imageDownloadMode), nil
}

func (enclaveCtx *EnclaveContext) uploadStarlarkPackage(packageId string, packageRootPath string) error {
Expand Down
7 changes: 7 additions & 0 deletions api/golang/core/lib/shared_utils/proto_default_value.go
Expand Up @@ -14,3 +14,10 @@ func GetOrDefaultString(optionalValue *string, defaultValue string) string {
}
return *optionalValue
}

func GetOrDefault[T any](optionalValue *T, defaultValue T) T {
if optionalValue == nil {
return defaultValue
}
return *optionalValue
}
13 changes: 11 additions & 2 deletions api/golang/core/lib/starlark_run_config/starlark_run_config.go
Expand Up @@ -10,6 +10,7 @@ const (
defaultParallelism = 4
defaultCloudInstanceId = ""
defaultCloudUserId = ""
defaultImageDownload = kurtosis_core_rpc_api_bindings.ImageDownloadMode_missing
)

var defaultExperimentalFeatureFlags = []kurtosis_core_rpc_api_bindings.KurtosisFeatureFlag(nil)
Expand All @@ -23,6 +24,7 @@ type StarlarkRunConfig struct {
ExperimentalFeatureFlags []kurtosis_core_rpc_api_bindings.KurtosisFeatureFlag
CloudInstanceId string
CloudUserId string
ImageDownload kurtosis_core_rpc_api_bindings.ImageDownloadMode
}

type starlarkRunConfigOption func(*StarlarkRunConfig)
Expand All @@ -35,8 +37,9 @@ func NewRunStarlarkConfig(opts ...starlarkRunConfigOption) *StarlarkRunConfig {
DryRun: defaultDryRun,
Parallelism: defaultParallelism,
ExperimentalFeatureFlags: defaultExperimentalFeatureFlags,
CloudInstanceId: defaultCloudUserId,
CloudUserId: defaultCloudInstanceId,
CloudInstanceId: defaultCloudInstanceId,
CloudUserId: defaultCloudUserId,
ImageDownload: defaultImageDownload,
}

for _, opt := range opts {
Expand Down Expand Up @@ -93,3 +96,9 @@ func WithCloudUserId(cloudUserId string) starlarkRunConfigOption {
config.CloudUserId = cloudUserId
}
}

func WithImageDownloadMode(imageDownloadMode kurtosis_core_rpc_api_bindings.ImageDownloadMode) starlarkRunConfigOption {
return func(config *StarlarkRunConfig) {
config.ImageDownload = imageDownloadMode
}
}

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 11 additions & 0 deletions api/protobuf/core/api_container_service.proto
Expand Up @@ -94,6 +94,11 @@ enum ServiceStatus {
UNKNOWN = 2;
}

enum ImageDownloadMode {
always = 0;
missing = 1;
}

message ServiceInfo {
// UUID of the service
string service_uuid = 1;
Expand Down Expand Up @@ -162,6 +167,9 @@ message RunStarlarkScriptArgs {

// Defaults to empty
optional string cloud_user_id = 8;

// Defaults to empty
optional ImageDownloadMode image_download_mode = 9;
}

message RunStarlarkPackageArgs {
Expand Down Expand Up @@ -204,6 +212,9 @@ message RunStarlarkPackageArgs {

// Defaults to empty
optional string cloud_user_id = 13;

// Defaults to empty
optional ImageDownloadMode image_download_mode = 14;
}

enum KurtosisFeatureFlag {
Expand Down
32 changes: 32 additions & 0 deletions api/rust/src/api_container_api.rs
Expand Up @@ -178,6 +178,9 @@ pub struct RunStarlarkScriptArgs {
/// Defaults to empty
#[prost(string, optional, tag = "8")]
pub cloud_user_id: ::core::option::Option<::prost::alloc::string::String>,
/// Defaults to empty
#[prost(enumeration = "ImageDownloadMode", optional, tag = "9")]
pub image_download_mode: ::core::option::Option<i32>,
}
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
Expand Down Expand Up @@ -214,6 +217,9 @@ pub struct RunStarlarkPackageArgs {
/// Defaults to empty
#[prost(string, optional, tag = "13")]
pub cloud_user_id: ::core::option::Option<::prost::alloc::string::String>,
/// Defaults to empty
#[prost(enumeration = "ImageDownloadMode", optional, tag = "14")]
pub image_download_mode: ::core::option::Option<i32>,
/// Deprecated: If the package is local, it should have been uploaded with UploadStarlarkPackage prior to calling
/// RunStarlarkPackage. If the package is remote and must be cloned within the APIC, use the standalone boolean flag
/// clone_package below
Expand Down Expand Up @@ -690,6 +696,32 @@ impl ServiceStatus {
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
#[repr(i32)]
pub enum ImageDownloadMode {
Always = 0,
Missing = 1,
}
impl ImageDownloadMode {
/// String value of the enum field names used in the ProtoBuf definition.
///
/// The values are not transformed in any way and thus are considered stable
/// (if the ProtoBuf definition does not change) and safe for programmatic use.
pub fn as_str_name(&self) -> &'static str {
match self {
ImageDownloadMode::Always => "always",
ImageDownloadMode::Missing => "missing",
}
}
/// Creates an enum from field names used in the ProtoBuf definition.
pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
match value {
"always" => Some(Self::Always),
"missing" => Some(Self::Missing),
_ => None,
}
}
}
/// User services port forwarding
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
#[repr(i32)]
Expand Down
Expand Up @@ -170,6 +170,11 @@ export class RunStarlarkScriptArgs extends jspb.Message {
hasCloudUserId(): boolean;
clearCloudUserId(): RunStarlarkScriptArgs;

getImageDownloadMode(): ImageDownloadMode;
setImageDownloadMode(value: ImageDownloadMode): RunStarlarkScriptArgs;
hasImageDownloadMode(): boolean;
clearImageDownloadMode(): RunStarlarkScriptArgs;

serializeBinary(): Uint8Array;
toObject(includeInstance?: boolean): RunStarlarkScriptArgs.AsObject;
static toObject(includeInstance: boolean, msg: RunStarlarkScriptArgs): RunStarlarkScriptArgs.AsObject;
Expand All @@ -188,6 +193,7 @@ export namespace RunStarlarkScriptArgs {
experimentalFeaturesList: Array<KurtosisFeatureFlag>,
cloudInstanceId?: string,
cloudUserId?: string,
imageDownloadMode?: ImageDownloadMode,
}

export enum DryRunCase {
Expand All @@ -209,6 +215,11 @@ export namespace RunStarlarkScriptArgs {
_CLOUD_USER_ID_NOT_SET = 0,
CLOUD_USER_ID = 8,
}

export enum ImageDownloadModeCase {
_IMAGE_DOWNLOAD_MODE_NOT_SET = 0,
IMAGE_DOWNLOAD_MODE = 9,
}
}

export class RunStarlarkPackageArgs extends jspb.Message {
Expand Down Expand Up @@ -262,6 +273,11 @@ export class RunStarlarkPackageArgs extends jspb.Message {
hasCloudUserId(): boolean;
clearCloudUserId(): RunStarlarkPackageArgs;

getImageDownloadMode(): ImageDownloadMode;
setImageDownloadMode(value: ImageDownloadMode): RunStarlarkPackageArgs;
hasImageDownloadMode(): boolean;
clearImageDownloadMode(): RunStarlarkPackageArgs;

getStarlarkPackageContentCase(): RunStarlarkPackageArgs.StarlarkPackageContentCase;

serializeBinary(): Uint8Array;
Expand All @@ -286,6 +302,7 @@ export namespace RunStarlarkPackageArgs {
experimentalFeaturesList: Array<KurtosisFeatureFlag>,
cloudInstanceId?: string,
cloudUserId?: string,
imageDownloadMode?: ImageDownloadMode,
}

export enum StarlarkPackageContentCase {
Expand Down Expand Up @@ -318,6 +335,11 @@ export namespace RunStarlarkPackageArgs {
_CLOUD_USER_ID_NOT_SET = 0,
CLOUD_USER_ID = 13,
}

export enum ImageDownloadModeCase {
_IMAGE_DOWNLOAD_MODE_NOT_SET = 0,
IMAGE_DOWNLOAD_MODE = 14,
}
}

export class StarlarkRunResponseLine extends jspb.Message {
Expand Down Expand Up @@ -1280,6 +1302,10 @@ export enum ServiceStatus {
RUNNING = 1,
UNKNOWN = 2,
}
export enum ImageDownloadMode {
ALWAYS = 0,
MISSING = 1,
}
export enum Connect {
CONNECT = 0,
NO_CONNECT = 1,
Expand Down