Skip to content

Commit

Permalink
feat: docker compose integration pt. 2 (#2043)
Browse files Browse the repository at this point in the history 
## Description:
This feature allows users to run basic docker compose's with Kurtosis.
Now, directories containing a `kurtosis.yml` OR a `docker-compose.yaml`
are treated as starlark packages. If a compose package is detected,
`APIC` converts the compose to Starlark, and executes the Starlark
within enclaves.

eg. 
`kurtosis run .`
`kurtosis run github.com/awesome-compose/django`

## Is this change user facing?
YES

## References:
Part 1 (Docker Compose Transpiler)
#2001
  • Loading branch information
@tedim52
tedim52 committed Jan 15, 2024
1 parent e8ed4be commit 2a2793b
Show file tree
Hide file tree
Showing 16 changed files with 416 additions and 97 deletions.
79 changes: 71 additions & 8 deletions api/golang/core/lib/enclaves/enclave_context.go
View file
Expand Up @@ -47,8 +47,22 @@ const (
osPathSeparatorString = string(os.PathSeparator)

dotRelativePathIndicatorString = "."

// required to get around "only Github URLs" validation
composePackageIdPlaceholder = "github.com/NOTIONAL_USER/USER_UPLOADED_COMPOSE_PACKAGE"
)

// TODO Remove this once package ID is detected ONLY the APIC side (i.e. the CLI doesn't need to tell the APIC what package ID it's using)
// Doing so requires that we upload completely anonymous packages to the APIC, and it figures things out from there
var supportedDockerComposeYmlFilenames = []string{
"compose.yml",
"compose.yaml",
"docker-compose.yml",
"docker-compose.yaml",
"docker_compose.yml",
"docker_compose.yaml",
}

// Docs available at https://docs.kurtosis.com/sdk/#enclavecontext
type EnclaveContext struct {
client kurtosis_core_rpc_api_bindings.ApiContainerServiceClient
Expand Down Expand Up @@ -143,12 +157,22 @@ func (enclaveCtx *EnclaveContext) RunStarlarkPackage(

starlarkResponseLineChan := make(chan *kurtosis_core_rpc_api_bindings.StarlarkRunResponseLine)

kurtosisYml, err := getKurtosisYaml(packageRootPath)
packageName, packageReplaceOptions, err := getPackageNameAndReplaceOptions(packageRootPath)
if err != nil {
return nil, nil, stacktrace.Propagate(err, "An error occurred getting Kurtosis yaml file from path '%s'", packageRootPath)
return nil, nil, err
}

executeStartosisPackageArgs, err := enclaveCtx.assembleRunStartosisPackageArg(kurtosisYml, runConfig.RelativePathToMainFile, runConfig.MainFunctionName, serializedParams, runConfig.DryRun, runConfig.Parallelism, runConfig.ExperimentalFeatureFlags, runConfig.CloudInstanceId, runConfig.CloudUserId, runConfig.ImageDownload)
executeStartosisPackageArgs, err := enclaveCtx.assembleRunStartosisPackageArg(
packageName,
runConfig.RelativePathToMainFile,
runConfig.MainFunctionName,
serializedParams,
runConfig.DryRun,
runConfig.Parallelism,
runConfig.ExperimentalFeatureFlags,
runConfig.CloudInstanceId,
runConfig.CloudUserId,
runConfig.ImageDownload)
if err != nil {
return nil, nil, stacktrace.Propagate(err, "Error preparing package '%s' for execution", packageRootPath)
}
Expand All @@ -158,9 +182,9 @@ func (enclaveCtx *EnclaveContext) RunStarlarkPackage(
return nil, nil, stacktrace.Propagate(err, "Error uploading package '%s' prior to executing it", packageRootPath)
}

if len(kurtosisYml.PackageReplaceOptions) > 0 {
if err = enclaveCtx.uploadLocalStarlarkPackageDependencies(packageRootPath, kurtosisYml.PackageReplaceOptions); err != nil {
return nil, nil, stacktrace.Propagate(err, "An error occurred while uploading the local starlark package dependencies from the replace options '%+v'", kurtosisYml.PackageReplaceOptions)
if len(packageReplaceOptions) > 0 {
if err = enclaveCtx.uploadLocalStarlarkPackageDependencies(packageRootPath, packageReplaceOptions); err != nil {
return nil, nil, stacktrace.Propagate(err, "An error occurred while uploading the local starlark package dependencies from the replace options '%+v'", packageReplaceOptions)
}
}

Expand All @@ -174,6 +198,45 @@ func (enclaveCtx *EnclaveContext) RunStarlarkPackage(
return starlarkResponseLineChan, cancelCtxFunc, nil
}

// Determines the package name and replace options based on [packageRootPath]
// If a kurtosis.yml is detected, package is a kurtosis package
// If a valid [supportedDockerComposeYaml] is detected, package is a docker compose package
func getPackageNameAndReplaceOptions(packageRootPath string) (string, map[string]string, error) {
var packageName string
var packageReplaceOptions map[string]string

// use kurtosis package if it exists
if _, err := os.Stat(path.Join(packageRootPath, kurtosisYamlFilename)); err == nil {
kurtosisYml, err := getKurtosisYaml(packageRootPath)
if err != nil {
return "", map[string]string{}, stacktrace.Propagate(err, "An error occurred getting Kurtosis yaml file from path '%s'", packageRootPath)
}
packageName = kurtosisYml.PackageName
packageReplaceOptions = kurtosisYml.PackageReplaceOptions
} else {
// use compose package if it exists
composeAbsFilepath := ""
for _, candidateComposeFilename := range supportedDockerComposeYmlFilenames {
candidateComposeAbsFilepath := path.Join(packageRootPath, candidateComposeFilename)
if _, err := os.Stat(candidateComposeAbsFilepath); err == nil {
composeAbsFilepath = candidateComposeAbsFilepath
break
}
}
if composeAbsFilepath == "" {
return "", map[string]string{}, stacktrace.NewError(
"Neither a '%s' file nor one of the default Compose files (%s) was found in the package root; at least one of these is required",
kurtosisYamlFilename,
strings.Join(supportedDockerComposeYmlFilenames, ", "),
)
}
packageName = composePackageIdPlaceholder
packageReplaceOptions = map[string]string{}
}

return packageName, packageReplaceOptions, nil
}

func (enclaveCtx *EnclaveContext) uploadLocalStarlarkPackageDependencies(packageRootPath string, packageReplaceOptions map[string]string) error {
for dependencyPackageId, replaceOption := range packageReplaceOptions {
if isLocalDependencyReplace(replaceOption) {
Expand Down Expand Up @@ -523,7 +586,7 @@ func getErrFromStarlarkRunResult(result *StarlarkRunResult) error {
}

func (enclaveCtx *EnclaveContext) assembleRunStartosisPackageArg(
kurtosisYaml *KurtosisYaml,
packageName string,
relativePathToMainFile string,
mainFunctionName string,
serializedParams string,
Expand All @@ -535,7 +598,7 @@ func (enclaveCtx *EnclaveContext) assembleRunStartosisPackageArg(
imageDownloadMode kurtosis_core_rpc_api_bindings.ImageDownloadMode,
) (*kurtosis_core_rpc_api_bindings.RunStarlarkPackageArgs, error) {

return binding_constructors.NewRunStarlarkPackageArgs(kurtosisYaml.PackageName, relativePathToMainFile, mainFunctionName, serializedParams, dryRun, parallelism, experimentalFeatures, cloudInstanceId, cloudUserId, imageDownloadMode), nil
return binding_constructors.NewRunStarlarkPackageArgs(packageName, relativePathToMainFile, mainFunctionName, serializedParams, dryRun, parallelism, experimentalFeatures, cloudInstanceId, cloudUserId, imageDownloadMode), nil
}

func (enclaveCtx *EnclaveContext) uploadStarlarkPackage(packageId string, packageRootPath string) error {
Expand Down
85 changes: 71 additions & 14 deletions api/typescript/src/core/lib/enclaves/enclave_context.ts
View file
Expand Up @@ -3,7 +3,7 @@
* All Rights Reserved.
*/

import {ok, err, Result} from "neverthrow";
import {ok, err, Result, Err} from "neverthrow";
import * as jspb from "google-protobuf";
import type {
Port,
Expand Down Expand Up @@ -34,6 +34,7 @@ import {
GetStarlarkRunResponse,
} from "../../kurtosis_core_rpc_api_bindings/api_container_service_pb";
import * as path from "path";
import * as fs from 'fs';
import {parseKurtosisYaml, KurtosisYaml} from "./kurtosis_yaml";
import {Readable} from "stream";
import {readStreamContentUntilClosed, StarlarkRunResult} from "./starlark_run_blocking";
Expand All @@ -48,6 +49,20 @@ const OS_PATH_SEPARATOR_STRING = "/"

const DOT_RELATIVE_PATH_INDICATOR_STRING = "."

// required to get around the "only Github URLs" validation
const composePackageIdPlaceholder = 'github.com/NOTIONAL_USER/COMPOSE-PACKAGE'


// TODO Remove this once package ID is detected ONLY the APIC side (i.e. the CLI doesn't need to tell the APIC what package ID it's using)
// Doing so requires that we upload completely anonymous packages to the APIC, and it figures things out from there
let supportedDockerComposeYmlFilenames = [
"compose.yml",
"compose.yaml",
"docker-compose.yml",
"docker-compose.yaml",
"docker_compose.yml",
"docker_compose.yaml",
]

// Docs available at https://docs.kurtosis.com/sdk/#enclavecontext
export class EnclaveContext {
Expand Down Expand Up @@ -146,16 +161,20 @@ export class EnclaveContext {
packageRootPath: string,
runConfig: StarlarkRunConfig,
): Promise<Result<Readable, Error>> {
const kurtosisYmlResult = await this.getKurtosisYaml(packageRootPath)
if (kurtosisYmlResult.isErr()) {
return err(new Error(`Unexpected error while getting the Kurtosis yaml file from path '${packageRootPath}'`))
const packageNameAndReplaceOptionsResult = await this.getPackageNameAndReplaceOptions(packageRootPath);
if (packageNameAndReplaceOptionsResult.isErr()) {
return err(new Error(`Unexpected error occurred while trying to get package name and replace options:\n${packageNameAndReplaceOptionsResult.error}`))
}

const kurtosisYaml: KurtosisYaml = kurtosisYmlResult.value
const packageId: string = kurtosisYaml.name
const packageReplaceOptions: Map<string, string> = kurtosisYaml.packageReplaceOptions

const args = await this.assembleRunStarlarkPackageArg(kurtosisYaml, runConfig.relativePathToMainFile, runConfig.mainFunctionName, runConfig.serializedParams, runConfig.dryRun, runConfig.cloudInstanceId, runConfig.cloudUserId)
const [packageName, packageReplaceOptions] = packageNameAndReplaceOptionsResult.value;

const args = await this.assembleRunStarlarkPackageArg(
packageName,
runConfig.relativePathToMainFile,
runConfig.mainFunctionName,
runConfig.serializedParams,
runConfig.dryRun,
runConfig.cloudInstanceId,
runConfig.cloudUserId)
if (args.isErr()) {
return err(new Error(`Unexpected error while assembling arguments to pass to the Starlark executor \n${args.error}`))
}
Expand All @@ -165,9 +184,9 @@ export class EnclaveContext {
return err(new Error(`Unexpected error while creating the package's tgs file from '${packageRootPath}'\n${archiverResponse.error}`))
}

const uploadStarlarkPackageResponse = await this.backend.uploadStarlarkPackage(packageId, archiverResponse.value)
const uploadStarlarkPackageResponse = await this.backend.uploadStarlarkPackage(packageName, archiverResponse.value)
if (uploadStarlarkPackageResponse.isErr()){
return err(new Error(`Unexpected error while uploading Starlark package '${packageId}'\n${uploadStarlarkPackageResponse.error}`))
return err(new Error(`Unexpected error while uploading Starlark package '${packageName}'\n${uploadStarlarkPackageResponse.error}`))
}

if (packageReplaceOptions !== undefined && packageReplaceOptions.size > 0) {
Expand Down Expand Up @@ -388,6 +407,44 @@ export class EnclaveContext {
// ====================================================================================================
// Private helper functions
// ====================================================================================================
// Determines the package name and replace options based on [packageRootPath]
// If a kurtosis.yml is detected, package is a kurtosis package
// If a valid [supportedDockerComposeYaml] is detected, package is a docker compose package
private async getPackageNameAndReplaceOptions(packageRootPath: string): Promise<Result<[string, Map<string, string>], Error>>
{
let packageName: string;
let packageReplaceOptions: Map<string, string>;

// Use kurtosis package if it exists
if (fs.existsSync(path.join(packageRootPath, KURTOSIS_YAML_FILENAME))) {
const kurtosisYmlResult = await this.getKurtosisYaml(packageRootPath)
if (kurtosisYmlResult.isErr()) {
return err(new Error(`Unexpected error while getting the Kurtosis yaml file from path '${packageRootPath}'`))
}
const kurtosisYml: KurtosisYaml = kurtosisYmlResult.value
packageName = kurtosisYml.name;
packageReplaceOptions = kurtosisYml.packageReplaceOptions;
} else {
// Use compose package if it exists
let composeAbsFilepath = '';
for (const candidateComposeFilename of supportedDockerComposeYmlFilenames) {
const candidateComposeAbsFilepath = path.join(packageRootPath, candidateComposeFilename);
if (fs.existsSync(candidateComposeAbsFilepath)) {
composeAbsFilepath = candidateComposeAbsFilepath;
break;
}
}
if (composeAbsFilepath === '') {
return err(new Error(
`Neither a '${KURTOSIS_YAML_FILENAME}' file nor one of the default Compose files (${supportedDockerComposeYmlFilenames.join(', ')}) was found in the package root; at least one of these is required`,
));
}
packageName = composePackageIdPlaceholder;
packageReplaceOptions = new Map<string, string>();
}

return ok([packageName, packageReplaceOptions]);
}

// convertApiPortsToServiceContextPorts returns a converted map where Port objects associated with strings in [apiPorts] are
// properly converted to PortSpec objects.
Expand All @@ -412,7 +469,7 @@ export class EnclaveContext {
}

private async assembleRunStarlarkPackageArg(
kurtosisYaml: KurtosisYaml,
packageName: string,
relativePathToMainFile: string,
mainFunctionName: string,
serializedParams: string,
Expand All @@ -422,7 +479,7 @@ export class EnclaveContext {
): Promise<Result<RunStarlarkPackageArgs, Error>> {

const args = new RunStarlarkPackageArgs;
args.setPackageId(kurtosisYaml.name)
args.setPackageId(packageName)
args.setSerializedParams(serializedParams)
args.setDryRun(dryRun)
args.setRelativePathToMainFile(relativePathToMainFile)
Expand Down

0 comments on commit 2a2793b

Please sign in to comment.