From 1e3dce0ff8ae8353144f7402b91f2f3d2a4b8023 Mon Sep 17 00:00:00 2001 From: Jon Edvald Date: Thu, 8 Oct 2020 14:27:52 +0200 Subject: [PATCH] improvement(core): resolve remote sources in parallel (#2097) * improvement(core): resolve remote sources in parallel * TBS: Fix tests Co-authored-by: Thorarinn Sigurdsson --- core/src/garden.ts | 17 ++++++++--------- core/src/resolve-module.ts | 3 +++ core/test/unit/src/garden.ts | 22 ++++++++++++---------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/core/src/garden.ts b/core/src/garden.ts index 13c70311fc..90ab49e2c5 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -42,7 +42,7 @@ import { loadConfigResources, findProjectConfig, prepareModuleResource } from ". import { DeepPrimitiveMap, StringMap, PrimitiveMap } from "./config/common" import { validateSchema } from "./config/validation" import { BaseTask } from "./tasks/base" -import { LocalConfigStore, ConfigStore, GlobalConfigStore } from "./config-store" +import { LocalConfigStore, ConfigStore, GlobalConfigStore, LinkedSource } from "./config-store" import { getLinkedSources, ExternalSourceType } from "./util/ext-source-util" import { BuildDependencyConfig, ModuleConfig, ModuleResource } from "./config/module" import { resolveModuleConfig } from "./resolve-module" @@ -999,18 +999,17 @@ export class Garden { this.log.silly(`Scanning for modules and workflows`) - let extSourcePaths: string[] = [] - // Add external sources that are defined at the project level. External sources are either kept in // the .garden/sources dir (and cloned there if needed), or they're linked to a local path via the link command. - for (const { name, repositoryUrl } of this.projectSources) { - const path = await this.loadExtSourcePath({ + const linkedSources = await getLinkedSources(this, "project") + const extSourcePaths = await Bluebird.map(this.projectSources, ({ name, repositoryUrl }) => { + return this.loadExtSourcePath({ name, + linkedSources, repositoryUrl, sourceType: "project", }) - extSourcePaths.push(path) - } + }) const dirsToScan = [this.projectRoot, ...extSourcePaths] const configPaths = flatten(await Bluebird.map(dirsToScan, (path) => this.scanForConfigs(path))) @@ -1117,15 +1116,15 @@ export class Garden { */ public async loadExtSourcePath({ name, + linkedSources, repositoryUrl, sourceType, }: { name: string + linkedSources: LinkedSource[] repositoryUrl: string sourceType: ExternalSourceType }): Promise { - const linkedSources = await getLinkedSources(this, sourceType) - const linked = findByName(linkedSources, name) if (linked) { diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index 495ec263ed..d78ff86d55 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -18,6 +18,7 @@ import { getModuleKey } from "./types/module" import { getModuleTypeBases } from "./plugins" import { ModuleConfig, moduleConfigSchema } from "./config/module" import { profileAsync } from "./util/profiling" +import { getLinkedSources } from "./util/ext-source-util" export interface ModuleConfigResolveOpts extends ContextResolveOpts { configContext: ModuleConfigContext @@ -81,8 +82,10 @@ export const resolveModuleConfig = profileAsync(async function $resolveModuleCon }) if (config.repositoryUrl) { + const linkedSources = await getLinkedSources(garden, "module") config.path = await garden.loadExtSourcePath({ name: config.name, + linkedSources, repositoryUrl: config.repositoryUrl, sourceType: "module", }) diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index ddb1473928..e8cb01c367 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -45,6 +45,7 @@ import { realpath, writeFile } from "fs-extra" import { dedent, deline } from "../../../src/util/string" import { ServiceState } from "../../../src/types/service" import execa from "execa" +import { getLinkedSources } from "../../../src/util/ext-source-util" describe("Garden", () => { let tmpDir: tmp.DirectoryResult @@ -3519,19 +3520,22 @@ describe("Garden", () => { describe("loadExtSourcePath", () => { let garden: TestGarden + let linkedSources: LinkedSource[] + + afterEach(async () => { + await resetLocalConfig(garden.gardenDirPath) + }) context("external project sources", () => { before(async () => { garden = await makeExtProjectSourcesGarden() - }) - - afterEach(async () => { - await resetLocalConfig(garden.gardenDirPath) + linkedSources = await getLinkedSources(garden, "module") }) it("should return the path to the project source if source type is project", async () => { const projectRoot = getDataDir("test-project-ext-project-sources") const path = await garden.loadExtSourcePath({ + linkedSources, repositoryUrl: testGitUrl, name: "source-a", sourceType: "project", @@ -3549,10 +3553,10 @@ describe("Garden", () => { path: linkedSourcePath, }, ] - await garden.configStore.set(["linkedProjectSources"], linked) const path = await garden.loadExtSourcePath({ name: "source-a", + linkedSources: linked, repositoryUrl: testGitUrl, sourceType: "project", }) @@ -3564,15 +3568,13 @@ describe("Garden", () => { context("external module sources", () => { before(async () => { garden = await makeExtModuleSourcesGarden() - }) - - afterEach(async () => { - await resetLocalConfig(garden.gardenDirPath) + linkedSources = await getLinkedSources(garden, "module") }) it("should return the path to the module source if source type is module", async () => { const projectRoot = getDataDir("test-project-ext-module-sources") const path = await garden.loadExtSourcePath({ + linkedSources, repositoryUrl: testGitUrl, name: "module-a", sourceType: "module", @@ -3590,10 +3592,10 @@ describe("Garden", () => { path: linkedModulePath, }, ] - await garden.configStore.set(["linkedModuleSources"], linked) const path = await garden.loadExtSourcePath({ name: "module-a", + linkedSources: linked, repositoryUrl: testGitUrl, sourceType: "module", })