improvement(core): resolve remote sources in parallel (#2097)

* improvement(core): resolve remote sources in parallel

* TBS: Fix tests

Co-authored-by: Thorarinn Sigurdsson <thorarinnsigurdsson@gmail.com>

core/src/garden.ts

@@ -42,7 +42,7 @@ import { loadConfigResources, findProjectConfig, prepareModuleResource } from ".
 import { DeepPrimitiveMap, StringMap, PrimitiveMap } from "./config/common"
 import { validateSchema } from "./config/validation"
 import { BaseTask } from "./tasks/base"
-import { LocalConfigStore, ConfigStore, GlobalConfigStore } from "./config-store"
+import { LocalConfigStore, ConfigStore, GlobalConfigStore, LinkedSource } from "./config-store"
 import { getLinkedSources, ExternalSourceType } from "./util/ext-source-util"
 import { BuildDependencyConfig, ModuleConfig, ModuleResource } from "./config/module"
 import { resolveModuleConfig } from "./resolve-module"
@@ -999,18 +999,17 @@ export class Garden {
 
       this.log.silly(`Scanning for modules and workflows`)
 
-      let extSourcePaths: string[] = []
-
       // Add external sources that are defined at the project level. External sources are either kept in
       // the .garden/sources dir (and cloned there if needed), or they're linked to a local path via the link command.
-      for (const { name, repositoryUrl } of this.projectSources) {
-        const path = await this.loadExtSourcePath({
+      const linkedSources = await getLinkedSources(this, "project")
+      const extSourcePaths = await Bluebird.map(this.projectSources, ({ name, repositoryUrl }) => {
+        return this.loadExtSourcePath({
           name,
+          linkedSources,
           repositoryUrl,
           sourceType: "project",
         })
-        extSourcePaths.push(path)
-      }
+      })
 
       const dirsToScan = [this.projectRoot, ...extSourcePaths]
       const configPaths = flatten(await Bluebird.map(dirsToScan, (path) => this.scanForConfigs(path)))
@@ -1117,15 +1116,15 @@ export class Garden {
    */
   public async loadExtSourcePath({
     name,
+    linkedSources,
     repositoryUrl,
     sourceType,
   }: {
     name: string
+    linkedSources: LinkedSource[]
     repositoryUrl: string
     sourceType: ExternalSourceType
   }): Promise<string> {
-    const linkedSources = await getLinkedSources(this, sourceType)
-
     const linked = findByName(linkedSources, name)
 
     if (linked) {

core/src/resolve-module.ts

@@ -18,6 +18,7 @@ import { getModuleKey } from "./types/module"
 import { getModuleTypeBases } from "./plugins"
 import { ModuleConfig, moduleConfigSchema } from "./config/module"
 import { profileAsync } from "./util/profiling"
+import { getLinkedSources } from "./util/ext-source-util"
 
 export interface ModuleConfigResolveOpts extends ContextResolveOpts {
   configContext: ModuleConfigContext
@@ -81,8 +82,10 @@ export const resolveModuleConfig = profileAsync(async function $resolveModuleCon
   })
 
   if (config.repositoryUrl) {
+    const linkedSources = await getLinkedSources(garden, "module")
     config.path = await garden.loadExtSourcePath({
       name: config.name,
+      linkedSources,
       repositoryUrl: config.repositoryUrl,
       sourceType: "module",
     })

core/test/unit/src/garden.ts

@@ -45,6 +45,7 @@ import { realpath, writeFile } from "fs-extra"
 import { dedent, deline } from "../../../src/util/string"
 import { ServiceState } from "../../../src/types/service"
 import execa from "execa"
+import { getLinkedSources } from "../../../src/util/ext-source-util"
 
 describe("Garden", () => {
   let tmpDir: tmp.DirectoryResult
@@ -3519,19 +3520,22 @@ describe("Garden", () => {
 
   describe("loadExtSourcePath", () => {
     let garden: TestGarden
+    let linkedSources: LinkedSource[]
+
+    afterEach(async () => {
+      await resetLocalConfig(garden.gardenDirPath)
+    })
 
     context("external project sources", () => {
       before(async () => {
         garden = await makeExtProjectSourcesGarden()
-      })
-
-      afterEach(async () => {
-        await resetLocalConfig(garden.gardenDirPath)
+        linkedSources = await getLinkedSources(garden, "module")
       })
 
       it("should return the path to the project source if source type is project", async () => {
         const projectRoot = getDataDir("test-project-ext-project-sources")
         const path = await garden.loadExtSourcePath({
+          linkedSources,
           repositoryUrl: testGitUrl,
           name: "source-a",
           sourceType: "project",
@@ -3549,10 +3553,10 @@ describe("Garden", () => {
             path: linkedSourcePath,
           },
         ]
-        await garden.configStore.set(["linkedProjectSources"], linked)
 
         const path = await garden.loadExtSourcePath({
           name: "source-a",
+          linkedSources: linked,
           repositoryUrl: testGitUrl,
           sourceType: "project",
         })
@@ -3564,15 +3568,13 @@ describe("Garden", () => {
     context("external module sources", () => {
       before(async () => {
         garden = await makeExtModuleSourcesGarden()
-      })
-
-      afterEach(async () => {
-        await resetLocalConfig(garden.gardenDirPath)
+        linkedSources = await getLinkedSources(garden, "module")
       })
 
       it("should return the path to the module source if source type is module", async () => {
         const projectRoot = getDataDir("test-project-ext-module-sources")
         const path = await garden.loadExtSourcePath({
+          linkedSources,
           repositoryUrl: testGitUrl,
           name: "module-a",
           sourceType: "module",
@@ -3590,10 +3592,10 @@ describe("Garden", () => {
             path: linkedModulePath,
           },
         ]
-        await garden.configStore.set(["linkedModuleSources"], linked)
 
         const path = await garden.loadExtSourcePath({
           name: "module-a",
+          linkedSources: linked,
           repositoryUrl: testGitUrl,
           sourceType: "module",
         })