-
Notifications
You must be signed in to change notification settings - Fork 267
/
garden.ts
2172 lines (1879 loc) · 71.7 KB
/
garden.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* Copyright (C) 2018-2023 Garden Technologies, Inc. <info@garden.io>
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
import fsExtra from "fs-extra"
const { ensureDir } = fsExtra
import { platform, arch } from "os"
import { relative, resolve } from "path"
import cloneDeep from "fast-copy"
import { flatten, sortBy, keyBy, mapValues, groupBy, set } from "lodash-es"
import AsyncLock from "async-lock"
import { TreeCache } from "./cache.js"
import { getBuiltinPlugins } from "./plugins/plugins.js"
import type { GardenModule, ModuleConfigMap, ModuleTypeMap } from "./types/module.js"
import { getModuleCacheContext } from "./types/module.js"
import type { SourceConfig, ProjectConfig, OutputSpec, ProxyConfig } from "./config/project.js"
import {
resolveProjectConfig,
pickEnvironment,
parseEnvironment,
getDefaultEnvironmentName,
projectSourcesSchema,
defaultNamespace,
defaultEnvironment,
} from "./config/project.js"
import { getCloudDistributionName, getCloudLogSectionName } from "./util/cloud.js"
import { findByName, pickKeys, getPackageVersion, getNames, findByNames, duplicatesByKey } from "./util/util.js"
import type { GardenError } from "./exceptions.js"
import {
ConfigurationError,
PluginError,
RuntimeError,
InternalError,
toGardenError,
CircularDependenciesError,
CloudApiError,
} from "./exceptions.js"
import type { VcsHandler, ModuleVersion, VcsInfo } from "./vcs/vcs.js"
import { getModuleVersionString } from "./vcs/vcs.js"
import { GitHandler } from "./vcs/git.js"
import { BuildStaging } from "./build-staging/build-staging.js"
import type { ConfigGraph } from "./graph/config-graph.js"
import { ResolvedConfigGraph } from "./graph/config-graph.js"
import { getRootLogger } from "./logger/logger.js"
import type { GardenPluginSpec } from "./plugin/plugin.js"
import type { GardenResource } from "./config/base.js"
import { loadConfigResources, findProjectConfig, configTemplateKind, renderTemplateKind } from "./config/base.js"
import type { DeepPrimitiveMap, StringMap, PrimitiveMap } from "./config/common.js"
import { treeVersionSchema, joi, allowUnknown } from "./config/common.js"
import { GlobalConfigStore } from "./config-store/global.js"
import type { LinkedSource } from "./config-store/local.js"
import { LocalConfigStore } from "./config-store/local.js"
import type { ExternalSourceType } from "./util/ext-source-util.js"
import { getLinkedSources } from "./util/ext-source-util.js"
import type { ModuleConfig } from "./config/module.js"
import { convertModules, ModuleResolver } from "./resolve-module.js"
import type { CommandInfo, PluginEventBroker } from "./plugin-context.js"
import { createPluginContext } from "./plugin-context.js"
import type { RegisterPluginParam } from "./plugin/plugin.js"
import {
SUPPORTED_PLATFORMS,
DEFAULT_GARDEN_DIR_NAME,
gardenEnv,
SUPPORTED_ARCHITECTURES,
GardenApiVersion,
DOCS_BASE_URL,
} from "./constants.js"
import type { Log } from "./logger/log-entry.js"
import { EventBus } from "./events/events.js"
import { Watcher } from "./watch.js"
import {
findConfigPathsInPath,
getWorkingCopyId,
fixedProjectExcludes,
defaultConfigFilename,
defaultDotIgnoreFile,
} from "./util/fs.js"
import type { Provider, GenericProviderConfig, ProviderMap } from "./config/provider.js"
import { getAllProviderDependencyNames, defaultProvider } from "./config/provider.js"
import { ResolveProviderTask } from "./tasks/resolve-provider.js"
import { ActionRouter } from "./router/router.js"
import type { ActionDefinitionMap, ActionTypeMap } from "./plugins.js"
import {
loadAndResolvePlugins,
getDependencyOrder,
getModuleTypes,
loadPlugin,
getActionTypes,
getActionTypeBases,
} from "./plugins.js"
import { dedent, deline, naturalList, wordWrap } from "./util/string.js"
import { DependencyGraph } from "./graph/common.js"
import { Profile, profileAsync } from "./util/profiling.js"
import { username } from "username"
import {
throwOnMissingSecretKeys,
resolveTemplateString,
resolveTemplateStrings,
} from "./template-string/template-string.js"
import type { WorkflowConfig, WorkflowConfigMap } from "./config/workflow.js"
import { resolveWorkflowConfig, isWorkflowConfig } from "./config/workflow.js"
import type { PluginTools } from "./util/ext-tools.js"
import { PluginTool } from "./util/ext-tools.js"
import type { ConfigTemplateResource, ConfigTemplateConfig } from "./config/config-template.js"
import { resolveConfigTemplate } from "./config/config-template.js"
import type { TemplatedModuleConfig } from "./plugins/templated.js"
import { BuildStagingRsync } from "./build-staging/rsync.js"
import {
DefaultEnvironmentContext,
ProjectConfigContext,
RemoteSourceConfigContext,
} from "./config/template-contexts/project.js"
import type { CloudApi, CloudProject } from "./cloud/api.js"
import { getGardenCloudDomain } from "./cloud/api.js"
import { OutputConfigContext } from "./config/template-contexts/module.js"
import { ProviderConfigContext } from "./config/template-contexts/provider.js"
import type { ConfigContext } from "./config/template-contexts/base.js"
import { validateSchema, validateWithPath } from "./config/validation.js"
import { pMemoizeDecorator } from "./lib/p-memoize.js"
import { ModuleGraph } from "./graph/modules.js"
import {
actionKinds,
type Action,
type ActionConfigMap,
type ActionConfigsByKey,
type ActionKind,
type ActionModeMap,
type BaseActionConfig,
} from "./actions/types.js"
import { actionIsDisabled, actionReferenceToString, isActionConfig } from "./actions/base.js"
import type { SolveOpts, SolveParams, SolveResult } from "./graph/solver.js"
import { GraphSolver } from "./graph/solver.js"
import {
actionConfigsToGraph,
actionFromConfig,
executeAction,
resolveAction,
resolveActions,
} from "./graph/actions.js"
import type { ActionTypeDefinition } from "./plugin/action-types.js"
import type { Task } from "./tasks/base.js"
import type { GraphResultFromTask, GraphResults } from "./graph/results.js"
import { uuidv4 } from "./util/random.js"
import type { RenderTemplateConfig } from "./config/render-template.js"
import { convertTemplatedModuleToRender, renderConfigTemplate } from "./config/render-template.js"
import { MonitorManager } from "./monitors/manager.js"
import { AnalyticsHandler } from "./analytics/analytics.js"
import { getGardenInstanceKey } from "./server/helpers.js"
import type { SuggestedCommand } from "./commands/base.js"
import { OtelTraced } from "./util/open-telemetry/decorators.js"
import { wrapActiveSpan } from "./util/open-telemetry/spans.js"
import { GitRepoHandler } from "./vcs/git-repo.js"
import { configureNoOpExporter } from "./util/open-telemetry/tracing.js"
import { detectModuleOverlap, makeOverlapErrors } from "./util/module-overlap.js"
import { GotHttpError } from "./util/http.js"
import { styles } from "./logger/styles.js"
import { renderDuration } from "./logger/util.js"
const defaultLocalAddress = "localhost"
export interface GardenOpts {
commandInfo: CommandInfo
config?: ProjectConfig
environmentString?: string // Note: This is the string, as e.g. passed with the --env flag
forceRefresh?: boolean
gardenDirPath?: string
globalConfigStore?: GlobalConfigStore
legacyBuildSync?: boolean
log?: Log
/**
* Log context for logging the start and finish of the Garden class
* initialization with duration.
*/
gardenInitLog?: Log
monitors?: MonitorManager
noEnterprise?: boolean
persistent?: boolean
plugins?: RegisterPluginParam[]
sessionId?: string
variableOverrides?: PrimitiveMap
cloudApi?: CloudApi
}
export interface GardenParams {
artifactsPath: string
vcsInfo: VcsInfo
projectId?: string
cloudDomain?: string
cache: TreeCache
dotIgnoreFile: string
proxy: ProxyConfig
environmentName: string
resolvedDefaultNamespace: string | null
namespace: string
gardenDirPath: string
globalConfigStore?: GlobalConfigStore
localConfigStore?: LocalConfigStore
log: Log
gardenInitLog?: Log
moduleIncludePatterns?: string[]
moduleExcludePatterns?: string[]
monitors?: MonitorManager
opts: GardenOpts
outputs: OutputSpec[]
plugins: RegisterPluginParam[]
production: boolean
projectConfig: ProjectConfig
projectName: string
projectRoot: string
projectSources?: SourceConfig[]
providerConfigs: GenericProviderConfig[]
variables: DeepPrimitiveMap
variableOverrides: DeepPrimitiveMap
secrets: StringMap
sessionId: string
username: string | undefined
workingCopyId: string
forceRefresh?: boolean
cloudApi?: CloudApi | null
projectApiVersion: ProjectConfig["apiVersion"]
}
interface GardenInstanceState {
configsScanned: boolean
needsReload: boolean
}
@Profile()
export class Garden {
public log: Log
private gardenInitLog?: Log
private loadedPlugins?: GardenPluginSpec[]
protected actionConfigs: ActionConfigMap
protected moduleConfigs: ModuleConfigMap
protected workflowConfigs: WorkflowConfigMap
protected configPaths: Set<string>
private resolvedProviders: { [key: string]: Provider }
protected readonly state: GardenInstanceState
protected registeredPlugins: RegisterPluginParam[]
private readonly solver: GraphSolver
private asyncLock: AsyncLock
public readonly projectId?: string
public readonly cloudDomain?: string
public sessionId: string
public readonly localConfigStore: LocalConfigStore
public globalConfigStore: GlobalConfigStore
public readonly vcs: VcsHandler
public readonly treeCache: TreeCache
public events: EventBus
private tools?: { [key: string]: PluginTool }
public configTemplates: { [name: string]: ConfigTemplateConfig }
private actionTypeBases: ActionTypeMap<ActionTypeDefinition<any>[]>
private emittedWarnings: Set<string>
public cloudApi: CloudApi | null
public readonly production: boolean
public readonly projectRoot: string
public readonly projectName: string
public readonly projectApiVersion: string
public readonly environmentName: string
/**
* The resolved default namespace as defined in the Project config for the current environment.
*/
public readonly resolvedDefaultNamespace: string | null
/**
* The actual namespace for the Garden instance. This is by default the namespace defined in the Project config
* for the current environment but can be overwritten with the `--env` flag.
*/
public readonly namespace: string
public readonly variables: DeepPrimitiveMap
// Any variables passed via the `--var` CLI option (maintained here so that they can be used during module resolution
// to override module variables and module varfiles).
public readonly variableOverrides: DeepPrimitiveMap
public readonly secrets: StringMap
private readonly projectSources: SourceConfig[]
public readonly buildStaging: BuildStaging
public readonly gardenDirPath: string
public readonly artifactsPath: string
public readonly vcsInfo: VcsInfo
public readonly opts: GardenOpts
private readonly projectConfig: ProjectConfig
private readonly providerConfigs: GenericProviderConfig[]
public readonly workingCopyId: string
public readonly dotIgnoreFile: string
public readonly proxy: ProxyConfig
public readonly moduleIncludePatterns?: string[]
public readonly moduleExcludePatterns: string[]
public readonly persistent: boolean
public readonly rawOutputs: OutputSpec[]
public readonly username?: string
public readonly version: string
private readonly forceRefresh: boolean
public readonly commandInfo: CommandInfo
public readonly monitors: MonitorManager
public readonly nestedSessions: Map<string, Garden>
// Used internally for introspection
public readonly isGarden: true
constructor(params: GardenParams) {
this.projectId = params.projectId
this.cloudDomain = params.cloudDomain
this.sessionId = params.sessionId
this.environmentName = params.environmentName
this.resolvedDefaultNamespace = params.resolvedDefaultNamespace
this.namespace = params.namespace
this.gardenDirPath = params.gardenDirPath
this.log = params.log
this.gardenInitLog = params.gardenInitLog
this.artifactsPath = params.artifactsPath
this.vcsInfo = params.vcsInfo
this.opts = params.opts
this.rawOutputs = params.outputs
this.production = params.production
this.projectConfig = params.projectConfig
this.projectName = params.projectName
this.projectRoot = params.projectRoot
this.projectSources = params.projectSources || []
this.projectApiVersion = params.projectApiVersion
this.providerConfigs = params.providerConfigs
this.variables = params.variables
this.variableOverrides = params.variableOverrides
this.secrets = params.secrets
this.workingCopyId = params.workingCopyId
this.dotIgnoreFile = params.dotIgnoreFile
this.proxy = params.proxy
this.moduleIncludePatterns = params.moduleIncludePatterns
this.moduleExcludePatterns = params.moduleExcludePatterns || []
this.persistent = !!params.opts.persistent
this.username = params.username
this.forceRefresh = !!params.forceRefresh
this.cloudApi = params.cloudApi || null
this.commandInfo = params.opts.commandInfo
this.treeCache = params.cache
this.isGarden = true
this.configTemplates = {}
this.emittedWarnings = new Set()
this.state = { configsScanned: false, needsReload: false }
this.nestedSessions = new Map()
this.asyncLock = new AsyncLock()
const gitMode = gardenEnv.GARDEN_GIT_SCAN_MODE || params.projectConfig.scan?.git?.mode
const handlerCls = gitMode === "repo" ? GitRepoHandler : GitHandler
this.vcs = new handlerCls({
garden: this,
projectRoot: params.projectRoot,
gardenDirPath: params.gardenDirPath,
ignoreFile: params.dotIgnoreFile,
cache: params.cache,
})
// Use the legacy build sync mode if
// A) GARDEN_LEGACY_BUILD_STAGE=true is set or
// B) if running Windows and GARDEN_EXPERIMENTAL_BUILD_STAGE != true (until #2299 is properly fixed)
const legacyBuildSync =
params.opts.legacyBuildSync === undefined
? gardenEnv.GARDEN_LEGACY_BUILD_STAGE || (platform() === "win32" && !gardenEnv.GARDEN_EXPERIMENTAL_BUILD_STAGE)
: params.opts.legacyBuildSync
const buildDirCls = legacyBuildSync ? BuildStagingRsync : BuildStaging
if (legacyBuildSync) {
this.log.silly(() => `Using rsync build staging mode`)
}
this.buildStaging = new buildDirCls(params.projectRoot, params.gardenDirPath)
// make sure we're on a supported platform
const currentPlatform = platform()
const currentArch = arch() as NodeJS.Architecture
if (!SUPPORTED_PLATFORMS.includes(currentPlatform)) {
throw new RuntimeError({
message: `Unsupported platform: ${currentPlatform}`,
})
}
if (!SUPPORTED_ARCHITECTURES.includes(currentArch)) {
throw new RuntimeError({
message: `Unsupported CPU architecture: ${currentArch}`,
})
}
this.state.configsScanned = false
// TODO: Support other VCS options.
this.localConfigStore = params.localConfigStore || new LocalConfigStore(this.gardenDirPath)
this.globalConfigStore = params.globalConfigStore || new GlobalConfigStore()
this.actionConfigs = {
Build: {},
Deploy: {},
Run: {},
Test: {},
}
this.actionTypeBases = {
Build: {},
Deploy: {},
Run: {},
Test: {},
}
this.moduleConfigs = {}
this.workflowConfigs = {}
this.configPaths = new Set<string>()
this.registeredPlugins = [...getBuiltinPlugins(), ...params.plugins]
this.resolvedProviders = {}
this.events = new EventBus({ gardenKey: this.getInstanceKey() })
// TODO: actually resolve version, based on the VCS version of the plugin and its dependencies
this.version = getPackageVersion()
this.monitors = params.monitors || new MonitorManager(this.log, this.events)
this.solver = new GraphSolver(this)
// In order not to leak memory, we should ensure that there's always a collector for the OTEL data
// Here we check if the otel-collector was configured and we set a NoOp exporter if it was not
// This is of course not entirely ideal since this puts into this class some level of coupling
// with the plugin based otel-collector.
// Since we don't have the ability to hook into the post provider init stage from within the provider plugin
// especially because it's the absence of said provider that needs to trigger this case,
// there isn't really a cleaner way around this for now.
const providerConfigs = this.getRawProviderConfigs()
const hasOtelCollectorProvider = providerConfigs.some((providerConfig) => {
return providerConfig.name === "otel-collector"
})
if (!hasOtelCollectorProvider) {
this.log.silly(() => "No OTEL collector configured, setting no-op exporter")
configureNoOpExporter()
}
}
static async factory<T extends typeof Garden>(
this: T,
currentDirectory: string,
opts: GardenOpts
): Promise<InstanceType<T>> {
const garden = new this(await resolveGardenParams(currentDirectory, opts)) as InstanceType<T>
// Make sure the project root is in a git repo
await garden.getRepoRoot()
return garden
}
/**
* Clean up before shutting down.
*/
close() {
this.events.removeAllListeners()
Watcher.getInstance({ log: this.log }).unsubscribe(this.events)
}
/**
* Returns a shallow clone of this instance. Useful if you need to override properties for a specific context.
*/
clone(): Garden {
return Object.assign(Object.create(Object.getPrototypeOf(this)), this)
}
cloneForCommand(sessionId: string, cloudApi?: CloudApi): Garden {
// Make an instance clone to override anything that needs to be scoped to a specific command run
// TODO: this could be made more elegant
const garden = this.clone()
const parentSessionId = this.sessionId
this.nestedSessions.set(sessionId, garden)
garden.sessionId = sessionId
garden.log = garden.log.createLog()
garden.log.context.sessionId = sessionId
garden.log.context.parentSessionId = parentSessionId
if (cloudApi) {
garden.cloudApi = cloudApi
}
const parentEvents = garden.events
garden.events = new EventBus({ gardenKey: garden.getInstanceKey(), sessionId })
// We make sure events emitted in the context of the command are forwarded to the parent Garden event bus.
garden.events.onAny((name, payload) => {
parentEvents.emit(name, payload)
})
return garden
}
needsReload(v?: true) {
if (v) {
this.state.needsReload = true
}
return this.state.needsReload
}
/**
* Get the repository root for the project.
*/
async getRepoRoot() {
return this.vcs.getRepoRoot(this.log, this.projectRoot)
}
/**
* Returns a new PluginContext, i.e. the `ctx` object that's passed to plugin handlers.
*
* The object contains a helper to resolve template strings. By default the templating context is set to the
* provider template context. Callers should specify the appropriate templating for the handler that will be
* called with the PluginContext.
*/
async getPluginContext({
provider,
templateContext,
events,
}: {
provider: Provider
templateContext: ConfigContext | undefined
events: PluginEventBroker | undefined
}) {
return createPluginContext({
garden: this,
provider,
command: this.opts.commandInfo,
templateContext: templateContext || new ProviderConfigContext(this, provider.dependencies, this.variables),
events,
})
}
getProjectConfigContext() {
const loggedIn = this.isLoggedIn()
const enterpriseDomain = this.cloudApi?.domain
return new ProjectConfigContext({ ...this, loggedIn, enterpriseDomain })
}
async clearBuilds() {
return this.buildStaging.clear()
}
clearCaches() {
this.treeCache.clear()
this.solver.clearCache()
}
async emitWarning({ key, log, message }: { key: string; log: Log; message: string }) {
await this.asyncLock.acquire("emitWarning", async () => {
// Only emit a warning once per instance
if (this.emittedWarnings.has(key)) {
return
}
const existing = await this.localConfigStore.get("warnings", key)
if (!existing || !existing.hidden) {
this.emittedWarnings.add(key)
log.warn(message + `\n→ Run ${styles.underline(`garden util hide-warning ${key}`)} to disable this warning.`)
}
})
}
async hideWarning(key: string) {
await this.localConfigStore.set("warnings", key, { hidden: true })
}
@pMemoizeDecorator()
async getAnalyticsHandler() {
return AnalyticsHandler.init(this, this.log)
}
// TODO: would be nice if this returned a type based on the input tasks
async processTasks(params: SolveParams): Promise<SolveResult> {
return this.solver.solve(params)
}
async processTask<T extends Task>(task: T, log: Log, opts: SolveOpts): Promise<GraphResultFromTask<T> | null> {
const { results } = await this.solver.solve({ tasks: [task], log, ...opts })
return results.getResult(task)
}
/**
* Subscribes to watch events for config paths.
*/
watchPaths() {
const watcher = Watcher.getInstance({ log: this.log })
watcher.unsubscribe(this.events)
watcher.subscribe(this.events, [
// Watch config files
...Array.from(this.configPaths.values()).map((path) => ({ type: "config" as const, path })),
// TODO: watch source directories when on Windows or Mac (watching on linux is too expensive atm)
])
}
getProjectConfig() {
return this.projectConfig
}
async getRegisteredPlugins(): Promise<GardenPluginSpec[]> {
return Promise.all(this.registeredPlugins.map((p) => loadPlugin(this.log, this.projectRoot, p)))
}
@pMemoizeDecorator()
async getPlugin(pluginName: string): Promise<GardenPluginSpec> {
const plugins = await this.getAllPlugins()
const plugin = findByName(plugins, pluginName)
if (!plugin) {
const availablePlugins = getNames(plugins)
throw new PluginError({
message: dedent`
Could not find plugin '${pluginName}'. Are you missing a provider configuration?
Currently configured plugins: ${availablePlugins.join(", ")}`,
})
}
return plugin
}
/**
* Returns all registered plugins, loading them if necessary.
*/
@pMemoizeDecorator()
@OtelTraced({
name: "loadPlugins",
})
async getAllPlugins() {
// The duplicated check is a small optimization to avoid the async lock when possible,
// since this is called quite frequently.
if (this.loadedPlugins) {
return this.loadedPlugins
}
return this.asyncLock.acquire("load-plugins", async () => {
// This check is necessary since we could in theory have two calls waiting for the lock at the same time.
if (this.loadedPlugins) {
return this.loadedPlugins
}
this.log.silly(() => `Loading plugins`)
const rawConfigs = this.getRawProviderConfigs()
this.loadedPlugins = await loadAndResolvePlugins(this.log, this.projectRoot, this.registeredPlugins, rawConfigs)
this.log.silly(`Loaded plugins: ${this.loadedPlugins.map((c) => c.name).join(", ")}`)
return this.loadedPlugins
})
}
/**
* Returns plugins that are currently configured in provider configs.
*/
@pMemoizeDecorator()
async getConfiguredPlugins() {
const plugins = await this.getAllPlugins()
const configNames = keyBy(this.getRawProviderConfigs(), "name")
return plugins.filter((p) => configNames[p.name])
}
/**
* Returns a mapping of all configured module types in the project and their definitions.
*/
@pMemoizeDecorator()
async getModuleTypes(): Promise<ModuleTypeMap> {
const configuredPlugins = await this.getConfiguredPlugins()
return getModuleTypes(configuredPlugins)
}
/**
* Returns a mapping of all configured action types in the project and their definitions.
*/
@pMemoizeDecorator()
async getActionTypes(): Promise<ActionDefinitionMap> {
const configuredPlugins = await this.getConfiguredPlugins()
return getActionTypes(configuredPlugins)
}
/**
* Get the bases for the given action kind/type, with schemas modified to allow any unknown fields.
* Used to validate actions whose types inherit from others.
*
* Implemented here so that we can cache the modified schemas.
*/
async getActionTypeBases(kind: ActionKind, type: string) {
const definitions = await this.getActionTypes()
if (this.actionTypeBases[kind][type]) {
return this.actionTypeBases[kind][type] || []
}
const bases = getActionTypeBases(definitions[kind][type].spec, definitions[kind])
this.actionTypeBases[kind][type] = bases.map((b) => ({ ...b, schema: allowUnknown(b.schema) }))
return this.actionTypeBases[kind][type] || []
}
getRawProviderConfigs({ names, allowMissing = false }: { names?: string[]; allowMissing?: boolean } = {}) {
return names
? findByNames({ names, entries: this.providerConfigs, description: "provider", allowMissing })
: this.providerConfigs
}
async resolveProvider(log: Log, name: string) {
if (name === "_default") {
return defaultProvider
}
if (this.resolvedProviders[name]) {
return cloneDeep(this.resolvedProviders[name])
}
this.log.silly(() => `Resolving provider ${name}`)
const providers = await this.resolveProviders(log, false, [name])
const provider = providers[name]
if (!provider) {
const providerNames = Object.keys(providers)
throw new PluginError({
message: dedent`
Could not find provider '${name}' in environment '${this.environmentName}'
(configured providers: ${providerNames.join(", ") || "<none>"})
`,
})
}
return provider
}
@OtelTraced({
name: "resolveProviders",
})
async resolveProviders(log: Log, forceInit = false, names?: string[]): Promise<ProviderMap> {
// TODO: split this out of the Garden class
let providers: Provider[] = []
await this.asyncLock.acquire("resolve-providers", async () => {
const rawConfigs = this.getRawProviderConfigs({ names })
if (!names) {
names = getNames(rawConfigs)
}
throwOnMissingSecretKeys(rawConfigs, this.secrets, "Provider", log)
// As an optimization, we return immediately if all requested providers are already resolved
const alreadyResolvedProviders = names.map((name) => this.resolvedProviders[name]).filter(Boolean)
if (alreadyResolvedProviders.length === names.length) {
providers = cloneDeep(alreadyResolvedProviders)
return
}
const providerLog = log.createLog({ name: "providers", showDuration: true })
if (this.forceRefresh) {
providerLog.info("Resolving providers (will force refresh statuses)...")
} else {
providerLog.info("Resolving providers...")
}
const plugins = keyBy(await this.getAllPlugins(), "name")
// Detect circular dependencies here
const validationGraph = new DependencyGraph()
await Promise.all(
rawConfigs.map(async (config) => {
const plugin = plugins[config.name]
if (!plugin) {
throw new ConfigurationError({
message: dedent`
Configured provider '${config.name}' has not been registered.
Available plugins: ${Object.keys(plugins).join(", ")}
`,
})
}
validationGraph.addNode(plugin.name)
for (const dep of await getAllProviderDependencyNames(plugin!, config!)) {
validationGraph.addNode(dep)
validationGraph.addDependency(plugin.name, dep)
}
})
)
const cycles = validationGraph.detectCircularDependencies()
if (cycles.length > 0) {
const cyclesSummary = validationGraph.cyclesToString(cycles)
throw new CircularDependenciesError({
messagePrefix: "One or more circular dependencies found between providers or their configurations",
cycles,
cyclesSummary,
})
}
const tasks = rawConfigs.map((config) => {
const plugin = plugins[config.name]
return new ResolveProviderTask({
garden: this,
log: providerLog,
plugin,
config,
force: false,
forceRefresh: this.forceRefresh,
forceInit,
allPlugins: Object.values(plugins),
})
})
// Process as many providers in parallel as possible
const taskResults = await this.processTasks({ tasks, log })
const providerResults = Object.values(taskResults.results.getMap())
const failed = providerResults.filter((r) => r && r.error)
if (failed.length) {
const failedNames = failed.map((r) => r!.name)
const wrappedErrors: GardenError[] = failed.flatMap((f) => {
return f && f.error ? [toGardenError(f.error)] : []
})
// we do not include the error messages in the message, because we already log those errors in the solver.
throw new PluginError({
message: `Failed resolving one or more providers:\n- ${failedNames.join("\n- ")}`,
wrappedErrors,
})
}
providers = providerResults.map((result) => result!.result)
const allCached = providers.every((p) => p.status.cached)
const someCached = providers.some((p) => p.status.cached)
await Promise.all(
providers.flatMap((provider) =>
provider.moduleConfigs.map(async (moduleConfig) => {
// Make sure module and all nested entities are scoped to the plugin
moduleConfig.plugin = provider.name
return this.addModuleConfig(moduleConfig)
})
)
)
for (const provider of providers) {
this.resolvedProviders[provider.name] = provider
}
providerLog.success("Finished resolving providers")
if (someCached || allCached) {
const msg = allCached ? "All" : "Some"
providerLog.info(
`${msg} provider statuses were cached. Run with --force-refresh to force a refresh of provider statuses.`
)
}
providerLog.silly(() => `Resolved providers: ${providers.map((p) => p.name).join(", ")}`)
})
return keyBy(providers, "name")
}
@pMemoizeDecorator()
async getTools() {
if (!this.tools) {
const plugins = await this.getAllPlugins()
const tools: PluginTools = {}
for (const plugin of Object.values(plugins)) {
for (const tool of plugin.tools || []) {
tools[`${plugin.name}.${tool.name}`] = new PluginTool(tool)
}
}
this.tools = tools
}
return this.tools
}
/**
* When running workflows via the `workflow` command, we only resolve the workflow being executed.
*/
async getWorkflowConfig(name: string): Promise<WorkflowConfig> {
return resolveWorkflowConfig(this, await this.getRawWorkflowConfig(name))
}
async getRawWorkflowConfig(name: string): Promise<WorkflowConfig> {
return (await this.getRawWorkflowConfigs([name]))[0]
}
async getRawWorkflowConfigs(names?: string[]): Promise<WorkflowConfig[]> {
if (!this.state.configsScanned) {
await this.scanAndAddConfigs()
}
if (names) {
return Object.values(pickKeys(this.workflowConfigs, names, "workflow"))
} else {
return Object.values(this.workflowConfigs)
}
}
/**
* Returns the reported status from all configured providers.
*/
async getEnvironmentStatus(log: Log) {
const providers = await this.resolveProviders(log)
return mapValues(providers, (p) => p.status)
}
@pMemoizeDecorator()
async getActionRouter() {
const loadedPlugins = await this.getAllPlugins()
const moduleTypes = await this.getModuleTypes()
const plugins = keyBy(loadedPlugins, "name")
// We only pass configured plugins to the router (others won't have the required configuration to call handlers)
const configuredPlugins = this.getRawProviderConfigs().map((c) => plugins[c.name])
return new ActionRouter(this, configuredPlugins, loadedPlugins, moduleTypes)
}
/**
* Returns module configs that are registered in this context, before template resolution and validation.
* Scans for modules in the project root and remote/linked sources if it hasn't already been done.
*/
async getRawModuleConfigs(keys?: string[]): Promise<ModuleConfig[]> {
if (!this.state.configsScanned) {
await this.scanAndAddConfigs()
}
return Object.values(keys ? pickKeys(this.moduleConfigs, keys, "module config") : this.moduleConfigs)
}
/**
* Returns action configs that are registered in this context, before template resolution and validation.
* Scans for configs in the project root and remote/linked sources if it hasn't already been done.
*/
async getRawActionConfigs() {
if (!this.state.configsScanned) {
await this.scanAndAddConfigs()
}
return this.actionConfigs
}
async getOutputConfigContext(log: Log, modules: GardenModule[], graphResults: GraphResults) {
const providers = await this.resolveProviders(log)
return new OutputConfigContext({
garden: this,
resolvedProviders: providers,
variables: this.variables,
modules,
graphResults,
partialRuntimeResolution: false,
})
}
/**
* Resolve the raw module and action configs and return a new instance of ConfigGraph.
* The graph instance is immutable and represents the configuration at the point of calling this method.
* For long-running processes, you need to call this again when any module or configuration has been updated.
*
* If `emit = true` is passed, a `stackGraph` event with a rendered DAG representation of the graph will be emitted.
* When implementing a new command that calls this method and also streams events, make sure that the first
* call to `getConfigGraph` in the command uses `emit = true` to ensure that the graph event gets streamed.
*/
@OtelTraced({
name: "getConfigGraph",
})
async getConfigGraph({ log, graphResults, emit, actionModes = {} }: GetConfigGraphParams): Promise<ConfigGraph> {
// TODO: split this out of the Garden class
await this.scanAndAddConfigs()
const resolvedProviders = await this.resolveProviders(log)
const rawModuleConfigs = await this.getRawModuleConfigs()
const graphLog = log.createLog({ name: "graph", showDuration: true }).info(`Resolving actions and modules...`)
// Resolve the project module configs
const resolver = new ModuleResolver({
garden: this,
log: graphLog,
rawConfigs: rawModuleConfigs,
resolvedProviders,
graphResults,
})
const resolvedModules = await resolver.resolveAll()
// Validate the module dependency structure. This will throw on failure.
const router = await this.getActionRouter()
const moduleTypes = await this.getModuleTypes()
const moduleGraph = new ModuleGraph(resolvedModules, moduleTypes)
// Require include/exclude on modules if their paths overlap
const overlaps = detectModuleOverlap({