-
Notifications
You must be signed in to change notification settings - Fork 266
/
vcs.ts
476 lines (412 loc) · 15.3 KB
/
vcs.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
/*
* Copyright (C) 2018-2023 Garden Technologies, Inc. <info@garden.io>
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
import Joi from "@hapi/joi"
import normalize from "normalize-path"
import { sortBy, pick } from "lodash"
import { createHash } from "crypto"
import { validateSchema } from "../config/validation"
import { join, relative, isAbsolute, sep } from "path"
import { DOCS_BASE_URL, GARDEN_VERSIONFILE_NAME as GARDEN_TREEVERSION_FILENAME } from "../constants"
import { pathExists, readFile, writeFile } from "fs-extra"
import { ConfigurationError } from "../exceptions"
import { ExternalSourceType, getRemoteSourceLocalPath, getRemoteSourcesPath } from "../util/ext-source-util"
import { ModuleConfig, serializeConfig } from "../config/module"
import type { Log } from "../logger/log-entry"
import { treeVersionSchema } from "../config/common"
import { dedent, splitLast } from "../util/string"
import { fixedProjectExcludes } from "../util/fs"
import { pathToCacheContext, TreeCache } from "../cache"
import type { ServiceConfig } from "../config/service"
import type { TaskConfig } from "../config/task"
import type { TestConfig } from "../config/test"
import type { GardenModule } from "../types/module"
import { validateInstall } from "../util/validateInstall"
import { getSourceAbsPath, isActionConfig } from "../actions/base"
import type { BaseActionConfig } from "../actions/types"
import { Garden } from "../garden"
import chalk from "chalk"
import { Profile } from "../util/profiling"
import AsyncLock = require("async-lock")
const scanLock = new AsyncLock()
export const versionStringPrefix = "v-"
export const NEW_RESOURCE_VERSION = "0000000000"
const fileCountWarningThreshold = 10000
const minGitVersion = "2.14.0"
export const gitVersionRegex = /git\s+version\s+v?(\d+.\d+.\d+)/
/**
* throws if no git is installed or version is too old
*/
export async function validateGitInstall() {
await validateInstall({
minVersion: minGitVersion,
name: "git",
versionCommand: { cmd: "git", args: ["--version"] },
versionRegex: gitVersionRegex,
})
}
export interface TreeVersion {
contentHash: string
/**
* Important! Do not use the files to determine if a file will exist when performing an action.
* Other mechanisms, e.g. the build command itself and `copyFrom` might affect available files at runtime.
*
* See also https://github.com/garden-io/garden/issues/5201
*/
files: string[]
}
export interface TreeVersions {
[moduleName: string]: TreeVersion
}
// TODO: rename, maybe to ResourceVersion
export interface ModuleVersion extends TreeVersion {
versionString: string
dependencyVersions: DependencyVersions
}
export interface ActionVersion {
versionString: string
dependencyVersions: DependencyVersions
configVersion: string
sourceVersion: string
files: string[]
}
export interface NamedModuleVersion extends ModuleVersion {
name: string
}
export interface DependencyVersions {
[key: string]: string
}
export interface NamedTreeVersion extends TreeVersion {
name: string
}
export interface VcsInfo {
branch: string
commitHash: string
originUrl: string
}
export interface GetFilesParams {
log: Log
path: string
pathDescription?: string
include?: string[]
exclude?: string[]
filter?: (path: string) => boolean
failOnPrompt?: boolean
scanRoot: string | undefined
}
export interface GetTreeVersionParams {
log: Log
projectName: string
config: ModuleConfig | BaseActionConfig
scanRoot?: string // Set the scanning root instead of detecting, in order to optimize the scanning.
}
export interface RemoteSourceParams {
url: string
name: string
sourceType: ExternalSourceType
log: Log
failOnPrompt?: boolean
}
export interface VcsFile {
path: string
hash: string
}
export interface VcsHandlerParams {
garden?: Garden
projectRoot: string
gardenDirPath: string
ignoreFile: string
cache: TreeCache
}
@Profile()
export abstract class VcsHandler {
protected garden?: Garden
protected projectRoot: string
protected gardenDirPath: string
protected ignoreFile: string
protected cache: TreeCache
constructor(params: VcsHandlerParams) {
this.garden = params.garden
this.projectRoot = params.projectRoot
this.gardenDirPath = params.gardenDirPath
this.ignoreFile = params.ignoreFile
this.cache = params.cache
}
abstract name: string
abstract getRepoRoot(log: Log, path: string): Promise<string>
abstract getFiles(params: GetFilesParams): Promise<VcsFile[]>
abstract ensureRemoteSource(params: RemoteSourceParams): Promise<string>
abstract updateRemoteSource(params: RemoteSourceParams): Promise<void>
abstract getPathInfo(log: Log, path: string): Promise<VcsInfo>
clearTreeCache() {
this.cache.clear()
}
async getTreeVersion({
log,
projectName,
config,
force = false,
scanRoot,
}: {
log: Log
projectName: string
config: ModuleConfig | BaseActionConfig
force?: boolean
scanRoot?: string
}): Promise<TreeVersion> {
const cacheKey = getResourceTreeCacheKey(config)
const description = describeConfig(config)
// Note: duplicating this as an optimization (avoid the async lock)
if (!force) {
const cached = this.cache.get(log, cacheKey)
if (cached) {
log.silly(`Got cached tree version for ${description} (key ${cacheKey})`)
return cached
}
}
const configPath = getConfigFilePath(config)
const path = getSourcePath(config)
let result: TreeVersion = { contentHash: NEW_RESOURCE_VERSION, files: [] }
// Make sure we don't concurrently scan the exact same context
await scanLock.acquire(cacheKey.join(":"), async () => {
if (!force) {
const cached = this.cache.get(log, cacheKey)
if (cached) {
log.silly(`Got cached tree version for ${description} (key ${cacheKey})`)
result = cached
return
}
}
// Apply project root excludes if the module config is in the project root and `include` isn't set
const exclude =
path === this.projectRoot && !config.include
? [...(config.exclude || []), ...fixedProjectExcludes]
: config.exclude
// No need to scan for files if nothing should be included
if (!(config.include && config.include.length === 0)) {
let files = await this.getFiles({
log,
path,
pathDescription: description + " root",
include: config.include,
exclude,
scanRoot,
})
if (files.length > fileCountWarningThreshold) {
// TODO-0.13.0: This will be repeated for modules and actions resulting from module conversion
await this.garden?.emitWarning({
key: `${projectName}-filecount-${config.name}`,
log,
message: chalk.yellow(dedent`
Large number of files (${files.length}) found in ${description}. You may need to configure file exclusions.
See ${DOCS_BASE_URL}/using-garden/configuration-overview#including-excluding-files-and-directories for details.
`),
})
}
files = sortBy(files, "path")
// Don't include the config file in the file list
.filter((f) => !configPath || f.path !== configPath)
// compute hash using <file-relative-path>-<file-hash> to cater for path changes (e.g. renaming)
result.contentHash = hashStrings(files.map((f) => `${relative(this.projectRoot, f.path)}-${f.hash}`))
result.files = files.map((f) => f.path)
}
this.cache.set(log, cacheKey, result, pathToCacheContext(path))
})
return result
}
/**
* Write a file and ensure relevant caches are invalidated after writing.
*/
async writeFile(log: Log, path: string, data: string | Buffer) {
await writeFile(path, data)
this.cache.invalidateUp(log, pathToCacheContext(path))
}
async resolveTreeVersion(params: GetTreeVersionParams): Promise<TreeVersion> {
// the version file is used internally to specify versions outside of source control
const path = getSourcePath(params.config)
const versionFilePath = join(path, GARDEN_TREEVERSION_FILENAME)
const fileVersion = await readTreeVersionFile(versionFilePath)
return fileVersion || (await this.getTreeVersion(params))
}
/**
* Returns a map of the optimal paths for each of the given action/module source path.
* This is used to avoid scanning more of each git repository than necessary, and
* reduces duplicate scanning of the same directories (since fewer unique roots mean
* more tree cache hits).
*/
async getMinimalRoots(log: Log, paths: string[]) {
const repoRoots: { [path: string]: string } = {}
const outputs: { [path: string]: string } = {}
const rootsToPaths: { [repoRoot: string]: string[] } = {}
await Promise.all(
paths.map(async (path) => {
const repoRoot = await this.getRepoRoot(log, path)
repoRoots[path] = repoRoot
if (rootsToPaths[repoRoot]) {
rootsToPaths[repoRoot].push(path)
} else {
rootsToPaths[repoRoot] = [path]
}
})
)
for (const path of paths) {
const repoRoot = repoRoots[path]
const repoPaths = rootsToPaths[repoRoot]
for (const repoPath of repoPaths) {
if (!outputs[path]) {
// No path set so far
outputs[path] = repoPath
} else if (outputs[path].startsWith(repoPath)) {
// New path is prefix of prior path
outputs[path] = repoPath
} else {
// Find common prefix
let p = repoPath
while (true) {
p = splitLast(p, sep)[0]
if (p.length < repoRoot.length) {
// Don't go past the actual git repo root
outputs[path] = repoRoot
break
} else if (outputs[path].startsWith(p)) {
// Found a common prefix
outputs[path] = p
break
}
}
}
}
}
return outputs
}
/**
* Returns the absolute path to the local directory for all remote sources
*/
getRemoteSourcesLocalPath(type: ExternalSourceType) {
return getRemoteSourcesPath({ gardenDirPath: this.gardenDirPath, type })
}
/**
* Returns the absolute path to the local directory for the remote source
*/
getRemoteSourceLocalPath(name: string, url: string, type: ExternalSourceType) {
return getRemoteSourceLocalPath({ gardenDirPath: this.gardenDirPath, name, url, type })
}
}
async function readVersionFile(path: string, schema: Joi.Schema): Promise<any> {
if (!(await pathExists(path))) {
return null
}
// this is used internally to specify version outside of source control
const versionFileContents = (await readFile(path)).toString().trim()
if (!versionFileContents) {
return null
}
try {
return validateSchema(JSON.parse(versionFileContents), schema)
} catch (error) {
throw new ConfigurationError({
message: `Unable to parse ${path} as valid version file: ${error}`,
})
}
}
export async function readTreeVersionFile(path: string): Promise<TreeVersion | null> {
return readVersionFile(path, treeVersionSchema())
}
/**
* Writes a normalized TreeVersion file to the specified directory
*
* @param dir The directory to write the file to
* @param version The TreeVersion for the directory
*/
export async function writeTreeVersionFile(dir: string, version: TreeVersion) {
const processed = {
...version,
files: version.files
// Always write relative paths, normalized to POSIX style
.map((f) => normalize(isAbsolute(f) ? relative(dir, f) : f))
.filter((f) => f !== GARDEN_TREEVERSION_FILENAME),
}
const path = join(dir, GARDEN_TREEVERSION_FILENAME)
await writeFile(path, JSON.stringify(processed, null, 4) + "\n")
}
/**
* We prefix with "v-" to prevent this.version from being read as a number when only a prefix of the
* commit hash is used, and that prefix consists of only numbers. This can cause errors in certain contexts
* when the version string is used in template variables in configuration files.
*/
export function getModuleVersionString(
moduleConfig: ModuleConfig,
treeVersion: NamedTreeVersion,
dependencyModuleVersions: NamedModuleVersion[]
) {
// TODO: allow overriding the prefix
return `${versionStringPrefix}${hashModuleVersion(moduleConfig, treeVersion, dependencyModuleVersions)}`
}
/**
* Compute the version of the given module, based on its configuration and the versions of its build dependencies.
* The versions argument should consist of moduleConfig's tree version, and the tree versions of its dependencies.
*/
export function hashModuleVersion(
moduleConfig: ModuleConfig,
treeVersion: NamedTreeVersion,
dependencyModuleVersions: NamedModuleVersion[]
) {
// If a build config is provided, we use that.
// Otherwise, we use the full module config, omitting the configPath, path, and outputs fields, as well as individual
// entity configuration fields, as these often vary between environments and runtimes but are unlikely to impact the
// build output.
const configToHash =
moduleConfig.buildConfig ||
pick(moduleConfig, ["apiVersion", "name", "spec", "type", "variables", "varfile", "inputs"])
const configString = serializeConfig(configToHash)
const versionStrings = sortBy(
[[treeVersion.name, treeVersion.contentHash], ...dependencyModuleVersions.map((v) => [v.name, v.versionString])],
(vs) => vs[0]
).map((vs) => vs[1])
return hashStrings([configString, ...versionStrings])
}
/**
* Return the version string for the given Stack Graph entity (i.e. service, task or test).
* It is simply a hash of the module version and the configuration of the entity.
*
* @param module The module containing the entity in question
* @param entityConfig The configuration of the entity
*/
export function getEntityVersion(module: GardenModule, entityConfig: ServiceConfig | TaskConfig | TestConfig) {
const configString = serializeConfig(entityConfig)
return `${versionStringPrefix}${hashStrings([module.version.versionString, configString])}`
}
export function hashStrings(hashes: string[]) {
const versionHash = createHash("sha256")
versionHash.update(hashes.join("."))
return versionHash.digest("hex").slice(0, 10)
}
export function getResourceTreeCacheKey(config: ModuleConfig | BaseActionConfig) {
const cacheKey = ["source", getSourcePath(config)]
if (config.include) {
cacheKey.push("include", hashStrings(config.include.sort()))
}
if (config.exclude) {
cacheKey.push("exclude", hashStrings(config.exclude.sort()))
}
return cacheKey
}
export function getConfigFilePath(config: ModuleConfig | BaseActionConfig) {
return isActionConfig(config) ? config.internal?.configFilePath : config.configPath
}
export function getSourcePath(config: ModuleConfig | BaseActionConfig) {
if (isActionConfig(config)) {
const basePath = config.internal.basePath
const sourceRelPath = config.source?.path
return sourceRelPath ? getSourceAbsPath(basePath, sourceRelPath) : basePath
} else {
return config.path
}
}
export function describeConfig(config: ModuleConfig | BaseActionConfig) {
return isActionConfig(config) ? `${config.kind} action ${config.name}` : `module ${config.name}`
}