From c83f815fe1f993327e012c98739d99b3f9035934 Mon Sep 17 00:00:00 2001 From: Thorarinn Sigurdsson Date: Tue, 27 Feb 2024 14:48:29 +0100 Subject: [PATCH] feat: added profile-project command (#5780) **What this PR does / why we need it**: Added a simple utility command for profiling projects, the `profile-project` command. This command logs the number of files included in each action and module in the project (omitting actions that were converted from modules to avoid duplication), their include/exclude configs (if any), and finally the number of modules, actions and tracked files in the project. This should be useful when helping users with large projects diagnose slow init performance and how to best structure their includes, excludes and `.gardenignore`. --- core/src/commands/util/profile-project.ts | 107 ++++++++++++++++++++++ core/src/commands/util/util.ts | 3 +- core/src/vcs/git-repo.ts | 37 ++++++-- docs/reference/commands.md | 12 +++ docs/reference/dockerhub-containers.md | 1 - 5 files changed, 148 insertions(+), 12 deletions(-) create mode 100644 core/src/commands/util/profile-project.ts diff --git a/core/src/commands/util/profile-project.ts b/core/src/commands/util/profile-project.ts new file mode 100644 index 0000000000..4db8c45883 --- /dev/null +++ b/core/src/commands/util/profile-project.ts @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2018-2023 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { CommandParams, CommandResult } from "../base.js" +import { Command } from "../base.js" +import { printEmoji, printHeader, renderDivider } from "../../logger/util.js" +import { dedent } from "../../util/string.js" +import { styles } from "../../logger/styles.js" +import type { ConfigGraph } from "../../graph/config-graph.js" +import indentString from "indent-string" +import type { BaseActionConfig } from "../../actions/types.js" + +export class ProfileProjectCommand extends Command { + name = "profile-project" + help = "Renders a high-level sumamry of actions and modules in your project." + emoji = "📊" + + override description = dedent` + Useful for diagnosing slow init performance for projects with lots of actions and modules and/or lots of files. + ` + + override printHeader({ log }) { + printHeader(log, "Profile Project", "️📊") + } + + async action({ garden, log }: CommandParams): Promise { + const graph = await garden.getConfigGraph({ log, emit: false }) + summarizeGraph(log, garden, graph) + + log.info(renderDivider()) + log.info("Summary") + log.info("") + log.info("Module config count: " + styles.highlight(Object.keys(graph.moduleGraph.getModules()).length)) + const actionConfigCount = Object.values(graph.getActions()).filter( + (a) => a.getInternal().moduleName === undefined + ).length + log.info("Action config count (excluding those converted from modules): " + styles.highlight(actionConfigCount)) + const trackedFilesInProjectRoot = await garden.vcs.getFiles({ + log, + path: garden.projectRoot, + pathDescription: `project root`, + scanRoot: garden.projectRoot, + }) + log.info("Total tracked files in project root:" + styles.highlight(trackedFilesInProjectRoot.length)) + log.info("") + log.info(styles.success("OK") + " " + printEmoji("✔️", log)) + + return {} + } +} + +function summarizeGraph(log: CommandParams["log"], garden: CommandParams["garden"], graph: ConfigGraph) { + if (Object.keys(graph.moduleGraph).length > 0) { + summarizeModuleGraph(log, graph.moduleGraph) + } + summarizeActionGraph(log, graph) +} + +const indent = 2 + +function summarizeModuleGraph(log: CommandParams["log"], moduleGraph: ConfigGraph["moduleGraph"]) { + const sortedModules = Object.values(moduleGraph.getModules()).sort( + // We sort the modules by path and then name, so that modules at the same path appear together. + (m1, m2) => m1.path.localeCompare(m2.path) || m1.name.localeCompare(m2.name) + ) + for (const module of sortedModules) { + log.info("Module: " + styles.highlight(module.name) + styles.primary(" (at " + module.path + ")")) + if (module.include && module.include.length > 0) { + log.info(indentString(styles.primary("Include: " + JSON.stringify(module.include, null, 2)), indent)) + } + if (module.exclude && module.exclude.length > 0) { + log.info(indentString(styles.primary(" Exclude: " + module.exclude), indent)) + } + + log.info(indentString(styles.primary("Tracked file count: ") + module.version.files.length, indent)) + log.info("") + } +} + +function summarizeActionGraph(log: CommandParams["log"], graph: ConfigGraph) { + const sortedActions = Object.values(graph.getActions()) + // We sort the actions by path and then name, so that actions at the same path appear together. + .sort((a1, a2) => a1.sourcePath().localeCompare(a2.sourcePath()) || a1.name.localeCompare(a2.name)) + // We only want to show actions that are not converted from modules (since the file scanning cost for converted + // actions was incurred when scanning files for their parent module). + .filter((a) => a.getInternal().moduleName === undefined) + for (const action of sortedActions) { + const { include, exclude } = action._config as BaseActionConfig + log.info("Action: " + styles.highlight(action.name) + styles.primary(" (at " + action.sourcePath() + ")")) + if (action.getInternal().moduleName) { + log.info(indentString(styles.primary("From module: " + action.getInternal().moduleName), indent)) + } + if (include && include.length > 0) { + log.info(indentString(styles.primary("Include: " + JSON.stringify(include, null, 2)), indent)) + } + if (exclude && exclude.length > 0) { + log.info(indentString(styles.primary(" Exclude: " + exclude), indent)) + } + log.info(indentString(styles.primary("Tracked file count: ") + action.getFullVersion().files.length, indent)) + log.info("") + } +} diff --git a/core/src/commands/util/util.ts b/core/src/commands/util/util.ts index dbefa23f18..10838da334 100644 --- a/core/src/commands/util/util.ts +++ b/core/src/commands/util/util.ts @@ -10,10 +10,11 @@ import { CommandGroup } from "../base.js" import { FetchToolsCommand } from "./fetch-tools.js" import { HideWarningCommand } from "./hide-warning.js" import { MutagenCommand } from "./mutagen.js" +import { ProfileProjectCommand } from "./profile-project.js" export class UtilCommand extends CommandGroup { name = "util" help = "Misc utility commands." - subCommands = [FetchToolsCommand, HideWarningCommand, MutagenCommand] + subCommands = [FetchToolsCommand, HideWarningCommand, MutagenCommand, ProfileProjectCommand] } diff --git a/core/src/vcs/git-repo.ts b/core/src/vcs/git-repo.ts index 6eea2c30da..4b75e4c1a0 100644 --- a/core/src/vcs/git-repo.ts +++ b/core/src/vcs/git-repo.ts @@ -53,6 +53,7 @@ const getIncludeExcludeFiles: IncludeExcludeFilesHandler `Include globs: ${augmentedIncludes.join(", ")}`) - log.silly(() => + log.debug(() => `Include globs: ${augmentedIncludes.join(", ")}`) + log.debug(() => augmentedExcludes.length > 0 ? `Exclude globs: ${augmentedExcludes.join(", ")}` : "No exclude globs" ) - const filtered = filesAtPath.filter(({ path: p }) => { + const filtered = this.filterPaths({ files: filesAtPath, log, path, augmentedIncludes, augmentedExcludes, filter }) + log.debug(`Found ${filtered.length} files in path ${path} after glob matching`) + this.cache.set(log, filteredFilesCacheKey, filtered, pathToCacheContext(path)) + + return filtered + } + + filterPaths({ + log, + files, + path, + augmentedIncludes, + augmentedExcludes, + filter, + }: { + log: GetFilesParams["log"] + files: VcsFile[] + path: string + augmentedIncludes: string[] + augmentedExcludes: string[] + filter: GetFilesParams["filter"] + }): VcsFile[] { + return files.filter(({ path: p }) => { if (filter && !filter(p)) { return false } @@ -131,12 +154,6 @@ export class GitRepoHandler extends GitHandler { log.silly(() => `Checking if ${relativePath} matches include/exclude globs`) return matchPath(relativePath, augmentedIncludes, augmentedExcludes) }) - - log.debug(`Found ${filtered.length} files in module path after glob matching`) - - this.cache.set(log, filteredFilesCacheKey, filtered, pathToCacheContext(path)) - - return filtered } /** diff --git a/docs/reference/commands.md b/docs/reference/commands.md index fe746968a5..9a3810b9c6 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -5852,6 +5852,18 @@ Examples: +### garden util profile-project + +**Renders a high-level sumamry of actions and modules in your project.** + +Useful for diagnosing slow init performance for projects with lots of actions and modules and/or lots of files. + +#### Usage + + garden util profile-project + + + ### garden validate **Check your garden configuration for errors.** diff --git a/docs/reference/dockerhub-containers.md b/docs/reference/dockerhub-containers.md index c87c6a56c6..198155dc00 100644 --- a/docs/reference/dockerhub-containers.md +++ b/docs/reference/dockerhub-containers.md @@ -19,7 +19,6 @@ For your convenience, we build and publish Docker containers that contain the Ga | [`gardendev/garden-gcloud`](https://hub.docker.com/r/gardendev/garden-gcloud) | Contains the Garden CLI, and the Google Cloud CLI | | [`gardendev/garden-aws-gcloud`](https://hub.docker.com/r/gardendev/garden-aws-gcloud) | Contains the Garden CLI, the Google Cloud CLI and the AWS CLI v2 | | [`gardendev/garden-aws-gcloud-azure`](https://hub.docker.com/r/gardendev/garden-aws-gcloud-azure) | Contains the Garden CLI, the Google Cloud CLI, the AWS CLI v2, and the Azure CLI | -| [`gardendev/garden-full`](https://hub.docker.com/r/gardendev/garden-full) | DEPRECATED: This container image is not being maintained anymore and will be removed in the future. | ### Tags