Add needs option for actions

.vscode/launch.json

@@ -0,0 +1,19 @@
+{
+	// Use IntelliSense to learn about possible attributes.
+	// Hover to view descriptions of existing attributes.
+	// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+	"version": "0.2.0",
+	"configurations": [
+		{
+			"name": "Run ts-node debug",
+			"type": "node",
+			"request": "launch",
+			"runtimeExecutable": "node",
+			"runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
+			"skipFiles": ["<node_internals>/**", "node_modules/**"],
+			"internalConsoleOptions": "openOnSessionStart",
+			"cwd": "${workspaceRoot}",
+			"args": ["bin/main.ts", "run", "down", "ht", "--cwd", "../gldtest/"]
+		}
+	]
+}

README.md

@@ -72,3 +72,17 @@ All projects on custom branch, will attempt to rebase `origin/<default_branch>`
 After git operations are done, scripts matching cli inputs will be executed.
 
 In this example only `"bash", "-c", "start-docker-stack.sh"` will be executed in `~/git-local-devops/cego/example` checkout
+
+## Execution order
+
+You may specify either a priority or a needs array for each action, but never both.
+The needs array must point to other project names and must be acyclic.
+
+If there is no priority or needs, the action has a default priority of 0.
+
+Execution order is as follows:
+
+1. Execute the lowest priority actions (Will execute in parallel if same priority)
+2. When these actions finish, remove their needs from other action that needs these actions
+   - If this result in an action with an empty needs array, it will start execution of that action, then go back to step 2.
+3. Remove the lowest priority actions and go back to step 1.

src/actions.ts

@@ -1,82 +1,138 @@
 import { getProjectDirFromRemote } from "./project";
 import chalk from "chalk";
-import { default as to } from "await-to-js";
-import { Config } from "./types/config";
+import { Config, ProjectAction } from "./types/config";
 import * as pcp from "promisify-child-process";
-import { GroupKey, ToChildProcessOutput } from "./types/utils";
-import { getPriorityRange } from "./priority";
-import { searchOutputForHints } from "./search_output";
+import { GroupKey } from "./types/utils";
+import { logActionOutput, searchOutputForHints } from "./search_output";
 import { printHeader } from "./utils";
+import { getProgressBar, waitingOnToString } from "./progress";
+import { SingleBar } from "cli-progress";
+import { topologicalSortActionGraph } from "./graph";
+
+export type ActionOutput = GroupKey & pcp.Output & { dir: string; cmd: string[] };
 
 export async function actions(
 	config: Config,
 	cwd: string,
 	actionToRun: string,
 	groupToRun: string,
-	runActionFn: (opts: RunActionOpts) => Promise<(GroupKey & pcp.Output) | undefined> = runAction,
-): Promise<(GroupKey & pcp.Output)[]> {
-	const prioRange = getPriorityRange(Object.values(config.projects));
-
-	const stdoutBuffer: (GroupKey & pcp.Output)[] = [];
-	for (let currentPrio = prioRange.min; currentPrio <= prioRange.max; currentPrio++) {
-		const runActionPromises = Object.keys(config.projects).map((project) =>
-			runActionFn({
-				cwd,
-				config,
-				keys: { project: project, action: actionToRun, group: groupToRun },
-				currentPrio,
-			}),
+	runActionFn: (opts: RunActionOpts) => Promise<ActionOutput> = runAction,
+): Promise<ActionOutput[]> {
+	const uniquePriorities = getUniquePriorities(config, actionToRun, groupToRun);
+	const actionsToRun = getActions(config, actionToRun, groupToRun);
+	const blockedActions = actionsToRun.filter((action) => (action.needs?.length ?? 0) > 0);
+
+	const progressBar = getProgressBar(`Running ${actionToRun} ${groupToRun}`);
+	const waitingOn = [] as string[];
+
+	progressBar.start(actionsToRun.length, 0, { status: waitingOnToString(waitingOn) });
+
+	// Go through the sorted priority groups, and run the actions
+	// After an action is run, the runActionPromiseWrapper will handle calling any actions that needs the completed action.
+	const stdoutBuffer: ActionOutput[] = [];
+	for (const priority of uniquePriorities) {
+		const runActionPromises = actionsToRun
+			.filter((action) => (action.priority ?? 0) === priority && (action.needs?.length ?? 0) === 0)
+			.map((action) => {
+				return runActionPromiseWrapper(
+					{ cwd, config, keys: { project: action.project, action: action.action, group: action.group } },
+					runActionFn,
+					progressBar,
+					blockedActions,
+					waitingOn,
+				);
+			});
+
+		(await Promise.all(runActionPromises)).forEach((outputArr) =>
+			outputArr.forEach((output) => stdoutBuffer.push(output)),
 		);
-		(await Promise.all(runActionPromises))
-			.filter((p) => p)
-			.forEach((p) => stdoutBuffer.push(p as GroupKey & { stdout: string }));
 	}
+
+	progressBar.update({ status: waitingOnToString([]) });
+	progressBar.stop();
+	console.log();
+
+	logActionOutput(stdoutBuffer);
 	return stdoutBuffer;
 }
+export function getUniquePriorities(config: Config, actionToRun: string, groupToRun: string): Set<number> {
+	return Object.values(config.projects).reduce((carry, project) => {
+		if (project.actions[actionToRun]?.groups[groupToRun]) {
+			carry.add(project.actions[actionToRun].priority ?? 0);
+		}
+		return carry;
+	}, new Set<number>());
+}
+
+export async function runActionPromiseWrapper(
+	runActionOpts: RunActionOpts,
+	runActionFn: (opts: RunActionOpts) => Promise<ActionOutput>,
+	progressBar: SingleBar,
+	blockedActions: (GroupKey & ProjectAction)[],
+	waitingOn: string[],
+): Promise<ActionOutput[]> {
+	waitingOn.push(runActionOpts.keys.project);
+	progressBar.update({ status: waitingOnToString(waitingOn) });
+	return runActionFn(runActionOpts)
+		.then((res) => {
+			waitingOn.splice(waitingOn.indexOf(runActionOpts.keys.project), 1);
+			progressBar.increment();
+			return res;
+		})
+		.then(async (res) => {
+			blockedActions.forEach((action) => {
+				action.needs = action.needs?.filter((need) => need !== runActionOpts.keys.project);
+			});
+
+			const runBlockedActionPromises = blockedActions
+				.filter((action) => action.needs?.length === 0)
+				.map((action) => {
+					const newBlockedActions = blockedActions.filter((blockedAction) => blockedAction.needs?.length !== 0);
+					return runActionPromiseWrapper(
+						{ ...runActionOpts, keys: { ...runActionOpts.keys, project: action.project } },
+						runActionFn,
+						progressBar,
+						newBlockedActions,
+						waitingOn,
+					);
+				});
+
+			const blockedActionsResult = (await Promise.all(runBlockedActionPromises)).reduce(
+				(carry, blockedActionResult) => {
+					return [...carry, ...blockedActionResult];
+				},
+				[] as ActionOutput[],
+			);
+			return [res, ...blockedActionsResult];
+		});
+}
 
 interface RunActionOpts {
 	cwd: string;
 	config: Config;
 	keys: GroupKey;
-	currentPrio: number;
 }
 
-export async function runAction(options: RunActionOpts): Promise<(GroupKey & pcp.Output) | undefined> {
-	if (!(options.keys.project in options.config.projects)) return;
+export async function runAction(options: RunActionOpts): Promise<ActionOutput> {
 	const project = options.config.projects[options.keys.project];
-
+	const group = project.actions[options.keys.action].groups[options.keys.group];
 	const dir = getProjectDirFromRemote(options.cwd, project.remote);
 
-	if (!(options.keys.action in project.actions)) return;
-	const action = project.actions[options.keys.action];
-
-	if (!(options.keys.group in action.groups)) return;
-	const group = action.groups[options.keys.group];
-
-	const priority = action.priority ?? project.priority ?? 0;
-
-	if (options.currentPrio !== priority) return;
-
-	console.log(chalk`{blue ${group.join(" ")}} is running in {cyan ${dir}}`);
-	const [err, res]: ToChildProcessOutput = await to(
-		pcp.spawn(group[0], group.slice(1), {
+	const res = await pcp
+		.spawn(group[0], group.slice(1), {
 			cwd: dir,
 			env: process.env,
 			encoding: "utf8",
-		}),
-	);
-
-	if (err) {
-		console.error(
-			chalk`"${options.keys.action}" "${options.keys.group}" {red failed}, ` +
-				chalk`goto {cyan ${dir}} and run {blue ${group.join(" ")}} manually`,
-		);
-	}
+		})
+		.catch((err) => err);
 
 	return {
 		...options.keys,
-		stdout: res?.stdout?.toString() ?? "",
-		stderr: res?.stderr?.toString() ?? "",
+		stdout: res.stdout?.toString() ?? "",
+		stderr: res.stderr?.toString() ?? "",
+		code: res.code,
+		dir,
+		cmd: group,
 	};
 }
 
@@ -88,3 +144,50 @@ export async function fromConfig(cwd: string, cnf: Config, actionToRun: string,
 		console.log(chalk`{yellow No groups found for action {cyan ${actionToRun}} and group {cyan ${groupToRun}}}`);
 	}
 }
+
+export function getActions(config: Config, actionToRun: string, groupToRun: string): (GroupKey & ProjectAction)[] {
+	// get all actions from all projects with actionToRun key
+	const actionsToRun = Object.entries(config.projects)
+		.filter(([, project]) => project.actions[actionToRun]?.groups[groupToRun])
+		.reduce((carry, [projectName, project]) => {
+			carry.push({
+				project: projectName,
+				action: actionToRun,
+				group: groupToRun,
+				...project.actions[actionToRun],
+			});
+
+			return carry;
+		}, [] as (GroupKey & ProjectAction)[]);
+
+	/**
+	 * Sometime an action will not have the specific group it needs to run.
+	 * If another action needs such action, we have to rearrange dependencies, as such action cannot be run without the specific group.
+	 * This is done by adding the needs from the action without the specific group, to the actions that need the action witohut the specific group
+	 *
+	 * For example:
+	 *
+	 * A needs B, B needs C
+	 *
+	 * We want to run group X, but only A and C have group X.
+	 * In this case we rewrite A to needs C.
+	 *
+	 * In order to avoid having multiple iterations, we sort the actions topologically first to ensure the dependencies are always resolved.
+	 */
+	topologicalSortActionGraph(config, actionToRun)
+		.filter((action) => !config.projects[action].actions[actionToRun]?.groups[groupToRun])
+		.forEach((actionNoGroup) => {
+			// Find all actions that needs this action, remove this action from the needs list, and replace it with the needs of this action
+			actionsToRun
+				.filter((action) => action.needs?.includes(actionNoGroup))
+				.forEach((action) => {
+					action.needs = action.needs?.filter((need) => need !== actionNoGroup);
+					action.needs = [
+						...(action.needs ?? []),
+						...(config.projects[actionNoGroup]?.actions[actionToRun]?.needs ?? []),
+					];
+				});
+		});
+
+	return actionsToRun;
+}

src/config_loader.ts

@@ -48,5 +48,5 @@ export async function loadConfig(cwd: string): Promise<Config> {
 	const yml: any = yaml.load(fileContent);
 	assert(validateYaml(yml), "Invalid .git-local-devops.yml file");
 
-	return yml as Config;
+	return yml;
 }

src/graph.ts

@@ -0,0 +1,74 @@
+import assert from "assert";
+import { Config } from "./types/config";
+
+export type ActionGraphs = { [actionName: string]: Map<string, string[]> };
+
+export function createActionGraphs(obj: Config): ActionGraphs {
+	// find all unique action names
+	const actionNames = new Set<string>();
+	Object.values(obj.projects).forEach((project) => {
+		Object.keys(project.actions).forEach((actionKey) => {
+			actionNames.add(actionKey);
+		});
+	});
+
+	// create a graph for each action
+	return [...actionNames.keys()].reduce((acc, actionName) => {
+		return { ...acc, [actionName]: topologicalSortActionGraph(obj, actionName) };
+	}, {});
+}
+
+export function topologicalSortActionGraph(obj: Config, actionName: string, sorter = topologicalSort): string[] {
+	const edges = new Map<string, string[]>();
+
+	// Explore edges:
+	Object.entries(obj.projects)
+		.filter(([, project]) => project.actions[actionName])
+		.forEach(([projectKey, project]) => {
+			const needs = [...(project.actions[actionName]?.needs ?? [])];
+			if (project.actions[actionName]?.priority !== undefined) {
+				assert(needs.length === 0, `Priority actions cannot have needs: ${projectKey}/${actionName}`);
+			}
+			edges.set(projectKey, needs);
+		});
+
+	return sorter(edges, actionName);
+}
+
+/**
+ * https://stackoverflow.com/a/4577/17466122
+ *
+ * @param edges
+ * @returns
+ */
+export function topologicalSort(edges: Map<string, string[]>, actionName: string): string[] {
+	// Copy map to avoid mutations
+	edges = new Map(edges);
+
+	const sorted = [] as string[];
+	const leaves = [...edges.entries()].filter(([, mapsTo]) => mapsTo.length === 0).map(([mapsFrom]) => mapsFrom);
+	while (leaves.length > 0) {
+		const leaf = leaves.shift() as string; // We just checked length, so this is safe
+		sorted.push(leaf);
+		edges.delete(leaf);
+		edges.forEach((mapsTo, mapsFrom) => {
+			if (mapsTo.includes(leaf)) {
+				mapsTo.splice(mapsTo.indexOf(leaf), 1);
+				if (mapsTo.length === 0) {
+					leaves.push(mapsFrom);
+				}
+			}
+		});
+	}
+
+	// If there are any edges left, there is a cycle
+	if (edges.size > 0) {
+		console.log(`Unreachable projects for action "${actionName}":`);
+		// rename columns to make it easier to read
+		const edgesWithNames = [...edges.entries()].map(([mapsFrom, mapsTo]) => ({ project: mapsFrom, needs: mapsTo }));
+		console.table(edgesWithNames);
+		assert(false, "Cycle detected in action dependencies or an action dependency is not defined");
+	}
+
+	return sorted;
+}

src/progress.ts

@@ -13,7 +13,7 @@ export function waitingOnToString(waitingOn: string[]): string {
 	if (waitingOn.length === 0) return "Finished all tasks";
 	let str = "";
 	for (const [i, waitingOnStr] of waitingOn.entries()) {
-		if (i !== 0 && str.length + waitingOnStr.length > 40) {
+		if (i !== 0 && str.length + waitingOnStr.length > 80) {
 			return `${str} and ${waitingOn.length - i} more`;
 		}
 		str += `${str ? ", " : ""}${waitingOnStr}`;