Try parallel app processing

.gitignore

@@ -71,6 +71,7 @@ cached-requests.json
 /build-tools/dist/
 /apps/*/.cache/
 /desktop/.cache/
+/apps/*/release-files/
 
 # webpack assets
 /assets*.json

.teamcity/_self/projects/WPComPlugins.kt

@@ -92,9 +92,6 @@ object CalypsoApps: BuildType({
 
 				# Install dependencies
 				yarn
-
-				# Set execution permission for additional scripts.
-				chmod +x .teamcity/scripts/WPComPlugins/
 			"""
 		}
 
@@ -106,6 +103,24 @@ object CalypsoApps: BuildType({
 				yarn workspaces foreach --verbose --parallel --include '{happy-blocks,@automattic/notifications}' run build-ci
 			"""
 		}
+
+		bashNodeScript {
+			name = "Process artifact"
+			scriptContent = """
+				export tc_auth="%system.teamcity.auth.userId%:%system.teamcity.auth.password%"
+				export git_branch="%teamcity.build.branch%"
+				export build_id="%teamcity.build.id%"
+				export GH_TOKEN="%matticbot_oauth_token%"
+				export is_default_branch="%teamcity.build.branch.is_default%"
+				export skip_build_diff="%skip_release_diff%"
+				export mc_auth_secret="%mc_auth_secret%"
+				export commit_sha="%build.vcs.number%"
+				export mc_post_root="%mc_post_root%"
+				export tc_sever_url="%teamcity.serverUrl%"
+
+				node ./bin/process-calypso-app-artifacts.mjs
+			"""
+		}
 	}
 })
 

apps/happy-blocks/.gitignore

@@ -1,4 +1,3 @@
 /dist
 /block-library/*/build
-/release-files
 translations-manifest.json

apps/happy-blocks/bin/prepArtifactForCI.sh

@@ -1,17 +1,18 @@
 #!/bin/bash
-
-set -x
+set -o errexit
+set -o nounset
+set -o pipefail
 
 # Copy build directories to the release directory.
-find ./block-library/* -type d -name "*" -prune |\
+find ./block-library/* -type d -name "*" -prune ! -name "shared" |\
 while read -r block;
 do
-	mkdir -p ./release-files/${block//\.\.\//}; 
-	cp -r $block/build/* ./release-files/${block//\.\.\//}/; 
+	mkdir -p "./release-files/${block//\.\.\//}"; 
+	cp -r $block/build/* "./release-files/${block//\.\.\//}/"; 
 done
 
 # Add the index.php file
-cp ./index.php ./README.md ./dist/* ./release-files/
+cp ./index.php ./README.md ./release-files/
 cp ./translations-manifest.json ./release-files/
 
 printf "Finished configuration of the @automattic/happy-blocks plugin artifacts directory.\n"

apps/happy-blocks/package.json

@@ -17,7 +17,7 @@
 		"build:universal-header": "calypso-build --env block='universal-header'",
 		"build:universal-footer": "calypso-build --env block='universal-footer'",
 		"build-translations-manifest": "yarn run build-calypso-strings && node bin/build-translations-manifest.js",
-		"clean": "rm -rf dist block-library/*/build || true",
+		"clean": "rm -r release-files block-library/*/build || true",
 		"dev": "yarn run calypso-apps-builder --localPath / --remotePath /home/wpcom/public_html/wp-content/a8c-plugins/happy-blocks"
 	},
 	"dependencies": {

apps/notifications/package.json

@@ -22,7 +22,7 @@
 		"start": "yarn run clean && yarn run build:notifications && yarn run dev-server",
 		"dev": "yarn run calypso-apps-builder --localPath dist --remotePath /home/wpcom/public_html/widgets.wp.com/notifications",
 		"build": "NODE_ENV=production yarn dev",
-		"build-ci": "yarn build && ./bin/prepArtifactForCI.sh"
+		"build-ci": "yarn build"
 	},
 	"dependencies": {
 		"@automattic/calypso-color-schemes": "workspace:^",

apps/odyssey-stats/webpack.config.js

@@ -41,9 +41,10 @@ const excludedPackages = [
 ];
 
 const excludedPackagePlugins = excludedPackages.map(
-	( package ) =>
+	// Note: apparently the word "package" is a reserved keyword here for some reason
+	( pkg ) =>
 		new webpack.NormalModuleReplacementPlugin(
-			package,
+			pkg,
 			path.resolve( __dirname, 'src/components/nothing' )
 		)
 );

bin/did-calypso-app-change.mjs

@@ -0,0 +1,45 @@
+import { exec as _exec } from 'node:child_process';
+import { createWriteStream } from 'node:fs';
+import { Readable } from 'node:stream';
+import { finished } from 'node:stream/promises';
+import util from 'node:util';
+const exec = util.promisify( _exec );
+
+export default async function didCalypsoAppChange( { slug, dir, newReleaseDir, customNormalize } ) {
+	await downloadPrevBuild( slug, dir );
+	await customNormalize?.();
+	try {
+		await exec(
+			`diff -rq --exclude="*.js.map" --exclude="*.asset.php" --exclude="build_meta.json" --exclude="README.md" ${ newReleaseDir } ${ dir }/prev-release/`,
+			{ encoding: 'UTF-8', cwd: dir, stdio: 'inherit' }
+		);
+		return false;
+	} catch ( { code, stdout, stderr } ) {
+		if ( code === 1 ) {
+			console.info( `The build for ${ slug } changed. Cause:` );
+			console.info( stdout );
+			return true;
+		}
+		throw new Error( `Unexpected error code ${ code } while diffing ${ slug } build: ${ stderr }` );
+	}
+}
+
+async function downloadPrevBuild( appSlug, dir ) {
+	const prevBuildZip = `${ dir }/prev-archive-download.zip`;
+	const stream = createWriteStream( prevBuildZip );
+
+	const prevBuildUrl = `${ process.env.tc_sever_url }/repository/download/calypso_calypso_WPComPlugins_Build_Plugins/${ appSlug }-release-build.tcbuildtag/${ appSlug }.zip?guest=1&branch=try-parallel-app-processing`;
+	console.info( `Fetching previous release build for ${ appSlug } from ${ prevBuildUrl }` );
+
+	const { body, status } = await fetch( prevBuildUrl );
+	if ( status !== 200 ) {
+		throw new Error( `Could not fetch previous build! Response code ${ status }.` );
+	}
+
+	console.info( `Extracting downloaded archive for ${ appSlug }...` );
+	await finished( Readable.fromWeb( body ).pipe( stream ) );
+	await exec( `unzip -q ${ prevBuildZip } -d ${ dir }/prev-release`, {
+		encoding: 'UTF-8',
+		stdio: 'inherit',
+	} );
+}

bin/process-calypso-app-artifacts.mjs

@@ -0,0 +1,165 @@
+import { exec as _exec } from 'node:child_process';
+import { createHmac } from 'node:crypto';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import util from 'node:util';
+import didCalypsoAppChange from './did-calypso-app-change.mjs';
+
+checkEnvVars();
+
+const exec = util.promisify( _exec );
+const SKIP_BUILD_DIFF = process.env.skip_build_diff === 'true';
+const IS_DEFAULT_BRANCH = process.env.is_default_branch === 'true';
+const dirname = fileURLToPath( new URL( '.', import.meta.url ) );
+const appRoot = path.resolve( dirname, '../apps' );
+
+const apps = [
+	{
+		slug: 'happy-blocks',
+		dir: path.resolve( appRoot, 'happy-blocks' ),
+		newReleaseDir: path.resolve( appRoot, 'happy-blocks/release-files' ),
+		slackNotify: true,
+	},
+];
+
+// STEP 1: Check if any apps have changed. If skipping the diff, continue as if all apps have changed.
+const changedApps = SKIP_BUILD_DIFF
+	? apps
+	: (
+			await Promise.all(
+				apps.map( async ( app ) => ( ( await didCalypsoAppChange( app ) ) ? app : null ) )
+			)
+	   ).filter( Boolean );
+
+if ( changedApps.length ) {
+	console.info(
+		'The following apps changed: ',
+		changedApps.map( ( { slug } ) => slug ).join( ', ' )
+	);
+} else {
+	console.info( 'No apps changed.' );
+}
+
+// STEP 2: Tag the build in TeamCity. This will let future builds identify the previous release.
+const finalTasks = [];
+if ( changedApps.length ) {
+	console.info( 'Tagging build...' );
+	finalTasks.push( tagBuild( changedApps ) );
+}
+
+// STEP 3: Notify the author. On trunk, send a Slack notification. On a PR, a GitHub commnent.
+if ( ! IS_DEFAULT_BRANCH ) {
+	console.info( 'Running GitHub comment...' );
+	finalTasks.push( addGitHubComment( changedApps ) );
+} else {
+	console.info( 'Running Slack notification...' );
+	finalTasks.push( sendSlackNotification( changedApps ) );
+}
+
+await Promise.all( finalTasks );
+console.log( 'Success!' );
+
+async function tagBuild( _changedApps ) {
+	const tags = _changedApps.map( ( app ) => `${ app.slug }-release-build` );
+
+	const tagurl = `https://teamcity.a8c.com/httpAuth/app/rest/builds/id:${ process.env.build_id }/tags/`;
+	console.info( `Adding tags (${ tags }) to current build at URL ${ tagurl }` );
+
+	const jsonTags = JSON.stringify( {
+		count: tags.length,
+		tag: tags.map( ( tag ) => ( {
+			name: tag,
+		} ) ),
+	} );
+
+	const res = await fetch( tagurl, {
+		method: 'POST',
+		headers: new Headers( {
+			'Content-Type': 'application/json',
+			Authorization: `Basic ${ Buffer.from( process.env.tc_auth ).toString( 'base64' ) }`,
+		} ),
+		body: jsonTags,
+	} );
+	if ( res.status !== 200 ) {
+		console.error( 'Tagging the build failed!' );
+	}
+}
+
+async function addGitHubComment( _changedApps ) {
+	const notifyApps = _changedApps.filter( ( { ghNotify = true } ) => ghNotify );
+
+	const commentWatermark = 'calypso-app-artifacts';
+	const ghCommentCmd = `./bin/add-pr-comment.sh ${ process.env.git_branch } ${ commentWatermark }`;
+
+	if ( ! notifyApps.length ) {
+		console.info( 'No apps to notify about. Deleting existing comment if exists.' );
+		// Delete the existing comment, since there are no apps to notify about.
+		return await exec( `${ ghCommentCmd } delete <<< "" || true`, {
+			encoding: 'UTF-8',
+			stdio: 'inherit',
+		} );
+	}
+
+	const header = '**This PR modifies the release build for the following Calypso Apps:**';
+	const docsMsg = '_For info about this notification, see here: PCYsg-OT6-p2_';
+	const changedAppsMsg = notifyApps.map( ( { slug } ) => `* ${ slug }` ).join( '\n' );
+	const testMsg = `To test WordPress.com changes, run "install-plugin.sh $pluginSlug ${ process.env.git_branch }" on your sandbox.`;
+
+	const appMsg = `${ header }\n\n${ docsMsg }\n\n${ changedAppsMsg }\n\n${ testMsg }`;
+
+	await exec( `${ ghCommentCmd } <<- EOF || true\n${ appMsg }\nEOF`, {
+		encoding: 'UTF-8',
+	} );
+}
+
+async function sendSlackNotification( _changedApps ) {
+	const notifyApps = _changedApps.filter( ( { slackNotify = false } ) => slackNotify );
+
+	if ( ! notifyApps.length ) {
+		console.info( 'No apps to notify about. Skipping Slack notification.' );
+		return;
+	}
+
+	// TODO: move from one to multiple plugins!
+	const body = `commit=${ process.env.commit_sha }&plugin=$pluginSlug`;
+
+	const signature = createHmac( 'sha256', process.env.mc_auth_secret )
+		.update( body )
+		.digest( 'hex' );
+
+	console.log( `Sending data to slack endpoint: ${ body }` );
+	const res = await fetch( `${ process.env.mc_post_root }?plugin-deploy-reminder`, {
+		method: 'POST',
+		headers: new Headers( {
+			'Content-Type': 'application/x-www-form-urlencoded',
+			'TEAMCITY-SIGNATURE': signature,
+		} ),
+		body,
+	} );
+	if ( res.status !== 200 ) {
+		console.error( 'Slack notification failed!' );
+		console.error( 'Details: ', await res.text() );
+	}
+}
+
+function checkEnvVars() {
+	const requiredVars = [
+		'tc_auth',
+		'git_branch',
+		'build_id',
+		'GH_TOKEN',
+		'is_default_branch',
+		'skip_build_diff',
+		'mc_auth_secret',
+		'commit_sha',
+		'mc_post_root',
+		'tc_sever_url',
+	];
+
+	// Undefined and empty strings will be detected.
+	const missingVars = requiredVars.filter( ( varName ) => ! process.env[ varName ] );
+	if ( missingVars.length > 0 ) {
+		console.error( `Missing required environment variables: ${ missingVars.join( ', ' ) }` );
+		process.exit( 1 );
+	}
+}