add source usage tracking evergreen issue

.github/workflows/source-tracker.yml

@@ -0,0 +1,34 @@
+name: source package usage tracker
+on:
+  schedule:
+    # Every work day of the week at 08:08
+    - cron: '8 8 * * MON-FRI'
+
+  # Allows you to run this workflow manually from the Actions tab.
+  workflow_dispatch:
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  scheduled:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # https://github.com/denoland/setup-deno#latest-stable-for-a-major
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
+
+      - name: Source tracker
+        run: |
+          deno run \
+            --allow-read \
+            --allow-net \
+            --allow-env=HOME,GITHUB_TOKEN \
+            scripts/deno/source-tracker/mod.ts
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

scripts/deno/deno.json

@@ -0,0 +1,6 @@
+{
+	"fmt": {
+		"useTabs": true,
+		"singleQuote": true
+	}
+}

scripts/deno/octokit.ts

@@ -0,0 +1,20 @@
+import { load } from 'https://deno.land/std@0.210.0/dotenv/mod.ts';
+import { Octokit } from 'https://esm.sh/@octokit/rest@20.0.2';
+
+export { type RestEndpointMethodTypes } from 'https://esm.sh/@octokit/rest@20.0.2';
+
+// this should be provided by the environment (i.e. GitHub Actions)
+let token = Deno.env.get('GITHUB_TOKEN');
+
+if (!token) {
+	// we're probably running in a local dev environment
+	// Create a personal access token at https://github.com/settings/tokens/new?scopes=repo
+	// and add it to your .env file as GITHUB_TOKEN
+	const env = await load();
+	token = env.GITHUB_TOKEN;
+}
+if (!token) console.warn('Missing GITHUB_TOKEN');
+
+export const octokit = new Octokit({
+	auth: token,
+});

scripts/deno/source-tracker/README.md

@@ -0,0 +1,21 @@
+# `@guardian/source-*` package usage tracker
+
+This Deno script is used to track the usage of the `@guardian/source-*` packages in the Guardian GitHub organisation.
+
+It keeps https://github.com/guardian/csnx/issues/1058 up to date.
+
+## Development
+
+You can run it locally by running `deno run -A scripts/deno/source-tracker/mod.ts`.
+
+You will need to create a personal access token at https://github.com/settings/tokens/new?scopes=repo and add it to an `.env` file as `GITHUB_TOKEN`.
+
+### Local caching
+
+To avoid hitting the rate limit in dev, the script saves the responses from the APIs it hits in Deno's `localStorage`.
+
+If you want to clear/disable that, uncomment the `localStorage.clear()` line in `mod.ts`.
+
+## Production
+
+The script is run once a day by a GitHub Action. See `.github/workflows/source-tracker.yml`.

scripts/deno/source-tracker/get-installation-data.ts

@@ -0,0 +1,21 @@
+import { octokit, type RestEndpointMethodTypes } from '../octokit.ts';
+
+export const getInstallationData = async (
+	installation: RestEndpointMethodTypes['search']['code']['response']['data']['items'][number],
+): Promise<
+	RestEndpointMethodTypes['repos']['getContent']['response']['data']
+> => {
+	const key = `${installation.git_url}-data`;
+
+	const stored = localStorage.getItem(key);
+	if (stored) return JSON.parse(stored);
+
+	const { data } = await octokit.rest.repos.getContent({
+		owner: installation.repository.owner.login,
+		repo: installation.repository.name,
+		path: installation.path,
+	});
+	localStorage.setItem(key, JSON.stringify(data));
+
+	return data;
+};

scripts/deno/source-tracker/get-installations.ts

@@ -0,0 +1,26 @@
+import { octokit, type RestEndpointMethodTypes } from '../octokit.ts';
+
+export const getInstallations = async (
+	packageName: string,
+): Promise<
+	RestEndpointMethodTypes['search']['code']['response']['data']['items']
+> => {
+	const key = `${packageName}-result`;
+
+	const stored = localStorage.getItem(key);
+	if (stored) return JSON.parse(stored);
+
+	const {
+		data: { items },
+	} = await octokit.rest.search.code({
+		headers: {
+			'X-GitHub-Api-Version': '2022-11-28',
+		},
+		q: `filename:package.json+org:guardian+"${packageName}"`,
+		per_page: 100,
+	});
+
+	localStorage.setItem(key, JSON.stringify(items));
+
+	return items;
+};

scripts/deno/source-tracker/get-latest-version.ts

@@ -0,0 +1,15 @@
+import getLatestOnNpm from 'npm:latest-version@7.0.0';
+
+export const getLatestVersion = async (packageName: string) => {
+	const key = `${packageName}-latest`;
+
+	const stored = localStorage.getItem(key);
+	if (stored) {
+		return stored;
+	}
+
+	const result = await getLatestOnNpm(packageName);
+	localStorage.setItem(key, result);
+
+	return result;
+};

scripts/deno/source-tracker/get-markdown.ts

@@ -0,0 +1,45 @@
+import { Markdown } from 'https://deno.land/x/deno_markdown@v0.2/mod.ts';
+import { depTypes } from './get-pkg-versions-in-use.ts';
+import { UsageData } from './mod.ts';
+
+export const getMarkdown = (usageData: UsageData) => {
+	const issue = new Markdown();
+
+	for (const [packageName, { usage, latestVersion }] of Object.entries(
+		usageData,
+	)) {
+		issue.header(`${packageName}`, 1);
+		issue.paragraph(
+			`Latest version: [\`v${latestVersion}\`](https://www.npmjs.com/package/${packageName}).`,
+		);
+		issue.header(`Versions in use`, 3);
+
+		const versions = usage
+			.sort((a, b) => b.version - a.version)
+			.flatMap(({ version, installations }) => {
+				const x = installations
+					.sort((a, b) => a.project.localeCompare(b.project))
+					.map(
+						({
+							project,
+							pkgUrl,
+							dependencies,
+							devDependencies,
+							peerDependencies,
+						}) => [
+							`[${project}](${pkgUrl})`,
+							dependencies ? `\`${dependencies}\`` : '',
+							devDependencies ? `\`${devDependencies}\`` : '',
+							peerDependencies ? `\`${peerDependencies}\`` : '',
+						],
+					);
+				return [[`**${version}**`], ...x];
+			});
+
+		issue.table([['Installation', ...depTypes], ...versions], {
+			align: ['l', 'r', 'r', 'r'],
+		});
+	}
+
+	return issue.content;
+};

scripts/deno/source-tracker/get-pkg-versions-in-use.ts

@@ -0,0 +1,82 @@
+import semver from 'npm:semver@7.5.4';
+import { getInstallationData } from './get-installation-data.ts';
+import { getInstallations } from './get-installations.ts';
+
+export type PkgVersionsInUse = {
+	version: number;
+	installations: Array<{
+		project: string;
+		pkgUrl: string;
+		dependencies?: string;
+		devDependencies?: string;
+		peerDependencies?: string;
+	}>;
+}[];
+
+export const depTypes = [
+	'dependencies',
+	'devDependencies',
+	'peerDependencies',
+] as const;
+
+export const getPkgVersionsInUse = async (
+	packageName: string,
+): Promise<PkgVersionsInUse> => {
+	const pkgVersionsInUse: PkgVersionsInUse = [];
+
+	console.log(`Searching for installations of ${packageName}:`);
+	const installations = await getInstallations(packageName);
+	console.log(`- found ${installations.length} installations`);
+
+	console.log(`Getting versions in:`);
+
+	for (const installation of installations) {
+		const installationData = await getInstallationData(installation);
+		const project =
+			`${installation.repository.name}/${installation.path}`.replace(
+				'/package.json',
+				'',
+			);
+
+		console.log(`- ${project}`);
+
+		if (!Array.isArray(installationData) && 'content' in installationData) {
+			const contents = atob(installationData.content);
+			const installationPkg = JSON.parse(contents);
+
+			if (installationPkg.name === packageName) continue;
+
+			const instance: PkgVersionsInUse[number]['installations'][number] = {
+				project,
+				pkgUrl: installation.html_url,
+			};
+
+			for (const depType of depTypes) {
+				instance[depType] = installationPkg[depType]?.[packageName];
+			}
+
+			const versions = depTypes.map(
+				(depType) => installationPkg[depType]?.[packageName],
+			);
+
+			const minVersions = versions
+				.filter(Boolean)
+				.map((version) => semver.minVersion(version).version);
+			const lowestVersion = semver.sort(minVersions)[0];
+			const lowestVersionMajor = semver.major(lowestVersion);
+
+			const existingVersion = pkgVersionsInUse.find(
+				(_) => _.version === lowestVersionMajor,
+			);
+			if (existingVersion) {
+				existingVersion.installations.push(instance);
+			} else {
+				pkgVersionsInUse.push({
+					version: lowestVersionMajor,
+					installations: [instance],
+				});
+			}
+		}
+	}
+	return pkgVersionsInUse;
+};

scripts/deno/source-tracker/mod.ts

@@ -0,0 +1,57 @@
+import { getLatestVersion } from './get-latest-version.ts';
+import { getMarkdown } from './get-markdown.ts';
+import {
+	getPkgVersionsInUse,
+	type PkgVersionsInUse,
+} from './get-pkg-versions-in-use.ts';
+import { octokit } from '../octokit.ts';
+
+// localStorage.clear();
+
+const packages = [
+	'@guardian/source-foundations',
+	'@guardian/source-react-components',
+] as const;
+
+export type UsageData = {
+	[name in (typeof packages)[number]]: {
+		latestVersion: string;
+		usage: PkgVersionsInUse;
+	};
+};
+
+const usageData: UsageData = {} as UsageData;
+
+for (const packageName of packages) {
+	const versionsInUse = await getPkgVersionsInUse(packageName);
+
+	console.log(`Getting latest version of ${packageName}:`);
+	const latestVersion = await getLatestVersion(packageName);
+	console.log(`- ${latestVersion}`);
+
+	usageData[packageName] = { usage: versionsInUse, latestVersion };
+}
+
+const markdown = getMarkdown(usageData);
+
+// const content = new TextEncoder().encode(markdown);
+// await Deno.writeFile('markdown.md', content);
+
+const issue_number = 1058;
+
+try {
+	const {
+		data: { html_url },
+	} = await octokit.rest.issues.update({
+		owner: 'guardian',
+		repo: 'csnx',
+		issue_number,
+		body: markdown,
+	});
+
+	console.info(`Successfully updated issue #${issue_number}`);
+	console.info(html_url);
+} catch (error) {
+	console.warn(`Failed to update issue #${issue_number}`);
+	console.error(error);
+}