diff --git a/.github/workflows/source-tracker.yml b/.github/workflows/source-tracker.yml new file mode 100644 index 000000000..8f97b5baf --- /dev/null +++ b/.github/workflows/source-tracker.yml @@ -0,0 +1,34 @@ +name: source package usage tracker +on: + schedule: + # Every work day of the week at 08:08 + - cron: '8 8 * * MON-FRI' + + # Allows you to run this workflow manually from the Actions tab. + workflow_dispatch: + +permissions: + issues: write + pull-requests: write + +jobs: + scheduled: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + # https://github.com/denoland/setup-deno#latest-stable-for-a-major + - uses: denoland/setup-deno@v1 + with: + deno-version: v1.x + + - name: Thrasher tracker + run: | + deno run \ + --allow-read \ + --allow-net \ + --allow-env=HOME,GITHUB_TOKEN \ + scripts/deno/source-tracker/mod.ts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/scripts/deno/deno.json b/scripts/deno/deno.json new file mode 100644 index 000000000..0f4b781ba --- /dev/null +++ b/scripts/deno/deno.json @@ -0,0 +1,6 @@ +{ + "fmt": { + "useTabs": true, + "singleQuote": true + } +} diff --git a/scripts/deno/octokit.ts b/scripts/deno/octokit.ts new file mode 100644 index 000000000..886c6c6d9 --- /dev/null +++ b/scripts/deno/octokit.ts @@ -0,0 +1,20 @@ +import { load } from 'https://deno.land/std@0.210.0/dotenv/mod.ts'; +import { Octokit } from 'https://esm.sh/@octokit/rest@20.0.2'; + +export { type RestEndpointMethodTypes } from 'https://esm.sh/@octokit/rest@20.0.2'; + +// this should be provided by the environment (i.e. GitHub Actions) +let token = Deno.env.get('GITHUB_TOKEN'); + +// we're probably running in a local dev environment +// Create a personal access token at https://github.com/settings/tokens/new?scopes=repo +// and add it to your .env file as GITHUB_TOKEN +if (!token) { + const env = await load(); + token = env.GITHUB_TOKEN; +} +if (!token) console.warn('Missing GITHUB_TOKEN'); + +export const octokit = new Octokit({ + auth: token, +}); diff --git a/scripts/deno/source-tracker/README.md b/scripts/deno/source-tracker/README.md new file mode 100644 index 000000000..cd28d9132 --- /dev/null +++ b/scripts/deno/source-tracker/README.md @@ -0,0 +1,21 @@ +# `@guardian/source-*` package usage tracker + +This Deno script is used to track the usage of the `@guardian/source-*` packages in the Guardian GitHub organisation. + +It keeps https://github.com/guardian/csnx/issues/1058 up to date. + +## Development + +You can run it locally by running `deno run -A scripts/deno/source-tracker/mod.ts`. + +You will need to create a personal access token at https://github.com/settings/tokens/new?scopes=repo and add it to an `.env` file as `GITHUB_TOKEN`. + +### Local caching + +To avoid hitting the rate limit in dev, the script saves the responses from the APIs it hits in Deno's `localStorage`. + +If you want to clear/disable that, uncomment the `localStorage.clear()` line in `mod.ts`. + +## Production + +The script is run once a day by a GitHub Action. See `.github/workflows/source-tracker.yml`. diff --git a/scripts/deno/source-tracker/get-installation-data.ts b/scripts/deno/source-tracker/get-installation-data.ts new file mode 100644 index 000000000..878570811 --- /dev/null +++ b/scripts/deno/source-tracker/get-installation-data.ts @@ -0,0 +1,21 @@ +import { octokit, type RestEndpointMethodTypes } from '../octokit.ts'; + +export const getInstallationData = async ( + installation: RestEndpointMethodTypes['search']['code']['response']['data']['items'][number], +): Promise< + RestEndpointMethodTypes['repos']['getContent']['response']['data'] +> => { + const key = `${installation.git_url}-data`; + + const stored = localStorage.getItem(key); + if (stored) return JSON.parse(stored); + + const { data } = await octokit.rest.repos.getContent({ + owner: installation.repository.owner.login, + repo: installation.repository.name, + path: installation.path, + }); + localStorage.setItem(key, JSON.stringify(data)); + + return data; +}; diff --git a/scripts/deno/source-tracker/get-installations.ts b/scripts/deno/source-tracker/get-installations.ts new file mode 100644 index 000000000..f17801ed9 --- /dev/null +++ b/scripts/deno/source-tracker/get-installations.ts @@ -0,0 +1,26 @@ +import { octokit, type RestEndpointMethodTypes } from '../octokit.ts'; + +export const getInstallations = async ( + packageName: string, +): Promise< + RestEndpointMethodTypes['search']['code']['response']['data']['items'] +> => { + const key = `${packageName}-result`; + + const stored = localStorage.getItem(key); + if (stored) return JSON.parse(stored); + + const { + data: { items }, + } = await octokit.rest.search.code({ + headers: { + 'X-GitHub-Api-Version': '2022-11-28', + }, + q: `filename:package.json+org:guardian+"${packageName}"`, + per_page: 100, + }); + + localStorage.setItem(key, JSON.stringify(items)); + + return items; +}; diff --git a/scripts/deno/source-tracker/get-latest-version.ts b/scripts/deno/source-tracker/get-latest-version.ts new file mode 100644 index 000000000..ce35d0af0 --- /dev/null +++ b/scripts/deno/source-tracker/get-latest-version.ts @@ -0,0 +1,15 @@ +import getLatestOnNpm from 'npm:latest-version@7.0.0'; + +export const getLatestVersion = async (packageName: string) => { + const key = `${packageName}-latest`; + + const stored = localStorage.getItem(key); + if (stored) { + return stored; + } + + const result = await getLatestOnNpm(packageName); + localStorage.setItem(key, result); + + return result; +}; diff --git a/scripts/deno/source-tracker/get-markdown.ts b/scripts/deno/source-tracker/get-markdown.ts new file mode 100644 index 000000000..cd527e176 --- /dev/null +++ b/scripts/deno/source-tracker/get-markdown.ts @@ -0,0 +1,45 @@ +import { Markdown } from 'https://deno.land/x/deno_markdown@v0.2/mod.ts'; +import { depTypes } from './get-pkg-versions-in-use.ts'; +import { UsageData } from './mod.ts'; + +export const getMarkdown = (usageData: UsageData) => { + const issue = new Markdown(); + + for (const [packageName, { usage, latestVersion }] of Object.entries( + usageData, + )) { + issue.header(`${packageName}`, 1); + issue.paragraph( + `Latest version: [\`v${latestVersion}\`](https://www.npmjs.com/package/${packageName}).`, + ); + issue.header(`Versions in use`, 3); + + const versions = usage + .sort((a, b) => b.version - a.version) + .flatMap(({ version, installations }) => { + const x = installations + .sort((a, b) => a.project.localeCompare(b.project)) + .map( + ({ + project, + pkgUrl, + dependencies, + devDependencies, + peerDependencies, + }) => [ + `[${project}](${pkgUrl})`, + dependencies ? `\`${dependencies}\`` : '', + devDependencies ? `\`${devDependencies}\`` : '', + peerDependencies ? `\`${peerDependencies}\`` : '', + ], + ); + return [[`**${version}**`], ...x]; + }); + + issue.table([['Installation', ...depTypes], ...versions], { + align: ['l', 'r', 'r', 'r'], + }); + } + + return issue.content; +}; diff --git a/scripts/deno/source-tracker/get-pkg-versions-in-use.ts b/scripts/deno/source-tracker/get-pkg-versions-in-use.ts new file mode 100644 index 000000000..c5e7d45e2 --- /dev/null +++ b/scripts/deno/source-tracker/get-pkg-versions-in-use.ts @@ -0,0 +1,82 @@ +import semver from 'npm:semver@7.5.4'; +import { getInstallationData } from './get-installation-data.ts'; +import { getInstallations } from './get-installations.ts'; + +export type PkgVersionsInUse = { + version: number; + installations: Array<{ + project: string; + pkgUrl: string; + dependencies?: string; + devDependencies?: string; + peerDependencies?: string; + }>; +}[]; + +export const depTypes = [ + 'dependencies', + 'devDependencies', + 'peerDependencies', +] as const; + +export const getPkgVersionsInUse = async ( + packageName: string, +): Promise => { + const pkgVersionsInUse: PkgVersionsInUse = []; + + console.log(`Searching for installations of ${packageName}:`); + const installations = await getInstallations(packageName); + console.log(`- found ${installations.length} installations`); + + console.log(`Getting versions in:`); + + for (const installation of installations) { + const installationData = await getInstallationData(installation); + const project = + `${installation.repository.name}/${installation.path}`.replace( + '/package.json', + '', + ); + + console.log(`- ${project}`); + + if (!Array.isArray(installationData) && 'content' in installationData) { + const contents = atob(installationData.content); + const installationPkg = JSON.parse(contents); + + if (installationPkg.name === packageName) continue; + + const instance: PkgVersionsInUse[number]['installations'][number] = { + project, + pkgUrl: installation.html_url, + }; + + for (const depType of depTypes) { + instance[depType] = installationPkg[depType]?.[packageName]; + } + + const versions = depTypes.map( + (depType) => installationPkg[depType]?.[packageName], + ); + + const minVersions = versions + .filter(Boolean) + .map((version) => semver.minVersion(version).version); + const lowestVersion = semver.sort(minVersions)[0]; + const lowestVersionMajor = semver.major(lowestVersion); + + const existingVersion = pkgVersionsInUse.find( + (_) => _.version === lowestVersionMajor, + ); + if (existingVersion) { + existingVersion.installations.push(instance); + } else { + pkgVersionsInUse.push({ + version: lowestVersionMajor, + installations: [instance], + }); + } + } + } + return pkgVersionsInUse; +}; diff --git a/scripts/deno/source-tracker/mod.ts b/scripts/deno/source-tracker/mod.ts new file mode 100644 index 000000000..bdac4d2c9 --- /dev/null +++ b/scripts/deno/source-tracker/mod.ts @@ -0,0 +1,57 @@ +import { getLatestVersion } from './get-latest-version.ts'; +import { getMarkdown } from './get-markdown.ts'; +import { + getPkgVersionsInUse, + type PkgVersionsInUse, +} from './get-pkg-versions-in-use.ts'; +import { octokit } from '../octokit.ts'; + +// localStorage.clear(); + +const packages = [ + '@guardian/source-foundations', + '@guardian/source-react-components', +] as const; + +export type UsageData = { + [name in (typeof packages)[number]]: { + latestVersion: string; + usage: PkgVersionsInUse; + }; +}; + +const usageData: UsageData = {} as UsageData; + +for (const packageName of packages) { + const versionsInUse = await getPkgVersionsInUse(packageName); + + console.log(`Getting latest version of ${packageName}:`); + const latestVersion = await getLatestVersion(packageName); + console.log(`- ${latestVersion}`); + + usageData[packageName] = { usage: versionsInUse, latestVersion }; +} + +const markdown = getMarkdown(usageData); + +// const content = new TextEncoder().encode(markdown); +// await Deno.writeFile('markdown.md', content); + +const issue_number = 1058; + +try { + const { + data: { html_url }, + } = await octokit.rest.issues.update({ + owner: 'guardian', + repo: 'csnx', + issue_number, + body: markdown, + }); + + console.info(`Successfully updated issue #${issue_number}`); + console.info(html_url); +} catch (error) { + console.warn(`Failed to update issue #${issue_number}`); + console.error(error); +}