-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add source usage tracking evergreen issue
- Loading branch information
Showing
10 changed files
with
327 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
name: source package usage tracker | ||
on: | ||
schedule: | ||
# Every work day of the week at 08:08 | ||
- cron: '8 8 * * MON-FRI' | ||
|
||
# Allows you to run this workflow manually from the Actions tab. | ||
workflow_dispatch: | ||
|
||
permissions: | ||
issues: write | ||
pull-requests: write | ||
|
||
jobs: | ||
scheduled: | ||
runs-on: ubuntu-latest | ||
steps: | ||
- name: Checkout | ||
uses: actions/checkout@v4 | ||
|
||
# https://github.com/denoland/setup-deno#latest-stable-for-a-major | ||
- uses: denoland/setup-deno@v1 | ||
with: | ||
deno-version: v1.x | ||
|
||
- name: Thrasher tracker | ||
run: | | ||
deno run \ | ||
--allow-read \ | ||
--allow-net \ | ||
--allow-env=HOME,GITHUB_TOKEN \ | ||
scripts/deno/source-tracker/mod.ts | ||
env: | ||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
{ | ||
"fmt": { | ||
"useTabs": true, | ||
"singleQuote": true | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
import { load } from 'https://deno.land/std@0.210.0/dotenv/mod.ts'; | ||
import { Octokit } from 'https://esm.sh/@octokit/rest@20.0.2'; | ||
|
||
export { type RestEndpointMethodTypes } from 'https://esm.sh/@octokit/rest@20.0.2'; | ||
|
||
// this should be provided by the environment (i.e. GitHub Actions) | ||
let token = Deno.env.get('GITHUB_TOKEN'); | ||
|
||
// we're probably running in a local dev environment | ||
// Create a personal access token at https://github.com/settings/tokens/new?scopes=repo | ||
// and add it to your .env file as GITHUB_TOKEN | ||
if (!token) { | ||
const env = await load(); | ||
token = env.GITHUB_TOKEN; | ||
} | ||
if (!token) console.warn('Missing GITHUB_TOKEN'); | ||
|
||
export const octokit = new Octokit({ | ||
auth: token, | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
# `@guardian/source-*` package usage tracker | ||
|
||
This Deno script is used to track the usage of the `@guardian/source-*` packages in the Guardian GitHub organisation. | ||
|
||
It keeps https://github.com/guardian/csnx/issues/1058 up to date. | ||
|
||
## Development | ||
|
||
You can run it locally by running `deno run -A scripts/deno/source-tracker/mod.ts`. | ||
|
||
You will need to create a personal access token at https://github.com/settings/tokens/new?scopes=repo and add it to an `.env` file as `GITHUB_TOKEN`. | ||
|
||
### Local caching | ||
|
||
To avoid hitting the rate limit in dev, the script saves the responses from the APIs it hits in Deno's `localStorage`. | ||
|
||
If you want to clear/disable that, uncomment the `localStorage.clear()` line in `mod.ts`. | ||
|
||
## Production | ||
|
||
The script is run once a day by a GitHub Action. See `.github/workflows/source-tracker.yml`. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
import { octokit, type RestEndpointMethodTypes } from '../octokit.ts'; | ||
|
||
export const getInstallationData = async ( | ||
installation: RestEndpointMethodTypes['search']['code']['response']['data']['items'][number], | ||
): Promise< | ||
RestEndpointMethodTypes['repos']['getContent']['response']['data'] | ||
> => { | ||
const key = `${installation.git_url}-data`; | ||
|
||
const stored = localStorage.getItem(key); | ||
if (stored) return JSON.parse(stored); | ||
|
||
const { data } = await octokit.rest.repos.getContent({ | ||
owner: installation.repository.owner.login, | ||
repo: installation.repository.name, | ||
path: installation.path, | ||
}); | ||
localStorage.setItem(key, JSON.stringify(data)); | ||
|
||
return data; | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
import { octokit, type RestEndpointMethodTypes } from '../octokit.ts'; | ||
|
||
export const getInstallations = async ( | ||
packageName: string, | ||
): Promise< | ||
RestEndpointMethodTypes['search']['code']['response']['data']['items'] | ||
> => { | ||
const key = `${packageName}-result`; | ||
|
||
const stored = localStorage.getItem(key); | ||
if (stored) return JSON.parse(stored); | ||
|
||
const { | ||
data: { items }, | ||
} = await octokit.rest.search.code({ | ||
headers: { | ||
'X-GitHub-Api-Version': '2022-11-28', | ||
}, | ||
q: `filename:package.json+org:guardian+"${packageName}"`, | ||
per_page: 100, | ||
}); | ||
|
||
localStorage.setItem(key, JSON.stringify(items)); | ||
|
||
return items; | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
import getLatestOnNpm from 'npm:latest-version@7.0.0'; | ||
|
||
export const getLatestVersion = async (packageName: string) => { | ||
const key = `${packageName}-latest`; | ||
|
||
const stored = localStorage.getItem(key); | ||
if (stored) { | ||
return stored; | ||
} | ||
|
||
const result = await getLatestOnNpm(packageName); | ||
localStorage.setItem(key, result); | ||
|
||
return result; | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
import { Markdown } from 'https://deno.land/x/deno_markdown@v0.2/mod.ts'; | ||
import { depTypes } from './get-pkg-versions-in-use.ts'; | ||
import { UsageData } from './mod.ts'; | ||
|
||
export const getMarkdown = (usageData: UsageData) => { | ||
const issue = new Markdown(); | ||
|
||
for (const [packageName, { usage, latestVersion }] of Object.entries( | ||
usageData, | ||
)) { | ||
issue.header(`${packageName}`, 1); | ||
issue.paragraph( | ||
`Latest version: [\`v${latestVersion}\`](https://www.npmjs.com/package/${packageName}).`, | ||
); | ||
issue.header(`Versions in use`, 3); | ||
|
||
const versions = usage | ||
.sort((a, b) => b.version - a.version) | ||
.flatMap(({ version, installations }) => { | ||
const x = installations | ||
.sort((a, b) => a.project.localeCompare(b.project)) | ||
.map( | ||
({ | ||
project, | ||
pkgUrl, | ||
dependencies, | ||
devDependencies, | ||
peerDependencies, | ||
}) => [ | ||
`[${project}](${pkgUrl})`, | ||
dependencies ? `\`${dependencies}\`` : '', | ||
devDependencies ? `\`${devDependencies}\`` : '', | ||
peerDependencies ? `\`${peerDependencies}\`` : '', | ||
], | ||
); | ||
return [[`**${version}**`], ...x]; | ||
}); | ||
|
||
issue.table([['Installation', ...depTypes], ...versions], { | ||
align: ['l', 'r', 'r', 'r'], | ||
}); | ||
} | ||
|
||
return issue.content; | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,82 @@ | ||
import semver from 'npm:semver@7.5.4'; | ||
import { getInstallationData } from './get-installation-data.ts'; | ||
import { getInstallations } from './get-installations.ts'; | ||
|
||
export type PkgVersionsInUse = { | ||
version: number; | ||
installations: Array<{ | ||
project: string; | ||
pkgUrl: string; | ||
dependencies?: string; | ||
devDependencies?: string; | ||
peerDependencies?: string; | ||
}>; | ||
}[]; | ||
|
||
export const depTypes = [ | ||
'dependencies', | ||
'devDependencies', | ||
'peerDependencies', | ||
] as const; | ||
|
||
export const getPkgVersionsInUse = async ( | ||
packageName: string, | ||
): Promise<PkgVersionsInUse> => { | ||
const pkgVersionsInUse: PkgVersionsInUse = []; | ||
|
||
console.log(`Searching for installations of ${packageName}:`); | ||
const installations = await getInstallations(packageName); | ||
console.log(`- found ${installations.length} installations`); | ||
|
||
console.log(`Getting versions in:`); | ||
|
||
for (const installation of installations) { | ||
const installationData = await getInstallationData(installation); | ||
const project = | ||
`${installation.repository.name}/${installation.path}`.replace( | ||
'/package.json', | ||
'', | ||
); | ||
|
||
console.log(`- ${project}`); | ||
|
||
if (!Array.isArray(installationData) && 'content' in installationData) { | ||
const contents = atob(installationData.content); | ||
const installationPkg = JSON.parse(contents); | ||
|
||
if (installationPkg.name === packageName) continue; | ||
|
||
const instance: PkgVersionsInUse[number]['installations'][number] = { | ||
project, | ||
pkgUrl: installation.html_url, | ||
}; | ||
|
||
for (const depType of depTypes) { | ||
instance[depType] = installationPkg[depType]?.[packageName]; | ||
} | ||
|
||
const versions = depTypes.map( | ||
(depType) => installationPkg[depType]?.[packageName], | ||
); | ||
|
||
const minVersions = versions | ||
.filter(Boolean) | ||
.map((version) => semver.minVersion(version).version); | ||
const lowestVersion = semver.sort(minVersions)[0]; | ||
const lowestVersionMajor = semver.major(lowestVersion); | ||
|
||
const existingVersion = pkgVersionsInUse.find( | ||
(_) => _.version === lowestVersionMajor, | ||
); | ||
if (existingVersion) { | ||
existingVersion.installations.push(instance); | ||
} else { | ||
pkgVersionsInUse.push({ | ||
version: lowestVersionMajor, | ||
installations: [instance], | ||
}); | ||
} | ||
} | ||
} | ||
return pkgVersionsInUse; | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
import { getLatestVersion } from './get-latest-version.ts'; | ||
import { getMarkdown } from './get-markdown.ts'; | ||
import { | ||
getPkgVersionsInUse, | ||
type PkgVersionsInUse, | ||
} from './get-pkg-versions-in-use.ts'; | ||
import { octokit } from '../octokit.ts'; | ||
|
||
// localStorage.clear(); | ||
|
||
const packages = [ | ||
'@guardian/source-foundations', | ||
'@guardian/source-react-components', | ||
] as const; | ||
|
||
export type UsageData = { | ||
[name in (typeof packages)[number]]: { | ||
latestVersion: string; | ||
usage: PkgVersionsInUse; | ||
}; | ||
}; | ||
|
||
const usageData: UsageData = {} as UsageData; | ||
|
||
for (const packageName of packages) { | ||
const versionsInUse = await getPkgVersionsInUse(packageName); | ||
|
||
console.log(`Getting latest version of ${packageName}:`); | ||
const latestVersion = await getLatestVersion(packageName); | ||
console.log(`- ${latestVersion}`); | ||
|
||
usageData[packageName] = { usage: versionsInUse, latestVersion }; | ||
} | ||
|
||
const markdown = getMarkdown(usageData); | ||
|
||
// const content = new TextEncoder().encode(markdown); | ||
// await Deno.writeFile('markdown.md', content); | ||
|
||
const issue_number = 1058; | ||
|
||
try { | ||
const { | ||
data: { html_url }, | ||
} = await octokit.rest.issues.update({ | ||
owner: 'guardian', | ||
repo: 'csnx', | ||
issue_number, | ||
body: markdown, | ||
}); | ||
|
||
console.info(`Successfully updated issue #${issue_number}`); | ||
console.info(html_url); | ||
} catch (error) { | ||
console.warn(`Failed to update issue #${issue_number}`); | ||
console.error(error); | ||
} |