diff --git a/.github/actions-scripts/lib/issue-report.js b/.github/actions-scripts/lib/issue-report.js new file mode 100644 index 000000000000..ec6cdd4c8918 --- /dev/null +++ b/.github/actions-scripts/lib/issue-report.js @@ -0,0 +1,125 @@ +export async function createReportIssue({ + core, + octokit, + reportTitle, + reportBody, + reportRepository, + reportLabel, +}) { + const [owner, repo] = reportRepository.split('/') + // Create issue + let newReport + try { + const { data } = await octokit.request('POST /repos/{owner}/{repo}/issues', { + owner, + repo, + title: reportTitle, + body: reportBody, + labels: [reportLabel], + }) + newReport = data + core.info(`Created new report issue at ${newReport.html_url}\n`) + } catch (error) { + core.error(error) + core.setFailed('Error creating new issue') + throw error + } + + return newReport +} + +export async function linkReports({ + core, + octokit, + newReport, + reportRepository, + reportAuthor, + reportLabel, +}) { + const [owner, repo] = reportRepository.split('/') + + core.info('Attempting to link reports...') + // Find previous report issue + let previousReports + try { + previousReports = await octokit.rest.issues.listForRepo({ + owner, + repo, + creator: reportAuthor, + labels: reportLabel, + state: 'all', // We want to get the previous report, even if it is closed + sort: 'created', + direction: 'desc', + per_page: 25, + }) + previousReports = previousReports.data + } catch (error) { + core.setFailed('Error listing issues for repo') + throw error + } + core.info(`Found ${previousReports.length} previous reports`) + + if (previousReports.length <= 1) { + core.info('No previous reports to link to') + return + } + + // 2nd report should be most recent previous report + const previousReport = previousReports[1] + + // Comment the old report link on the new report + try { + await octokit.rest.issues.createComment({ + owner, + repo, + issue_number: newReport.number, + body: `⬅️ [Previous report](${previousReport.html_url})`, + }) + core.info(`Linked old report to new report via comment on new report, #${newReport.number}`) + } catch (error) { + core.setFailed(`Error commenting on newReport, #${newReport.number}`) + throw error + } + + // Comment on all previous reports that are still open + for (const previousReport of previousReports) { + if (previousReport.state === 'closed' || previousReport.html_url === newReport.html_url) { + continue + } + + // If an old report is not assigned to someone we close it + const shouldClose = !previousReport.assignees.length + let body = `➡️ [Newer report](${newReport.html_url})` + if (shouldClose) { + body += '\n\nClosing in favor of newer report since there are no assignees on this issue' + } + try { + await octokit.rest.issues.createComment({ + owner, + repo, + issue_number: previousReport.number, + body, + }) + core.info( + `Linked old report to new report via comment on old report: #${previousReport.number}.`, + ) + } catch (error) { + core.setFailed(`Error commenting on previousReport, #${previousReport.number}`) + throw error + } + if (shouldClose) { + try { + await octokit.rest.issues.update({ + owner, + repo, + issue_number: previousReport.number, + state: 'closed', + }) + core.info(`Closing old report: #${previousReport.number} because it doesn't have assignees`) + } catch (error) { + core.setFailed(`Error closing previousReport, #${previousReport.number}`) + throw error + } + } + } +} diff --git a/.github/actions-scripts/post-lints.js b/.github/actions-scripts/post-lints.js new file mode 100644 index 000000000000..d3aaed35af45 --- /dev/null +++ b/.github/actions-scripts/post-lints.js @@ -0,0 +1,69 @@ +#!/usr/bin/env node + +import { program } from 'commander' +import fs from 'fs' +import coreLib from '@actions/core' + +import github from '../../script/helpers/github.js' +import { getEnvInputs } from './lib/get-env-inputs.js' +import { createReportIssue, linkReports } from './lib/issue-report.js' + +// [start-readme] +// +// This script runs once a week via a scheduled GitHub Action to lint +// the entire content and data directories based on our +// markdownlint.js rules. +// +// If errors are found, it will open up a new issue in the +// docs-content repo with the label "broken content markdown report". +// +// The Content FR will go through the issue and update the content and +// data files accordingly. +// +// [end-readme] + +program + .description('Opens an issue for Content FR with the errors from the weekly content/data linter.') + .option( + '-p, --path ', + 'provide a path to the errors output json file that will be in the issue body', + ) + .parse(process.argv) + +const { path } = program.opts() + +main() +async function main() { + const errors = fs.readFileSync(`${path}`, 'utf8') + const core = coreLib + const { REPORT_REPOSITORY, REPORT_AUTHOR, REPORT_LABEL } = process.env + + const octokit = github() + // `GITHUB_TOKEN` is optional. If you need the token to post a comment + // or open an issue report, you might get cryptic error messages from Octokit. + getEnvInputs(['GITHUB_TOKEN']) + + core.info(`Creating issue for errors...`) + + const reportProps = { + core, + octokit, + reportTitle: `Error(s) in content markdown file(s)`, + reportBody: JSON.parse(errors), + reportRepository: REPORT_REPOSITORY, + reportLabel: REPORT_LABEL, + } + + await createReportIssue(reportProps) + + const linkProps = { + core, + octokit, + newReport: await createReportIssue(reportProps), + reportRepository: REPORT_REPOSITORY, + reportAuthor: REPORT_AUTHOR, + reportLabel: REPORT_LABEL, + } + + await linkReports(linkProps) +} diff --git a/.github/workflows/lint-entire-content-data-markdown.yml b/.github/workflows/lint-entire-content-data-markdown.yml new file mode 100644 index 000000000000..d08e343e1ecd --- /dev/null +++ b/.github/workflows/lint-entire-content-data-markdown.yml @@ -0,0 +1,40 @@ +name: 'Lint entire content and data markdown files' + +# **What it does**: Lints our content markdown weekly to ensure the content matches the specified styleguide. If errors exists, it opens a PR for the Docs content team to review. +# **Why we have it**: Extra precaution to run linter on the entire content/data directories. +# **Who does it impact**: Docs content. + +on: + workflow_dispatch: + schedule: + - cron: '20 16 * * 0' # Run every day at 16:20 UTC / 8:20 PST every Sunday + +permissions: + contents: read + issues: write + +jobs: + lint-entire-content-data: + name: Lint entire content and data directories + if: github.repository == 'github/docs-internal' + runs-on: ubuntu-20.04-xl + steps: + - name: Check that gh CLI is installed + run: gh --version + + - name: Check out repo's default branch + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + + - name: Set up Node and dependencies + uses: ./.github/actions/node-npm-setup + + - name: Run content linter + env: + GITHUB_TOKEN: ${{ secrets.DOCS_BOT_PAT_WRITEORG_PROJECT }} + REPORT_AUTHOR: docs-bot + REPORT_LABEL: broken content markdown report + REPORT_REPOSITORY: github/docs-content + timeout-minutes: 10 + run: | + node src/content-linter/scripts/markdownlint.js --errors-only --paths content data --output-file /tmp/error-lints.json + node .github/actions-scripts/post-lints.js --path /tmp/error-lints.json diff --git a/src/content-linter/scripts/markdownlint.js b/src/content-linter/scripts/markdownlint.js index 6f4ae5712b03..590c7d303b9d 100755 --- a/src/content-linter/scripts/markdownlint.js +++ b/src/content-linter/scripts/markdownlint.js @@ -2,7 +2,7 @@ import { program, Option } from 'commander' import markdownlint from 'markdownlint' import { applyFixes } from 'markdownlint-rule-helpers' -import { readFile, writeFile } from 'fs/promises' +import fs from 'fs' import ora from 'ora' import { extname } from 'path' import { execSync } from 'child_process' @@ -36,9 +36,12 @@ program `Specify rules to run. For example, by short name MD001 or long name heading-increment \n${listRules()}\n\n`, ).conflicts('error'), ) + .addOption( + new Option('-o, --output-file ', `Outputs the errors/warnings to the filepath.`), + ) .parse(process.argv) -const { fix, paths, errorsOnly, rules, summaryByRule, verbose } = program.opts() +const { fix, paths, errorsOnly, rules, summaryByRule, outputFile, verbose } = program.opts() const ALL_CONTENT_DIR = ['content', 'data'] main() @@ -65,21 +68,28 @@ async function main() { // Apply markdownlint fixes if available and rewrite the files if (fix) { for (const file of [...files.content, ...files.data]) { - const content = await readFile(file, 'utf8') + const content = fs.readFileSync(file, 'utf8') const applied = applyFixes(content, results[file]) - await writeFile(file, applied) + fs.writeFileSync(file, applied, 'utf-8') } } const errorFileCount = getErrorCountByFile(results) - // Used for a temparary way to allow us to see how many errors currently + // Used for a temporary way to allow us to see how many errors currently // exist for each rule in the content directory. if (summaryByRule && errorFileCount > 0) { reportSummaryByRule(results, config) } else if (errorFileCount > 0) { - reportResults(results) + const errorReport = getResults(results) + if (outputFile) { + fs.writeFileSync(`${outputFile}`, JSON.stringify(errorReport, undefined, 2), function (err) { + if (err) throw err + }) + console.log(`Output written to ${outputFile}`) + } else { + console.log(errorReport) + } } - const end = Date.now() console.log(`\n🕦 Markdownlint finished in ${(end - start) / 1000} s`) @@ -154,23 +164,23 @@ function reportSummaryByRule(results, config) { console.log(JSON.stringify(ruleCount, null, 2)) } -function reportResults(allResults) { - console.log('\n\nMarkdownlint results:\n') +function getResults(allResults) { + const output = {} Object.entries(allResults) // Each result key always has an array value, but it may be empty .filter(([, results]) => results.length) .forEach(([key, results]) => { - console.log(key) if (!verbose) { const formattedResults = results.map((flaw) => formatResult(flaw)) const errors = formattedResults.filter((result) => result.severity === 'error') const warnings = formattedResults.filter((result) => result.severity === 'warning') const sortedResult = [...errors, ...warnings] - console.log(sortedResult) + output[key] = [...sortedResult] } else { - console.log(results) + output[key] = [...results] } }) + return output } // Results are formatted with the key being the filepath diff --git a/src/links/scripts/rendered-content-link-checker.js b/src/links/scripts/rendered-content-link-checker.js index 5c2c3f656598..f6a7939e1bc2 100755 --- a/src/links/scripts/rendered-content-link-checker.js +++ b/src/links/scripts/rendered-content-link-checker.js @@ -26,6 +26,10 @@ import { uploadArtifact as uploadArtifactLib } from '../../../.github/actions-sc import github from '../../../script/helpers/github.js' import { getActionContext } from '../../../.github/actions-scripts/lib/action-context.js' import { createMinimalProcessor } from '#src/content-render/unified/processor.js' +import { + createReportIssue, + linkReports, +} from '../../../.github/actions-scripts/lib/issue-report.js' const STATIC_PREFIXES = { assets: path.resolve('assets'), @@ -187,6 +191,9 @@ async function main(core, octokit, uploadArtifact, opts = {}) { createReport = false, failOnFlaw = false, shouldComment = false, + reportRepository = 'github/docs-content', + reportAuthor = 'docs-bot', + reportLabel = 'broken link report', } = opts // Note! The reason we're using `warmServer()` in this script, @@ -265,9 +272,26 @@ async function main(core, octokit, uploadArtifact, opts = {}) { core.info(`All flaws written to artifact log.`) if (createReport) { core.info(`Creating issue for flaws...`) - const newReport = await createReportIssue(core, octokit, flaws, opts) + const reportProps = { + core, + octokit, + reportTitle: `${flaws.length + 1} broken links found`, + reportBody: flawIssueDisplay(flaws, opts), + reportRepository, + reportLabel, + } + const newReport = await createReportIssue(reportProps) + if (linkReports) { - await linkReports(core, octokit, newReport, opts) + const linkProps = { + core, + octokit, + newReport, + reportRepository, + reportAuthor, + reportLabel, + } + await linkReports(linkProps) } } if (shouldComment) { @@ -299,128 +323,6 @@ async function main(core, octokit, uploadArtifact, opts = {}) { } } -async function createReportIssue(core, octokit, flaws, opts) { - const { reportRepository = 'github/docs-content', reportLabel = 'broken link report' } = opts - const [owner, repo] = reportRepository.split('/') - - const brokenLinksDisplay = flawIssueDisplay(flaws, opts) - - // Create issue with broken links - let newReport - try { - const { data } = await octokit.request('POST /repos/{owner}/{repo}/issues', { - owner, - repo, - title: `${flaws.length + 1} broken links found`, - body: brokenLinksDisplay, - labels: [reportLabel], - }) - newReport = data - core.info(`Created broken links report at ${newReport.html_url}\n`) - } catch (error) { - core.error(error) - core.setFailed('Error creating new issue') - throw error - } - - return newReport -} - -async function linkReports(core, octokit, newReport, opts) { - const { - reportRepository = 'github/docs-content', - reportAuthor = 'docs-bot', - reportLabel = 'broken link report', - } = opts - - const [owner, repo] = reportRepository.split('/') - - core.info('Attempting to link reports...') - // Find previous broken link report issue - let previousReports - try { - previousReports = await octokit.rest.issues.listForRepo({ - owner, - repo, - creator: reportAuthor, - labels: reportLabel, - state: 'all', // We want to get the previous report, even if it is closed - sort: 'created', - direction: 'desc', - per_page: 25, - }) - previousReports = previousReports.data - } catch (error) { - core.setFailed('Error listing issues for repo') - throw error - } - core.info(`Found ${previousReports.length} previous reports`) - - if (previousReports.length <= 1) { - core.info('No previous reports to link to') - return - } - - // 2nd report should be most recent previous report - const previousReport = previousReports[1] - - // Comment the old report link on the new report - try { - await octokit.rest.issues.createComment({ - owner, - repo, - issue_number: newReport.number, - body: `⬅️ [Previous report](${previousReport.html_url})`, - }) - core.info(`Linked old report to new report via comment on new report, #${newReport.number}`) - } catch (error) { - core.setFailed(`Error commenting on newReport, #${newReport.number}`) - throw error - } - - // Comment on all previous reports that are still open - for (const previousReport of previousReports) { - if (previousReport.state === 'closed' || previousReport.html_url === newReport.html_url) { - continue - } - - // If an old report is not assigned to someone we close it - const shouldClose = !previousReport.assignees.length - let body = `➡️ [Newer report](${newReport.html_url})` - if (shouldClose) { - body += '\n\nClosing in favor of newer report since there are no assignees on this issue' - } - try { - await octokit.rest.issues.createComment({ - owner, - repo, - issue_number: previousReport.number, - body, - }) - core.info( - `Linked old report to new report via comment on old report: #${previousReport.number}.`, - ) - } catch (error) { - core.setFailed(`Error commenting on previousReport, #${previousReport.number}`) - throw error - } - if (shouldClose) { - try { - await octokit.rest.issues.update({ - owner, - repo, - issue_number: previousReport.number, - state: 'closed', - }) - core.info(`Closing old report: #${previousReport.number} because it doesn't have assignees`) - } catch (error) { - core.setFailed(`Error closing previousReport, #${previousReport.number}`) - throw error - } - } - } -} - async function commentOnPR(core, octokit, flaws, opts) { const { actionContext = {} } = opts const { owner, repo } = actionContext