Privacy 2024 #2498
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Progress Tracker | |
| env: | |
| # Update these environment variables every year | |
| # And check the tracker is enabled at: | |
| # https://github.com/HTTPArchive/almanac.httparchive.org/actions/workflows/progress-tracker.yml | |
| FILTER_LABEL: '2024 chapter' | |
| TRACKER_ISSUE_NUMBER: 3634 | |
| SQL_BASE_DIR: 'https://github.com/HTTPArchive/almanac.httparchive.org/tree/main/sql/2024/' | |
| on: | |
| workflow_dispatch: | |
| issues: | |
| types: | |
| - edited | |
| jobs: | |
| track-progress: | |
| name: Track Progress | |
| if: github.repository == 'HTTPArchive/almanac.httparchive.org' | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/github-script@v7 | |
| if: github.event_name == 'workflow_dispatch' || contains(github.event.issue.labels.*.name, env.FILTER_LABEL) | |
| with: | |
| # yamllint disable rule:line-length | |
| script: | | |
| const filterLabel = process.env.FILTER_LABEL | |
| const trackerIssueNumber = process.env.TRACKER_ISSUE_NUMBER | |
| const sqlBaseDir = process.env.SQL_BASE_DIR | |
| const icons = { | |
| Draft: '📄', | |
| SQL: '🔍', | |
| Results: '📊' | |
| } | |
| const linkMap = { | |
| Draft: /^\[~google-doc\]:\s*(?<link>\S+)/, | |
| SQL: /^\[~sql-files\]:\s*(?<link>\S+)/, | |
| Results: /^\[~google-sheets\]:\s*(?<link>\S+)/ | |
| } | |
| const teamMembers = { | |
| Leads: {}, | |
| Authors: {}, | |
| Reviewers: {}, | |
| Analysts: {}, | |
| Editors: {}, | |
| All: {} | |
| } | |
| const parseListItems = text => { | |
| const items = [] | |
| text.split('\n').forEach(line => { | |
| const m = line.trim().match(/^([*+-]|\d+\.)\s+(?<item>.+)/) | |
| if (m) { | |
| items.push(m.groups.item) | |
| } | |
| }) | |
| return items | |
| } | |
| const progressIssue = await github.rest.issues.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: trackerIssueNumber | |
| }) | |
| const progressContent = progressIssue.data.body | |
| const m = progressContent.match(/<!-- TASKS BEGIN -->(?<tasks>[\s\S]*)<!-- TASKS END -->/) | |
| if (!m) { | |
| core.info(`Tasks list marker not found in issue #${trackerIssueNumber}, add list of tracked tasks as: <!-- TASKS BEGIN -->TRACKED TASKS LIST<!-- TASKS END -->`) | |
| return | |
| } | |
| const trackedTasks = parseListItems(m.groups.tasks) | |
| const totalTrackedTasks = trackedTasks.length | |
| core.info(`Tracking progress of ${totalTrackedTasks} tasks across chapters`) | |
| const calcStats = nums => { | |
| const size = nums.length | |
| if (!size) { | |
| return [0, 0, 0, 0, 0, 0, 0] | |
| } | |
| nums.sort((a, b) => a - b) | |
| const sum = nums.reduce((a,b) => a + b, 0) | |
| return [ | |
| sum, | |
| nums[0], | |
| nums[size - 1], | |
| (sum / size).toFixed(2), | |
| nums[Math.floor(size / 2)] | |
| ] | |
| } | |
| const parseTasks = text => { | |
| const tasks = [] | |
| text.split('\n').forEach(line => { | |
| if (/^(\s{2,})?([*+-]|\d+\.)\s+\[ \]\s+\S+/.test(line)) { | |
| tasks.push('') | |
| } | |
| if (/^(\s{2,})?([*+-]|\d+\.)\s+\[[xX]\]\s+\S+/.test(line)) { | |
| tasks.push('☑') | |
| } | |
| }) | |
| if (tasks[tasks.length - 1] === '☑') { | |
| tasks[tasks.length - 1] = '🚀' | |
| } | |
| return tasks | |
| } | |
| const parseLinks = text => { | |
| const links = {} | |
| text.split('\n').forEach(line => { | |
| for (const [k, v] of Object.entries(linkMap)) { | |
| const m = line.match(v) | |
| if (m) { | |
| links[k] = m.groups.link | |
| } | |
| } | |
| }) | |
| return links | |
| } | |
| const parseHeading = text => { | |
| const m = text.match(/^#\s+Part\s+(?<part>\S+)\s+Chapter\s+(?<number>\S+)\s*:\s*(?<title>.+)/) | |
| return m ? m.groups : {part: '', number: '', title: ''} | |
| } | |
| const parseUserNames = members => { | |
| return [...members.matchAll(/@(?<username>[\w-]+)/g)].map(u => u.groups.username) | |
| } | |
| const parseTeam = text => { | |
| const team = {} | |
| text.split('\n').forEach(line => { | |
| if (/\|\s*\@[^|]*\s*\|/.test(line)) { | |
| const [, leads, authors, reviewers, analysts, editors] = line.split('|') | |
| team.Leads = parseUserNames(leads) | |
| team.Authors = parseUserNames(authors) | |
| team.Reviewers = parseUserNames(reviewers) | |
| team.Analysts = parseUserNames(analysts) | |
| team.Editors = parseUserNames(editors) | |
| team.All = [...new Set([...team.Leads,...team.Authors, ...team.Reviewers, ...team.Analysts, ...team.Editors])] | |
| } | |
| }) | |
| return team | |
| } | |
| const formatLinks = links => { | |
| return Object.entries(links).filter(l => !['#', sqlBaseDir].includes(l[1])).map(l => `[${icons[l[0]] || l[0]}](${l[1]})`).join(' ') | |
| } | |
| const formatTeam = team => { | |
| return Object.entries(team).map(([k, v]) => `<span title="${k}: ${v.length ? v.join(', ') : 'TBD'}">${v.length}</span>`).join('+').replace(/\+([^+]*)$/, ' ($1)').replace(/\+/g,' ') | |
| } | |
| const formatRow = chapter => { | |
| return `${chapter.number ? `<span title="Part: ${chapter.part}, Chapter: ${chapter.number}">${chapter.number}</span>. ` : ''}[${chapter.title}](${chapter.url}) | ${formatLinks(chapter.links)} | ${formatTeam(chapter.team)} | ${chapter.tasks.join(' | ')}` | |
| } | |
| const taskIssues = await github.paginate(github.rest.issues.listForRepo, { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| direction: 'asc', | |
| labels: filterLabel, | |
| state: 'all' | |
| }) | |
| let chapters = [] | |
| for (const issue of taskIssues) { | |
| const tasksStatus = parseTasks(issue.body) | |
| const tasksCount = tasksStatus.length | |
| if(tasksCount != totalTrackedTasks) { | |
| core.info(`Skipping issue #${issue.number} with ${tasksCount} instead of ${totalTrackedTasks} tasks`) | |
| continue | |
| } | |
| core.info(`Adding issue #${issue.number} with ${tasksCount} tasks`) | |
| const heading = parseHeading(issue.body) | |
| chapters.push({ | |
| id: issue.number, | |
| url: issue.html_url, | |
| title: heading.title || issue.title.replace(/\b20[0-9][0-9]\b/,'').replace(/ /,' ').trim(), | |
| part: heading.part, | |
| number: heading.number, | |
| links: parseLinks(issue.body), | |
| team: parseTeam(issue.body), | |
| tasks: tasksStatus | |
| }) | |
| } | |
| if(!chapters.length) { | |
| core.info('Nothing to report') | |
| return | |
| } | |
| const teamStats = {} | |
| Object.keys(teamMembers).forEach(t => { | |
| teamStats[t] = calcStats(chapters.map(c => (c.team[t] || []).length)) | |
| }) | |
| chapters.sort((a, b) => a.number.localeCompare(b.number, undefined, {numeric: true})) | |
| chapters.forEach(c => { | |
| Object.entries(c.team).forEach(([k, v]) => { | |
| v.forEach(m => { | |
| teamMembers[k][m] = (teamMembers[k][m] || []).concat(c.title) | |
| }) | |
| }) | |
| }) | |
| const report = [ | |
| `Chapters | <span title="${Object.entries(icons).map(([k, v]) => `${v} (${k})`).join(', ')}">Links</span> | <span title="${Object.keys(teamMembers).join('+').replace(/\+([^+]*)$/, ' ($1)').replace(/\+/g,' ')}">Team</span> | ${trackedTasks.map((v, i) => `<span title="${v}">${i}<span>`).join(' | ')}`, | |
| `:--------|:------|:----${'|:-:'.repeat(totalTrackedTasks)}` | |
| ].concat(chapters.map(formatRow)).join('\n') | |
| const stats = [ | |
| 'Contributor Stats | Total | Min | Max | Mean | Median', | |
| ':-----------------|------:|----:|----:|-----:|------:' | |
| ].concat(Object.entries(teamStats).map(([k, v]) => `${k} | ${v.join(' | ')}`)).join('\n') | |
| const teams = Object.entries(teamMembers).map(([team, members]) => { | |
| const memberList = Object.entries(members).sort(([x, a], [y, b]) => { | |
| return x.toLocaleLowerCase().localeCompare(y.toLocaleLowerCase()) | |
| }).sort(([x, a], [y, b]) => { | |
| return b.length - a.length | |
| }).map(([member, chapters]) => { | |
| return `<span title="Chapters: ${chapters.join(', ')}">${member} (${chapters.length})</span>` | |
| }).join(', ') | |
| return `### ${team} (${Object.keys(members).length})\n\n${memberList}` | |
| }).join('\n\n') | |
| github.rest.issues.update({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: trackerIssueNumber, | |
| body: progressContent.replace(/<!-- REPORT START -->[\s\S]*<!-- REPORT END -->/, `<!-- REPORT START -->\n${report}\n\n${stats}\n\n<details><summary><b>Contributor Details</b></summary>\n\n${teams}\n</details>\n<!-- REPORT END -->`) | |
| }) | |
| core.info(`Updated report in issue #${trackerIssueNumber} with ${chapters.length} chapters`) |