-
-
Notifications
You must be signed in to change notification settings - Fork 2.2k
ref(plugins/languages): indepth mode #1118
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
26 commits
Select commit
Hold shift + click to select a range
1798e05
refactor indepth
lowlighter b6059e4
continue
lowlighter a01d0a0
Update allow.txt
lowlighter 10ca229
Update allow.txt
lowlighter be568b9
improvements
lowlighter 253b36c
lint
lowlighter cabb989
custom repo and small improvements
lowlighter 78d8c12
continue
lowlighter 3200161
continue
lowlighter d4ec6d6
Update README.md
lowlighter 0a2c237
Merge branch 'master' into ref-languages-indepth
lowlighter a192fdf
fix: gpg armor not imported in indepth
lowlighter dfc4b49
fix: pass context for repository support, and fix mishandled timeout
lowlighter 1d55f39
fix: filter for repository
lowlighter fe7310f
fix: filter
lowlighter aaeced8
Update index.mjs
lowlighter 99cf07b
test
lowlighter 729fe7d
fix: remove condition
lowlighter 3b21d59
Update recent.mjs
lowlighter 21032a1
fix options
lowlighter 525ef2c
fix recent
lowlighter 098dd11
some fixes
lowlighter b9aaaed
add branch display
lowlighter fad58e8
Update README.md
lowlighter c21ba13
Update request.mjs
lowlighter e5543ea
test: add mocked data
lowlighter File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,10 @@ | ||
| gpgarmor | ||
| github | ||
| https | ||
| leetcode | ||
| pgn | ||
| scm | ||
| shas | ||
| ssh | ||
| ubuntu | ||
| yargsparser |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,182 @@ | ||
| //Imports | ||
| import fs from "fs/promises" | ||
| import os from "os" | ||
| import paths from "path" | ||
| import git from "simple-git" | ||
| import {filters} from "../../../app/metrics/utils.mjs" | ||
|
|
||
| /**Analyzer */ | ||
| export class Analyzer { | ||
|
|
||
| /**Constructor */ | ||
| constructor(login, {account = "bypass", authoring = [], uid = Math.random(), shell, rest = null, context = {mode:"user"}, skipped = [], categories = ["programming", "markup"], timeout = {global:NaN, repositories:NaN}}) { | ||
| //User informations | ||
| this.login = login | ||
| this.account = account | ||
| this.authoring = authoring | ||
| this.uid = uid | ||
| this.gpg = [] | ||
|
|
||
| //Utilities | ||
| this.shell = shell | ||
| this.rest = rest | ||
| this.context = context | ||
| this.markers = { | ||
| hash:/\b[0-9a-f]{40}\b/, | ||
| file:/^[+]{3}\sb[/](?<file>[\s\S]+)$/, | ||
| line:/^(?<op>[-+])\s*(?<content>[\s\S]+)$/, | ||
| } | ||
| this.parser = /^(?<login>[\s\S]+?)\/(?<name>[\s\S]+?)(?:@(?<branch>[\s\S]+?)(?::(?<ref>[\s\S]+))?)?$/ | ||
| this.consumed = false | ||
|
|
||
| //Options | ||
| this.skipped = skipped | ||
| this.categories = categories | ||
| this.timeout = timeout | ||
|
|
||
| //Results | ||
| this.results = {partial: {global:false, repositories:false}, total: 0, lines: {}, stats: {}, colors: {}, commits: 0, files: 0, missed: {lines: 0, bytes: 0, commits: 0}, elapsed:0} | ||
| this.debug(`instantiated a new ${this.constructor.name}`) | ||
| } | ||
|
|
||
| /**Run analyzer */ | ||
| async run(runner) { | ||
| if (this.consumed) | ||
| throw new Error("This analyzer has already been consumed, another instance needs to be created to perform a new analysis") | ||
| this.consumed = true | ||
| const results = await new Promise(async solve => { | ||
| let completed = false | ||
| if (Number.isFinite(this.timeout.global)) { | ||
| this.debug(`timeout set to ${this.timeout.global}m`) | ||
| setTimeout(() => { | ||
| if (!completed) { | ||
| try { | ||
| this.debug(`reached maximum execution time of ${this.timeout.global}m for analysis`) | ||
| this.results.partial.global = true | ||
| solve(this.results) | ||
| } | ||
| catch { | ||
| //Ignore errors | ||
| } | ||
| } | ||
| }, this.timeout.global * 60 * 1000) | ||
| } | ||
| await runner() | ||
| completed = true | ||
| solve(this.results) | ||
| }) | ||
| results.partial = (results.partial.global)||(results.partial.repositories) | ||
| return results | ||
| } | ||
|
|
||
| /**Parse repository */ | ||
| parse(repository) { | ||
| let branch = null, ref = null | ||
| if (typeof repository === "string") { | ||
| if (!this.parser.test(repository)) | ||
| throw new TypeError(`"${repository}" pattern is not supported`) | ||
| const {login, name, ...groups} = repository.match(this.parser)?.groups ?? {} | ||
| repository = {owner:{login}, name} | ||
| branch = groups.branch ?? null | ||
| ref = groups.ref ?? null | ||
| } | ||
| const repo = `${repository.owner.login}/${repository.name}` | ||
| const path = paths.join(os.tmpdir(), `${this.uid}-${repo.replace(/[^\w]/g, "_")}`) | ||
| return {repo, path, branch, ref} | ||
| } | ||
|
|
||
| /**Clone a repository */ | ||
| async clone(repository) { | ||
| const {repo, branch, path} = this.parse(repository) | ||
| let url = /^https?:\/\//.test(repo) ? repo : `https://github.com/${repo}` | ||
| try { | ||
| this.debug(`cloning ${url} to ${path}`) | ||
| await fs.rm(path, {recursive: true, force: true}) | ||
| await fs.mkdir(path, {recursive: true}) | ||
| await git(path).clone(url, ".", ["--single-branch"]).status() | ||
| this.debug(`cloned ${url} to ${path}`) | ||
| if (branch) { | ||
| this.debug(`switching to branch ${branch} for ${repo}`) | ||
| await git(path).branch(branch) | ||
| } | ||
| return true | ||
| } | ||
| catch (error) { | ||
| this.debug(`failed to clone ${url} (${error})`) | ||
| this.clean(path) | ||
| return false | ||
| } | ||
| } | ||
|
|
||
| /**Analyze a repository */ | ||
| async analyze(path, {commits = []} = {}) { | ||
| const cache = {files:{}, languages:{}} | ||
| const start = Date.now() | ||
| let elapsed = 0, processed = 0 | ||
| if (this.timeout.repositories) | ||
| this.debug(`timeout for repository analysis set to ${this.timeout.repositories}m`) | ||
| for (const commit of commits) { | ||
| elapsed = (Date.now() - start)/1000/60 | ||
| if ((this.timeout.repositories)&&(elapsed > this.timeout.repositories)) { | ||
| this.results.partial.repositories = true | ||
| this.debug(`reached maximum execution time of ${this.timeout.repositories}m for repository analysis (${elapsed}m elapsed)`) | ||
| break | ||
| } | ||
| try { | ||
| const {total, files, missed, lines, stats} = await this.linguist(path, {commit, cache}) | ||
| this.results.commits++ | ||
| this.results.total += total | ||
| this.results.files += files | ||
| this.results.missed.lines += missed.lines | ||
| this.results.missed.bytes += missed.bytes | ||
| for (const language in lines) { | ||
| if (this.categories.includes(cache.languages[language]?.type)) | ||
| this.results.lines[language] = (this.results.lines[language] ?? 0) + lines[language] | ||
| } | ||
| for (const language in stats) { | ||
| if (this.categories.includes(cache.languages[language]?.type)) | ||
| this.results.stats[language] = (this.results.stats[language] ?? 0) + stats[language] | ||
| } | ||
| } | ||
| catch (error) { | ||
| this.debug(`skipping commit ${commit.sha} (${error})`) | ||
| this.results.missed.commits++ | ||
| } | ||
| finally { | ||
| this.results.elapsed += elapsed | ||
| processed++ | ||
| if ((processed%50 === 0)||(processed === commits.length)) | ||
| this.debug(`at commit ${processed}/${commits.length} (${(100*processed/commits.length).toFixed(2)}%, ${elapsed.toFixed(2)}m elapsed)`) | ||
| } | ||
| } | ||
| this.results.colors = Object.fromEntries(Object.entries(cache.languages).map(([lang, {color}]) => [lang, color])) | ||
| } | ||
|
|
||
| /**Clean a path */ | ||
| async clean(path) { | ||
| try { | ||
| this.debug(`cleaning ${path}`) | ||
| await fs.rm(path, {recursive: true, force: true}) | ||
| this.debug(`cleaned ${path}`) | ||
| return true | ||
| } | ||
| catch (error) { | ||
| this.debug(`failed to clean (${error})`) | ||
| return false | ||
| } | ||
| } | ||
|
|
||
| /**Whether to skip a repository or not */ | ||
| ignore(repository) { | ||
| const ignored = !filters.repo(repository, this.skipped) | ||
| if (ignored) | ||
| this.debug(`skipping ${typeof repository === "string" ? repository : `${repository?.owner?.login}/${repository?.name}`} as it matches skipped repositories`) | ||
| return ignored | ||
| } | ||
|
|
||
| /**Debug log */ | ||
| debug(message) { | ||
| return console.debug(`metrics/compute/${this.login}/plugins > languages > ${this.constructor.name.replace(/([a-z])([A-Z])/, (_, a, b) => `${a} ${b.toLocaleLowerCase()}`).toLocaleLowerCase()} > ${message}`) | ||
| } | ||
|
|
||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Check failure
Code scanning / CodeQL
Insecure randomness