diff --git a/Dockerfile b/Dockerfile index 2e52f3b624..9de8c1eb01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,4 +101,4 @@ WORKDIR ${RUNNER_PATH} # COMMAND ENV IN_DOCKER=1 -CMD ["cml-runner"] +CMD ["cml"] diff --git a/bin/cml-pr.js b/bin/cml-pr.js deleted file mode 100755 index 2584947e40..0000000000 --- a/bin/cml-pr.js +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env node - -const print = console.log; -console.log = console.error; - -const yargs = require('yargs'); - -const CML = require('../src/cml').default; -const { GIT_REMOTE, GIT_USER_NAME, GIT_USER_EMAIL } = require('../src/cml'); - -const run = async (opts) => { - const globs = opts._.length ? opts._ : undefined; - const cml = new CML(opts); - print((await cml.prCreate({ ...opts, globs })) || ''); -}; - -const opts = yargs - .strict() - .usage('Usage: $0 ... ') - .describe('md', 'Output in markdown format [](url).') - .boolean('md') - .default('remote', GIT_REMOTE) - .describe('remote', 'Sets git remote.') - .default('user-email', GIT_USER_EMAIL) - .describe('user-email', 'Sets git user email.') - .default('user-name', GIT_USER_NAME) - .describe('user-name', 'Sets git user name.') - .default('repo') - .describe( - 'repo', - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - ) - .default('token') - .describe( - 'token', - 'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.' - ) - .default('driver') - .choices('driver', ['github', 'gitlab']) - .describe('driver', 'If not specify it infers it from the ENV.') - .help('h').argv; -run(opts).catch((e) => { - console.error(e); - process.exit(1); -}); diff --git a/bin/cml-pr.test.js b/bin/cml-pr.test.js deleted file mode 100644 index 060f8504d5..0000000000 --- a/bin/cml-pr.test.js +++ /dev/null @@ -1,25 +0,0 @@ -const { exec } = require('../src/utils'); - -describe('CML e2e', () => { - test('cml-publish -h', async () => { - const output = await exec(`echo none | node ./bin/cml-pr.js -h`); - - expect(output).toMatchInlineSnapshot(` - "Usage: cml-pr.js ... - - Options: - --version Show version number [boolean] - --md Output in markdown format [](url). [boolean] - --remote Sets git remote. [default: \\"origin\\"] - --user-email Sets git user email. [default: \\"olivaw@iterative.ai\\"] - --user-name Sets git user name. [default: \\"Olivaw[bot]\\"] - --repo Specifies the repo to be used. If not specified is extracted - from the CI ENV. - --token Personal access token to be used. If not specified in extracted - from ENV REPO_TOKEN. - --driver If not specify it infers it from the ENV. - [choices: \\"github\\", \\"gitlab\\"] - -h Show help [boolean]" - `); - }); -}); diff --git a/bin/cml-publish.js b/bin/cml-publish.js deleted file mode 100755 index a244f11383..0000000000 --- a/bin/cml-publish.js +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env node - -const print = console.log; -console.log = console.error; - -const fs = require('fs').promises; -const pipeArgs = require('../src/pipe-args'); -const yargs = require('yargs'); - -const CML = require('../src/cml').default; - -const run = async (opts) => { - const { data, file, repo, native } = opts; - - const path = opts._[0]; - const buffer = data ? Buffer.from(data, 'binary') : null; - - const cml = new CML({ ...opts, repo: native ? repo : 'cml' }); - - const output = await cml.publish({ - ...opts, - buffer, - path - }); - - if (!file) print(output); - else await fs.writeFile(file, output); -}; - -pipeArgs.load('binary'); -const data = pipeArgs.pipedArg(); -const opts = yargs - .strict() - .usage(`Usage: $0 `) - .describe('md', 'Output in markdown format [title || name](url).') - .boolean('md') - .describe('md', 'Output in markdown format [title || name](url).') - .default('title') - .describe('title', 'Markdown title [title](url) or ![](url title).') - .alias('title', 't') - .boolean('native') - .describe( - 'native', - "Uses driver's native capabilities to upload assets instead of CML's storage. Currently only available for GitLab CI." - ) - .alias('native', 'gitlab-uploads') - .boolean('rm-watermark') - .describe('rm-watermark', 'Avoid CML watermark.') - .default('mime-type') - .describe( - 'mime-type', - 'Specifies the mime-type. If not set guess it from the content.' - ) - .default('file') - .describe( - 'file', - 'Append the output to the given file. Create it if does not exist.' - ) - .alias('file', 'f') - .default('repo') - .describe( - 'repo', - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - ) - .default('token') - .describe( - 'token', - 'Personal access token to be used. If not specified, extracted from ENV REPO_TOKEN, GITLAB_TOKEN, GITHUB_TOKEN, or BITBUCKET_TOKEN.' - ) - .default('driver') - .choices('driver', ['github', 'gitlab']) - .describe('driver', 'If not specify it infers it from the ENV.') - .help('h') - .demand(data ? 0 : 1).argv; - -run({ ...opts, data }).catch((e) => { - console.error(e); - process.exit(1); -}); diff --git a/bin/cml-publish.test.js b/bin/cml-publish.test.js deleted file mode 100644 index 070a8e3dfc..0000000000 --- a/bin/cml-publish.test.js +++ /dev/null @@ -1,129 +0,0 @@ -const fs = require('fs'); -const { exec } = require('../src/utils'); - -describe('CML e2e', () => { - test('cml-publish -h', async () => { - const output = await exec(`echo none | node ./bin/cml-publish.js -h`); - - expect(output).toMatchInlineSnapshot(` - "Usage: cml-publish.js - - Options: - --version Show version number [boolean] - --md Output in markdown format [title || name](url). - [boolean] - --title, -t Markdown title [title](url) or ![](url title). - --native, --gitlab-uploads Uses driver's native capabilities to upload assets - instead of CML's storage. Currently only available - for GitLab CI. [boolean] - --rm-watermark Avoid CML watermark. [boolean] - --mime-type Specifies the mime-type. If not set guess it from - the content. - --file, -f Append the output to the given file. Create it if - does not exist. - --repo Specifies the repo to be used. If not specified is - extracted from the CI ENV. - --token Personal access token to be used. If not - specified, extracted from ENV REPO_TOKEN, - GITLAB_TOKEN, GITHUB_TOKEN, or BITBUCKET_TOKEN. - --driver If not specify it infers it from the ENV. - [choices: \\"github\\", \\"gitlab\\"] - -h Show help [boolean]" - `); - }); - - test('cml-publish assets/logo.png --md', async () => { - const output = await exec( - `echo none | node ./bin/cml-publish.js assets/logo.png --md` - ); - - expect(output.startsWith('![](')).toBe(true); - }); - - test('cml-publish assets/logo.png', async () => { - const output = await exec( - `echo none | node ./bin/cml-publish.js assets/logo.png` - ); - - expect(output.startsWith('https://')).toBe(true); - }); - - test('cml-publish assets/logo.pdf --md', async () => { - const title = 'this is awesome'; - const output = await exec( - `echo none | node ./bin/cml-publish.js assets/logo.pdf --md --title '${title}'` - ); - - expect(output.startsWith(`[${title}](`)).toBe(true); - }); - - test('cml-publish assets/logo.pdf', async () => { - const output = await exec( - `echo none | node ./bin/cml-publish.js assets/logo.pdf` - ); - - expect(output.startsWith('https://')).toBe(true); - }); - - test('cml-publish assets/test.svg --md', async () => { - const title = 'this is awesome'; - const output = await exec( - `echo none | node ./bin/cml-publish.js assets/test.svg --md --title '${title}'` - ); - - expect(output.startsWith('![](') && output.endsWith(`${title}")`)).toBe( - true - ); - }); - - test('cml-publish assets/test.svg', async () => { - const output = await exec( - `echo none | node ./bin/cml-publish.js assets/test.svg` - ); - - expect(output.startsWith('https://')).toBe(true); - }); - - test('cml-publish assets/logo.pdf to file', async () => { - const file = `cml-publish-test.md`; - - await exec( - `echo none | node ./bin/cml-publish.js assets/logo.pdf --file ${file}` - ); - - expect(fs.existsSync(file)).toBe(true); - await fs.promises.unlink(file); - }); - - test('cml-publish assets/vega-lite.json', async () => { - const output = await exec( - `echo none | node ./bin/cml-publish.js --mime-type=application/json assets/vega-lite.json` - ); - - expect(output.startsWith('https://')).toBe(true); - expect(output.endsWith('json')).toBe(true); - }); - - test('cml-publish assets/test.svg in Gitlab storage', async () => { - const { TEST_GITLAB_REPO: repo, TEST_GITLAB_TOKEN: token } = process.env; - - const output = await exec( - `echo none | node ./bin/cml-publish.js --repo=${repo} --token=${token} --gitlab-uploads assets/test.svg` - ); - - expect(output.startsWith('https://')).toBe(true); - }); - - test('cml-publish /nonexistent produces file error', async () => { - await expect( - exec('echo none | node ./bin/cml-publish.js /nonexistent') - ).rejects.toThrowError('ENOENT: no such file or directory, stat'); - }); - - test('echo text | cml-publish produces a plain text file', async () => { - const output = await exec(`echo none | node ./bin/cml-publish.js`); - - expect(output.startsWith('https://')).toBe(true); - expect(output.endsWith('plain')).toBe(true); - }); -}); diff --git a/bin/cml-send-comment.js b/bin/cml-send-comment.js deleted file mode 100755 index 266e463cc8..0000000000 --- a/bin/cml-send-comment.js +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env node - -const print = console.log; -console.log = console.error; - -const fs = require('fs').promises; -const yargs = require('yargs'); - -const CML = require('../src/cml').default; - -const run = async (opts) => { - const path = opts._[0]; - const report = await fs.readFile(path, 'utf-8'); - const cml = new CML(opts); - print(await cml.commentCreate({ ...opts, report })); -}; - -const opts = yargs - .strict() - .usage('Usage: $0 ') - .default('commit-sha') - .describe( - 'commit-sha', - 'Commit SHA linked to this comment. Defaults to HEAD.' - ) - .alias('commit-sha', 'head-sha') - .boolean('update') - .describe( - 'update', - 'Update the last CML comment (if any) instead of creating a new one' - ) - .boolean('rm-watermark') - .describe( - 'rm-watermark', - 'Avoid watermark. CML needs a watermark to be able to distinguish CML reports from other comments in order to provide extra functionality.' - ) - .default('repo') - .describe( - 'repo', - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - ) - .default('token') - .describe( - 'token', - 'Personal access token to be used. If not specified is extracted from ENV REPO_TOKEN.' - ) - .default('driver') - .choices('driver', ['github', 'gitlab', 'bitbucket']) - .describe('driver', 'If not specify it infers it from the ENV.') - .help('h') - .demand(1).argv; - -run(opts).catch((e) => { - console.error(e); - process.exit(1); -}); diff --git a/bin/cml-send-github-check.js b/bin/cml-send-github-check.js deleted file mode 100755 index db50148ed7..0000000000 --- a/bin/cml-send-github-check.js +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env node - -console.log = console.error; - -const fs = require('fs').promises; -const yargs = require('yargs'); - -const CML = require('../src/cml').default; -const CHECK_TITLE = 'CML Report'; - -const run = async (opts) => { - const path = opts._[0]; - const report = await fs.readFile(path, 'utf-8'); - const cml = new CML({ ...opts }); - await cml.checkCreate({ ...opts, report }); -}; - -const opts = yargs - .strict() - .usage('Usage: $0 ') - .describe( - 'commit-sha', - 'Commit SHA linked to this comment. Defaults to HEAD.' - ) - .alias('commit-sha', 'head-sha') - .default('conclusion', 'success', 'Sets the conclusion status of the check.') - .choices('conclusion', [ - 'success', - 'failure', - 'neutral', - 'cancelled', - 'skipped', - 'timed_out' - ]) - .default('title', CHECK_TITLE) - .describe('title', 'Sets title of the check.') - .default('repo') - .describe( - 'repo', - 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' - ) - .default('token') - .describe( - 'token', - 'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.' - ) - .help('h') - .demand(1).argv; - -run(opts).catch((e) => { - console.error(e); - process.exit(1); -}); diff --git a/bin/cml.js b/bin/cml.js new file mode 100755 index 0000000000..230dc451be --- /dev/null +++ b/bin/cml.js @@ -0,0 +1,68 @@ +#!/usr/bin/env node + +const { basename } = require('path'); +const { pseudoexec } = require('pseudoexec'); + +const which = require('which'); +const winston = require('winston'); +const yargs = require('yargs'); + +const configureLogger = (level) => { + winston.configure({ + format: process.stdout.isTTY + ? winston.format.combine( + winston.format.colorize({ all: true }), + winston.format.simple() + ) + : winston.format.json(), + transports: [ + new winston.transports.Console({ + handleExceptions: true, + handleRejections: true, + level + }) + ] + }); +}; + +const runPlugin = async ({ $0: executable, command }) => { + try { + if (command === undefined) throw new Error('no command'); + const path = which.sync(`${basename(executable)}-${command}`); + const parameters = process.argv.slice(process.argv.indexOf(command) + 1); // HACK + process.exit(await pseudoexec(path, parameters)); + } catch (error) { + winston.debug(error); + yargs.showHelp(); + } +}; + +const handleError = (message, error) => { + if (error) { + winston.error(error); + } else { + yargs.showHelp(); + console.error('\n' + message); + } + process.exit(1); +}; + +const options = { + log: { + describe: 'Maximum log level', + coerce: (value) => configureLogger(value) && value, + choices: ['error', 'warn', 'info', 'debug'], + default: 'info' + } +}; + +yargs + .fail(handleError) + .env('CML') + .options(options) + .commandDir('./cml', { exclude: /\.test\.js$/ }) + .command('$0 ', false, (builder) => builder.strict(false), runPlugin) + .recommendCommands() + .demandCommand() + .strict() + .parse(); diff --git a/bin/cml/pr.js b/bin/cml/pr.js new file mode 100755 index 0000000000..647d306d20 --- /dev/null +++ b/bin/cml/pr.js @@ -0,0 +1,35 @@ +const { GIT_REMOTE, GIT_USER_NAME, GIT_USER_EMAIL } = require('../../src/cml'); +const CML = require('../../src/cml').default; + +exports.command = 'pr '; +exports.desc = 'Create a pull request with the specified files'; + +exports.handler = async (opts) => { + const cml = new CML(opts); + const link = await cml.prCreate({ ...opts, globs: opts.globpath }); + if (link) console.log(link); +}; + +exports.builder = (yargs) => + yargs + .describe('md', 'Output in markdown format [](url).') + .boolean('md') + .default('remote', GIT_REMOTE) + .describe('remote', 'Sets git remote.') + .default('user-email', GIT_USER_EMAIL) + .describe('user-email', 'Sets git user email.') + .default('user-name', GIT_USER_NAME) + .describe('user-name', 'Sets git user name.') + .default('repo') + .describe( + 'repo', + 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' + ) + .default('token') + .describe( + 'token', + 'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.' + ) + .default('driver') + .choices('driver', ['github', 'gitlab']) + .describe('driver', 'If not specify it infers it from the ENV.'); diff --git a/bin/cml/pr.test.js b/bin/cml/pr.test.js new file mode 100644 index 0000000000..24942e9042 --- /dev/null +++ b/bin/cml/pr.test.js @@ -0,0 +1,29 @@ +const { exec } = require('../../src/utils'); + +describe('CML e2e', () => { + test('cml-pr --help', async () => { + const output = await exec(`echo none | node ./bin/cml.js pr --help`); + + expect(output).toMatchInlineSnapshot(` +"cml.js pr + +Create a pull request with the specified files + +Options: + --help Show help [boolean] + --version Show version number [boolean] + --log Maximum log level + [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --md Output in markdown format [](url). [boolean] + --remote Sets git remote. [default: \\"origin\\"] + --user-email Sets git user email. [default: \\"olivaw@iterative.ai\\"] + --user-name Sets git user name. [default: \\"Olivaw[bot]\\"] + --repo Specifies the repo to be used. If not specified is extracted + from the CI ENV. + --token Personal access token to be used. If not specified in extracted + from ENV REPO_TOKEN. + --driver If not specify it infers it from the ENV. + [choices: \\"github\\", \\"gitlab\\"]" +`); + }); +}); diff --git a/bin/cml/publish.js b/bin/cml/publish.js new file mode 100644 index 0000000000..fb56a88a54 --- /dev/null +++ b/bin/cml/publish.js @@ -0,0 +1,68 @@ +const fs = require('fs').promises; +const pipeArgs = require('../../src/pipe-args'); + +const CML = require('../../src/cml').default; + +pipeArgs.load('binary'); +const data = pipeArgs.pipedArg(); // HACK: see yargs/yargs#1312 + +exports.command = data ? 'publish' : 'publish '; +exports.desc = 'Upload an image to build a report'; + +exports.handler = async (opts) => { + const { file, repo, native } = opts; + + const path = opts.asset; + const buffer = data ? Buffer.from(data, 'binary') : null; + const cml = new CML({ ...opts, repo: native ? repo : 'cml' }); + + const output = await cml.publish({ + ...opts, + buffer, + path + }); + + if (!file) console.log(output); + else await fs.writeFile(file, output); +}; + +exports.builder = (yargs) => + yargs + .describe('md', 'Output in markdown format [title || name](url).') + .boolean('md') + .describe('md', 'Output in markdown format [title || name](url).') + .default('title') + .describe('title', 'Markdown title [title](url) or ![](url title).') + .alias('title', 't') + .boolean('native') + .describe( + 'native', + "Uses driver's native capabilities to upload assets instead of CML's storage. Currently only available for GitLab CI." + ) + .alias('native', 'gitlab-uploads') + .boolean('rm-watermark') + .describe('rm-watermark', 'Avoid CML watermark.') + .default('mime-type') + .describe( + 'mime-type', + 'Specifies the mime-type. If not set guess it from the content.' + ) + .default('file') + .describe( + 'file', + 'Append the output to the given file. Create it if does not exist.' + ) + .alias('file', 'f') + .default('repo') + .describe( + 'repo', + 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' + ) + .default('token') + .describe( + 'token', + 'Personal access token to be used. If not specified, extracted from ENV REPO_TOKEN, GITLAB_TOKEN, GITHUB_TOKEN, or BITBUCKET_TOKEN.' + ) + .default('driver') + .choices('driver', ['github', 'gitlab']) + .describe('driver', 'If not specify it infers it from the ENV.'); diff --git a/bin/cml/publish.test.js b/bin/cml/publish.test.js new file mode 100644 index 0000000000..4dca1b5d3b --- /dev/null +++ b/bin/cml/publish.test.js @@ -0,0 +1,124 @@ +const fs = require('fs'); +const { exec } = require('../../src/utils'); + +describe('CML e2e', () => { + test('cml publish --help', async () => { + const output = await exec(`node ./bin/cml.js publish --help`); + + expect(output).toMatchInlineSnapshot(` +"cml.js publish + +Upload an image to build a report + +Options: + --help Show help [boolean] + --version Show version number [boolean] + --log Maximum log level + [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --md Output in markdown format [title || + name](url). [boolean] + -t, --title Markdown title [title](url) or ![](url title). + --native, --gitlab-uploads Uses driver's native capabilities to upload + assets instead of CML's storage. Currently + only available for GitLab CI. [boolean] + --rm-watermark Avoid CML watermark. [boolean] + --mime-type Specifies the mime-type. If not set guess it + from the content. + -f, --file Append the output to the given file. Create it + if does not exist. + --repo Specifies the repo to be used. If not + specified is extracted from the CI ENV. + --token Personal access token to be used. If not + specified, extracted from ENV REPO_TOKEN, + GITLAB_TOKEN, GITHUB_TOKEN, or + BITBUCKET_TOKEN. + --driver If not specify it infers it from the ENV. + [choices: \\"github\\", \\"gitlab\\"]" +`); + }); + + test('cml publish assets/logo.png --md', async () => { + const output = await exec(`node ./bin/cml.js publish assets/logo.png --md`); + + expect(output.startsWith('![](')).toBe(true); + }); + + test('cml publish assets/logo.png', async () => { + const output = await exec(`eode ./bin/cml.js publish assets/logo.png`); + + expect(output.startsWith('https://')).toBe(true); + }); + + test('cml publish assets/logo.pdf --md', async () => { + const title = 'this is awesome'; + const output = await exec( + `node ./bin/cml.js publish assets/logo.pdf --md --title '${title}'` + ); + + expect(output.startsWith(`[${title}](`)).toBe(true); + }); + + test('cml publish assets/logo.pdf', async () => { + const output = await exec(`node ./bin/cml.js publish assets/logo.pdf`); + + expect(output.startsWith('https://')).toBe(true); + }); + + test('cml publish assets/test.svg --md', async () => { + const title = 'this is awesome'; + const output = await exec( + `node ./bin/cml.js publish assets/test.svg --md --title '${title}'` + ); + + expect(output.startsWith('![](') && output.endsWith(`${title}")`)).toBe( + true + ); + }); + + test('cml publish assets/test.svg', async () => { + const output = await exec(`node ./bin/cml.js publish assets/test.svg`); + + expect(output.startsWith('https://')).toBe(true); + }); + + test('cml publish assets/logo.pdf to file', async () => { + const file = `cml-publish-test.md`; + + await exec(`node ./bin/cml.js publish assets/logo.pdf --file ${file}`); + + expect(fs.existsSync(file)).toBe(true); + await fs.promises.unlink(file); + }); + + test('cml publish assets/vega-lite.json', async () => { + const output = await exec( + `node ./bin/cml.js publish --mime-type=application/json assets/vega-lite.json` + ); + + expect(output.startsWith('https://')).toBe(true); + expect(output.endsWith('json')).toBe(true); + }); + + test('cml publish assets/test.svg in Gitlab storage', async () => { + const { TEST_GITLAB_REPO: repo, TEST_GITLAB_TOKEN: token } = process.env; + + const output = await exec( + `node ./bin/cml.js publish --repo=${repo} --token=${token} --gitlab-uploads assets/test.svg` + ); + + expect(output.startsWith('https://')).toBe(true); + }); + + test('cml publish /nonexistent produces file error', async () => { + await expect( + exec('node ./bin/cml.js publish /nonexistent') + ).rejects.toThrowError('ENOENT: no such file or directory, stat'); + }); + + test('echo text | cml publish produces a plain text file', async () => { + const output = await exec(`echo none | node ./bin/cml.js publish`); + + expect(output.startsWith('https://')).toBe(true); + expect(output.endsWith('plain')).toBe(true); + }); +}); diff --git a/bin/cml-runner.js b/bin/cml/runner.js similarity index 65% rename from bin/cml-runner.js rename to bin/cml/runner.js index 07c782e768..aeb78e6add 100755 --- a/bin/cml-runner.js +++ b/bin/cml/runner.js @@ -1,15 +1,13 @@ -#!/usr/bin/env node - const { join } = require('path'); const { homedir } = require('os'); const fs = require('fs').promises; -const yargs = require('yargs'); const { SpotNotifier } = require('ec2-spot-notification'); -const { exec, randid, sleep } = require('../src/utils'); -const tf = require('../src/terraform'); -const CML = require('../src/cml').default; +const winston = require('winston'); +const CML = require('../../src/cml').default; +const { exec, randid, sleep } = require('../../src/utils'); +const tf = require('../../src/terraform'); const NAME = `cml-${randid()}`; const WORKDIR_BASE = `${homedir()}/.cml`; @@ -48,12 +46,12 @@ const shutdown = async (opts) => { if (!RUNNER) return; try { - console.log(`Unregistering runner ${name}...`); + winston.info(`Unregistering runner ${name}...`); RUNNER && RUNNER.kill('SIGINT'); await cml.unregisterRunner({ name }); - console.log('\tSuccess'); + winston.info('\tSuccess'); } catch (err) { - console.error(`\tFailed: ${err.message}`); + winston.error(`\tFailed: ${err.message}`); } }; @@ -67,21 +65,21 @@ const shutdown = async (opts) => { ); } } catch (err) { - console.error(err); + winston.error(err); } }; const destroyDockerMachine = async () => { if (!DOCKER_MACHINE) return; - console.log('docker-machine destroy...'); - console.log( + winston.info('docker-machine destroy...'); + winston.warning( 'Docker machine is deprecated and will be removed!! Check how to deploy using our tf provider.' ); try { await exec(`echo y | docker-machine rm ${DOCKER_MACHINE}`); } catch (err) { - console.error(`\tFailed shutting down docker machine: ${err.message}`); + winston.error(`\tFailed shutting down docker machine: ${err.message}`); } }; @@ -89,14 +87,14 @@ const shutdown = async (opts) => { if (!tfResource) return; try { - console.log(await tf.destroy({ dir: tfPath })); + winston.debug(await tf.destroy({ dir: tfPath })); } catch (err) { - console.error(`\tFailed destroying terraform: ${err.message}`); + winston.error(`\tFailed destroying terraform: ${err.message}`); } }; - if (error) console.log(error); - console.log( + if (error) winston.error(error); + winston.info( JSON.stringify({ level: error ? 'error' : 'info', status: 'terminated', @@ -120,7 +118,7 @@ const shutdown = async (opts) => { const runCloud = async (opts) => { const runTerraform = async (opts) => { - console.log('Terraform apply...'); + winston.info('Terraform apply...'); const { token, repo, driver } = cml; const { @@ -150,7 +148,7 @@ const runCloud = async (opts) => { tpl = await fs.writeFile(tfMainPath, await fs.readFile(tfFile)); } else { if (gpu === 'tesla') - console.log( + winston.warn( 'GPU model "tesla" has been deprecated; please use "v100" instead.' ); tpl = tf.iterativeCmlRunnerTpl({ @@ -184,7 +182,7 @@ const runCloud = async (opts) => { return tfstate; }; - console.log('Deploying cloud runner plan...'); + winston.info('Deploying cloud runner plan...'); const tfstate = await runTerraform(opts); const { resources } = tfstate; for (const resource of resources) { @@ -211,14 +209,14 @@ const runCloud = async (opts) => { spotPrice: attributes.spot_price, timeouts: attributes.timeouts }; - console.log(JSON.stringify(nonSensitiveValues)); + winston.info(JSON.stringify(nonSensitiveValues)); } } } }; const runLocal = async (opts) => { - console.log(`Launching ${cml.driver} runner`); + winston.info(`Launching ${cml.driver} runner`); const { workdir, name, labels, single, idleTimeout, noRetry } = opts; const proc = await cml.startRunner({ @@ -231,7 +229,7 @@ const runLocal = async (opts) => { const dataHandler = async (data) => { const log = await cml.parseRunnerLog({ data }); - log && console.log(JSON.stringify(log)); + log && winston.debug(JSON.stringify(log)); if (log && log.status === 'job_started') { RUNNER_JOBS_RUNNING.push({ id: log.job, date: log.date }); @@ -285,13 +283,13 @@ const runLocal = async (opts) => { if (!noRetry) { try { - console.log(`EC2 id ${await SpotNotifier.instanceId()}`); + winston.info(`EC2 id ${await SpotNotifier.instanceId()}`); SpotNotifier.on('termination', () => shutdown({ ...opts, reason: 'spot_termination' }) ); SpotNotifier.start(); } catch (err) { - console.log('SpotNotifier can not be started.'); + winston.warn('SpotNotifier can not be started.'); } } @@ -371,7 +369,7 @@ const run = async (opts) => { throw new Error( `Runner name ${name} is already in use. Please change the name or terminate the other runner.` ); - console.log(`Reusing existing runner named ${name}...`); + winston.info(`Reusing existing runner named ${name}...`); process.exit(0); } @@ -381,12 +379,14 @@ const run = async (opts) => { (runner) => runner.online ) ) { - console.log(`Reusing existing online runners with the ${labels} labels...`); + winston.info( + `Reusing existing online runners with the ${labels} labels...` + ); process.exit(0); } try { - console.log(`Preparing workdir ${workdir}...`); + winston.info(`Preparing workdir ${workdir}...`); await fs.mkdir(workdir, { recursive: true }); } catch (err) {} @@ -394,98 +394,107 @@ const run = async (opts) => { else await runLocal(opts); }; -const opts = yargs - .strict() - .usage(`Usage: $0`) - .default('labels', RUNNER_LABELS) - .describe( - 'labels', - 'One or more user-defined labels for this runner (delimited with commas)' - ) - .default('idle-timeout', RUNNER_IDLE_TIMEOUT) - .describe( - 'idle-timeout', - 'Time in seconds for the runner to be waiting for jobs before shutting down. Setting it to 0 disables automatic shutdown' - ) - .default('name') - .describe('name', 'Name displayed in the repository once registered cml-{ID}') - .coerce('name', (val) => val || RUNNER_NAME) - .boolean('no-retry') - .default('no-retry', RUNNER_NO_RETRY) - .describe( - 'no-retry', - 'Do not restart workflow terminated due to instance disposal or GitHub Actions timeout' - ) - .boolean('single') - .default('single', RUNNER_SINGLE) - .describe('single', 'Exit after running a single job') - .boolean('reuse') - .default('reuse', RUNNER_REUSE) - .describe( - 'reuse', - "Don't launch a new runner if an existing one has the same name or overlapping labels" - ) - - .default('driver', RUNNER_DRIVER) - .describe( - 'driver', - 'Platform where the repository is hosted. If not specified, it will be inferred from the environment' - ) - .choices('driver', ['github', 'gitlab']) - .default('repo', RUNNER_REPO) - .describe( - 'repo', - 'Repository to be used for registering the runner. If not specified, it will be inferred from the environment' - ) - .default('token', REPO_TOKEN) - .describe( - 'token', - 'Personal access token to register a self-hosted runner on the repository. If not specified, it will be inferred from the environment' - ) - .default('cloud') - .describe('cloud', 'Cloud to deploy the runner') - .choices('cloud', ['aws', 'azure', 'gcp', 'kubernetes']) - .default('cloud-region', 'us-west') - .describe( - 'cloud-region', - 'Region where the instance is deployed. Choices: [us-east, us-west, eu-west, eu-north]. Also accepts native cloud regions' - ) - .default('cloud-type') - .describe( - 'cloud-type', - 'Instance type. Choices: [m, l, xl]. Also supports native types like i.e. t2.micro' - ) - .default('cloud-gpu') - .describe('cloud-gpu', 'GPU type.') - .choices('cloud-gpu', ['nogpu', 'k80', 'v100', 'tesla']) - .coerce('cloud-gpu-type', (val) => (val === 'nogpu' ? null : val)) - .default('cloud-hdd-size') - .describe('cloud-hdd-size', 'HDD size in GB') - .default('cloud-ssh-private', '') - .describe( - 'cloud-ssh-private', - 'Custom private RSA SSH key. If not provided an automatically generated throwaway key will be used' - ) - .coerce('cloud-ssh-private', (val) => val.replace(/\n/g, '\\n')) - .boolean('cloud-spot') - .describe('cloud-spot', 'Request a spot instance') - .default('cloud-spot-price', '-1') - .describe( - 'cloud-spot-price', - 'Maximum spot instance bidding price in USD. Defaults to the current spot bidding price' - ) - .default('cloud-startup-script', '') - .describe( - 'cloud-startup-script', - 'Run the provided Base64-encoded Linux shell script during the instance initialization' - ) - .default('cloud-aws-security-group', '') - .describe('cloud-aws-security-group', 'Specifies the security group in AWS') - .default('tf-resource') - .hide('tf-resource') - .alias('tf-resource', 'tf_resource') - .help('h').argv; - -run(opts).catch((error) => { - shutdown({ ...opts, error }); -}); +exports.command = 'runner'; +exports.desc = 'Launch and register a self-hosted runner'; + +exports.handler = async (opts) => { + try { + await run(opts); + } catch (error) { + await shutdown({ ...opts, error }); + throw error; + } +}; + +exports.builder = (yargs) => + yargs + .default('labels', RUNNER_LABELS) + .describe( + 'labels', + 'One or more user-defined labels for this runner (delimited with commas)' + ) + .default('idle-timeout', RUNNER_IDLE_TIMEOUT) + .describe( + 'idle-timeout', + 'Time in seconds for the runner to be waiting for jobs before shutting down. Setting it to 0 disables automatic shutdown' + ) + .default('name') + .describe( + 'name', + 'Name displayed in the repository once registered cml-{ID}' + ) + .coerce('name', (val) => val || RUNNER_NAME) + .boolean('no-retry') + .default('no-retry', RUNNER_NO_RETRY) + .describe( + 'no-retry', + 'Do not restart workflow terminated due to instance disposal or GitHub Actions timeout' + ) + .boolean('single') + .default('single', RUNNER_SINGLE) + .describe('single', 'Exit after running a single job') + .boolean('reuse') + .default('reuse', RUNNER_REUSE) + .describe( + 'reuse', + "Don't launch a new runner if an existing one has the same name or overlapping labels" + ) + + .default('driver', RUNNER_DRIVER) + .describe( + 'driver', + 'Platform where the repository is hosted. If not specified, it will be inferred from the environment' + ) + .choices('driver', ['github', 'gitlab']) + .default('repo', RUNNER_REPO) + .describe( + 'repo', + 'Repository to be used for registering the runner. If not specified, it will be inferred from the environment' + ) + .default('token', REPO_TOKEN) + .describe( + 'token', + 'Personal access token to register a self-hosted runner on the repository. If not specified, it will be inferred from the environment' + ) + .default('cloud') + .describe('cloud', 'Cloud to deploy the runner') + .choices('cloud', ['aws', 'azure', 'gcp', 'kubernetes']) + .default('cloud-region', 'us-west') + .describe( + 'cloud-region', + 'Region where the instance is deployed. Choices: [us-east, us-west, eu-west, eu-north]. Also accepts native cloud regions' + ) + .default('cloud-type') + .describe( + 'cloud-type', + 'Instance type. Choices: [m, l, xl]. Also supports native types like i.e. t2.micro' + ) + .default('cloud-gpu') + .describe('cloud-gpu', 'GPU type.') + .choices('cloud-gpu', ['nogpu', 'k80', 'v100', 'tesla']) + .coerce('cloud-gpu-type', (val) => (val === 'nogpu' ? null : val)) + .default('cloud-hdd-size') + .describe('cloud-hdd-size', 'HDD size in GB') + .default('cloud-ssh-private', '') + .describe( + 'cloud-ssh-private', + 'Custom private RSA SSH key. If not provided an automatically generated throwaway key will be used' + ) + .coerce('cloud-ssh-private', (val) => val.replace(/\n/g, '\\n')) + .boolean('cloud-spot') + .describe('cloud-spot', 'Request a spot instance') + .default('cloud-spot-price', '-1') + .describe( + 'cloud-spot-price', + 'Maximum spot instance bidding price in USD. Defaults to the current spot bidding price' + ) + .default('cloud-startup-script', '') + .describe( + 'cloud-startup-script', + 'Run the provided Base64-encoded Linux shell script during the instance initialization' + ) + .default('cloud-aws-security-group', '') + .describe('cloud-aws-security-group', 'Specifies the security group in AWS') + .default('tf-resource') + .hide('tf-resource') + .alias('tf-resource', 'tf_resource'); diff --git a/bin/cml/runner.test.js b/bin/cml/runner.test.js new file mode 100644 index 0000000000..bc59659ced --- /dev/null +++ b/bin/cml/runner.test.js @@ -0,0 +1,63 @@ +const { exec } = require('../../src/utils'); + +describe('CML e2e', () => { + test('cml-runner --help', async () => { + const output = await exec(`echo none | node ./bin/cml.js runner --help`); + + expect(output).toMatchInlineSnapshot(` +"cml.js runner + +Launch and register a self-hosted runner + +Options: + --help Show help [boolean] + --version Show version number [boolean] + --log Maximum log level + [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --labels One or more user-defined labels for this runner + (delimited with commas) [default: \\"cml\\"] + --idle-timeout Time in seconds for the runner to be waiting for + jobs before shutting down. Setting it to 0 + disables automatic shutdown [default: 300] + --name Name displayed in the repository once registered + cml-{ID} + --no-retry Do not restart workflow terminated due to instance + disposal or GitHub Actions timeout + [boolean] [default: false] + --single Exit after running a single job + [boolean] [default: false] + --reuse Don't launch a new runner if an existing one has + the same name or overlapping labels + [boolean] [default: false] + --driver Platform where the repository is hosted. If not + specified, it will be inferred from the + environment [choices: \\"github\\", \\"gitlab\\"] + --repo Repository to be used for registering the runner. + If not specified, it will be inferred from the + environment + --token Personal access token to register a self-hosted + runner on the repository. If not specified, it + will be inferred from the environment + --cloud Cloud to deploy the runner + [choices: \\"aws\\", \\"azure\\", \\"gcp\\", \\"kubernetes\\"] + --cloud-region Region where the instance is deployed. Choices: + [us-east, us-west, eu-west, eu-north]. Also + accepts native cloud regions [default: \\"us-west\\"] + --cloud-type Instance type. Choices: [m, l, xl]. Also supports + native types like i.e. t2.micro + --cloud-gpu GPU type. + [choices: \\"nogpu\\", \\"k80\\", \\"v100\\", \\"tesla\\"] + --cloud-hdd-size HDD size in GB + --cloud-ssh-private Custom private RSA SSH key. If not provided an + automatically generated throwaway key will be used + [default: \\"\\"] + --cloud-spot Request a spot instance [boolean] + --cloud-spot-price Maximum spot instance bidding price in USD. + Defaults to the current spot bidding price + [default: \\"-1\\"] + --cloud-startup-script Run the provided Base64-encoded Linux shell script + during the instance initialization [default: \\"\\"] + --cloud-aws-security-group Specifies the security group in AWS [default: \\"\\"]" +`); + }); +}); diff --git a/bin/cml/send-comment.js b/bin/cml/send-comment.js new file mode 100644 index 0000000000..11924680bb --- /dev/null +++ b/bin/cml/send-comment.js @@ -0,0 +1,45 @@ +const fs = require('fs').promises; + +const CML = require('../../src/cml').default; + +exports.command = 'send-comment '; +exports.desc = 'Comment on a commit'; + +exports.handler = async (opts) => { + const path = opts.markdownfile; + const report = await fs.readFile(path, 'utf-8'); + const cml = new CML(opts); + console.log(await cml.commentCreate({ ...opts, report })); +}; + +exports.builder = (yargs) => + yargs + .default('commit-sha') + .describe( + 'commit-sha', + 'Commit SHA linked to this comment. Defaults to HEAD.' + ) + .alias('commit-sha', 'head-sha') + .boolean('update') + .describe( + 'update', + 'Update the last CML comment (if any) instead of creating a new one' + ) + .boolean('rm-watermark') + .describe( + 'rm-watermark', + 'Avoid watermark. CML needs a watermark to be able to distinguish CML reports from other comments in order to provide extra functionality.' + ) + .default('repo') + .describe( + 'repo', + 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' + ) + .default('token') + .describe( + 'token', + 'Personal access token to be used. If not specified is extracted from ENV REPO_TOKEN.' + ) + .default('driver') + .choices('driver', ['github', 'gitlab', 'bitbucket']) + .describe('driver', 'If not specify it infers it from the ENV.'); diff --git a/bin/cml-send-comment.test.js b/bin/cml/send-comment.test.js similarity index 82% rename from bin/cml-send-comment.test.js rename to bin/cml/send-comment.test.js index a8dc887863..541f8a85f5 100644 --- a/bin/cml-send-comment.test.js +++ b/bin/cml/send-comment.test.js @@ -1,4 +1,4 @@ -const { exec } = require('../src/utils'); +const { exec } = require('../../src/utils'); const fs = require('fs').promises; describe('Comment integration tests', () => { @@ -10,8 +10,8 @@ describe('Comment integration tests', () => { } catch (err) {} }); - test('cml-send-comment -h', async () => { - const output = await exec(`node ./bin/cml-send-comment.js -h`); + test('cml send-comment --help', async () => { + const output = await exec(`node ./bin/cml.js send-comment --help`); expect(output).toMatchInlineSnapshot(` "Usage: cml-send-comment.js @@ -34,7 +34,7 @@ describe('Comment integration tests', () => { `); }); - test('cml-send-comment to specific repo', async () => { + test('cml send-comment to specific repo', async () => { const { TEST_GITHUB_REPO: repo, TEST_GITHUB_TOKEN: token, @@ -45,14 +45,14 @@ describe('Comment integration tests', () => { await fs.writeFile(path, report); await exec( - `node ./bin/cml-send-comment.js --repo=${repo} --token=${token} --commit-sha=${sha} ${path}` + `node ./bin/cml.js send-comment --repo=${repo} --token=${token} --commit-sha=${sha} ${path}` ); }); - test('cml-send-comment to current repo', async () => { + test('cml send-comment to current repo', async () => { const report = `## Test Comment`; await fs.writeFile(path, report); - await exec(`node ./bin/cml-send-comment.js ${path}`); + await exec(`node ./bin/cml.js send-comment ${path}`); }); }); diff --git a/bin/cml/send-github-check.js b/bin/cml/send-github-check.js new file mode 100755 index 0000000000..8573b5c367 --- /dev/null +++ b/bin/cml/send-github-check.js @@ -0,0 +1,46 @@ +const fs = require('fs').promises; +const CML = require('../../src/cml').default; +const CHECK_TITLE = 'CML Report'; + +exports.command = 'send-github-check '; +exports.desc = 'Create a check report'; + +exports.handler = async (opts) => { + const path = opts.markdownfile; + const report = await fs.readFile(path, 'utf-8'); + const cml = new CML({ ...opts }); + await cml.checkCreate({ ...opts, report }); +}; + +exports.builder = (yargs) => + yargs + .describe( + 'commit-sha', + 'Commit SHA linked to this comment. Defaults to HEAD.' + ) + .alias('commit-sha', 'head-sha') + .default( + 'conclusion', + 'success', + 'Sets the conclusion status of the check.' + ) + .choices('conclusion', [ + 'success', + 'failure', + 'neutral', + 'cancelled', + 'skipped', + 'timed_out' + ]) + .default('title', CHECK_TITLE) + .describe('title', 'Sets title of the check.') + .default('repo') + .describe( + 'repo', + 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' + ) + .default('token') + .describe( + 'token', + 'Personal access token to be used. If not specified in extracted from ENV REPO_TOKEN.' + ); diff --git a/bin/cml-send-github-check.test.js b/bin/cml/send-github-check.test.js similarity index 79% rename from bin/cml-send-github-check.test.js rename to bin/cml/send-github-check.test.js index ea76d90d7d..20408705c5 100644 --- a/bin/cml-send-github-check.test.js +++ b/bin/cml/send-github-check.test.js @@ -1,4 +1,4 @@ -const { exec } = require('../src/utils'); +const { exec } = require('../../src/utils'); const fs = require('fs').promises; describe('CML e2e', () => { @@ -10,15 +10,15 @@ describe('CML e2e', () => { } catch (err) {} }); - test('cml-send-github-check', async () => { + test('cml send-github-check', async () => { const report = `## Test Check Report`; await fs.writeFile(path, report); process.env.GITHUB_ACTIONS && - (await exec(`node ./bin/cml-send-github-check.js ${path}`)); + (await exec(`node ./bin/cml.js send-github-check ${path}`)); }); - test('cml-send-github-check failure with tile "CML neutral test"', async () => { + test('cml send-github-check failure with tile "CML neutral test"', async () => { const report = `## Hi this check should be neutral`; const title = 'CML neutral test'; const conclusion = 'neutral'; @@ -26,12 +26,12 @@ describe('CML e2e', () => { await fs.writeFile(path, report); process.env.GITHUB_ACTIONS && (await exec( - `node ./bin/cml-send-github-check.js ${path} --title "${title}" --conclusion "${conclusion}"` + `node ./bin/cml.js send-github-check ${path} --title "${title}" --conclusion "${conclusion}"` )); }); - test('cml-send-github-check -h', async () => { - const output = await exec(`node ./bin/cml-send-github-check.js -h`); + test('cml send-github-check --help', async () => { + const output = await exec(`node ./bin/cml.js send-github-check --help`); expect(output).toMatchInlineSnapshot(` "Usage: cml-send-github-check.js diff --git a/bin/cml-tensorboard-dev.js b/bin/cml/tensorboard-dev.js old mode 100755 new mode 100644 similarity index 83% rename from bin/cml-tensorboard-dev.js rename to bin/cml/tensorboard-dev.js index 4b2c924716..357fe2142c --- a/bin/cml-tensorboard-dev.js +++ b/bin/cml/tensorboard-dev.js @@ -1,28 +1,22 @@ -#!/usr/bin/env node - -const print = console.log; -console.log = console.error; - -const yargs = require('yargs'); const fs = require('fs').promises; const { spawn } = require('child_process'); const { homedir } = require('os'); const tempy = require('tempy'); -const { exec, watermarkUri, sleep } = require('../src/utils'); +const winston = require('winston'); +const { exec, watermarkUri, sleep } = require('../../src/utils'); const { TB_CREDENTIALS } = process.env; -const isCLI = require.main === module; const closeFd = (fd) => { try { fd.close(); } catch (err) { - console.error(err.message); + winston.error(err.message); } }; -const tbLink = async (opts = {}) => { +exports.tbLink = async (opts = {}) => { const { stdout, stderror, title, name, rmWatermark, md, timeout = 60 } = opts; let chrono = 0; @@ -48,7 +42,10 @@ const tbLink = async (opts = {}) => { throw new Error(`Tensorboard took too long. ${error}`); }; -const run = async (opts) => { +exports.command = 'tensorboard-dev'; +exports.desc = 'Get a tensorboard link'; + +exports.handler = async (opts) => { const { md, file, @@ -89,12 +86,12 @@ const run = async (opts) => { proc.on('exit', async (code) => { if (code) { const error = await fs.readFile(stderrPath, 'utf8'); - print(`Tensorboard failed with error: ${error}`); + winston.error(`Tensorboard failed with error: ${error}`); } process.exit(code); }); - const url = await tbLink({ + const url = await exports.tbLink({ stdout: stdoutPath, stderror: stderrPath, title, @@ -102,17 +99,15 @@ const run = async (opts) => { rmWatermark, md }); - if (!file) print(url); + if (!file) winston.info(url); else await fs.appendFile(file, url); closeFd(stdoutFd) && closeFd(stderrFd); process.exit(0); }; -if (isCLI) { - const argv = yargs - .strict() - .usage(`Usage: $0`) +exports.builder = (yargs) => + yargs .default('credentials') .describe( 'credentials', @@ -143,15 +138,4 @@ if (isCLI) { 'Append the output to the given file. Create it if does not exist.' ) .describe('rm-watermark', 'Avoid CML watermark.') - .alias('file', 'f') - .help('h').argv; - - run(argv).catch((e) => { - console.error(e); - process.exit(1); - }); -} - -module.exports = { - tbLink -}; + .alias('file', 'f'); diff --git a/bin/cml-tensorboard-dev.test.js b/bin/cml/tensorboard-dev.test.js similarity index 87% rename from bin/cml-tensorboard-dev.test.js rename to bin/cml/tensorboard-dev.test.js index e84121566a..bc545f4051 100644 --- a/bin/cml-tensorboard-dev.test.js +++ b/bin/cml/tensorboard-dev.test.js @@ -1,7 +1,7 @@ const fs = require('fs').promises; const tempy = require('tempy'); -const { exec, isProcRunning, sleep } = require('../src/utils'); -const { tbLink } = require('./cml-tensorboard-dev'); +const { exec, isProcRunning, sleep } = require('../../src/utils'); +const { tbLink } = require('./tensorboard-dev'); const CREDENTIALS = '{"refresh_token": "1//03FiVnGk2xhnNCgYIARAAGAMSNwF-L9IrPH8FOOVWEYUihFDToqxyLArxfnbKFmxEfhzys_KYVVzBisYlAy225w4HaX3ais5TV_Q", "token_uri": "https://oauth2.googleapis.com/token", "client_id": "373649185512-8v619h5kft38l4456nm2dj4ubeqsrvh6.apps.googleusercontent.com", "client_secret": "pOyAuU2yq2arsM98Bw5hwYtr", "scopes": ["openid", "https://www.googleapis.com/auth/userinfo.email"], "type": "authorized_user"}'; @@ -51,8 +51,8 @@ describe('tbLink', () => { }); describe('CML e2e', () => { - test('cml-tensorboard-dev.js -h', async () => { - const output = await exec(`node ./bin/cml-tensorboard-dev.js -h`); + test('cml tensorboard-dev --help', async () => { + const output = await exec(`node ./bin/cml.js tensorboard-dev --help`); expect(output).toMatchInlineSnapshot(` "Usage: cml-tensorboard-dev.js @@ -77,12 +77,12 @@ describe('CML e2e', () => { `); }); - test('cml-tensorboard-dev.js --md returns md and after command TB is still up', async () => { + test('cml tensorboard-dev --md returns md and after command TB is still up', async () => { const name = 'My experiment'; const desc = 'Test experiment'; const title = 'go to the experiment'; const output = await exec( - `node ./bin/cml-tensorboard-dev.js --credentials '${CREDENTIALS}' \ + `node ./bin/cml.js tensorboard-dev --credentials '${CREDENTIALS}' \ --md --title '${title}' \ --logdir logs --name '${name}' --description '${desc}'` ); @@ -95,9 +95,9 @@ describe('CML e2e', () => { expect(output.includes('cml=tb')).toBe(true); }); - test('cml-tensorboard-dev.js invalid creds', async () => { + test('cml tensorboard-dev invalid creds', async () => { try { - await exec(`node ./bin/cml-tensorboard-dev.js --credentials 'invalid'`); + await exec(`node ./bin/cml.js tensorboard-dev --credentials 'invalid'`); } catch (err) { expect(err.message.includes('json.decoder.JSONDecodeError')).toBe(true); } diff --git a/bin/legacy.js b/bin/legacy.js new file mode 100755 index 0000000000..9659ab7a19 --- /dev/null +++ b/bin/legacy.js @@ -0,0 +1,14 @@ +#!/usr/bin/env node + +// This file provides backwards compatibility with the legacy cml-commands +// specified in package.json by acting as a single BusyBox-like entrypoint +// that detects the name of the executed symbolic link and invokes in turn +// the main command. E.g. cml-command should be redirected to cml command. + +const { basename } = require('path'); +const { pseudoexec } = require('pseudoexec'); + +const [, file, ...parameters] = process.argv; +const [, base, command] = basename(file).match(/^(\w+)-(.+)$/); + +(async () => process.exit(await pseudoexec(base, [command, ...parameters])))(); diff --git a/package-lock.json b/package-lock.json index 8ae7486d69..30d8b9e31f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -532,6 +532,16 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "@dabh/diagnostics": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz", + "integrity": "sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q==", + "requires": { + "colorspace": "1.1.x", + "enabled": "2.0.x", + "kuler": "^2.0.0" + } + }, "@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -1584,6 +1594,11 @@ "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, + "async": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.1.tgz", + "integrity": "sha512-XdD5lRO/87udXCMC9meWdYiR+Nq6ZjUfXidViUZGu2F1MO4T3XwZ1et0hb2++BgLfhyJwy44BGB/yx80ABx8hg==" + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -1826,7 +1841,8 @@ "camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true }, "caniuse-lite": { "version": "1.0.30001251", @@ -1957,7 +1973,6 @@ "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "dev": true, "requires": { "string-width": "^4.2.0", "strip-ansi": "^6.0.0", @@ -1976,11 +1991,19 @@ "integrity": "sha512-iBPtljfCNcTKNAto0KEtDfZ3qzjJvqE3aTGZsbhjSBlorqpXJlaWWtPO35D+ZImoC3KWejX64o+yPGxhWSTzfg==", "dev": true }, + "color": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz", + "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==", + "requires": { + "color-convert": "^1.9.1", + "color-string": "^1.5.2" + } + }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, "requires": { "color-name": "1.1.3" } @@ -1988,8 +2011,16 @@ "color-name": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", - "dev": true + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "color-string": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.6.0.tgz", + "integrity": "sha512-c/hGS+kRWJutUBEngKKmk4iH3sD59MBkoxVapS/0wgpCz2u7XsNloxknyvBhzwEs1IbV36D9PwqLPJ2DTu3vMA==", + "requires": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } }, "colorette": { "version": "1.2.2", @@ -1997,6 +2028,20 @@ "integrity": "sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w==", "dev": true }, + "colors": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", + "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==" + }, + "colorspace": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.2.tgz", + "integrity": "sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ==", + "requires": { + "color": "3.0.x", + "text-hex": "1.0.x" + } + }, "combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -2088,6 +2133,15 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", "dev": true + }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } } } }, @@ -2146,11 +2200,6 @@ "ms": "2.1.2" } }, - "decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" - }, "decimal.js": { "version": "10.3.1", "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.3.1.tgz", @@ -2298,6 +2347,11 @@ "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", "dev": true }, + "enabled": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" + }, "end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -2372,8 +2426,7 @@ "escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", - "dev": true + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, "escape-string-regexp": { "version": "1.0.5", @@ -3016,6 +3069,11 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, + "fast-safe-stringify": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.8.tgz", + "integrity": "sha512-lXatBjf3WPjmWD6DpIZxkeSsCOwqI0maYMpgDlx8g4U2qi4lbjA9oH/HD2a87G+KfsUmo5WbJFmqBZlPxtptag==" + }, "fastq": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.11.1.tgz", @@ -3033,6 +3091,11 @@ "bser": "2.1.1" } }, + "fecha": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.1.tgz", + "integrity": "sha512-MMMQ0ludy/nBs1/o0zVOiKTpG7qMbonKUzjJgQFEuvq6INZ1OraKPRAWkBq5vlKLOUMpmNYG1JoN3oDPUQ9m3Q==" + }, "figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -3120,6 +3183,11 @@ "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", "dev": true }, + "fn.name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" + }, "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", @@ -3614,8 +3682,7 @@ "inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "inquirer": { "version": "7.1.0", @@ -3900,8 +3967,7 @@ "is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" }, "is-string": { "version": "1.0.7", @@ -3940,11 +4006,15 @@ "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", "dev": true }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", - "dev": true + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" }, "isstream": { "version": "0.1.2", @@ -5629,6 +5699,11 @@ "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", "dev": true }, + "kuler": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" + }, "leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -5995,6 +6070,18 @@ } } }, + "logform": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.2.0.tgz", + "integrity": "sha512-N0qPlqfypFx7UHNn4B3lzS/b0uLqt2hmuoa+PpuXNYgozdJYAyauF5Ky0BWVjrxDlMWiT3qN4zPq3vVAfZy7Yg==", + "requires": { + "colors": "^1.2.1", + "fast-safe-stringify": "^2.0.4", + "fecha": "^4.2.0", + "ms": "^2.1.1", + "triple-beam": "^1.3.0" + } + }, "lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", @@ -6308,6 +6395,14 @@ "wrappy": "1" } }, + "one-time": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", + "requires": { + "fn.name": "1.x.x" + } + }, "onetime": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz", @@ -6353,6 +6448,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, "requires": { "p-try": "^2.0.0" } @@ -6378,7 +6474,8 @@ "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true }, "parent-module": { "version": "1.0.1", @@ -6653,6 +6750,11 @@ } } }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, "progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", @@ -6674,6 +6776,11 @@ "resolved": "https://registry.npmjs.org/protocols/-/protocols-1.4.8.tgz", "integrity": "sha512-IgjKyaUSjsROSO8/D49Ab7hP8mJgTYcqApOqdPhLoPxAplXmkp+zRvsrSQjFn5by0rhm4VH0GAUELIPpx7B1yg==" }, + "pseudoexec": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/pseudoexec/-/pseudoexec-0.1.4.tgz", + "integrity": "sha512-4FRBAbe36HjS4eocRH9yALjHwsiXHOt+OPl42Ss5UcaK82jvUZCWdkv5jBhhCXSBQL/aDx9qapX0taEHP5dZpg==" + }, "psl": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", @@ -6787,6 +6894,16 @@ } } }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, "regexpp": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.1.tgz", @@ -6837,11 +6954,6 @@ "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" }, - "require-main-filename": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", - "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" - }, "resolve": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.17.0.tgz", @@ -6973,11 +7085,6 @@ "integrity": "sha512-bXWyL6EAKOJa81XG1OZ/Yyuq+oT0b2YLlxx7c+mrdYPaPbnj6WgVULXhinMIeZGufuUBu/eVRqXEhiv4imfwxA==", "dev": true }, - "set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" - }, "shebang-command": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", @@ -7031,6 +7138,21 @@ "debug": "^4.3.1" } }, + "simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=", + "requires": { + "is-arrayish": "^0.3.1" + }, + "dependencies": { + "is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + } + } + }, "sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -7154,6 +7276,11 @@ "tweetnacl": "~0.14.0" } }, + "stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" + }, "stack-utils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.3.tgz", @@ -7234,6 +7361,21 @@ "define-properties": "^1.1.3" } }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + }, + "dependencies": { + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + } + } + }, "stringify-object": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz", @@ -7433,6 +7575,11 @@ "minimatch": "^3.0.4" } }, + "text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" + }, "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -7499,6 +7646,11 @@ "punycode": "^2.1.1" } }, + "triple-beam": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", + "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" + }, "tsconfig-paths": { "version": "3.10.1", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.10.1.tgz", @@ -7610,6 +7762,11 @@ "punycode": "^2.1.0" } }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, "uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", @@ -7720,10 +7877,9 @@ } }, "which": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", - "dev": true, + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "requires": { "isexe": "^2.0.0" } @@ -7741,17 +7897,61 @@ "is-symbol": "^1.0.3" } }, - "which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" - }, "which-pm-runs": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz", "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=", "dev": true }, + "winston": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.3.3.tgz", + "integrity": "sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw==", + "requires": { + "@dabh/diagnostics": "^2.0.2", + "async": "^3.1.0", + "is-stream": "^2.0.0", + "logform": "^2.2.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.4.0" + } + }, + "winston-transport": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.4.0.tgz", + "integrity": "sha512-Lc7/p3GtqtqPBYYtS6KCN3c77/2QCev51DvcJKbkFPQNoj1sinkGwLGFDxkXY9J6p9+EPnYs+D90uwbnaiURTw==", + "requires": { + "readable-stream": "^2.3.7", + "triple-beam": "^1.2.0" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", @@ -7762,7 +7962,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "requires": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -7773,7 +7972,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "requires": { "color-convert": "^2.0.1" } @@ -7782,7 +7980,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "requires": { "color-name": "~1.1.4" } @@ -7790,8 +7987,7 @@ "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" } } }, @@ -7840,9 +8036,9 @@ "dev": true }, "y18n": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.1.tgz", - "integrity": "sha512-wNcy4NvjMYL8gogWWYAO7ZFWFfHcbdbE57tZO8e4cbpj8tfUcwrwqSl3ad8HxpYWCdXcJUCeKKZS62Av1affwQ==" + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" }, "yallist": { "version": "4.0.0", @@ -7856,104 +8052,23 @@ "dev": true }, "yargs": { - "version": "15.4.1", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", - "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", + "version": "17.1.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.1.1.tgz", + "integrity": "sha512-c2k48R0PwKIqKhPMWjeiF6y2xY/gPMUlro0sgxqXpbOIohWiLNXWslsootttv7E1e73QPAMQSg5FeySbVcpsPQ==", "requires": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" - }, - "dependencies": { - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "requires": { - "color-convert": "^2.0.1" - } - }, - "cliui": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", - "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" - } - }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - } - }, - "locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "requires": { - "p-locate": "^4.1.0" - } - }, - "p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "requires": { - "p-limit": "^2.2.0" - } - }, - "path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" - }, - "wrap-ansi": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", - "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - } + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" } }, "yargs-parser": { - "version": "18.1.3", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", - "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", - "requires": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - } + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==" }, "yocto-queue": { "version": "0.1.0", diff --git a/package.json b/package.json index 9d7b105aab..7f22d79139 100644 --- a/package.json +++ b/package.json @@ -30,13 +30,14 @@ ], "main": "index.js", "bin": { - "cml-send-github-check": "bin/cml-send-github-check.js", - "cml-send-comment": "bin/cml-send-comment.js", - "cml-publish": "bin/cml-publish.js", - "cml-tensorboard-dev": "bin/cml-tensorboard-dev.js", - "cml-runner": "bin/cml-runner.js", - "cml-cloud-runner-entrypoint": "bin/cml-runner.js", - "cml-pr": "bin/cml-pr.js" + "cml": "bin/cml.js", + "cml-send-github-check": "bin/legacy.js", + "cml-send-comment": "bin/legacy.js", + "cml-publish": "bin/legacy.js", + "cml-tensorboard-dev": "bin/legacy.js", + "cml-runner": "bin/legacy.js", + "cml-cloud-runner-entrypoint": "bin/legacy.js", + "cml-pr": "bin/legacy.js" }, "scripts": { "lintfix": "eslint --fix ./ && prettier --write '**/*.{js,json,md,yaml,yml}'", @@ -73,12 +74,15 @@ "node-fetch": "^2.6.1", "node-forge": "^0.10.0", "node-ssh": "^11.1.1", + "pseudoexec": "^0.1.4", "semver": "^7.3.5", "simple-git": "^2.44.0", "strip-url-auth": "^1.0.1", "tar": "^6.1.8", "tempy": "^0.6.0", - "yargs": "^15.4.1" + "which": "^2.0.2", + "winston": "^3.3.3", + "yargs": "^17.1.1" }, "devDependencies": { "eslint": "^6.8.0", diff --git a/src/cml.js b/src/cml.js old mode 100644 new mode 100755 index a12f183a71..d330f89c47 --- a/src/cml.js +++ b/src/cml.js @@ -4,6 +4,8 @@ const stripAuth = require('strip-url-auth'); const globby = require('globby'); const git = require('simple-git/promise')('./'); +const winston = require('winston'); + const Gitlab = require('./drivers/gitlab'); const Github = require('./drivers/github'); const BitbucketCloud = require('./drivers/bitbucket_cloud'); @@ -198,8 +200,8 @@ class CML { return log; } } catch (err) { - console.log(`Failed parsing log: ${err.message}`); - console.log(`Original log bytes, as Base64: ${data.toString('base64')}`); + winston.warn(`Failed parsing log: ${err.message}`); + winston.warn(`Original log bytes, as Base64: ${data.toString('base64')}`); } } @@ -268,7 +270,7 @@ class CML { const { files } = await git.status(); if (!files.length) { - console.log('No files changed. Nothing to do.'); + winston.warn('No files changed. Nothing to do.'); return; } @@ -276,7 +278,7 @@ class CML { files.map((file) => file.path).includes(path) ); if (!paths.length) { - console.log('Input files are not affected. Nothing to do.'); + winston.warn('Input files are not affected. Nothing to do.'); return; } @@ -347,7 +349,7 @@ Automated commits for ${this.repo}/commit/${sha} created by CML. } logError(e) { - console.error(e.message); + winston.error(e.message); } } diff --git a/src/drivers/github.js b/src/drivers/github.js index ee2f6aae10..365734fe47 100644 --- a/src/drivers/github.js +++ b/src/drivers/github.js @@ -9,6 +9,7 @@ const { Octokit } = require('@octokit/rest'); const { throttling } = require('@octokit/plugin-throttling'); const tar = require('tar'); +const winston = require('winston'); const { download, exec } = require('../utils'); const CHECK_TITLE = 'CML Report'; @@ -47,7 +48,7 @@ const octokit = (token, repo) => { const throttleHandler = (retryAfter, options) => { if (options.request.retryCount <= 5) { - console.log(`Retrying after ${retryAfter} seconds!`); + winston.log(`Retrying after ${retryAfter} seconds!`); return true; } };