diff --git a/bin/cml-pr.js b/bin/cml-pr.js new file mode 100644 index 000000000..04c4121f9 --- /dev/null +++ b/bin/cml-pr.js @@ -0,0 +1,41 @@ +#!/usr/bin/env node + +const print = console.log; +console.log = console.error; + +const yargs = require('yargs'); +const decamelize = require('decamelize-keys'); + +const CML = require('../src/cml'); + +const run = async (opts) => { + const globs = opts._.length ? opts._ : undefined; + const cml = new CML(opts); + print(await cml.pr_create({ ...opts, globs })); +}; + +const opts = decamelize( + yargs + .usage('Usage: $0 ') + .describe('md', 'Output in markdown format [](url).') + .boolean('md') + .default('repo') + .describe( + 'repo', + 'Specifies the repo to be used. If not specified is extracted from the CI ENV.' + ) + .default('token') + .describe( + 'token', + 'Personal access token to be used. If not specified in extracted from ENV repo_token.' + ) + .default('driver') + .choices('driver', ['github', 'gitlab']) + .describe('driver', 'If not specify it infers it from the ENV.') + .help('h').argv +); + +run(opts).catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/package.json b/package.json index 3def2629e..4b9fc2fa8 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,8 @@ "cml-publish": "bin/cml-publish.js", "cml-tensorboard-dev": "bin/cml-tensorboard-dev.js", "cml-runner": "bin/cml-runner.js", - "cml-cloud-runner-entrypoint": "bin/cml-runner.js" + "cml-cloud-runner-entrypoint": "bin/cml-runner.js", + "test-cml-pr": "bin/cml-pr.js" }, "scripts": { "lintfix": "eslint --fix ./", diff --git a/src/cml.js b/src/cml.js index 9fe359f65..4f0f94915 100644 --- a/src/cml.js +++ b/src/cml.js @@ -1,6 +1,8 @@ const { execSync } = require('child_process'); const git_url_parse = require('git-url-parse'); const strip_auth = require('strip-url-auth'); +const git = require('simple-git/promise')('./'); +const globby = require('globby'); const Gitlab = require('./drivers/gitlab'); const Github = require('./drivers/github'); @@ -234,6 +236,89 @@ class CML { } } + async pr_create(opts = {}) { + const { globs = ['dvc.lock', '.gitignore'], md } = opts; + + const { files } = await git.status(); + if (!files.length) { + console.log('No files changed. Nothing to do.'); + return; + } + + const driver = get_driver(this); + const paths = (await globby(globs)).filter((path) => + files.map((item) => item.path).includes(path) + ); + + const render_pr = (url) => { + if (md) + return `[CML's ${ + this.driver === 'gitlab' ? 'Merge' : 'Pull' + } Request](${url})`; + return url; + }; + + const sha = await exec(`git rev-parse HEAD`); + const sha_short = sha.substr(0, 7); + let target = await exec(`git branch --show-current`); + if (!target) { + if (this.driver === 'gitlab') { + target = await exec('echo $CI_BUILD_REF_NAME'); + } + } + const source = `${target}-cmlpr-${sha_short}`; + + await exec(`git fetch origin`); + + const branch_exists = (await exec(`git branch -r`)).includes(source); + if (branch_exists) { + const prs = await driver.prs(); + const { url } = + prs.find((pr) => pr.source === source && pr.target === target) || {}; + + if (url) return render_pr(url); + } else { + try { + await exec(`git config --local user.email "david@iterative.ai"`); + await exec(`git config --local user.name "cml-bot"`); + await exec('git config advice.addIgnoredFile false'); + + if (this.driver !== 'github') { + const repo = new URL(this.repo); + repo.password = this.token; + repo.username = driver.user_name; + + await exec(`git remote rm origin`); + await exec(`git remote add origin "${repo.toString()}.git"`); + } + + await exec(`git checkout -B ${target} ${sha}`); + await exec(`git checkout -b ${source}`); + await exec(`git add ${paths.join(' ')}`); + await exec(`git commit -m "CML [skip ci]"`); + await exec(`git push --set-upstream origin ${source}`); + await exec(`git checkout -B ${target} ${sha}`); + } catch (err) { + await exec(`git checkout -B ${target} ${sha}`); + throw err; + } + } + + const title = `CML commits ${target} ${sha_short}`; + const description = ` + Automated commits for ${this.repo}/commit/${sha} created by CML. + `; + + const url = await driver.pr_create({ + source, + target, + title, + description + }); + + return render_pr(url); + } + log_error(e) { console.error(e.message); } diff --git a/src/drivers/github.js b/src/drivers/github.js index a0be2f233..9330dbca0 100644 --- a/src/drivers/github.js +++ b/src/drivers/github.js @@ -231,6 +231,58 @@ class Github { ) .map((runner) => ({ id: runner.id, name: runner.name })); } + + async pr_create(opts = {}) { + const { source: head, target: base, title, description: body } = opts; + const { owner, repo } = owner_repo({ uri: this.repo }); + const { pulls } = octokit(this.token, this.repo); + + const { + data: { url } + } = await pulls.create({ + owner, + repo, + head, + base, + title, + body + }); + + return url; + } + + async prs(opts = {}) { + const { state = 'open' } = opts; + const { owner, repo } = owner_repo({ uri: this.repo }); + const { pulls } = octokit(this.token, this.repo); + + const { data: prs } = await pulls.list({ + owner, + repo, + state + }); + + return prs.map((pr) => { + const { + url, + head: { ref: source }, + base: { ref: target } + } = pr; + return { + url, + source, + target + }; + }); + } + + get user_email() { + return 'action@github.com'; + } + + get user_name() { + return 'GitHub Action'; + } } module.exports = Github; diff --git a/src/drivers/gitlab.js b/src/drivers/gitlab.js index c7bbf1188..68d59fe08 100644 --- a/src/drivers/gitlab.js +++ b/src/drivers/gitlab.js @@ -8,7 +8,7 @@ const { resolve } = require('path'); const { fetch_upload_data, download, exec } = require('../utils'); -const { IN_DOCKER } = process.env; +const { IN_DOCKER, GITLAB_USER_EMAIL, GITLAB_USER_NAME } = process.env; const API_VER = 'v4'; class Gitlab { constructor(opts = {}) { @@ -180,6 +180,39 @@ class Gitlab { return runners.map((runner) => ({ id: runner.id, name: runner.name })); } + async pr_create(opts = {}) { + const { project_path } = this; + const { source, target, title, description } = opts; + + const endpoint = `/projects/${project_path}/merge_requests`; + const body = new URLSearchParams(); + body.append('source_branch', source); + body.append('target_branch', target); + body.append('title', title); + body.append('description', description); + + const { web_url } = await this.request({ endpoint, method: 'POST', body }); + + return web_url; + } + + async prs(opts = {}) { + const { project_path } = this; + const { state = 'opened' } = opts; + + const endpoint = `/projects/${project_path}/merge_requests?state=${state}`; + const prs = await this.request({ endpoint, method: 'GET' }); + + return prs.map((pr) => { + const { web_url: url, source_branch: source, target_branch: target } = pr; + return { + url, + source, + target + }; + }); + } + async request(opts = {}) { const { token } = this; const { endpoint, method = 'GET', body, raw } = opts; @@ -198,6 +231,14 @@ class Gitlab { return await response.json(); } + + get user_email() { + return GITLAB_USER_EMAIL; + } + + get user_name() { + return GITLAB_USER_NAME; + } } module.exports = Gitlab;