feat(envs): add abstruse defined ENV variables (closes #311)

jkuri · jkuri · commit 717d78b84803 · 2017-12-16T16:03:43.000+01:00
diff --git a/docs/ENV_VARIABLES.md b/docs/ENV_VARIABLES.md
@@ -0,0 +1,71 @@
+# Environment Variables
+
+A common way to customize the build process is to define environment variables, which can be accessed from any stage in your build process.
+
+### Public ENV variables reference
+
+- `ABSTRUSE_BRANCH`
+  - for push builds, or builds not triggered by a pull request, this is the name of the branch.
+  - for builds triggered by a pull request this is the name of the branch targeted by the pull request.
+  - for builds triggered by a tag, this is the same as the name of the tag (`ABSTRUSE_TAG`)
+
+- `ABSTRUSE_BUILD_DIR`
+  - The absolute path to the directory where the repository being built has been copied on the worker.
+
+- `ABSTRUSE_BUILD_ID`
+  - The id of the current build that Abstruse CI uses internally.
+
+- `ABSTRUSE_JOB_ID`
+   - The if of the current job that Abstruse CI uses internally.
+
+- `ABSTRUSE_COMMIT`
+   - The commit that the current build is testing.
+
+- `ABSTRUSE_EVENT_TYPE`
+   - Indicates how the build was triggered. One of `push` or `pull_request`
+
+- `ABSTRUSE_PULL_REQUEST`
+   - The pull request number if the current job is a pull request, “false” if it’s not a pull request.
+
+- `ABSTRUSE_PULL_REQUEST_BRANCH`
+   - if the current job is a pull request, the name of the branch from which the PR originated.
+   - if the current job is a push build, this variable is empty (`""`)
+
+- `ABSTRUSE_TAG`
+   -  If the current build is for a git tag, this variable is set to the tag’s name.
+
+- `ABSTRUSE_PULL_REQUEST_SHA`
+   - if the current job is a pull request, the commit SHA of the HEAD commit of the PR.
+   - if the current job is a push build, this variable is empty (`""`)
+
+- `ABSTRUSE_SECURE_ENV_VARS`
+   - Set to `true` if there are any encrypted environment variables.
+   - Set to `false` if no encrypted environment variables are available.
+
+- `ABSTRUSE_TEST_RESULT`
+   - is set to `0` if the build is successful and `1-255` if the build is broken.
+   - this variable is available only since `test` command is executed
+
+### Define public ENV variables in .abstruse.yml
+
+You can define multiple ENV variables per item.
+
+```yml
+matrix:
+  - env: SCRIPT=lint NODE_VERSION=8
+  - env: SCRIPT=test NODE_VERSION=8
+  - env: SCRIPT=test:e2e NODE_VERSION=8
+  - env: SCRIPT=test:protractor NODE_VERSION=8
+  - env: SCRIPT=test:karma NODE_VERSION=8
+```
+
+### Define variables public and encrypted variables under repository
+
+Variables defined in repository settings are the same for all builds, and when you restart an old build, it uses the latest values. These variables are not automatically available to forks.
+
+Define variables in the Repository Settings that:
+
+- differ per repository.
+- contain sensitive data, such as third-party credentials (encrypted variables).
+
+<img src="https://user-images.githubusercontent.com/1796022/34071301-9d4e4d04-e274-11e7-8be7-57f411d3f93f.png">
diff --git a/src/api/db/job.ts b/src/api/db/job.ts
@@ -11,8 +11,7 @@ export function getJob(jobId: number, userId?: number): Promise<any> {
           .andWhere('permissions.permission', true)
           .orWhere('public', true);
         }
-      }},
-      'runs']})
+      }}, 'runs']})
       .then(job => {
         if (!job) {
           reject();
diff --git a/src/api/docker.ts b/src/api/docker.ts
@@ -5,6 +5,7 @@ import * as dockerode from 'dockerode';
 import { Writable } from 'stream';
 import { CommandType } from './config';
 import { ProcessOutput } from './process';
+import * as envVars from './env-variables';
 import chalk from 'chalk';
 import * as style from 'ansi-styles';
 
@@ -59,9 +60,8 @@ export function startContainer(id: string): Promise<dockerode.Container> {
   return docker.getContainer(id).start();
 }
 
-export function dockerExec(id: string, cmd: any, env: string[] = []): Observable<any> {
+export function dockerExec(id: string, cmd: any, env: envVars.EnvVariables = {}): Observable<any> {
   return new Observable(observer => {
-    const startTime = new Date().getTime();
     let exitCode = 255;
     let command;
 
@@ -89,7 +89,7 @@ export function dockerExec(id: string, cmd: any, env: string[] = []): Observable
     const container = docker.getContainer(id);
     const execOptions = {
       Cmd: ['/usr/bin/abstruse-pty', cmd.command],
-      Env: env,
+      Env: envVars.serialize(env),
       AttachStdout: true,
       AttachStderr: true,
       Tty: true
@@ -102,7 +102,11 @@ export function dockerExec(id: string, cmd: any, env: string[] = []): Observable
         ws.setDefaultEncoding('utf8');
 
         ws.on('finish', () => {
-          const duration = new Date().getTime() - startTime;
+          if (cmd.type === CommandType.script) {
+            envVars.set(env, 'ABSTRUSE_TEST_RESULT', exitCode);
+          }
+
+          observer.next({ type: 'env', data: env });
           observer.next({ type: 'exit', data: exitCode });
           observer.complete();
         });
diff --git a/src/api/env-variables.ts b/src/api/env-variables.ts
@@ -0,0 +1,68 @@
+export interface EnvVariables {
+  [key: string]: string | number | boolean;
+}
+
+export function set(envs: EnvVariables, key: string, value: string | number | boolean): void {
+  envs[key] = value;
+}
+
+export function unset(envs: EnvVariables, key: string): void {
+  envs[key] = null;
+}
+
+export function serialize(envs: EnvVariables): string[] {
+  return Object.keys(envs).map(key => `${key}=${envs[key]}`);
+}
+
+export function unserialize(envs: string[]): EnvVariables {
+  return envs.reduce((acc, curr) => {
+    const splitted = curr.split('=');
+    acc = Object.assign({}, acc, { [splitted[0]]: splitted[1] });
+    return acc;
+  }, {});
+}
+
+export function generate(data: any): EnvVariables {
+  const envs = init();
+  const request = data.requestData;
+  const commit = request.data.pull_request && request.data.pull_request.head
+                 && request.data.pull_request.head.sha ||
+                 request.data.head_commit && request.data.head_commit.id ||
+                 request.data.sha ||
+                 request.data.object_attributes && request.data.object_attributes.last_commit &&
+                 request.data.object_attributes.last_commit.id ||
+                 request.data.push && request.data.push.changes[0].commits[0].hash ||
+                 request.data.pullrequest && request.data.pullrequest.source &&
+                 request.data.pullrequest.source.commit &&
+                 request.data.pullrequest.source.commit.hash ||
+                 request.data.commit || '';
+  const tag = request.ref && request.ref.startsWith('refs/tags/') ?
+    request.ref.replace('refs/tags/', '') : null;
+
+  set(envs, 'ABSTRUSE_BRANCH', request.branch);
+  set(envs, 'ABSTRUSE_BUILD_ID', data.build_id);
+  set(envs, 'ABSTRUSE_JOB_ID', data.job_id);
+  set(envs, 'ABSTRUSE_COMMIT', commit);
+  set(envs, 'ABSTRUSE_EVENT_TYPE', request.pr ? 'pull_request' : 'push');
+  set(envs, 'ABSTRUSE_PULL_REQUEST', request.pr ? request.pr : false);
+  set(envs, 'ABSTRUSE_PULL_REQUEST_BRANCH', request.pr ? request.branch : '');
+  set(envs, 'ABSTRUSE_TAG', tag);
+
+  const prSha = request.pr ? commit : '';
+  set(envs, 'ABSTRUSE_PULL_REQUEST_SHA', prSha);
+
+  return envs;
+}
+
+function init(): EnvVariables {
+  return [
+    'ABSTRUSE_BRANCH', 'ABSTRUSE_BUILD_DIR', 'ABSTRUSE_BUILD_ID',
+    'ABSTRUSE_JOB_ID', 'ABSTRUSE_COMMIT', 'ABSTRUSE_EVENT_TYPE',
+    'ABSTRUSE_PULL_REQUEST', 'ABSTRUSE_PULL_REQUEST_BRANCH',
+    'ABSTRUSE_TAG', 'ABSTRUSE_PULL_REQEUST_SHA', 'ABSTRUSE_SECURE_ENV_VARS',
+    'ABSTRUSE_TEST_RESULT'
+  ].reduce((acc, curr) => {
+    acc = Object.assign(acc, { [curr]: null });
+    return acc;
+  }, {});
+}
diff --git a/src/api/process-manager.ts b/src/api/process-manager.ts
@@ -40,6 +40,7 @@ export interface JobProcess {
   status?: 'queued' | 'running' | 'cancelled' | 'errored' | 'success';
   image_name?: string;
   log?: string;
+  requestData: any;
   commands?: { command: string, type: CommandType }[];
   cache?: string[];
   repo_name?: string;
@@ -564,14 +565,19 @@ export function stopBuild(buildId: number): Promise<any> {
 
 function queueJob(jobId: number): Promise<void> {
   let job = null;
+  let requestData = null;
+
   return dbJob.getJob(jobId)
     .then(jobData => job = jobData)
+    .then(() => getBuild(job.builds_id))
+    .then(build => requestData = { branch: build.branch, pr: build.pr, data: build.data })
     .then(() => {
       const data = JSON.parse(job.data);
       const jobProcess: JobProcess = {
         build_id: job.builds_id,
         job_id: jobId,
         status: 'queued',
+        requestData: requestData,
         commands: data.commands,
         cache: data.cache || null,
         repo_name: job.build.repository.full_name || null,
diff --git a/src/api/process.ts b/src/api/process.ts
@@ -6,6 +6,7 @@ import { getRepositoryByBuildId } from './db/repository';
 import { Observable } from 'rxjs';
 import { CommandType, Command, CommandTypePriority } from './config';
 import { JobProcess } from './process-manager';
+import * as envVars from './env-variables';
 import chalk from 'chalk';
 import * as style from 'ansi-styles';
 import { deploy } from './deploy';
@@ -25,7 +26,8 @@ export interface SpawnedProcessOutput {
 }
 
 export interface ProcessOutput {
-  type: 'data' | 'exit' | 'container' | 'exposed ports' | 'containerInfo' | 'containerError';
+  type: 'data' | 'exit' | 'container' | 'exposed ports' | 'containerInfo' | 'containerError' |
+        'env';
   data: any;
 }
 
@@ -37,9 +39,10 @@ export function startBuildProcess(
 ): Observable<ProcessOutput> {
   return new Observable(observer => {
     const image = proc.image_name;
-
     const name = 'abstruse_' + proc.build_id + '_' + proc.job_id;
-    const envs = proc.commands.filter(cmd => {
+
+    let envs = envVars.generate(proc);
+    const initEnvs = proc.commands.filter(cmd => {
         return typeof cmd.command === 'string' && cmd.command.startsWith('export');
       })
       .map(cmd => cmd.command.replace('export', ''))
@@ -51,7 +54,9 @@ export function startBuildProcess(
     const gitTypes = [CommandType.git];
     const installTypes = [CommandType.before_install, CommandType.install];
     const scriptTypes = [CommandType.before_script, CommandType.script,
-      CommandType.after_success, CommandType.after_failure, CommandType.after_script];
+                         CommandType.after_success, CommandType.after_failure,
+                         CommandType.after_script];
+
     const gitCommands = prepareCommands(proc, gitTypes);
     const installCommands = prepareCommands(proc, installTypes);
     const scriptCommands = prepareCommands(proc, scriptTypes);
@@ -83,7 +88,7 @@ export function startBuildProcess(
 
       restoreCache = Observable.concat(...[
         executeOutsideContainer(copyRestoreCmd),
-        docker.dockerExec(name, { command: restoreCmd, type: CommandType.restore_cache })
+        docker.dockerExec(name, { command: restoreCmd, type: CommandType.restore_cache, env: envs })
       ]);
 
       let cacheFolders = proc.cache.map(folder => {
@@ -104,27 +109,31 @@ export function startBuildProcess(
       ].join('');
 
       saveCache = Observable.concat(...[
-        docker.dockerExec(name, { command: tarCmd, type: CommandType.store_cache }),
+        docker.dockerExec(name, { command: tarCmd, type: CommandType.store_cache, env: envs }),
         executeOutsideContainer(saveTarCmd)
       ]);
     }
 
-    const sub = docker.createContainer(name, image, envs)
-      .concat(...gitCommands.map(cmd => docker.dockerExec(name, cmd)))
+    const sub = docker.createContainer(name, image, initEnvs)
+      .concat(...gitCommands.map(cmd => docker.dockerExec(name, cmd, envs)))
       .concat(restoreCache)
-      .concat(...installCommands.map(cmd => docker.dockerExec(name, cmd)))
+      .concat(...installCommands.map(cmd => docker.dockerExec(name, cmd, envs)))
       .concat(saveCache)
-      .concat(...scriptCommands.map(cmd => docker.dockerExec(name, cmd)))
-      .concat(...beforeDeployCommands.map(cmd => docker.dockerExec(name, cmd)))
-      .concat(...deployCommands.map(cmd => docker.dockerExec(name, cmd)))
-      .concat(deploy(deployPreferences, name, envs))
-      .concat(...afterDeployCommands.map(cmd => docker.dockerExec(name, cmd)))
+      .concat(...scriptCommands.map(cmd => docker.dockerExec(name, cmd, envs)))
+      .concat(...beforeDeployCommands.map(cmd => docker.dockerExec(name, cmd, envs)))
+      .concat(...deployCommands.map(cmd => docker.dockerExec(name, cmd, envs)))
+      .concat(deploy(deployPreferences, name, initEnvs))
+      .concat(...afterDeployCommands.map(cmd => docker.dockerExec(name, cmd, envs)))
       .timeoutWith(idleTimeout, Observable.throw(new Error('command timeout')))
       .takeUntil(Observable.timer(jobTimeout).timeInterval().mergeMap(() => {
         return Observable.throw('job timeout');
       }))
       .subscribe((event: ProcessOutput) => {
-        if (event.type === 'containerError') {
+        if (event.type === 'env') {
+          if (Object.keys(event.data).length) {
+            envs = event.data;
+          }
+        } else if (event.type === 'containerError') {
           const msg = chalk.red((event.data.json && event.data.json.message) || event.data);
           observer.next({ type: 'exit', data: msg });
           observer.error(msg);
@@ -140,7 +149,7 @@ export function startBuildProcess(
               `last executed command exited with code ${event.data}`
             ].join(' ');
             const tmsg = style.red.open + style.bold.open +
-              `[error]: executed command returned exit code ${event.data}` +
+              `\r\n[error]: executed command returned exit code ${event.data}` +
               style.bold.close + style.red.close;
             observer.next({ type: 'exit', data: chalk.red(tmsg) });
             observer.error(msg);
@@ -164,8 +173,8 @@ export function startBuildProcess(
           .catch(err => console.error(err));
       }, () => {
         const msg = style.green.open + style.bold.open +
-          '[success]: build returned exit code 0' +
-          style.bold.close + style.green.close;
+                    '\r\n[success]: build returned exit code 0' +
+                    style.bold.close + style.green.close;
         observer.next({ type: 'exit', data: chalk.green(msg) });
         docker.killContainer(name)
           .then(() => {
