Permalink
Browse files

feat(cache): enable caching files and directories (closes #132)

  • Loading branch information...
jkuri committed Sep 11, 2017
1 parent cfec72c commit 161014674c96b1eb3cd355a81c755253324794fc
@@ -26,6 +26,8 @@ export enum CommandType {
before_script = 'before_script',
script = 'script',
before_cache = 'before_cache',
restore_cache = 'restore_cache',
store_cache = 'store_cache',
after_success = 'after_success',
after_failure = 'after_failure',
before_deploy = 'before_deploy',
@@ -68,6 +70,7 @@ export interface JobsAndEnv {
display_version: string | null;
language: Language;
version?: string;
cache?: string[];
}

export interface Repository {
@@ -84,7 +87,7 @@ export interface Config {
language?: Language;
os?: string;
stage?: JobStage;
cache?: { [key: string]: string }[] | null;
cache?: string[] | null;
branches?: { test: string[], ignore: string[] };
env?: { global: string[], matrix: string[] };
before_install?: { command: string, type: CommandType }[];
@@ -232,33 +235,33 @@ function parseOS(os: string | null): string {
return 'linux'; // since we are compatible with travis configs, hardcode this to Linux
}

function parseCache(cache: any | null): { [key: string]: string }[] | null {
function parseCache(cache: any | null): string[] | null {
if (!cache) {
return null;
} else {
if (typeof cache === 'string') {
if (cache in CacheType) {
switch (cache) {
case 'bundler':
return [ { bundler: 'vendor/bundle' } ];
return [ 'vendor/bundle' ];
case 'yarn':
return [ { yarn: '$HOME/.cache/yarn' } ];
return [ '$HOME/.cache/yarn' ];
case 'pip':
return [ { pip: '$HOME/.cache/pip' } ];
return [ '$HOME/.cache/pip' ];
case 'ccache':
return [ { ccache: '$HOME/.ccache' } ];
return [ '$HOME/.ccache' ];
case 'packages':
return [ { packages: '$HOME/R/Library' } ];
return [ '$HOME/R/Library' ];
case 'cargo':
return [ { cargo: '$HOME/.cargo' } ];
return [ '$HOME/.cargo' ];
}
} else {
throw new Error(`${cache} is not a known option for caching.`);
}
} else if (typeof cache === 'object') {
if (cache && cache.directories) {
if (Array.isArray(cache.directories)) {
return cache.directories.map(dir => ({ dir }));
return cache.directories;
} else {
throw new Error(`${cache.directories} is not a type of array.`);
}
@@ -560,6 +563,9 @@ export function generateJobsAndEnv(repo: Repository, config: Config): JobsAndEnv
{ command: checkout, type: CommandType.git }
]);

// apply cache to each job
d.cache = config.cache;

d.commands = d.commands.filter(cmd => !!cmd.command);

return d;
@@ -40,6 +40,9 @@ export interface JobProcess {
image_name?: string;
log?: string[];
commands?: { command: string, type: CommandType }[];
cache?: string[];
repo_name?: string;
branch?: string;
env?: string[];
sshAndVnc?: boolean;
job?: Observable<any>;
@@ -134,7 +137,6 @@ export function startBuild(data: any): Promise<any> {
.then(() => data = Object.assign(data, { branch: branch, pr: pr }))
.then(() => insertBuild(data))
.then(build => {
console.log(config);
data = Object.assign(data, { build_id: build.id });
delete data.repositories_id;
delete data.pr;
@@ -457,6 +459,9 @@ function queueJob(buildId: number, jobId: number, sshAndVnc = false): Promise<vo
job_id: jobId,
status: 'queued',
commands: jobData.commands,
cache: jobData.cache || null,
repo_name: job.build.repository.full_name || null,
branch: job.build.branch || null,
env: jobData.env,
sshAndVnc: sshAndVnc,
log: []
@@ -1,7 +1,7 @@
import * as docker from './docker';
import { PtyInstance } from './pty';
import * as child_process from 'child_process';
import { generateRandomId } from './utils';
import { generateRandomId, getFilePath } from './utils';
import { getRepositoryByBuildId } from './db/repository';
import { Observable } from 'rxjs';
import { green, red, bold, yellow, blue, cyan } from 'chalk';
@@ -43,6 +43,54 @@ export function startBuildProcess(

proc.commands = proc.commands.filter(cmd => !cmd.command.startsWith('export'));

const gitCommands = proc.commands.filter(command => command.type === CommandType.git);
const installCommands = proc.commands.filter(command => {
return command.type === CommandType.before_install || command.type === CommandType.install;
});
const scriptCommands = proc.commands.filter(command => {
return command.type === CommandType.before_script || command.type === CommandType.script ||
command.type === CommandType.after_success || command.type === CommandType.after_failure;
});
const deployCommands = proc.commands.filter(command => {
return command.type === CommandType.before_deploy || command.type === CommandType.deploy ||
command.type === CommandType.after_deploy || command.type === CommandType.after_script;
});

let restoreCache: Observable<any> = Observable.empty();
let saveCache: Observable<any> = Observable.empty();
if (proc.repo_name && proc.branch && proc.cache) {
let cacheFile = `cache_${proc.repo_name.replace('/', '-')}_${proc.branch}.tar.bz2`;
let cacheHostPath = getFilePath(`cache/${cacheFile}`);
let cacheContainerPath = `/home/abstruse/${cacheFile}`;
let copyRestoreCmd = [
`if [ -f ${cacheHostPath} ];`,
`then docker cp ${cacheHostPath} ${name}:/home/abstruse`,
`; fi`
].join(' ');
let restoreCmd = [
`if [ -f /home/abstruse/${cacheFile} ];`,
`then tar xjf /home/abstruse/${cacheFile} -C .`,
`; fi`
].join(' ');

restoreCache = Observable.concat(...[
executeOutsideContainer(copyRestoreCmd),
executeInContainer(name, { command: restoreCmd, type: CommandType.restore_cache })
]);

let tarCmd = [
`if [ ! -f /home/abstruse/${cacheFile} ];`,
`then tar cjSf /home/abstruse/${cacheFile} ${proc.cache.join(' ')}`,
`; fi`
].join(' ');
let saveTarCmd = `docker cp ${name}:/home/abstruse/${cacheFile} ${cacheHostPath}`;

saveCache = Observable.concat(...[
executeInContainer(name, { command: tarCmd, type: CommandType.store_cache }),
executeOutsideContainer(saveTarCmd)
]);
}

let debug: Observable<any> = Observable.empty();
if (proc.sshAndVnc) {
const ssh = `sudo /etc/init.d/ssh start`;
@@ -69,7 +117,13 @@ export function startBuildProcess(

const sub = startContainer(name, image, envs)
.concat(debug)
.concat(...proc.commands.map(cmd => executeInContainer(name, cmd)))
// .concat(...proc.commands.map(cmd => executeInContainer(name, cmd)))
.concat(...gitCommands.map(cmd => executeInContainer(name, cmd)))
.concat(...[restoreCache])
.concat(...installCommands.map(cmd => executeInContainer(name, cmd)))
.concat(...[saveCache])
.concat(...scriptCommands.map(cmd => executeInContainer(name, cmd)))
.concat(...deployCommands.map(cmd => executeInContainer(name, cmd)))
.subscribe((event: ProcessOutput) => {
observer.next(event);
}, err => {
@@ -122,12 +176,18 @@ function executeInContainer(name: string, cmd: Command): Observable<ProcessOutpu
const exec = `/usr/bin/abstruse '${cmd.command}'\r`;
attach.write(exec);

// don't show access token
// don't show access token on UI
if (cmd.command.includes('http') && cmd.command.includes('@')) {
cmd.command = cmd.command.replace(/\/\/(.*)@/, '//');
}

observer.next({ type: 'data', data: yellow('==> ' + cmd.command) + '\r' });
if (cmd.type === CommandType.store_cache) {
observer.next({ type: 'data', data: yellow('==> Storing cache ...') + '\r' });
} else if (cmd.type === CommandType.restore_cache) {
observer.next({ type: 'data', data: yellow('==> Restoring cache ...') + '\r' });
} else {
observer.next({ type: 'data', data: yellow('==> ' + cmd.command) + '\r' });
}
executed = true;
} else {
if (data.includes('[success]')) {
@@ -170,6 +230,20 @@ function executeInContainer(name: string, cmd: Command): Observable<ProcessOutpu
});
}

function executeOutsideContainer(cmd: string): Observable<ProcessOutput> {
return new Observable(observer => {
const proc = child_process.exec(cmd);

proc.stdout.on('data', data => console.log(data.toString()));
proc.stderr.on('data', data => console.log(data.toString()));

proc.on('close', code => {
observer.next({ type: 'exit', data: code.toString() });
observer.complete();
});
});
}

function startContainer(name: string, image: string, vars = []): Observable<ProcessOutput> {
return new Observable(observer => {
docker.killContainer(name)
@@ -1,5 +1,5 @@
{
"action": "opened",
"action": "synchronize",
"number": 152,
"pull_request": {
"url": "https://api.github.com/repos/bleenco/abstruse/pulls/152",
@@ -33,10 +33,10 @@
},
"body": "",
"created_at": "2017-09-10T22:55:47Z",
"updated_at": "2017-09-10T22:55:47Z",
"updated_at": "2017-09-10T23:00:11Z",
"closed_at": null,
"merged_at": null,
"merge_commit_sha": null,
"merge_commit_sha": "d03751425b5d6fc9c25c371b694d72423d806658",
"assignee": {
"login": "jkuri",
"id": 1796022,
@@ -119,11 +119,11 @@
"review_comments_url": "https://api.github.com/repos/bleenco/abstruse/pulls/152/comments",
"review_comment_url": "https://api.github.com/repos/bleenco/abstruse/pulls/comments{/number}",
"comments_url": "https://api.github.com/repos/bleenco/abstruse/issues/152/comments",
"statuses_url": "https://api.github.com/repos/bleenco/abstruse/statuses/8dbd404ec683a79a06f93db2258d0e301c2b4423",
"statuses_url": "https://api.github.com/repos/bleenco/abstruse/statuses/88ca1940202e65529a6f039fc072fd15fe58781a",
"head": {
"label": "jkuri:caching",
"ref": "caching",
"sha": "8dbd404ec683a79a06f93db2258d0e301c2b4423",
"sha": "88ca1940202e65529a6f039fc072fd15fe58781a",
"user": {
"login": "jkuri",
"id": 1796022,
@@ -209,7 +209,7 @@
"deployments_url": "https://api.github.com/repos/jkuri/abstruse/deployments",
"created_at": "2017-05-05T18:47:09Z",
"updated_at": "2017-05-15T00:14:08Z",
"pushed_at": "2017-09-10T22:55:15Z",
"pushed_at": "2017-09-10T23:00:10Z",
"git_url": "git://github.com/jkuri/abstruse.git",
"ssh_url": "git@github.com:jkuri/abstruse.git",
"clone_url": "https://github.com/jkuri/abstruse.git",
@@ -322,7 +322,7 @@
"deployments_url": "https://api.github.com/repos/bleenco/abstruse/deployments",
"created_at": "2017-03-13T22:31:58Z",
"updated_at": "2017-09-07T19:22:31Z",
"pushed_at": "2017-09-10T16:18:30Z",
"pushed_at": "2017-09-10T22:55:47Z",
"git_url": "git://github.com/bleenco/abstruse.git",
"ssh_url": "git@github.com:bleenco/abstruse.git",
"clone_url": "https://github.com/bleenco/abstruse.git",
@@ -369,7 +369,7 @@
"href": "https://api.github.com/repos/bleenco/abstruse/pulls/152/commits"
},
"statuses": {
"href": "https://api.github.com/repos/bleenco/abstruse/statuses/8dbd404ec683a79a06f93db2258d0e301c2b4423"
"href": "https://api.github.com/repos/bleenco/abstruse/statuses/88ca1940202e65529a6f039fc072fd15fe58781a"
}
},
"author_association": "OWNER",
@@ -382,10 +382,12 @@
"review_comments": 0,
"maintainer_can_modify": true,
"commits": 3,
"additions": 5,
"deletions": 346,
"changed_files": 5
"additions": 520,
"deletions": 325,
"changed_files": 6
},
"before": "8dbd404ec683a79a06f93db2258d0e301c2b4423",
"after": "88ca1940202e65529a6f039fc072fd15fe58781a",
"repository": {
"id": 84880847,
"name": "abstruse",
@@ -452,7 +454,7 @@
"deployments_url": "https://api.github.com/repos/bleenco/abstruse/deployments",
"created_at": "2017-03-13T22:31:58Z",
"updated_at": "2017-09-07T19:22:31Z",
"pushed_at": "2017-09-10T16:18:30Z",
"pushed_at": "2017-09-10T22:55:47Z",
"git_url": "git://github.com/bleenco/abstruse.git",
"ssh_url": "git@github.com:bleenco/abstruse.git",
"clone_url": "https://github.com/bleenco/abstruse.git",
@@ -94,35 +94,35 @@ describe('Common Configuration Options', () => {
it(`should return array of single appropriate dir if 'bundler' is specified`, () => {
data.cache = 'bundler';
const parsed = parseConfig(data);
const expected = [ { bundler: 'vendor/bundle' } ];
const expected = [ 'vendor/bundle' ];
expect(parsed.cache).to.deep.equal(expected);
});

it(`should return array of single appropriate dir if 'yarn' is specified`, () => {
data.cache = 'yarn';
const parsed = parseConfig(data);
const expected = [ { yarn: '$HOME/.cache/yarn' } ];
const expected = [ '$HOME/.cache/yarn' ];
expect(parsed.cache).to.deep.equal(expected);
});

it(`should return array of single appropriate dir if 'pip' is specified`, () => {
data.cache = 'pip';
const parsed = parseConfig(data);
const expected = [ { pip: '$HOME/.cache/pip' } ];
const expected = [ '$HOME/.cache/pip' ];
expect(parsed.cache).to.deep.equal(expected);
});

it(`should return array of single appropriate dir if 'packages' is specified`, () => {
data.cache = 'packages';
const parsed = parseConfig(data);
const expected = [ { packages: '$HOME/R/Library' } ];
const expected = [ '$HOME/R/Library' ];
expect(parsed.cache).to.deep.equal(expected);
});

it(`should return array of single appropriate dir if 'cargo' is specified`, () => {
data.cache = 'cargo';
const parsed = parseConfig(data);
const expected = [ { cargo: '$HOME/.cargo' } ];
const expected = [ '$HOME/.cargo' ];
expect(parsed.cache).to.deep.equal(expected);
});

@@ -135,8 +135,8 @@ describe('Common Configuration Options', () => {
data.cache = { directories: ['node_modules', 'vendor/packages'] };
const parsed = parseConfig(data);
const expected = [
{ dir: 'node_modules' },
{ dir: 'vendor/packages' }
'node_modules',
'vendor/packages'
];
expect(parsed.cache).to.deep.equal(expected);
});

0 comments on commit 1610146

Please sign in to comment.