-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
32 changed files
with
724 additions
and
100 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,4 +3,5 @@ npm-debug.log | |
*.obj | ||
*.out | ||
node_modules/ | ||
out/ | ||
out/ | ||
docker-test/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,7 +1,3 @@ | ||
# PAIFlow | ||
|
||
[![Documentation Status](https://readthedocs.org/projects/paiflow/badge/?version=latest)](https://paiflow.readthedocs.io/en/latest/?badge=latest) | ||
[![Build Status](https://travis-ci.com/debuggy/PAIFlow.svg?branch=master)](https://travis-ci.com/debuggy/PAIFlow) | ||
[![Coverage Status](https://coveralls.io/repos/github/debuggy/PAIFlow/badge.svg?branch=master)](https://coveralls.io/github/debuggy/PAIFlow?branch=master) | ||
# gelato | ||
|
||
Work flow of Microsoft OpenPAI |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,19 +1,39 @@ | ||
# version: 0.1 | ||
|
||
From ubuntu | ||
From tensorflow/tensorflow | ||
|
||
SHELL ["/bin/bash", "-c"] | ||
ENV BASH_ENV=~/.bashrc | ||
|
||
RUN apt-get update | ||
RUN mv ~/.bashrc ~/.bashrc.bak | ||
RUN apt-get install -y curl wget gnupg bzip2 | ||
RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh | ||
RUN /bin/bash ~/miniconda.sh -b -p /opt/conda | ||
RUN rm ~/miniconda.sh | ||
|
||
## docker does not start bash as login session | ||
# RUN echo "echo bash_profile" >> ~/.bash_profile | ||
# RUN echo "export PATH=/opt/conda/bin:$PATH" >> ~/.bash_profile | ||
# RUN echo "echo bash_login" >> ~/.bash_login | ||
# RUN echo "export PATH=/opt/conda/bin:$PATH" >> ~/.bash_login | ||
# RUN echo "echo .profile" >> ~/.profile | ||
# RUN echo "export PATH=/opt/conda/bin:$PATH" >> ~/.profile | ||
|
||
RUN echo 'export PATH=/opt/conda/bin:$PATH' >> ~/.bashrc && source ~/.bashrc | ||
RUN conda -V && \ | ||
conda create --yes -n docker python=3.6 | ||
RUN echo 'source activate docker' >> ~/.bashrc && source ~/.bashrc | ||
RUN echo 'export PATH=/opt/conda/envs/docker/bin:$PATH' >> ~/.bashrc && source ~/.bashrc | ||
RUN conda install numpy | ||
|
||
RUN cat ~/.bashrc.bak >> ~/.bashrc | ||
|
||
CMD ["/bin/bash", "-c", "source activate docker"] | ||
|
||
|
||
ENV TINI_VERSION v0.9.0 | ||
RUN apt-get update && \ | ||
apt-get install -y curl wget gnupg bzip2 | ||
|
||
RUN wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O ~/miniconda.sh | ||
RUN chmod 0755 ~/miniconda.sh | ||
RUN ~/miniconda.sh -b -p ~/conda | ||
ENV PATH "~/conda/bin:$PATH" | ||
RUN ["/bin/bash", "-c", "conda create -n build_test python=3.4 && source activate build_test && conda install numpy"] | ||
|
||
# RUN conda activate bulid_test | ||
# RUN conda install scikit-learn | ||
|
||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
const dockerfileGenerator = require('../../dockerfile-generator') | ||
const SchemaValidationError = require('../../schema-validation-error') | ||
const fs = require('fs-extra') | ||
const path = require('path') | ||
|
||
/* eslint-env jest */ | ||
it('throws schema validation error', async () => { | ||
expect.assertions(1) | ||
const templateFile = path.resolve(__dirname, '..', 'data', 'config-negative-example.json') | ||
const template = await fs.readJson(templateFile) | ||
await expect(dockerfileGenerator(template)).rejects.toThrow(SchemaValidationError) | ||
}) | ||
|
||
it('generates dockerfile successfully', async () => { | ||
expect.assertions(2) | ||
const templateFile = path.resolve(__dirname, '..', '..', '..', 'examples', 'config-example.json') | ||
const template = await fs.readJson(templateFile) | ||
const dockerFilePath = path.resolve(__dirname, '..', '..', '..', 'out', 'dockerfile') | ||
await fs.remove(dockerFilePath) | ||
const result = await dockerfileGenerator(template, dockerFilePath) | ||
expect(result).toEqual(dockerFilePath) | ||
expect(await fs.pathExists(dockerFilePath)).toBeTruthy() | ||
}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
const fs = require('fs-extra') | ||
const path = require('path') | ||
|
||
const dockerfileGenerator = require('../../dockerfile-generator') | ||
|
||
/* eslint-env jest */ | ||
it('generate dockerfile with conda', async () => { | ||
const templateFile = path.resolve(__dirname, '..', 'data', 'install-conda.json') | ||
const dockerFile = path.resolve(__dirname, '..', '..', '..', 'out', 'dockerfile') | ||
const template = await fs.readJSON(templateFile) | ||
expect.assertions(2) | ||
await fs.remove(dockerFile) | ||
const result = await dockerfileGenerator(template, dockerFile) | ||
expect(result).toEqual(dockerFile) | ||
expect(await fs.pathExists(dockerFile)).toBeTruthy() | ||
}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
const fs = require('fs-extra') | ||
const path = require('path') | ||
|
||
const dockerfileGenerator = require('../../dockerfile-generator') | ||
|
||
/* eslint-env jest */ | ||
it('generate dockerfile with conda', async () => { | ||
const templateFile = path.resolve(__dirname, '..', 'data', 'install-python.json') | ||
const dockerFile = path.resolve(__dirname, '..', '..', '..', 'out', 'dockerfile') | ||
const template = await fs.readJSON(templateFile) | ||
expect.assertions(2) | ||
await fs.remove(dockerFile) | ||
const result = await dockerfileGenerator(template, dockerFile) | ||
expect(result).toEqual(dockerFile) | ||
expect(await fs.pathExists(dockerFile)).toBeTruthy() | ||
}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
const schemaValidator = require('../../schema-validator') | ||
const fs = require('fs-extra') | ||
const path = require('path') | ||
|
||
/* eslint-env jest */ | ||
it('schema validation succeeds', async () => { | ||
expect.assertions(1) | ||
const schemaFile = path.resolve(__dirname, '..', '..', '..', 'schemas', 'config-schema.json') | ||
const schema = await fs.readJson(schemaFile) | ||
const exampleFile = path.resolve(__dirname, '..', '..', '..', 'examples', 'config-example.json') | ||
const example = await fs.readJson(exampleFile) | ||
const validation = await schemaValidator(schema, example) | ||
expect(validation.valid).toBe(true) | ||
}) | ||
|
||
it('schema validation fails', async () => { | ||
expect.assertions(1) | ||
const schemaFile = path.resolve(__dirname, '..', '..', '..', 'schemas', 'config-schema.json') | ||
const schema = await fs.readJson(schemaFile) | ||
const exampleFile = path.resolve(__dirname, '..', 'data', 'config-negative-example.json') | ||
const example = await fs.readJson(exampleFile) | ||
const validation = await schemaValidator(schema, example) | ||
expect(validation.valid).toBe(false) | ||
}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
{ | ||
"env_variables": { | ||
"TENSORFLOW_VERSION": "1.4.0", | ||
"LC_ALL": "C" | ||
}, | ||
"base_docker": "pai.build.base:hadoop2.7.2-cuda9.0-cudnn7-devel-ubuntu16.04", | ||
"steps": [ | ||
{ | ||
"name": "install_python", | ||
"type": "components", | ||
"config": { | ||
"name": "python", | ||
"version": "3.6" | ||
} | ||
}, | ||
{ | ||
"name": "install_tensorflow", | ||
"type": "conda_install", | ||
"config": { | ||
"name": "tensorflow-gpu", | ||
"version": "1.4.0" | ||
} | ||
}, | ||
{ | ||
"name": "prepare_data", | ||
"type": "local_copy", | ||
"config": { | ||
"source": "./data", | ||
"dest": "/data" | ||
} | ||
}, | ||
{ | ||
"name": "prepare_code", | ||
"type": "git_clone", | ||
"config": { | ||
"type": "github", | ||
"url": "https://github.com/debuggy/DockerForPAI_init.git", | ||
"access_token": "asdflkjasdflkjsdf", | ||
"branch": "master", | ||
"tag": "v0.1", | ||
"commit": "asdfasdgf1234" | ||
} | ||
}, | ||
{ | ||
"name": "python_command", | ||
"type": "python_script", | ||
"config": { | ||
"python_code": "print(\"hello world!\")" | ||
} | ||
}, | ||
{ | ||
"name": "custom_command", | ||
"type": "custom", | ||
"config": { | ||
"command": "python hello.py" | ||
} | ||
} | ||
] | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
{ | ||
"version": 0.1, | ||
"name": "install_conda", | ||
"author": "example", | ||
"env_variables": {}, | ||
"base_docker": { | ||
"image_url": "tensorflow/tensorflow", | ||
"os": "ubuntu", | ||
"bit": "64" | ||
}, | ||
"run_steps": [ | ||
{ | ||
"name": "install conda", | ||
"type": "install_conda", | ||
"config": { | ||
"python_version": "3.6", | ||
"conda_version": "4.5.11", | ||
"bit": "64" | ||
} | ||
}, | ||
{ | ||
"name": "conda install", | ||
"type": "conda_install", | ||
"config": { | ||
"packages": [ | ||
"scipy" | ||
] | ||
} | ||
} | ||
], | ||
"entrypoint_steps": [ | ||
{ | ||
"name": "check_version", | ||
"type": "custom_command", | ||
"config": { | ||
"command": "conda -V" | ||
} | ||
} | ||
] | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
{ | ||
"version": 0.1, | ||
"name": "install_python", | ||
"author": "noname", | ||
"env_variables": {}, | ||
"base_docker": { | ||
"image_url": "tensorflow/tensorflow", | ||
"os": "ubuntu" | ||
}, | ||
"run_steps": [ | ||
{ | ||
"name": "install python", | ||
"type": "install_python", | ||
"config": { | ||
"version": "3.4" | ||
} | ||
}, | ||
{ | ||
"name": "install_git", | ||
"type": "install_git", | ||
"config": {} | ||
} | ||
], | ||
"entrypoint_steps": [ | ||
{ | ||
"name": "prepare_code", | ||
"type": "git_clone", | ||
"config": { | ||
"type": "github", | ||
"url": "https://github.com/debuggy/DockerForPAI_init.git", | ||
"branch": "master" | ||
} | ||
}, | ||
{ | ||
"name": "check_version", | ||
"type": "custom_command", | ||
"config": { | ||
"command": "python -V" | ||
} | ||
}, | ||
{ | ||
"name": "hello_world", | ||
"type": "custom_command", | ||
"config": { | ||
"command": "python src/cmd.py" | ||
} | ||
} | ||
] | ||
} | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
const execa = require('execa') | ||
const fs = require('fs-extra') | ||
const path = require('path') | ||
|
||
const dockerfileGenerator = require('../../dockerfile-generator') | ||
|
||
const testVersions = [ | ||
{ condaVerson: '4.5.11', pythonVersion: '2.7' }, | ||
{ condaVerson: 'latest', pythonVersion: '3.6' } | ||
] | ||
|
||
async function buildAndRun (condaVersion, pythonVersion) { | ||
const templateFile = path.resolve(__dirname, '..', 'data', 'install-conda.json') | ||
const dockerFile = path.resolve(__dirname, '..', '..', '..', 'out', 'dockerfile') | ||
const imageName = `test_conda_install_${condaVersion}` | ||
const containerName = `test_conda_install_container_${condaVersion}` | ||
const template = await fs.readJSON(templateFile) | ||
template.run_steps[0].config.conda_version = condaVersion | ||
template.run_steps[0].config.python_version = pythonVersion | ||
await dockerfileGenerator(template, dockerFile) | ||
await execa.shell(`docker build -t ${imageName} ${path.dirname(dockerFile)}`) | ||
await execa.shell(`docker run --rm --name ${containerName} ${imageName}`) | ||
await execa.shell(`docker rmi ${imageName}`) | ||
} | ||
|
||
/* eslint-env jest */ | ||
it('install conda', async () => { | ||
for (const { condaVersion, pythonVersion } of testVersions) { | ||
await buildAndRun(condaVersion, pythonVersion) | ||
} | ||
}, 500 * 1000) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
const execa = require('execa') | ||
const fs = require('fs-extra') | ||
const path = require('path') | ||
|
||
const dockerfileGenerator = require('../../dockerfile-generator') | ||
|
||
async function test (version) { | ||
if (!process.env['GELATO_DOCKER_TEST']) { | ||
return | ||
} | ||
const templateFile = path.resolve(__dirname, '..', 'data', 'install-python.json') | ||
const dockerFile = path.resolve(__dirname, '..', '..', '..', 'out', 'dockerfile') | ||
const imageName = `test_python_install_${version}` | ||
const containerName = `test_python_install_container_${version}` | ||
const template = await fs.readJSON(templateFile) | ||
template.run_steps[0].config.version = version | ||
await dockerfileGenerator(template, dockerFile) | ||
await execa.shell(`docker build -t ${imageName} ${path.dirname(dockerFile)}`) | ||
await execa.shell(`docker run --rm --name ${containerName} ${imageName}`) | ||
await execa.shell(`docker rmi ${imageName}`) | ||
} | ||
|
||
beforeAll(() => { | ||
if (!process.env['GELATO_DOCKER_TEST']) { | ||
console.log('Set environment variable GELATO_DOCKER_TEST to enable docker run/build tests') | ||
} | ||
}) | ||
|
||
/* eslint-env jest */ | ||
it.each(['2', '2.6'])('string version', async (version) => { | ||
await test(version) | ||
}, 500 * 1000) | ||
|
||
/* eslint-env jest */ | ||
it.each([3, 3.7])('number version', async (version) => { | ||
await test(version) | ||
}, 500 * 1000) | ||
|
||
/* eslint-env jest */ | ||
it.each([3, 3.7])('invalid version', async (version) => { | ||
await test(version) | ||
}, 500 * 1000) |
Oops, something went wrong.