From 9880df7c1ac0f1986b490a3375ac4e3b04d3b80f Mon Sep 17 00:00:00 2001 From: debuggy Date: Wed, 21 Nov 2018 17:07:03 +0800 Subject: [PATCH 1/5] refactor config parser --- examples/config-example.json | 15 +++------ .../{base_docker.js => base-docker.js} | 2 +- lib/config-parser/entrypoint-steps.js | 30 ++++++++++++++++++ .../{env_variables.js => env-variables.js} | 4 ++- lib/config-parser/index.js | 17 ++++++---- lib/config-parser/run-steps.js | 31 +++++++++++++++++++ .../custom-command.js} | 2 +- .../step-components/custom-install.js | 11 +++++++ .../{steps => step-components}/default.js | 2 +- .../install-python.js} | 2 +- lib/config-parser/steps/index.js | 30 ------------------ lib/config-parser/version.js | 2 +- schemas/config-schema.json | 3 +- 13 files changed, 98 insertions(+), 53 deletions(-) rename lib/config-parser/{base_docker.js => base-docker.js} (73%) create mode 100644 lib/config-parser/entrypoint-steps.js rename lib/config-parser/{env_variables.js => env-variables.js} (54%) create mode 100644 lib/config-parser/run-steps.js rename lib/config-parser/{steps/custom_command.js => step-components/custom-command.js} (74%) create mode 100644 lib/config-parser/step-components/custom-install.js rename lib/config-parser/{steps => step-components}/default.js (70%) rename lib/config-parser/{steps/install_python.js => step-components/install-python.js} (76%) delete mode 100644 lib/config-parser/steps/index.js diff --git a/examples/config-example.json b/examples/config-example.json index 06304c6..a341dc7 100644 --- a/examples/config-example.json +++ b/examples/config-example.json @@ -5,7 +5,7 @@ "LC_ALL": "C" }, "base_docker": "pai.build.base:hadoop2.7.2-cuda9.0-cudnn7-devel-ubuntu16.04", - "steps": [ + "run_steps": [ { "name": "install_python", "type": "install_python", @@ -41,16 +41,11 @@ "tag": "v0.1", "commit": "asdfasdgf1234" } - }, - { - "name": "python_command", - "type": "python_script", - "config": { - "python_code": "print(\"hello world!\")" - } - }, + } + ], + "entrypoint_steps": [ { - "name": "custom_command", + "name": "custom_install", "type": "custom_command", "config": { "command": "python hello.py" diff --git a/lib/config-parser/base_docker.js b/lib/config-parser/base-docker.js similarity index 73% rename from lib/config-parser/base_docker.js rename to lib/config-parser/base-docker.js index d5aad1a..82fc2ee 100644 --- a/lib/config-parser/base_docker.js +++ b/lib/config-parser/base-docker.js @@ -1,7 +1,7 @@ function buildBase (config) { let dockerContent = '' - dockerContent += `From ${config.base_docker}\n` + dockerContent += `From ${config.base_docker}\n \n` return dockerContent } diff --git a/lib/config-parser/entrypoint-steps.js b/lib/config-parser/entrypoint-steps.js new file mode 100644 index 0000000..ecd87bb --- /dev/null +++ b/lib/config-parser/entrypoint-steps.js @@ -0,0 +1,30 @@ +const _ = require('lodash') + +const customCommand = require('./step-components/custom-command') +const defaultStep = require('./step-components/default') + +const entrypointsList = [customCommand] + +function selectEntrypointBuilder (stepType) { + return entrypointsList.find(item => item.type === stepType) +} + +function buildSteps (config) { + let dockerContent = '' + for (let step of config.entrypoint_steps) { + let entrypointBuilder = selectEntrypointBuilder(step.type) + if (_.isNil(entrypointBuilder)) { + // throw new Error(`cannot find step builder: '${step.type}'`) + entrypointBuilder = defaultStep + } + dockerContent += ' ' + entrypointBuilder.build(step) + } + dockerContent = 'ENTRYPOINT' + dockerContent.substring(10, dockerContent.length - 6) + '\n\n' + + return dockerContent +} + +module.exports = { + type: 'entrypoint_steps', + build: buildSteps +} diff --git a/lib/config-parser/env_variables.js b/lib/config-parser/env-variables.js similarity index 54% rename from lib/config-parser/env_variables.js rename to lib/config-parser/env-variables.js index b05ad0d..e369bf2 100644 --- a/lib/config-parser/env_variables.js +++ b/lib/config-parser/env-variables.js @@ -2,8 +2,10 @@ function buildEnv (config) { let dockerContent = '' for (const envKey in config.env_variables) { - dockerContent += `ENV ${envKey}=${config.env_variables[envKey]}\n` + dockerContent += ` ${envKey}=${config.env_variables[envKey]} \\ \n` } + dockerContent = 'ENV' + dockerContent.substring(3, dockerContent.length - 4) + '\n\n' + return dockerContent } diff --git a/lib/config-parser/index.js b/lib/config-parser/index.js index e0f7f1f..472fdb1 100644 --- a/lib/config-parser/index.js +++ b/lib/config-parser/index.js @@ -1,10 +1,11 @@ const version = require('./version') -const baseDocker = require('./base_docker') -const envVariables = require('./env_variables') -const steps = require('./steps') +const baseDocker = require('./base-docker') +const envVariables = require('./env-variables') +const runSteps = require('./run-steps') +const entrypointSteps = require('./entrypoint-steps') const _ = require('lodash') -const componentsList = [version, baseDocker, envVariables, steps] +const componentsList = [version, baseDocker, envVariables, runSteps, entrypointSteps] function selectComponent (type) { const component = componentsList.find(item => item.type === type) @@ -30,8 +31,12 @@ function buildConfig (config) { dockerContent += selectComponent('env_variables').build(config) } - if (!_.isNil(config.steps)) { - dockerContent += selectComponent('steps').build(config) + if (!_.isNil(config.run_steps)) { + dockerContent += selectComponent('run_steps').build(config) + } + + if (!_.isNil(config.entrypoint_steps)) { + dockerContent += selectComponent('entrypoint_steps').build(config) } return dockerContent diff --git a/lib/config-parser/run-steps.js b/lib/config-parser/run-steps.js new file mode 100644 index 0000000..7705383 --- /dev/null +++ b/lib/config-parser/run-steps.js @@ -0,0 +1,31 @@ +const _ = require('lodash') + +const customCommand = require('./step-components/custom-command') +const installPython = require('./step-components/install-python') +const defaultStep = require('./step-components/default') + +const runStepsList = [customCommand, installPython] + +function selectRunBuilder (stepType) { + return runStepsList.find(item => item.type === stepType) +} + +function buildSteps (config) { + let dockerContent = '' + for (let step of config.run_steps) { + let runBuilder = selectRunBuilder(step.type) + if (_.isNil(runBuilder)) { + // throw new Error(`cannot find step builder: '${step.type}'`) + runBuilder = defaultStep + } + dockerContent += ' ' + runBuilder.build(step) + } + dockerContent = 'RUN' + dockerContent.substring(3, dockerContent.length - 6) + '\n\n' + + return dockerContent +} + +module.exports = { + type: 'run_steps', + build: buildSteps +} diff --git a/lib/config-parser/steps/custom_command.js b/lib/config-parser/step-components/custom-command.js similarity index 74% rename from lib/config-parser/steps/custom_command.js rename to lib/config-parser/step-components/custom-command.js index cff2310..ffbc60f 100644 --- a/lib/config-parser/steps/custom_command.js +++ b/lib/config-parser/step-components/custom-command.js @@ -1,7 +1,7 @@ function buildCustom (step) { let dockerContent = '' - dockerContent += `CMD ${step.config.command}\n` + dockerContent += `${step.config.command} && \\ \n` return dockerContent } diff --git a/lib/config-parser/step-components/custom-install.js b/lib/config-parser/step-components/custom-install.js new file mode 100644 index 0000000..a1f5673 --- /dev/null +++ b/lib/config-parser/step-components/custom-install.js @@ -0,0 +1,11 @@ + +function buildCustomInstall (step) { + let dockerContent = '' + dockerContent += `${step.config.command} && \\ \n` + return dockerContent +} + +module.exports = { + 'type': 'custom_command', + 'build': buildCustomInstall +} diff --git a/lib/config-parser/steps/default.js b/lib/config-parser/step-components/default.js similarity index 70% rename from lib/config-parser/steps/default.js rename to lib/config-parser/step-components/default.js index 955a4ac..92ac176 100644 --- a/lib/config-parser/steps/default.js +++ b/lib/config-parser/step-components/default.js @@ -1,7 +1,7 @@ function buildDefault (step) { let dockerContent = '' - dockerContent += `no processing for ${step.type}\n` + dockerContent += `no processing for ${step.type} && \\ \n` return dockerContent } diff --git a/lib/config-parser/steps/install_python.js b/lib/config-parser/step-components/install-python.js similarity index 76% rename from lib/config-parser/steps/install_python.js rename to lib/config-parser/step-components/install-python.js index ffb7701..bf86165 100644 --- a/lib/config-parser/steps/install_python.js +++ b/lib/config-parser/step-components/install-python.js @@ -1,6 +1,6 @@ function installPython (config) { - return 'install python\n' + return 'install python && \\ \n' } module.exports = { diff --git a/lib/config-parser/steps/index.js b/lib/config-parser/steps/index.js deleted file mode 100644 index a1af9eb..0000000 --- a/lib/config-parser/steps/index.js +++ /dev/null @@ -1,30 +0,0 @@ -const _ = require('lodash') - -const customCommand = require('./custom_command') -const installPython = require('./install_python') -const defaultStep = require('./default') - -const stepsList = [customCommand, installPython] - -function selectStepBuilder (stepType) { - return stepsList.find(item => item.type === stepType) -} - -function buildSteps (config) { - let dockerContent = '' - for (let step of config.steps) { - let stepBuilder = selectStepBuilder(step.type) - if (_.isNil(stepBuilder)) { - // throw new Error(`cannot find step builder: '${step.type}'`) - stepBuilder = defaultStep - } - dockerContent += stepBuilder.build(step) - } - - return dockerContent -} - -module.exports = { - type: 'steps', - build: buildSteps -} diff --git a/lib/config-parser/version.js b/lib/config-parser/version.js index 5bd1779..6486fca 100644 --- a/lib/config-parser/version.js +++ b/lib/config-parser/version.js @@ -1,6 +1,6 @@ function buildVersion (config) { - return `# version: ${config.version}\n` + return `# version: ${config.version}\n \n` } module.exports = { diff --git a/schemas/config-schema.json b/schemas/config-schema.json index 228ea09..5eca7fc 100644 --- a/schemas/config-schema.json +++ b/schemas/config-schema.json @@ -7,7 +7,8 @@ "version", "env_variables", "base_docker", - "steps" + "run_steps", + "entrypoint_steps" ], "properties": { "version": { From a1b3e2f5ecf5e477c70181debdddbc7a8a65f6ba Mon Sep 17 00:00:00 2001 From: debuggy Date: Thu, 29 Nov 2018 15:47:12 +0800 Subject: [PATCH 2/5] test dockerfile --- docker-test/dockerfile | 19 +++++++++++ examples/config-example.json | 33 +++++++++---------- .../step-components/conda-install.js | 0 .../step-components/git-clone.js | 23 +++++++++++++ .../step-components/install-conda.js | 16 +++++++++ schemas/config-schema.json | 11 +++++-- 6 files changed, 81 insertions(+), 21 deletions(-) create mode 100644 docker-test/dockerfile create mode 100644 lib/config-parser/step-components/conda-install.js create mode 100644 lib/config-parser/step-components/git-clone.js create mode 100644 lib/config-parser/step-components/install-conda.js diff --git a/docker-test/dockerfile b/docker-test/dockerfile new file mode 100644 index 0000000..3967fcb --- /dev/null +++ b/docker-test/dockerfile @@ -0,0 +1,19 @@ +# version: 0.1 + +From ubuntu + +ENV TINI_VERSION v0.9.0 +RUN apt-get update && \ + apt-get install -y curl wget gnupg bzip2 + +RUN wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O ~/miniconda.sh +RUN chmod 0755 ~/miniconda.sh +RUN ~/miniconda.sh -b -p ~/conda +ENV PATH "~/conda/bin:$PATH" +RUN ["/bin/bash", "-c", "conda create -n build_test python=3.4 && source activate build_test && conda install numpy"] + +# RUN conda activate bulid_test +# RUN conda install scikit-learn + + + diff --git a/examples/config-example.json b/examples/config-example.json index a341dc7..3b72064 100644 --- a/examples/config-example.json +++ b/examples/config-example.json @@ -1,5 +1,7 @@ { "version": 0.1, + "name": "flow_test", + "author": "debuggy", "env_variables": { "TENSORFLOW_VERSION": "1.4.0", "LC_ALL": "C" @@ -7,27 +9,25 @@ "base_docker": "pai.build.base:hadoop2.7.2-cuda9.0-cudnn7-devel-ubuntu16.04", "run_steps": [ { - "name": "install_python", - "type": "install_python", + "name": "install_conda", + "type": "install_conda", "config": { - "name": "python", - "version": "3.6" + "env_name": "flow_example", + "python_version": "3.6" } }, { "name": "install_tensorflow", "type": "conda_install", "config": { - "name": "tensorflow-gpu", - "version": "1.4.0" - } - }, - { - "name": "prepare_data", - "type": "local_copy", - "config": { - "source": "./data", - "dest": "/data" + "packages": [ + { + "name": "scipy", + "version": "0.15.0" + }, + + + ] } }, { @@ -36,10 +36,7 @@ "config": { "type": "github", "url": "https://github.com/debuggy/DockerForPAI_init.git", - "access_token": "asdflkjasdflkjsdf", - "branch": "master", - "tag": "v0.1", - "commit": "asdfasdgf1234" + "branch": "master" } } ], diff --git a/lib/config-parser/step-components/conda-install.js b/lib/config-parser/step-components/conda-install.js new file mode 100644 index 0000000..e69de29 diff --git a/lib/config-parser/step-components/git-clone.js b/lib/config-parser/step-components/git-clone.js new file mode 100644 index 0000000..b924509 --- /dev/null +++ b/lib/config-parser/step-components/git-clone.js @@ -0,0 +1,23 @@ +const _ = require('lodash') + +function buildInstallConda (step) { + let dockerContent = `git clone ${step.config.url} && \\ \n` + + // get git repo dir + const repoNameReg = /\/([\w.@:-~_]+)\.git/ + const repoName = repoNameReg.exec(step.config.url)[1] + dockerContent += `cd ${repoName} && \\ \n` + if (step.config.branch !== 'master') { + dockerContent += `git checkout ${step.config.branch} && \\ \n` + } + if (!_.isNil(step.config.tag)) { + dockerContent += `git checkout tags/${step.config.tag} && \\ \n` + } + + return dockerContent +} + +module.exports = { + 'type': 'install_conda', + 'build': buildInstallConda +} diff --git a/lib/config-parser/step-components/install-conda.js b/lib/config-parser/step-components/install-conda.js new file mode 100644 index 0000000..50194ea --- /dev/null +++ b/lib/config-parser/step-components/install-conda.js @@ -0,0 +1,16 @@ +function buildInstallConda (step) { + let dockerContent = `wget https://repo.continuum.io/miniconda/Miniconda3-3.7.0-Linux-x86_64.sh -O ~/miniconda.sh && \\ \n` + + `bash ~/miniconda.sh -b -p $HOME/miniconda && \\ \n` + + `export PATH="$HOME/miniconda/bin:$PATH && \\ \n"` + + `conda -V && \\ \n` + + `export CONDA_ENV_NAME='${step.config.env_name}' && \\ \n` + + `conda create --yes -n $CONDA_ENV_NAME python=${step.config.python_version} && \\ \n` + + `source activate $CONDA_ENV_NAME && \\ \n` + + return dockerContent +} + +module.exports = { + 'type': 'install_conda', + 'build': buildInstallConda +} diff --git a/schemas/config-schema.json b/schemas/config-schema.json index 5eca7fc..ef82a47 100644 --- a/schemas/config-schema.json +++ b/schemas/config-schema.json @@ -5,6 +5,7 @@ "description": "Config schema of PAIFlow", "required": [ "version", + "name", "env_variables", "base_docker", "run_steps", @@ -14,8 +15,13 @@ "version": { "$id": "#/properties/version", "type": "number", - "description": "The version of the config definition.", - "default": 0.1 + "description": "The version of the job config.", + "default": 1.0 + }, + "name": { + "$id": "#/properties/name", + "type": "string", + "description": "The name of the job", }, "env_variables": { "$id": "#/properties/env_variables", @@ -114,4 +120,3 @@ } } } - \ No newline at end of file From 915b3dc240ce16915f1ca92845c2ad63a88cc78b Mon Sep 17 00:00:00 2001 From: debuggy Date: Thu, 29 Nov 2018 15:48:44 +0800 Subject: [PATCH 3/5] fix json --- examples/config-example.json | 4 +--- schemas/config-schema.json | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/config-example.json b/examples/config-example.json index 3b72064..0b92ddc 100644 --- a/examples/config-example.json +++ b/examples/config-example.json @@ -24,9 +24,7 @@ { "name": "scipy", "version": "0.15.0" - }, - - + } ] } }, diff --git a/schemas/config-schema.json b/schemas/config-schema.json index ef82a47..8ffa568 100644 --- a/schemas/config-schema.json +++ b/schemas/config-schema.json @@ -21,7 +21,7 @@ "name": { "$id": "#/properties/name", "type": "string", - "description": "The name of the job", + "description": "The name of the job" }, "env_variables": { "$id": "#/properties/env_variables", From 885002ea5390e0d62f28f2b2b238299c7e790a69 Mon Sep 17 00:00:00 2001 From: debuggy Date: Mon, 3 Dec 2018 19:36:53 +0800 Subject: [PATCH 4/5] temp --- docker-test/dockerfile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docker-test/dockerfile b/docker-test/dockerfile index 3967fcb..1cc8c1a 100644 --- a/docker-test/dockerfile +++ b/docker-test/dockerfile @@ -1,19 +1,19 @@ # version: 0.1 -From ubuntu +From tensorflow/tensorflow + + +RUN apt-get update +RUN apt-get install -y curl wget gnupg bzip2 +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh +RUN /bin/bash ~/miniconda.sh -b -p /opt/conda +RUN rm ~/miniconda.sh +RUN echo "export PATH=/opt/conda/bin:$PATH" >> ~/.bashrc +RUN export PATH=/opt/conda/bin:$PATH && echo $PATH && conda -V + -ENV TINI_VERSION v0.9.0 -RUN apt-get update && \ - apt-get install -y curl wget gnupg bzip2 -RUN wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O ~/miniconda.sh -RUN chmod 0755 ~/miniconda.sh -RUN ~/miniconda.sh -b -p ~/conda -ENV PATH "~/conda/bin:$PATH" -RUN ["/bin/bash", "-c", "conda create -n build_test python=3.4 && source activate build_test && conda install numpy"] -# RUN conda activate bulid_test -# RUN conda install scikit-learn From aff787309886aac3c0c77f344a4ab2654467563a Mon Sep 17 00:00:00 2001 From: debuggy Date: Wed, 5 Dec 2018 19:40:44 +0800 Subject: [PATCH 5/5] refactor: refine the infrastructure --- .gitignore | 3 +- README.md | 6 +- docker-test/dockerfile | 24 +++++++- examples/config-example.json | 5 +- .../common/dockerfile-generator.test.js | 23 +++++++ .../common/install-conda-dockerfile.test.js | 16 +++++ .../common/install-python-dockerfile.test.js | 16 +++++ lib/__tests__/common/schema-validator.test.js | 24 ++++++++ .../data/config-negative-example.json | 59 ++++++++++++++++++ lib/__tests__/data/install-conda.json | 40 +++++++++++++ lib/__tests__/data/install-python.json | 50 ++++++++++++++++ .../release/install-conda-build.test.js | 31 ++++++++++ lib/__tests__/release/install-python.test.js | 42 +++++++++++++ lib/config-parser/commands/entrypoint.js | 31 ++++++++++ lib/config-parser/commands/env.js | 15 +++++ lib/config-parser/commands/from.js | 13 ++++ lib/config-parser/commands/run.js | 39 ++++++++++++ lib/config-parser/commands/shell.js | 4 ++ lib/config-parser/commands/version.js | 13 ++++ lib/config-parser/index.js | 50 ++++------------ lib/config-parser/steps/conda-install.js | 11 ++++ lib/config-parser/steps/conda.js | 31 ++++++++++ lib/config-parser/steps/custom.js | 11 ++++ lib/config-parser/steps/git.js | 46 ++++++++++++++ lib/config-parser/steps/python.js | 60 +++++++++++++++++++ lib/config-parser/steps/step-list.js | 20 +++++++ lib/config-parser/util.js | 4 ++ lib/index.js | 8 --- package.json | 17 ++---- schemas/config-schema.json | 46 +++++++------- scripts/debug.js | 9 +++ yarn.lock | 39 +++++++++++- 32 files changed, 715 insertions(+), 91 deletions(-) create mode 100644 lib/__tests__/common/dockerfile-generator.test.js create mode 100644 lib/__tests__/common/install-conda-dockerfile.test.js create mode 100644 lib/__tests__/common/install-python-dockerfile.test.js create mode 100644 lib/__tests__/common/schema-validator.test.js create mode 100644 lib/__tests__/data/config-negative-example.json create mode 100644 lib/__tests__/data/install-conda.json create mode 100644 lib/__tests__/data/install-python.json create mode 100644 lib/__tests__/release/install-conda-build.test.js create mode 100644 lib/__tests__/release/install-python.test.js create mode 100644 lib/config-parser/commands/entrypoint.js create mode 100644 lib/config-parser/commands/env.js create mode 100644 lib/config-parser/commands/from.js create mode 100644 lib/config-parser/commands/run.js create mode 100644 lib/config-parser/commands/shell.js create mode 100644 lib/config-parser/commands/version.js create mode 100644 lib/config-parser/steps/conda-install.js create mode 100644 lib/config-parser/steps/conda.js create mode 100644 lib/config-parser/steps/custom.js create mode 100644 lib/config-parser/steps/git.js create mode 100644 lib/config-parser/steps/python.js create mode 100644 lib/config-parser/steps/step-list.js create mode 100644 lib/config-parser/util.js create mode 100644 scripts/debug.js diff --git a/.gitignore b/.gitignore index 393f589..e3876ff 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ npm-debug.log *.obj *.out node_modules/ -out/ \ No newline at end of file +out/ +docker-test/ diff --git a/README.md b/README.md index 0a814fc..138c4bf 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,3 @@ -# PAIFlow - -[![Documentation Status](https://readthedocs.org/projects/paiflow/badge/?version=latest)](https://paiflow.readthedocs.io/en/latest/?badge=latest) -[![Build Status](https://travis-ci.com/debuggy/PAIFlow.svg?branch=master)](https://travis-ci.com/debuggy/PAIFlow) -[![Coverage Status](https://coveralls.io/repos/github/debuggy/PAIFlow/badge.svg?branch=master)](https://coveralls.io/github/debuggy/PAIFlow?branch=master) +# gelato Work flow of Microsoft OpenPAI diff --git a/docker-test/dockerfile b/docker-test/dockerfile index 1cc8c1a..8f0b3f6 100644 --- a/docker-test/dockerfile +++ b/docker-test/dockerfile @@ -2,14 +2,34 @@ From tensorflow/tensorflow +SHELL ["/bin/bash", "-c"] +ENV BASH_ENV=~/.bashrc RUN apt-get update +RUN mv ~/.bashrc ~/.bashrc.bak RUN apt-get install -y curl wget gnupg bzip2 RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh RUN /bin/bash ~/miniconda.sh -b -p /opt/conda RUN rm ~/miniconda.sh -RUN echo "export PATH=/opt/conda/bin:$PATH" >> ~/.bashrc -RUN export PATH=/opt/conda/bin:$PATH && echo $PATH && conda -V + +## docker does not start bash as login session +# RUN echo "echo bash_profile" >> ~/.bash_profile +# RUN echo "export PATH=/opt/conda/bin:$PATH" >> ~/.bash_profile +# RUN echo "echo bash_login" >> ~/.bash_login +# RUN echo "export PATH=/opt/conda/bin:$PATH" >> ~/.bash_login +# RUN echo "echo .profile" >> ~/.profile +# RUN echo "export PATH=/opt/conda/bin:$PATH" >> ~/.profile + +RUN echo 'export PATH=/opt/conda/bin:$PATH' >> ~/.bashrc && source ~/.bashrc +RUN conda -V && \ + conda create --yes -n docker python=3.6 +RUN echo 'source activate docker' >> ~/.bashrc && source ~/.bashrc +RUN echo 'export PATH=/opt/conda/envs/docker/bin:$PATH' >> ~/.bashrc && source ~/.bashrc +RUN conda install numpy + +RUN cat ~/.bashrc.bak >> ~/.bashrc + +CMD ["/bin/bash", "-c", "source activate docker"] diff --git a/examples/config-example.json b/examples/config-example.json index 0b92ddc..ae58dc3 100644 --- a/examples/config-example.json +++ b/examples/config-example.json @@ -6,7 +6,10 @@ "TENSORFLOW_VERSION": "1.4.0", "LC_ALL": "C" }, - "base_docker": "pai.build.base:hadoop2.7.2-cuda9.0-cudnn7-devel-ubuntu16.04", + "base_docker": { + "image_url": "tensorflow/tensorflow", + "os": "ubuntu" + }, "run_steps": [ { "name": "install_conda", diff --git a/lib/__tests__/common/dockerfile-generator.test.js b/lib/__tests__/common/dockerfile-generator.test.js new file mode 100644 index 0000000..31bf413 --- /dev/null +++ b/lib/__tests__/common/dockerfile-generator.test.js @@ -0,0 +1,23 @@ +const dockerfileGenerator = require('../../dockerfile-generator') +const SchemaValidationError = require('../../schema-validation-error') +const fs = require('fs-extra') +const path = require('path') + +/* eslint-env jest */ +it('throws schema validation error', async () => { + expect.assertions(1) + const templateFile = path.resolve(__dirname, '..', 'data', 'config-negative-example.json') + const template = await fs.readJson(templateFile) + await expect(dockerfileGenerator(template)).rejects.toThrow(SchemaValidationError) +}) + +it('generates dockerfile successfully', async () => { + expect.assertions(2) + const templateFile = path.resolve(__dirname, '..', '..', '..', 'examples', 'config-example.json') + const template = await fs.readJson(templateFile) + const dockerFilePath = path.resolve(__dirname, '..', '..', '..', 'out', 'dockerfile') + await fs.remove(dockerFilePath) + const result = await dockerfileGenerator(template, dockerFilePath) + expect(result).toEqual(dockerFilePath) + expect(await fs.pathExists(dockerFilePath)).toBeTruthy() +}) diff --git a/lib/__tests__/common/install-conda-dockerfile.test.js b/lib/__tests__/common/install-conda-dockerfile.test.js new file mode 100644 index 0000000..09e4e3c --- /dev/null +++ b/lib/__tests__/common/install-conda-dockerfile.test.js @@ -0,0 +1,16 @@ +const fs = require('fs-extra') +const path = require('path') + +const dockerfileGenerator = require('../../dockerfile-generator') + +/* eslint-env jest */ +it('generate dockerfile with conda', async () => { + const templateFile = path.resolve(__dirname, '..', 'data', 'install-conda.json') + const dockerFile = path.resolve(__dirname, '..', '..', '..', 'out', 'dockerfile') + const template = await fs.readJSON(templateFile) + expect.assertions(2) + await fs.remove(dockerFile) + const result = await dockerfileGenerator(template, dockerFile) + expect(result).toEqual(dockerFile) + expect(await fs.pathExists(dockerFile)).toBeTruthy() +}) diff --git a/lib/__tests__/common/install-python-dockerfile.test.js b/lib/__tests__/common/install-python-dockerfile.test.js new file mode 100644 index 0000000..40f8efb --- /dev/null +++ b/lib/__tests__/common/install-python-dockerfile.test.js @@ -0,0 +1,16 @@ +const fs = require('fs-extra') +const path = require('path') + +const dockerfileGenerator = require('../../dockerfile-generator') + +/* eslint-env jest */ +it('generate dockerfile with conda', async () => { + const templateFile = path.resolve(__dirname, '..', 'data', 'install-python.json') + const dockerFile = path.resolve(__dirname, '..', '..', '..', 'out', 'dockerfile') + const template = await fs.readJSON(templateFile) + expect.assertions(2) + await fs.remove(dockerFile) + const result = await dockerfileGenerator(template, dockerFile) + expect(result).toEqual(dockerFile) + expect(await fs.pathExists(dockerFile)).toBeTruthy() +}) diff --git a/lib/__tests__/common/schema-validator.test.js b/lib/__tests__/common/schema-validator.test.js new file mode 100644 index 0000000..f6b37e5 --- /dev/null +++ b/lib/__tests__/common/schema-validator.test.js @@ -0,0 +1,24 @@ +const schemaValidator = require('../../schema-validator') +const fs = require('fs-extra') +const path = require('path') + +/* eslint-env jest */ +it('schema validation succeeds', async () => { + expect.assertions(1) + const schemaFile = path.resolve(__dirname, '..', '..', '..', 'schemas', 'config-schema.json') + const schema = await fs.readJson(schemaFile) + const exampleFile = path.resolve(__dirname, '..', '..', '..', 'examples', 'config-example.json') + const example = await fs.readJson(exampleFile) + const validation = await schemaValidator(schema, example) + expect(validation.valid).toBe(true) +}) + +it('schema validation fails', async () => { + expect.assertions(1) + const schemaFile = path.resolve(__dirname, '..', '..', '..', 'schemas', 'config-schema.json') + const schema = await fs.readJson(schemaFile) + const exampleFile = path.resolve(__dirname, '..', 'data', 'config-negative-example.json') + const example = await fs.readJson(exampleFile) + const validation = await schemaValidator(schema, example) + expect(validation.valid).toBe(false) +}) diff --git a/lib/__tests__/data/config-negative-example.json b/lib/__tests__/data/config-negative-example.json new file mode 100644 index 0000000..b520bdd --- /dev/null +++ b/lib/__tests__/data/config-negative-example.json @@ -0,0 +1,59 @@ +{ + "env_variables": { + "TENSORFLOW_VERSION": "1.4.0", + "LC_ALL": "C" + }, + "base_docker": "pai.build.base:hadoop2.7.2-cuda9.0-cudnn7-devel-ubuntu16.04", + "steps": [ + { + "name": "install_python", + "type": "components", + "config": { + "name": "python", + "version": "3.6" + } + }, + { + "name": "install_tensorflow", + "type": "conda_install", + "config": { + "name": "tensorflow-gpu", + "version": "1.4.0" + } + }, + { + "name": "prepare_data", + "type": "local_copy", + "config": { + "source": "./data", + "dest": "/data" + } + }, + { + "name": "prepare_code", + "type": "git_clone", + "config": { + "type": "github", + "url": "https://github.com/debuggy/DockerForPAI_init.git", + "access_token": "asdflkjasdflkjsdf", + "branch": "master", + "tag": "v0.1", + "commit": "asdfasdgf1234" + } + }, + { + "name": "python_command", + "type": "python_script", + "config": { + "python_code": "print(\"hello world!\")" + } + }, + { + "name": "custom_command", + "type": "custom", + "config": { + "command": "python hello.py" + } + } + ] +} diff --git a/lib/__tests__/data/install-conda.json b/lib/__tests__/data/install-conda.json new file mode 100644 index 0000000..a7f0d12 --- /dev/null +++ b/lib/__tests__/data/install-conda.json @@ -0,0 +1,40 @@ +{ + "version": 0.1, + "name": "install_conda", + "author": "example", + "env_variables": {}, + "base_docker": { + "image_url": "tensorflow/tensorflow", + "os": "ubuntu", + "bit": "64" + }, + "run_steps": [ + { + "name": "install conda", + "type": "install_conda", + "config": { + "python_version": "3.6", + "conda_version": "4.5.11", + "bit": "64" + } + }, + { + "name": "conda install", + "type": "conda_install", + "config": { + "packages": [ + "scipy" + ] + } + } + ], + "entrypoint_steps": [ + { + "name": "check_version", + "type": "custom_command", + "config": { + "command": "conda -V" + } + } + ] +} diff --git a/lib/__tests__/data/install-python.json b/lib/__tests__/data/install-python.json new file mode 100644 index 0000000..3b63538 --- /dev/null +++ b/lib/__tests__/data/install-python.json @@ -0,0 +1,50 @@ +{ + "version": 0.1, + "name": "install_python", + "author": "noname", + "env_variables": {}, + "base_docker": { + "image_url": "tensorflow/tensorflow", + "os": "ubuntu" + }, + "run_steps": [ + { + "name": "install python", + "type": "install_python", + "config": { + "version": "3.4" + } + }, + { + "name": "install_git", + "type": "install_git", + "config": {} + } + ], + "entrypoint_steps": [ + { + "name": "prepare_code", + "type": "git_clone", + "config": { + "type": "github", + "url": "https://github.com/debuggy/DockerForPAI_init.git", + "branch": "master" + } + }, + { + "name": "check_version", + "type": "custom_command", + "config": { + "command": "python -V" + } + }, + { + "name": "hello_world", + "type": "custom_command", + "config": { + "command": "python src/cmd.py" + } + } + ] +} + \ No newline at end of file diff --git a/lib/__tests__/release/install-conda-build.test.js b/lib/__tests__/release/install-conda-build.test.js new file mode 100644 index 0000000..1a19ed7 --- /dev/null +++ b/lib/__tests__/release/install-conda-build.test.js @@ -0,0 +1,31 @@ +const execa = require('execa') +const fs = require('fs-extra') +const path = require('path') + +const dockerfileGenerator = require('../../dockerfile-generator') + +const testVersions = [ + { condaVerson: '4.5.11', pythonVersion: '2.7' }, + { condaVerson: 'latest', pythonVersion: '3.6' } +] + +async function buildAndRun (condaVersion, pythonVersion) { + const templateFile = path.resolve(__dirname, '..', 'data', 'install-conda.json') + const dockerFile = path.resolve(__dirname, '..', '..', '..', 'out', 'dockerfile') + const imageName = `test_conda_install_${condaVersion}` + const containerName = `test_conda_install_container_${condaVersion}` + const template = await fs.readJSON(templateFile) + template.run_steps[0].config.conda_version = condaVersion + template.run_steps[0].config.python_version = pythonVersion + await dockerfileGenerator(template, dockerFile) + await execa.shell(`docker build -t ${imageName} ${path.dirname(dockerFile)}`) + await execa.shell(`docker run --rm --name ${containerName} ${imageName}`) + await execa.shell(`docker rmi ${imageName}`) +} + +/* eslint-env jest */ +it('install conda', async () => { + for (const { condaVersion, pythonVersion } of testVersions) { + await buildAndRun(condaVersion, pythonVersion) + } +}, 500 * 1000) diff --git a/lib/__tests__/release/install-python.test.js b/lib/__tests__/release/install-python.test.js new file mode 100644 index 0000000..3efef63 --- /dev/null +++ b/lib/__tests__/release/install-python.test.js @@ -0,0 +1,42 @@ +const execa = require('execa') +const fs = require('fs-extra') +const path = require('path') + +const dockerfileGenerator = require('../../dockerfile-generator') + +async function test (version) { + if (!process.env['GELATO_DOCKER_TEST']) { + return + } + const templateFile = path.resolve(__dirname, '..', 'data', 'install-python.json') + const dockerFile = path.resolve(__dirname, '..', '..', '..', 'out', 'dockerfile') + const imageName = `test_python_install_${version}` + const containerName = `test_python_install_container_${version}` + const template = await fs.readJSON(templateFile) + template.run_steps[0].config.version = version + await dockerfileGenerator(template, dockerFile) + await execa.shell(`docker build -t ${imageName} ${path.dirname(dockerFile)}`) + await execa.shell(`docker run --rm --name ${containerName} ${imageName}`) + await execa.shell(`docker rmi ${imageName}`) +} + +beforeAll(() => { + if (!process.env['GELATO_DOCKER_TEST']) { + console.log('Set environment variable GELATO_DOCKER_TEST to enable docker run/build tests') + } +}) + +/* eslint-env jest */ +it.each(['2', '2.6'])('string version', async (version) => { + await test(version) +}, 500 * 1000) + +/* eslint-env jest */ +it.each([3, 3.7])('number version', async (version) => { + await test(version) +}, 500 * 1000) + +/* eslint-env jest */ +it.each([3, 3.7])('invalid version', async (version) => { + await test(version) +}, 500 * 1000) diff --git a/lib/config-parser/commands/entrypoint.js b/lib/config-parser/commands/entrypoint.js new file mode 100644 index 0000000..9645e0f --- /dev/null +++ b/lib/config-parser/commands/entrypoint.js @@ -0,0 +1,31 @@ +const _ = require('lodash') + +const steps = require('../steps/step-list') + +function buildSteps (config) { + const result = [] + if (!config.entrypoint_steps) { + return result + } + + for (const step of config.entrypoint_steps) { + const builder = steps[step.type] + if (_.isNil(builder)) { + throw Error(`cannot find step builder: '${step.type}'`) + } + if (!builder.command.includes('entrypoint')) { + throw Error(`Step ${step.type} doesn't support ENTRYPOINT`) + } + if (!builder.os.includes(config.base_docker.os)) { + throw Error(`Step ${step.type} doesn't support ${config.base_docker.os} OS`) + } + result.push(...builder.build(step)) + } + + return [`ENTRYPOINT ${result.join(` && \\ \n${' '.repeat(11)}`)}`] +} + +module.exports = { + type: 'entrypoint_steps', + build: buildSteps +} diff --git a/lib/config-parser/commands/env.js b/lib/config-parser/commands/env.js new file mode 100644 index 0000000..e8437d4 --- /dev/null +++ b/lib/config-parser/commands/env.js @@ -0,0 +1,15 @@ + +function buildEnv (config) { + if (!config.env_variables) { + return [] + } + + return Object.entries(config.env_variables).map( + ([key, val]) => `ENV ${key}="${val}"` + ) +} + +module.exports = { + type: 'env_variables', + build: buildEnv +} diff --git a/lib/config-parser/commands/from.js b/lib/config-parser/commands/from.js new file mode 100644 index 0000000..8b44ddf --- /dev/null +++ b/lib/config-parser/commands/from.js @@ -0,0 +1,13 @@ + +function buildBase (config) { + if (!config.base_docker) { + throw new Error('No base docker in config file.') + } + + return [`From ${config.base_docker.image_url}`] +} + +module.exports = { + type: 'base_docker', + build: buildBase +} diff --git a/lib/config-parser/commands/run.js b/lib/config-parser/commands/run.js new file mode 100644 index 0000000..31dd266 --- /dev/null +++ b/lib/config-parser/commands/run.js @@ -0,0 +1,39 @@ +const _ = require('lodash') +const util = require('../util') +const steps = require('../steps/step-list') + +function buildSteps (config) { + if (!config.run_steps) { + return [] + } + + const result = [ + 'apt-get update', + util.aptInstall('apt-utils'), + 'mv ~/.bashrc ~/.bashrc.bak' // backup original bashrc to make user create new bashrc + ] + + for (const step of config.run_steps) { + const builder = steps[step.type] + if (_.isNil(builder)) { + throw Error(`cannot find step builder: '${step.type}'`) + } + if (!builder.command.includes('run')) { + throw Error(`Step ${step.type} doesn't support RUN`) + } + if (!builder.os.includes(config.base_docker.os)) { + throw Error(`Step ${step.type} doesn't support ${config.base_docker.os} OS`) + } + result.push(...builder.build(step)) + } + // append original bashrc to the end + result.push('cat ~/.bashrc.bak >> ~/.bashrc') + + return result.map((x) => `RUN ${x}`) + // return [`RUN ${result.join(` && \\ \n${' '.repeat(4)}`)}`] +} + +module.exports = { + type: 'run_steps', + build: buildSteps +} diff --git a/lib/config-parser/commands/shell.js b/lib/config-parser/commands/shell.js new file mode 100644 index 0000000..3bb3544 --- /dev/null +++ b/lib/config-parser/commands/shell.js @@ -0,0 +1,4 @@ +module.exports = { + type: 'shell', + build: (config) => ['SHELL ["/bin/bash", "-c"]', 'ENV BASH_ENV=~/.bashrc'] +} diff --git a/lib/config-parser/commands/version.js b/lib/config-parser/commands/version.js new file mode 100644 index 0000000..224ff57 --- /dev/null +++ b/lib/config-parser/commands/version.js @@ -0,0 +1,13 @@ + +function buildVersion (config) { + if (!config.version) { + return [] + } + + return [`# version: ${config.version}`] +} + +module.exports = { + type: 'version', + build: buildVersion +} diff --git a/lib/config-parser/index.js b/lib/config-parser/index.js index 472fdb1..34914a6 100644 --- a/lib/config-parser/index.js +++ b/lib/config-parser/index.js @@ -1,45 +1,21 @@ -const version = require('./version') -const baseDocker = require('./base-docker') -const envVariables = require('./env-variables') -const runSteps = require('./run-steps') -const entrypointSteps = require('./entrypoint-steps') -const _ = require('lodash') - -const componentsList = [version, baseDocker, envVariables, runSteps, entrypointSteps] - -function selectComponent (type) { - const component = componentsList.find(item => item.type === type) - if (_.isNil(component)) { - throw new Error(`cannot find component: '${type}'`) - } - - return component -} +const commands = [ + require('./commands/version'), + require('./commands/from'), + require('./commands/shell'), + require('./commands/env'), + require('./commands/run'), + require('./commands/entrypoint') +] function buildConfig (config) { - let dockerContent = '' - - if (!_.isNil(config.version)) { - dockerContent += selectComponent('version').build(config) - } - - if (!_.isNil(config.base_docker)) { - dockerContent += selectComponent('base_docker').build(config) - } - - if (!_.isNil(config.env_variables)) { - dockerContent += selectComponent('env_variables').build(config) - } - - if (!_.isNil(config.run_steps)) { - dockerContent += selectComponent('run_steps').build(config) - } + const result = [''] - if (!_.isNil(config.entrypoint_steps)) { - dockerContent += selectComponent('entrypoint_steps').build(config) + for (const builder of commands) { + result.push(...builder.build(config)) + result.push('') } - return dockerContent + return result.join('\n') } module.exports = buildConfig diff --git a/lib/config-parser/steps/conda-install.js b/lib/config-parser/steps/conda-install.js new file mode 100644 index 0000000..94caa49 --- /dev/null +++ b/lib/config-parser/steps/conda-install.js @@ -0,0 +1,11 @@ +function buildCondaInstall (step) { + const packages = step.config.packages + return [`conda install ${packages.join(' ')}`] +} + +module.exports = { + type: 'conda_install', + build: buildCondaInstall, + os: ['debian', 'ubuntu'], + command: ['run', 'entrypoint'] +} diff --git a/lib/config-parser/steps/conda.js b/lib/config-parser/steps/conda.js new file mode 100644 index 0000000..0ef1e30 --- /dev/null +++ b/lib/config-parser/steps/conda.js @@ -0,0 +1,31 @@ +const availableVersions = require('./python').availableVersions +const util = require('../util') + +function buildInstallConda (step) { + let pythonVersion = step.config.python_version || '3.6' + if (!availableVersions.includes(pythonVersion)) { + throw Error(`Invalid python version ${pythonVersion}`) + } + const condaVersion = step.config.conda_version || 'latest' + const osType = step.config.bit === '64' ? 'Linux-x86_64' : 'Linux-x86' + const result = [util.aptInstall('curl', 'wget', 'gnupg', 'bzip2')] + const envName = step.config.env_name || 'docker' + result.push( + `wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-${condaVersion}-${osType}.sh -O ~/miniconda.sh`, + '/bin/bash ~/miniconda.sh -b -p /opt/conda', + 'rm ~/miniconda.sh', + util.writeBashrc('export PATH=/opt/conda/bin:$PATH'), + `conda create --yes -n ${envName} python=${step.config.python_version}`, + util.writeBashrc('source activate docker'), + util.writeBashrc('export PATH=/opt/conda/envs/docker/bin:$PATH') + ) + + return result +} + +module.exports = { + type: 'install_conda', + build: buildInstallConda, + os: ['debian', 'ubuntu'], + command: ['run'] +} diff --git a/lib/config-parser/steps/custom.js b/lib/config-parser/steps/custom.js new file mode 100644 index 0000000..167a3db --- /dev/null +++ b/lib/config-parser/steps/custom.js @@ -0,0 +1,11 @@ + +function build (step) { + return [step.config.command] +} + +module.exports = { + type: 'custom_command', + build, + os: ['debian', 'ubuntu'], + command: ['run', 'entrypoint'] +} diff --git a/lib/config-parser/steps/git.js b/lib/config-parser/steps/git.js new file mode 100644 index 0000000..113f345 --- /dev/null +++ b/lib/config-parser/steps/git.js @@ -0,0 +1,46 @@ +const _ = require('lodash') + +function install (step) { + return [ + 'apt-get install -y --no-install-recommends git' + ] +} + +function clone (step) { + let result = install(step) + + // clone + result.push(`git clone ${step.config.url}`) + + // cd + const repoNameReg = /\/([\w.@:-~_]+?)(\/)?(\.git)?$/ + const repoName = repoNameReg.exec(step.config.url)[1] + result.push(`cd ${repoName}`) + + // branch + if (step.config.branch !== 'master') { + result.push(`git checkout ${step.config.branch}`) + } + + // tag + if (!_.isEmpty(step.config.tag)) { + result.push(`git checkout tags/${step.config.tag}`) + } + + return result +} + +module.exports = { + clone: { + type: 'git_clone', + build: clone, + os: ['debian', 'ubuntu'], + command: ['run', 'entrypoint'] + }, + install: { + type: 'install_git', + build: install, + os: ['debian', 'ubuntu'], + command: ['run'] + } +} diff --git a/lib/config-parser/steps/python.js b/lib/config-parser/steps/python.js new file mode 100644 index 0000000..cd00042 --- /dev/null +++ b/lib/config-parser/steps/python.js @@ -0,0 +1,60 @@ +const _ = require('lodash') +const util = require('../util') + +const availableVersions = [ + '2', + '2.6', + '2.7', + '3', + '3.4', + '3.5', + '3.6', + '3.7' +] + +function install (step) { + let version = _.toString(step.config.version) || '3.6' + if (!availableVersions.includes(version)) { + throw Error(`Invalid python version ${version}`) + } + + if (version === '2') { + version = '2.7' + } + + const python = `python${version}` + const result = [ + util.aptInstall('wget', 'software-properties-common'), + 'add-apt-repository ppa:deadsnakes/ppa', + 'apt-get update', + util.aptInstall(python) + ] + + if (version[0] === 3) { + result.push( + 'wget -O ~/get-pip.py https://bootstrap.pypa.io/get-pip.py', + `${python} ~/get-pip.py`, + 'pip install setuptools', + `ln -s /usr/bin/${python} /usr/local/bin/python`, + `ln -s /usr/bin/${python} /usr/local/bin/python3` + ) + } else { + result.push( + 'apt-get install -y --no-install-recommends python-pip', + 'pip install setuptools', + `ln -s /usr/bin/${python} /usr/local/bin/python` + ) + } + + return result +} + +module.exports = { + availableVersions, + install: { + type: 'install_python', + build: install, + os: ['ubuntu'], + command: ['run'] + } +} diff --git a/lib/config-parser/steps/step-list.js b/lib/config-parser/steps/step-list.js new file mode 100644 index 0000000..45a1fa6 --- /dev/null +++ b/lib/config-parser/steps/step-list.js @@ -0,0 +1,20 @@ +const steps = [ + require('./conda-install'), + require('./custom'), + require('./git').clone, + require('./git').install, + require('./conda'), + require('./python').install +] + +const result = {} + +for (const step of steps) { + if (result[step.type]) { + throw Error(`Step ID ${step.type} is not unique.`) + } + + result[step.type] = step +} + +module.exports = result diff --git a/lib/config-parser/util.js b/lib/config-parser/util.js new file mode 100644 index 0000000..f7cfe40 --- /dev/null +++ b/lib/config-parser/util.js @@ -0,0 +1,4 @@ +module.exports = { + writeBashrc: (content) => `echo "${content}" >> ~/.bashrc && source ~/.bashrc`, + aptInstall: (...packages) => `apt-get install -y --no-install-recommends ${packages.join(' ')}` +} diff --git a/lib/index.js b/lib/index.js index f7ed070..bb1e7a7 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,11 +1,3 @@ const dockerfileGenerator = require('./dockerfile-generator') -const fs = require('fs-extra') -const path = require('path') - -const dockerfilePath = path.join(__dirname, '../out/dockerfile') -fs.readJson('./examples/config-example.json') - .then(example => dockerfileGenerator(example, dockerfilePath)) - .then(data => console.log(data)) - .catch(err => console.log(err)) module.exports = dockerfileGenerator diff --git a/package.json b/package.json index 141ca6c..3ab3917 100644 --- a/package.json +++ b/package.json @@ -1,26 +1,20 @@ { - "name": "pai-flow", - "version": "0.2.0", + "name": "paiflow", + "version": "0.3.0", "description": "[![Documentation Status](https://readthedocs.org/projects/paiflow/badge/?version=latest)](https://paiflow.readthedocs.io/en/latest/?badge=latest)", "main": "./lib/index.js", - "repository": { - "type": "git", - "url": "git+https://github.com/debuggy/PAIFlow.git" - }, "author": "debuggy", "license": "MIT", - "bugs": { - "url": "https://github.com/debuggy/PAIFlow/issues" - }, - "homepage": "https://github.com/debuggy/PAIFlow#readme", "scripts": { - "debug": "node ./lib/index.js", + "debug": "node ./scripts/debug.js && docker build -t test ./out", "test": "jest", + "test-dev": "jest common/", "coveralls": "jest --coverage --coverageReporters=text-lcov | coveralls", "lint": "standard" }, "files": [ "lib/", + "!lib/__tests__", "schemas/" ], "dependencies": { @@ -30,6 +24,7 @@ }, "devDependencies": { "coveralls": "^3.0.2", + "execa": "^1.0.0", "jest": "^23.6.0", "standard": "^12.0.1" } diff --git a/schemas/config-schema.json b/schemas/config-schema.json index 8ffa568..de409f0 100644 --- a/schemas/config-schema.json +++ b/schemas/config-schema.json @@ -30,13 +30,27 @@ }, "base_docker": { "$id": "#/properties/base_docker", - "type": "string", + "type": "object", "description": "A base docker image that the dockerfile build from.", - "default": "", - "examples": [ - "pai.build.base:hadoop2.7.2-cuda9.0-cudnn7-devel-ubuntu16.04" - ], - "pattern": "^(.*)$" + "properties": { + "name": { + "type": "string" + }, + "image_url": { + "type": "string" + }, + "os": { + "type": "string", + "enum": ["debian", "ubuntu"] + } + }, + "required": ["image_url", "os"] + }, + "workdir": { + "$id": "#/properties/workdir", + "type": "string", + "description": "Working directory for any RUN, ENTRYPOINT instructions", + "default": "/" }, "run_steps": { "$id": "#/properties/run_steps", @@ -60,15 +74,7 @@ "type": { "$id": "#/properties/run_steps/items/properties/type", "type": "string", - "description": "The type of a certain build step.", - "oneof":[ - "components", - "python_requirements", - "dataset", - "code", - "custom" - ], - "pattern": "^(.*)$" + "description": "The type of a certain build step." }, "config": { "$id": "#/properties/run_steps/items/properties/config", @@ -100,15 +106,7 @@ "type": { "$id": "#/properties/entrypoint_steps/items/properties/type", "type": "string", - "description": "The type of a certain build step.", - "oneof":[ - "components", - "python_requirements", - "dataset", - "code", - "custom" - ], - "pattern": "^(.*)$" + "description": "The type of a certain build step." }, "config": { "$id": "#/properties/entrypoint_steps/items/properties/config", diff --git a/scripts/debug.js b/scripts/debug.js new file mode 100644 index 0000000..9dffc5d --- /dev/null +++ b/scripts/debug.js @@ -0,0 +1,9 @@ +const dockerfileGenerator = require('../lib/index') +const fs = require('fs-extra') +const path = require('path') + +const dockerfilePath = path.join(__dirname, '../out/dockerfile') +fs.readJson('./examples/install-conda.json') + .then(example => dockerfileGenerator(example, dockerfilePath)) + .then(data => console.log(data)) + .catch(err => console.log(err)) diff --git a/yarn.lock b/yarn.lock index 8348c31..8ead16d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -609,7 +609,7 @@ cross-spawn@^5.0.1: shebang-command "^1.2.0" which "^1.2.9" -cross-spawn@^6.0.5: +cross-spawn@^6.0.0, cross-spawn@^6.0.5: version "6.0.5" resolved "http://registry.npm.taobao.org/cross-spawn/download/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" dependencies: @@ -769,6 +769,13 @@ ecc-jsbn@~0.1.1: jsbn "~0.1.0" safer-buffer "^2.1.0" +end-of-stream@^1.1.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.1.tgz#ed29634d19baba463b6ce6b80a37213eab71ec43" + integrity sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q== + dependencies: + once "^1.4.0" + error-ex@^1.2.0, error-ex@^1.3.1: version "1.3.2" resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" @@ -993,6 +1000,19 @@ execa@^0.7.0: signal-exit "^3.0.0" strip-eof "^1.0.0" +execa@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/execa/-/execa-1.0.0.tgz#c6236a5bb4df6d6f15e88e7f017798216749ddd8" + integrity sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA== + dependencies: + cross-spawn "^6.0.0" + get-stream "^4.0.0" + is-stream "^1.1.0" + npm-run-path "^2.0.0" + p-finally "^1.0.0" + signal-exit "^3.0.0" + strip-eof "^1.0.0" + exit@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" @@ -1257,6 +1277,13 @@ get-stream@^3.0.0: version "3.0.0" resolved "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14" +get-stream@^4.0.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5" + integrity sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w== + dependencies: + pump "^3.0.0" + get-value@^2.0.3, get-value@^2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" @@ -2561,7 +2588,7 @@ object.pick@^1.3.0: dependencies: isobject "^3.0.1" -once@^1.3.0, once@^1.4.0: +once@^1.3.0, once@^1.3.1, once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" dependencies: @@ -2814,6 +2841,14 @@ psl@^1.1.24: version "1.1.29" resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.29.tgz#60f580d360170bb722a797cc704411e6da850c67" +pump@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" + integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww== + dependencies: + end-of-stream "^1.1.0" + once "^1.3.1" + punycode@^1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e"