refactor: refine the infrastructure

.gitignore

@@ -3,4 +3,5 @@ npm-debug.log
 *.obj
 *.out
 node_modules/
-out/
+out/
+docker-test/

README.md

@@ -1,7 +1,3 @@
-# PAIFlow
-
-[![Documentation Status](https://readthedocs.org/projects/paiflow/badge/?version=latest)](https://paiflow.readthedocs.io/en/latest/?badge=latest)
-[![Build Status](https://travis-ci.com/debuggy/PAIFlow.svg?branch=master)](https://travis-ci.com/debuggy/PAIFlow)
-[![Coverage Status](https://coveralls.io/repos/github/debuggy/PAIFlow/badge.svg?branch=master)](https://coveralls.io/github/debuggy/PAIFlow?branch=master)
+# gelato
 
 Work flow of Microsoft OpenPAI

docker-test/dockerfile

@@ -2,14 +2,34 @@
 
 From tensorflow/tensorflow
 
+SHELL ["/bin/bash", "-c"]
+ENV BASH_ENV=~/.bashrc
 
 RUN apt-get update
+RUN mv ~/.bashrc ~/.bashrc.bak
 RUN apt-get install -y curl wget gnupg bzip2
 RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh
 RUN /bin/bash ~/miniconda.sh -b -p /opt/conda
 RUN rm ~/miniconda.sh
-RUN echo "export PATH=/opt/conda/bin:$PATH" >> ~/.bashrc
-RUN export PATH=/opt/conda/bin:$PATH && echo $PATH && conda -V
+
+## docker does not start bash as login session
+# RUN echo "echo bash_profile" >> ~/.bash_profile
+# RUN echo "export PATH=/opt/conda/bin:$PATH" >> ~/.bash_profile
+# RUN echo "echo bash_login" >> ~/.bash_login
+# RUN echo "export PATH=/opt/conda/bin:$PATH" >> ~/.bash_login
+# RUN echo "echo .profile" >> ~/.profile
+# RUN echo "export PATH=/opt/conda/bin:$PATH" >> ~/.profile
+
+RUN echo 'export PATH=/opt/conda/bin:$PATH' >> ~/.bashrc && source ~/.bashrc
+RUN conda -V && \
+    conda create --yes -n docker python=3.6
+RUN echo 'source activate docker' >> ~/.bashrc && source ~/.bashrc
+RUN echo 'export PATH=/opt/conda/envs/docker/bin:$PATH' >> ~/.bashrc && source ~/.bashrc
+RUN conda install numpy
+
+RUN cat ~/.bashrc.bak >> ~/.bashrc
+
+CMD ["/bin/bash", "-c", "source activate docker"]
 
 
 

examples/config-example.json

@@ -6,7 +6,10 @@
     "TENSORFLOW_VERSION": "1.4.0",
     "LC_ALL": "C"
   },
-  "base_docker": "pai.build.base:hadoop2.7.2-cuda9.0-cudnn7-devel-ubuntu16.04",
+  "base_docker": {
+    "image_url": "tensorflow/tensorflow",
+    "os": "ubuntu"
+  },
   "run_steps": [
     {
       "name": "install_conda",

lib/__tests__/common/dockerfile-generator.test.js

@@ -0,0 +1,23 @@
+const dockerfileGenerator = require('../../dockerfile-generator')
+const SchemaValidationError = require('../../schema-validation-error')
+const fs = require('fs-extra')
+const path = require('path')
+
+/* eslint-env jest */
+it('throws schema validation error', async () => {
+  expect.assertions(1)
+  const templateFile = path.resolve(__dirname, '..', 'data', 'config-negative-example.json')
+  const template = await fs.readJson(templateFile)
+  await expect(dockerfileGenerator(template)).rejects.toThrow(SchemaValidationError)
+})
+
+it('generates dockerfile successfully', async () => {
+  expect.assertions(2)
+  const templateFile = path.resolve(__dirname, '..', '..', '..', 'examples', 'config-example.json')
+  const template = await fs.readJson(templateFile)
+  const dockerFilePath = path.resolve(__dirname, '..', '..', '..', 'out', 'dockerfile')
+  await fs.remove(dockerFilePath)
+  const result = await dockerfileGenerator(template, dockerFilePath)
+  expect(result).toEqual(dockerFilePath)
+  expect(await fs.pathExists(dockerFilePath)).toBeTruthy()
+})

lib/__tests__/common/install-conda-dockerfile.test.js

@@ -0,0 +1,16 @@
+const fs = require('fs-extra')
+const path = require('path')
+
+const dockerfileGenerator = require('../../dockerfile-generator')
+
+/* eslint-env jest */
+it('generate dockerfile with conda', async () => {
+  const templateFile = path.resolve(__dirname, '..', 'data', 'install-conda.json')
+  const dockerFile = path.resolve(__dirname, '..', '..', '..', 'out', 'dockerfile')
+  const template = await fs.readJSON(templateFile)
+  expect.assertions(2)
+  await fs.remove(dockerFile)
+  const result = await dockerfileGenerator(template, dockerFile)
+  expect(result).toEqual(dockerFile)
+  expect(await fs.pathExists(dockerFile)).toBeTruthy()
+})

lib/__tests__/common/install-python-dockerfile.test.js

@@ -0,0 +1,16 @@
+const fs = require('fs-extra')
+const path = require('path')
+
+const dockerfileGenerator = require('../../dockerfile-generator')
+
+/* eslint-env jest */
+it('generate dockerfile with conda', async () => {
+  const templateFile = path.resolve(__dirname, '..', 'data', 'install-python.json')
+  const dockerFile = path.resolve(__dirname, '..', '..', '..', 'out', 'dockerfile')
+  const template = await fs.readJSON(templateFile)
+  expect.assertions(2)
+  await fs.remove(dockerFile)
+  const result = await dockerfileGenerator(template, dockerFile)
+  expect(result).toEqual(dockerFile)
+  expect(await fs.pathExists(dockerFile)).toBeTruthy()
+})

lib/__tests__/common/schema-validator.test.js

@@ -0,0 +1,24 @@
+const schemaValidator = require('../../schema-validator')
+const fs = require('fs-extra')
+const path = require('path')
+
+/* eslint-env jest */
+it('schema validation succeeds', async () => {
+  expect.assertions(1)
+  const schemaFile = path.resolve(__dirname, '..', '..', '..', 'schemas', 'config-schema.json')
+  const schema = await fs.readJson(schemaFile)
+  const exampleFile = path.resolve(__dirname, '..', '..', '..', 'examples', 'config-example.json')
+  const example = await fs.readJson(exampleFile)
+  const validation = await schemaValidator(schema, example)
+  expect(validation.valid).toBe(true)
+})
+
+it('schema validation fails', async () => {
+  expect.assertions(1)
+  const schemaFile = path.resolve(__dirname, '..', '..', '..', 'schemas', 'config-schema.json')
+  const schema = await fs.readJson(schemaFile)
+  const exampleFile = path.resolve(__dirname, '..', 'data', 'config-negative-example.json')
+  const example = await fs.readJson(exampleFile)
+  const validation = await schemaValidator(schema, example)
+  expect(validation.valid).toBe(false)
+})

lib/__tests__/data/config-negative-example.json

@@ -0,0 +1,59 @@
+{
+  "env_variables": {
+    "TENSORFLOW_VERSION": "1.4.0",
+    "LC_ALL": "C"
+  },
+  "base_docker": "pai.build.base:hadoop2.7.2-cuda9.0-cudnn7-devel-ubuntu16.04",
+  "steps": [
+    {
+      "name": "install_python",
+      "type": "components",
+      "config": {
+        "name": "python",
+        "version": "3.6"
+      }
+    },
+    {
+      "name": "install_tensorflow",
+      "type": "conda_install",
+      "config": {
+        "name": "tensorflow-gpu",
+        "version": "1.4.0"
+      }
+    },
+    {
+      "name": "prepare_data",
+      "type": "local_copy",
+      "config": {
+        "source": "./data",
+        "dest": "/data"
+      }
+    },
+    {
+      "name": "prepare_code",
+      "type": "git_clone",
+      "config": {
+        "type": "github",
+        "url": "https://github.com/debuggy/DockerForPAI_init.git",
+        "access_token": "asdflkjasdflkjsdf",
+        "branch": "master",
+        "tag": "v0.1",
+        "commit": "asdfasdgf1234"
+      }
+    },
+    {
+      "name": "python_command",
+      "type": "python_script",
+      "config": {
+        "python_code": "print(\"hello world!\")"
+      }
+    },
+    {
+      "name": "custom_command",
+      "type": "custom",
+      "config": {
+        "command": "python hello.py"
+      }
+    }
+  ]
+}

lib/__tests__/data/install-conda.json

@@ -0,0 +1,40 @@
+{
+  "version": 0.1,
+  "name": "install_conda",
+  "author": "example",
+  "env_variables": {},
+  "base_docker": {
+    "image_url": "tensorflow/tensorflow",
+    "os": "ubuntu",
+    "bit": "64"
+  },
+  "run_steps": [
+    {
+      "name": "install conda",
+      "type": "install_conda",
+      "config": {
+        "python_version": "3.6",
+        "conda_version": "4.5.11",
+        "bit": "64"
+      }
+    },
+    {
+      "name": "conda install",
+      "type": "conda_install",
+      "config": {
+        "packages": [
+          "scipy"
+        ]
+      }
+    }
+  ],
+  "entrypoint_steps": [
+    {
+      "name": "check_version",
+      "type": "custom_command",
+      "config": {
+        "command": "conda -V"
+      }
+    }
+  ]
+}

lib/__tests__/data/install-python.json

@@ -0,0 +1,50 @@
+{
+  "version": 0.1,
+  "name": "install_python",
+  "author": "noname",
+  "env_variables": {},
+  "base_docker": {
+    "image_url": "tensorflow/tensorflow",
+    "os": "ubuntu"
+  },
+  "run_steps": [
+    {
+      "name": "install python",
+      "type": "install_python",
+      "config": {
+        "version": "3.4"
+      }
+    },
+    {
+      "name": "install_git",
+      "type": "install_git",
+      "config": {}
+    }
+  ],
+  "entrypoint_steps": [
+    {
+      "name": "prepare_code",
+      "type": "git_clone",
+      "config": {
+        "type": "github",
+        "url": "https://github.com/debuggy/DockerForPAI_init.git",
+        "branch": "master"
+      }
+    },
+    {
+      "name": "check_version",
+      "type": "custom_command",
+      "config": {
+        "command": "python -V"
+      }
+    },
+    {
+      "name": "hello_world",
+      "type": "custom_command",
+      "config": {
+        "command": "python src/cmd.py"
+      }
+    }
+  ]
+}
+

lib/__tests__/release/install-conda-build.test.js

@@ -0,0 +1,31 @@
+const execa = require('execa')
+const fs = require('fs-extra')
+const path = require('path')
+
+const dockerfileGenerator = require('../../dockerfile-generator')
+
+const testVersions = [
+  { condaVerson: '4.5.11', pythonVersion: '2.7' },
+  { condaVerson: 'latest', pythonVersion: '3.6' }
+]
+
+async function buildAndRun (condaVersion, pythonVersion) {
+  const templateFile = path.resolve(__dirname, '..', 'data', 'install-conda.json')
+  const dockerFile = path.resolve(__dirname, '..', '..', '..', 'out', 'dockerfile')
+  const imageName = `test_conda_install_${condaVersion}`
+  const containerName = `test_conda_install_container_${condaVersion}`
+  const template = await fs.readJSON(templateFile)
+  template.run_steps[0].config.conda_version = condaVersion
+  template.run_steps[0].config.python_version = pythonVersion
+  await dockerfileGenerator(template, dockerFile)
+  await execa.shell(`docker build -t ${imageName} ${path.dirname(dockerFile)}`)
+  await execa.shell(`docker run --rm --name ${containerName} ${imageName}`)
+  await execa.shell(`docker rmi ${imageName}`)
+}
+
+/* eslint-env jest */
+it('install conda', async () => {
+  for (const { condaVersion, pythonVersion } of testVersions) {
+    await buildAndRun(condaVersion, pythonVersion)
+  }
+}, 500 * 1000)

lib/__tests__/release/install-python.test.js

@@ -0,0 +1,42 @@
+const execa = require('execa')
+const fs = require('fs-extra')
+const path = require('path')
+
+const dockerfileGenerator = require('../../dockerfile-generator')
+
+async function test (version) {
+  if (!process.env['GELATO_DOCKER_TEST']) {
+    return
+  }
+  const templateFile = path.resolve(__dirname, '..', 'data', 'install-python.json')
+  const dockerFile = path.resolve(__dirname, '..', '..', '..', 'out', 'dockerfile')
+  const imageName = `test_python_install_${version}`
+  const containerName = `test_python_install_container_${version}`
+  const template = await fs.readJSON(templateFile)
+  template.run_steps[0].config.version = version
+  await dockerfileGenerator(template, dockerFile)
+  await execa.shell(`docker build -t ${imageName} ${path.dirname(dockerFile)}`)
+  await execa.shell(`docker run --rm --name ${containerName} ${imageName}`)
+  await execa.shell(`docker rmi ${imageName}`)
+}
+
+beforeAll(() => {
+  if (!process.env['GELATO_DOCKER_TEST']) {
+    console.log('Set environment variable GELATO_DOCKER_TEST to enable docker run/build tests')
+  }
+})
+
+/* eslint-env jest */
+it.each(['2', '2.6'])('string version', async (version) => {
+  await test(version)
+}, 500 * 1000)
+
+/* eslint-env jest */
+it.each([3, 3.7])('number version', async (version) => {
+  await test(version)
+}, 500 * 1000)
+
+/* eslint-env jest */
+it.each([3, 3.7])('invalid version', async (version) => {
+  await test(version)
+}, 500 * 1000)