feat: docker test (#11)

docker-test/dockerfile

@@ -0,0 +1,19 @@
+# version: 0.1
+
+From ubuntu
+
+ENV TINI_VERSION v0.9.0
+RUN apt-get update && \
+    apt-get install -y curl wget gnupg bzip2
+
+RUN wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O ~/miniconda.sh
+RUN chmod 0755 ~/miniconda.sh
+RUN ~/miniconda.sh -b -p ~/conda
+ENV PATH "~/conda/bin:$PATH"
+RUN ["/bin/bash", "-c", "conda create -n build_test python=3.4 && source activate build_test && conda install numpy"]
+
+# RUN conda activate bulid_test
+# RUN conda install scikit-learn
+
+
+

examples/config-example.json

@@ -1,33 +1,31 @@
 {
   "version": 0.1,
+  "name": "flow_test",
+  "author": "debuggy",
   "env_variables": {
     "TENSORFLOW_VERSION": "1.4.0",
     "LC_ALL": "C"
   },
   "base_docker": "pai.build.base:hadoop2.7.2-cuda9.0-cudnn7-devel-ubuntu16.04",
-  "steps": [
+  "run_steps": [
     {
-      "name": "install_python",
-      "type": "install_python",
+      "name": "install_conda",
+      "type": "install_conda",
       "config": {
-        "name": "python",
-        "version": "3.6"
+        "env_name": "flow_example",
+        "python_version": "3.6"
       }
     },
     {
       "name": "install_tensorflow",
       "type": "conda_install",
       "config": {
-        "name": "tensorflow-gpu",
-        "version": "1.4.0"
-      }
-    },
-    {
-      "name": "prepare_data",
-      "type": "local_copy",
-      "config": {
-        "source": "./data",
-        "dest": "/data"
+        "packages": [
+          {
+            "name": "scipy",
+            "version": "0.15.0"
+          }
+        ]
       }
     },
     {
@@ -36,21 +34,13 @@
       "config": {
         "type": "github",
         "url": "https://github.com/debuggy/DockerForPAI_init.git",
-        "access_token": "asdflkjasdflkjsdf",
-        "branch": "master",
-        "tag": "v0.1",
-        "commit": "asdfasdgf1234"
-      }
-    },
-    {
-      "name": "python_command",
-      "type": "python_script",
-      "config": {
-        "python_code": "print(\"hello world!\")"
+        "branch": "master"
       }
-    },
+    }
+  ],
+  "entrypoint_steps": [
     {
-      "name": "custom_command",
+      "name": "custom_install",
       "type": "custom_command",
       "config": {
         "command": "python hello.py"

lib/config-parser/base_docker.js → lib/config-parser/base-docker.js

@@ -1,7 +1,7 @@
 
 function buildBase (config) {
   let dockerContent = ''
-  dockerContent += `From ${config.base_docker}\n`
+  dockerContent += `From ${config.base_docker}\n \n`
   return dockerContent
 }
 

lib/config-parser/entrypoint-steps.js

@@ -0,0 +1,30 @@
+const _ = require('lodash')
+
+const customCommand = require('./step-components/custom-command')
+const defaultStep = require('./step-components/default')
+
+const entrypointsList = [customCommand]
+
+function selectEntrypointBuilder (stepType) {
+  return entrypointsList.find(item => item.type === stepType)
+}
+
+function buildSteps (config) {
+  let dockerContent = ''
+  for (let step of config.entrypoint_steps) {
+    let entrypointBuilder = selectEntrypointBuilder(step.type)
+    if (_.isNil(entrypointBuilder)) {
+      // throw new Error(`cannot find step builder: '${step.type}'`)
+      entrypointBuilder = defaultStep
+    }
+    dockerContent += '           ' + entrypointBuilder.build(step)
+  }
+  dockerContent = 'ENTRYPOINT' + dockerContent.substring(10, dockerContent.length - 6) + '\n\n'
+
+  return dockerContent
+}
+
+module.exports = {
+  type: 'entrypoint_steps',
+  build: buildSteps
+}

lib/config-parser/env_variables.js → lib/config-parser/env-variables.js

@@ -2,8 +2,10 @@
 function buildEnv (config) {
   let dockerContent = ''
   for (const envKey in config.env_variables) {
-    dockerContent += `ENV ${envKey}=${config.env_variables[envKey]}\n`
+    dockerContent += `    ${envKey}=${config.env_variables[envKey]} \\ \n`
   }
+  dockerContent = 'ENV' + dockerContent.substring(3, dockerContent.length - 4) + '\n\n'
+
   return dockerContent
 }
 

lib/config-parser/index.js

@@ -1,10 +1,11 @@
 const version = require('./version')
-const baseDocker = require('./base_docker')
-const envVariables = require('./env_variables')
-const steps = require('./steps')
+const baseDocker = require('./base-docker')
+const envVariables = require('./env-variables')
+const runSteps = require('./run-steps')
+const entrypointSteps = require('./entrypoint-steps')
 const _ = require('lodash')
 
-const componentsList = [version, baseDocker, envVariables, steps]
+const componentsList = [version, baseDocker, envVariables, runSteps, entrypointSteps]
 
 function selectComponent (type) {
   const component = componentsList.find(item => item.type === type)
@@ -30,8 +31,12 @@ function buildConfig (config) {
     dockerContent += selectComponent('env_variables').build(config)
   }
 
-  if (!_.isNil(config.steps)) {
-    dockerContent += selectComponent('steps').build(config)
+  if (!_.isNil(config.run_steps)) {
+    dockerContent += selectComponent('run_steps').build(config)
+  }
+
+  if (!_.isNil(config.entrypoint_steps)) {
+    dockerContent += selectComponent('entrypoint_steps').build(config)
   }
 
   return dockerContent

lib/config-parser/run-steps.js

@@ -0,0 +1,31 @@
+const _ = require('lodash')
+
+const customCommand = require('./step-components/custom-command')
+const installPython = require('./step-components/install-python')
+const defaultStep = require('./step-components/default')
+
+const runStepsList = [customCommand, installPython]
+
+function selectRunBuilder (stepType) {
+  return runStepsList.find(item => item.type === stepType)
+}
+
+function buildSteps (config) {
+  let dockerContent = ''
+  for (let step of config.run_steps) {
+    let runBuilder = selectRunBuilder(step.type)
+    if (_.isNil(runBuilder)) {
+      // throw new Error(`cannot find step builder: '${step.type}'`)
+      runBuilder = defaultStep
+    }
+    dockerContent += '    ' + runBuilder.build(step)
+  }
+  dockerContent = 'RUN' + dockerContent.substring(3, dockerContent.length - 6) + '\n\n'
+
+  return dockerContent
+}
+
+module.exports = {
+  type: 'run_steps',
+  build: buildSteps
+}

lib/config-parser/steps/custom_command.js → lib/config-parser/step-components/custom-command.js

@@ -1,7 +1,7 @@
 
 function buildCustom (step) {
   let dockerContent = ''
-  dockerContent += `CMD ${step.config.command}\n`
+  dockerContent += `${step.config.command} && \\ \n`
   return dockerContent
 }
 

lib/config-parser/step-components/custom-install.js

@@ -0,0 +1,11 @@
+
+function buildCustomInstall (step) {
+  let dockerContent = ''
+  dockerContent += `${step.config.command} && \\ \n`
+  return dockerContent
+}
+
+module.exports = {
+  'type': 'custom_command',
+  'build': buildCustomInstall
+}

lib/config-parser/steps/default.js → lib/config-parser/step-components/default.js

@@ -1,7 +1,7 @@
 
 function buildDefault (step) {
   let dockerContent = ''
-  dockerContent += `no processing for ${step.type}\n`
+  dockerContent += `no processing for ${step.type} && \\ \n`
   return dockerContent
 }
 

lib/config-parser/step-components/git-clone.js

@@ -0,0 +1,23 @@
+const _ = require('lodash')
+
+function buildInstallConda (step) {
+  let dockerContent = `git clone ${step.config.url} && \\ \n`
+
+  // get git repo dir
+  const repoNameReg = /\/([\w.@:-~_]+)\.git/
+  const repoName = repoNameReg.exec(step.config.url)[1]
+  dockerContent += `cd ${repoName} && \\ \n`
+  if (step.config.branch !== 'master') {
+    dockerContent += `git checkout ${step.config.branch} && \\ \n`
+  }
+  if (!_.isNil(step.config.tag)) {
+    dockerContent += `git checkout tags/${step.config.tag} && \\ \n`
+  }
+
+  return dockerContent
+}
+
+module.exports = {
+  'type': 'install_conda',
+  'build': buildInstallConda
+}

lib/config-parser/step-components/install-conda.js

@@ -0,0 +1,16 @@
+function buildInstallConda (step) {
+  let dockerContent = `wget https://repo.continuum.io/miniconda/Miniconda3-3.7.0-Linux-x86_64.sh -O ~/miniconda.sh && \\ \n` +
+                      `bash ~/miniconda.sh -b -p $HOME/miniconda && \\ \n` +
+                      `export PATH="$HOME/miniconda/bin:$PATH && \\ \n"` +
+                      `conda -V && \\ \n` +
+                      `export CONDA_ENV_NAME='${step.config.env_name}' && \\ \n` +
+                      `conda create --yes -n $CONDA_ENV_NAME python=${step.config.python_version} && \\ \n` +
+                      `source activate $CONDA_ENV_NAME && \\ \n`
+
+  return dockerContent
+}
+
+module.exports = {
+  'type': 'install_conda',
+  'build': buildInstallConda
+}

lib/config-parser/steps/install_python.js → lib/config-parser/step-components/install-python.js

@@ -1,6 +1,6 @@
 
 function installPython (config) {
-  return 'install python\n'
+  return 'install python && \\ \n'
 }
 
 module.exports = {

lib/config-parser/version.js

@@ -1,6 +1,6 @@
 
 function buildVersion (config) {
-  return `# version: ${config.version}\n`
+  return `# version: ${config.version}\n \n`
 }
 
 module.exports = {

schemas/config-schema.json

@@ -5,16 +5,23 @@
   "description": "Config schema of PAIFlow",
   "required": [
     "version",
+    "name",
     "env_variables",
     "base_docker",
-    "steps"
+    "run_steps",
+    "entrypoint_steps"
   ],
   "properties": {
     "version": {
       "$id": "#/properties/version",
       "type": "number",
-      "description": "The version of the config definition.",
-      "default": 0.1
+      "description": "The version of the job config.",
+      "default": 1.0
+    },
+    "name": {
+      "$id": "#/properties/name",
+      "type": "string",
+      "description": "The name of the job"
     },
     "env_variables": {
       "$id": "#/properties/env_variables",
@@ -113,4 +120,3 @@
     }
   }
 }
-