Merge 5802722 into f4b6f8c

.vscode/settings.json

@@ -1,4 +1,5 @@
 {
+  "standard.enable": true,
   "javascript.validate.enable": false,
   "standard.usePackageJson": true,
   "files.exclude": {

docs/config-parser.md

@@ -0,0 +1,23 @@
+# Config Parser Design
+
+- build-base
+
+  `From ${config.base_docker}\n`
+
+- build-env
+
+  `ENV ${config.env_variables}\n`
+
+- install
+  - export universal interface with a type and a function
+  - conda
+  - python os
+  - conda command
+  - pip command
+  - custom command
+  - hdfs copy
+
+- custom
+  - publish to npm with a tag
+  - expose a certain function
+  - get tag list to require

examples/config-example.json

@@ -8,7 +8,7 @@
   "steps": [
     {
       "name": "install_python",
-      "type": "components",
+      "type": "install_python",
       "config": {
         "name": "python",
         "version": "3.6"
@@ -51,7 +51,7 @@
     },
     {
       "name": "custom_command",
-      "type": "custom",
+      "type": "custom_command",
       "config": {
         "command": "python hello.py"
       }

lib/config-parser/base_docker.js

@@ -0,0 +1,11 @@
+
+function buildBase (config) {
+  let dockerContent = ''
+  dockerContent += `From ${config.base_docker}\n`
+  return dockerContent
+}
+
+module.exports = {
+  type: 'base_docker',
+  build: buildBase
+}

lib/config-parser/env_variables.js

@@ -0,0 +1,13 @@
+
+function buildEnv (config) {
+  let dockerContent = ''
+  for (const envKey in config.env_variables) {
+    dockerContent += `ENV ${envKey}=${config.env_variables[envKey]}\n`
+  }
+  return dockerContent
+}
+
+module.exports = {
+  type: 'env_variables',
+  build: buildEnv
+}

lib/config-parser/index.js

@@ -0,0 +1,40 @@
+const version = require('./version')
+const baseDocker = require('./base_docker')
+const envVariables = require('./env_variables')
+const steps = require('./steps')
+const _ = require('lodash')
+
+const componentsList = [version, baseDocker, envVariables, steps]
+
+function selectComponent (type) {
+  const component = componentsList.find(item => item.type === type)
+  if (_.isNil(component)) {
+    throw new Error(`cannot find component: '${type}'`)
+  }
+
+  return component
+}
+
+function buildConfig (config) {
+  let dockerContent = ''
+
+  if (!_.isNil(config.version)) {
+    dockerContent += selectComponent('version').build(config)
+  }
+
+  if (!_.isNil(config.base_docker)) {
+    dockerContent += selectComponent('base_docker').build(config)
+  }
+
+  if (!_.isNil(config.env_variables)) {
+    dockerContent += selectComponent('env_variables').build(config)
+  }
+
+  if (!_.isNil(config.steps)) {
+    dockerContent += selectComponent('steps').build(config)
+  }
+
+  return dockerContent
+}
+
+module.exports = buildConfig

lib/config-parser/steps/custom_command.js

@@ -0,0 +1,11 @@
+
+function buildCustom (step) {
+  let dockerContent = ''
+  dockerContent += `CMD ${step.config.command}\n`
+  return dockerContent
+}
+
+module.exports = {
+  'type': 'custom_command',
+  'build': buildCustom
+}

lib/config-parser/steps/default.js

@@ -0,0 +1,11 @@
+
+function buildDefault (step) {
+  let dockerContent = ''
+  dockerContent += `no processing for ${step.type}\n`
+  return dockerContent
+}
+
+module.exports = {
+  'type': 'default',
+  'build': buildDefault
+}

lib/config-parser/steps/index.js

@@ -0,0 +1,30 @@
+const _ = require('lodash')
+
+const customCommand = require('./custom_command')
+const installPython = require('./install_python')
+const defaultStep = require('./default')
+
+const stepsList = [customCommand, installPython]
+
+function selectStepBuilder (stepType) {
+  return stepsList.find(item => item.type === stepType)
+}
+
+function buildSteps (config) {
+  let dockerContent = ''
+  for (let step of config.steps) {
+    let stepBuilder = selectStepBuilder(step.type)
+    if (_.isNil(stepBuilder)) {
+      // throw new Error(`cannot find step builder: '${step.type}'`)
+      stepBuilder = defaultStep
+    }
+    dockerContent += stepBuilder.build(step)
+  }
+
+  return dockerContent
+}
+
+module.exports = {
+  type: 'steps',
+  build: buildSteps
+}

lib/config-parser/steps/install_python.js

@@ -0,0 +1,9 @@
+
+function installPython (config) {
+  return 'install python\n'
+}
+
+module.exports = {
+  'type': 'install_python',
+  'build': installPython
+}

lib/config-parser/version.js

@@ -0,0 +1,9 @@
+
+function buildVersion (config) {
+  return `# version: ${config.version}\n`
+}
+
+module.exports = {
+  type: 'version',
+  build: buildVersion
+}

lib/dockerfile-generator.js

@@ -1,36 +1,18 @@
 const fs = require('fs-extra')
-const _ = require('lodash')
 const SchemaValidationError = require('./schema-validation-error')
 const schemaValidator = require('./schema-validator')
 const path = require('path')
-
-async function configConverter (config) {
-  let dockerContent = ''
-  dockerContent += `From ${config.base_docker}\n`
-  if (!_.isNil(config.env_variables)) {
-    for (const envKey in config.env_variables) {
-      dockerContent += `ENV ${envKey}=${config.env_variables[envKey]}\n`
-    }
-  }
-  if (!_.isNil(config.steps)) {
-    for (const buildStep of config.steps) {
-      if (buildStep.type === 'custom') {
-        dockerContent += `CMD ${buildStep.config.command}\n`
-      }
-    }
-  }
-
-  return dockerContent
-}
+const configParser = require('./config-parser')
 
 async function dockerfileGenerator (config, dockerfilePath) {
   const configSchema = await fs.readJson(path.join(__dirname, '../schemas/config-schema.json'))
   const validation = await schemaValidator(configSchema, config)
   if (!validation.valid) {
     throw new SchemaValidationError(`The config validation is not passed.`, validation.errors)
   }
-  const dockerContent = await configConverter(config)
+  const dockerContent = configParser(config)
   await fs.outputFile(dockerfilePath, dockerContent)
+
   return dockerfilePath
 }
 

mkdocs.yml

@@ -1,6 +1,7 @@
 site_name: PAIFlow Docs
 nav:
     - Introduction: index.md
-    - Work Plan: work_plan.md
-    - Config Schema: config_schema.md
-theme: readthedocs
+    - Work Plan: work-plan.md
+    - Config Schema: config-schema.md
+    - Config Parser: config-parser.md
+theme: readthedocs

schemas/config-schema.json

@@ -31,12 +31,12 @@
       ],
       "pattern": "^(.*)$"
     },
-    "steps": {
-      "$id": "#/properties/steps",
+    "run_steps": {
+      "$id": "#/properties/run_steps",
       "type": "array",
       "description": "An array of steps that discribe the build process.",
       "items": {
-        "$id": "#/properties/steps/items",
+        "$id": "#/properties/run_steps/items",
         "type": "object",
         "required": [
           "name",
@@ -45,13 +45,13 @@
         ],
         "properties": {
           "name": {
-            "$id": "#/properties/steps/items/properties/name",
+            "$id": "#/properties/run_steps/items/properties/name",
             "type": "string",
             "description": "The name of a certain build step.",
             "pattern": "^(.*)$"
           },
           "type": {
-            "$id": "#/properties/steps/items/properties/type",
+            "$id": "#/properties/run_steps/items/properties/type",
             "type": "string",
             "description": "The type of a certain build step.",
             "oneof":[
@@ -64,7 +64,47 @@
             "pattern": "^(.*)$"
           },
           "config": {
-            "$id": "#/properties/steps/items/properties/config",
+            "$id": "#/properties/run_steps/items/properties/config",
+            "type": "object",
+            "description": "The config of a certain build step."
+          }
+        }
+      }
+    },
+    "entrypoint_steps": {
+      "$id": "#/properties/entrypoint_steps",
+      "type": "array",
+      "description": "An array of steps that discribe the entry point when running docker container.",
+      "items": {
+        "$id": "#/properties/entrypoint_steps/items",
+        "type": "object",
+        "required": [
+          "name",
+          "type",
+          "config"
+        ],
+        "properties": {
+          "name": {
+            "$id": "#/properties/entrypoint_steps/items/properties/name",
+            "type": "string",
+            "description": "The name of a certain build step.",
+            "pattern": "^(.*)$"
+          },
+          "type": {
+            "$id": "#/properties/entrypoint_steps/items/properties/type",
+            "type": "string",
+            "description": "The type of a certain build step.",
+            "oneof":[
+                "components",
+                "python_requirements",
+                "dataset",
+                "code",
+                "custom"
+            ],
+            "pattern": "^(.*)$"
+          },
+          "config": {
+            "$id": "#/properties/entrypoint_steps/items/properties/config",
             "type": "object",
             "description": "The config of a certain build step."
           }