Merge pull request #2619 from jeff1evesque/feature-2153

#2153: Create necessary docker containers for unit testing

.gitignore

@@ -17,6 +17,7 @@
 !Dockerfile
 !/test
 !/log/
+!/docker/
 
 #Ignore and don't commit the following directories
 /interface/static/js/

.travis.yml

@@ -6,15 +6,17 @@ language: python
 services:
   - docker
 
-## build docker container for unit testing.
+## build docker, install puppet (i.e. puppet parser validate)
 before_install:
-  - docker ps -a
-  - docker build -t ubuntu/trusty .
-  - wget http://apt.puppetlabs.com/puppetlabs-release-trusty.deb
-  - sudo dpkg -i puppetlabs-release-trusty.deb 
+  - docker build -f docker/default.dockerfile -t container-default .
+  - docker build -f docker/redis.dockerfile -t container-redis .
+  - docker build -f docker/database.dockerfile -t container-database .
+  - docker build -f docker/webserver.dockerfile -t container-webserver .
+  - wget https://apt.puppetlabs.com/puppetlabs-release-pc1-trusty.deb
+  - sudo dpkg -i puppetlabs-release-pc1-trusty.deb
   - sudo apt-get update
-  - sudo apt-get install puppet -y
-  - puppet -V
+  - sudo apt-get install puppet-agent -y
+  - sudo /opt/puppetlabs/bin/puppet -V
 
 ## install packages for linting
 #
@@ -39,8 +41,8 @@ install:
   - sudo apt-get install shellcheck=0.3.3-1~ubuntu14.04.1
   - npm install -g eslint babel-eslint@6.0.0-beta.6
   - npm install -g eslint-plugin-react@4.2.3
-  - gem install yaml-lint
-  - npm install -g travis-lint
+  - gem install yaml-lint -v 0.0.7
+  - npm install -g travis-lint@1.0.0
 
 ## implement linting
 #
@@ -50,6 +52,10 @@ install:
 #
 #      - https://github.com/rodjek/puppet-lint/issues/410
 #
+#  @docker run:
+#      -d, run container in background and print container ID
+#      -t, allocate pseudo-tty instead of default tty to running container
+#
 #  Note: unit testing is performed within the docker container, which is
 #        defined from the Dockerfile.
 script:
@@ -58,8 +64,8 @@ script:
   - jscs . --config test/config/.jscs.json
   - scss-lint src/scss/*.scss
   - bootlint -d W005 interface/templates/*.html
-  - puppet parser validate puppet/environment/development/manifests
-  - puppet parser validate puppet/environment/development/modules
+  - sudo find puppet/environment/development/manifests -name '*.pp' -type f -exec /opt/puppetlabs/bin/puppet parser validate {} \;
+  - sudo find puppet/environment/development/modules -name '*.pp' -type f -exec /opt/puppetlabs/bin/puppet parser validate {} \;
   - puppet-lint --no-variable_scope-check --no-ensure_first_param-check puppet/environment/development/modules
   - (cd puppet/environment/development/ && r10k puppetfile check)
   - (cd test/ && r10k puppetfile check)
@@ -75,3 +81,6 @@ script:
   - yaml-lint hiera/
   - yaml-lint *.yaml
   - travis-lint .travis.yml
+  - docker run -dt --name redis container-redis
+  - docker run -dt --name database container-database
+#  - docker run --link redis:container-redis --link database:container-database container-webserver bash -c "cd /var/machine-learning/test; py.test"

docker/database.dockerfile

@@ -0,0 +1,5 @@
+## custom container built locally
+FROM container-default
+
+## provision with puppet
+RUN /opt/puppetlabs/bin/puppet apply /var/machine-learning/puppet/environment/development/manifests/setup_database.pp --modulepath=/var/machine-learning/puppet/environment/development/modules_contrib:/var/machine-learning/puppet/environment/development/modules --confdir=/var/machine-learning/test

docker/default.dockerfile

@@ -0,0 +1,38 @@
+## container from dockerhub
+FROM ubuntu:14.04.4
+
+## copy files into container
+RUN mkdir /var/machine-learning
+COPY . /var/machine-learning
+
+## install git, and wget
+#
+#  Note: r10k requires 'git' installed
+RUN apt-get -y update
+RUN apt-get -y install git=1:1.9.1-1ubuntu0.3
+RUN apt-get -y install wget=1.15-1ubuntu1.14.04.2
+
+## install puppet
+RUN wget https://apt.puppetlabs.com/puppetlabs-release-pc1-trusty.deb
+RUN dpkg -i puppetlabs-release-pc1-trusty.deb
+RUN apt-get -y update
+RUN apt-get -y install puppet-agent
+
+## install r10k
+RUN apt-get -y install rubygems-integration=1.5
+RUN gem install r10k -v 2.2.0
+
+## install puppet modules using puppetfile with r10k
+RUN mkdir -p /var/machine-learning/puppet/environment/development/modules_contrib/
+RUN PUPPETFILE=/var/machine-learning/test/Puppetfile PUPPETFILE_DIR=/var/machine-learning/puppet/environment/development/modules_contrib/ r10k puppetfile install
+
+## provision with puppet
+RUN /opt/puppetlabs/bin/puppet apply /var/machine-learning/puppet/environment/development/manifests/install_packages.pp --modulepath=/var/machine-learning/puppet/environment/development/modules_contrib:/var/machine-learning/puppet/environment/development/modules --confdir=/var/machine-learning/test
+RUN /opt/puppetlabs/bin/puppet apply /var/machine-learning/puppet/environment/development/manifests/install_sklearn.pp --modulepath=/var/machine-learning/puppet/environment/development/modules_contrib:/var/machine-learning/puppet/environment/development/modules --confdir=/var/machine-learning/test
+RUN /opt/puppetlabs/bin/puppet apply /var/machine-learning/puppet/environment/development/manifests/configure_system.pp --modulepath=/var/machine-learning/puppet/environment/development/modules_contrib:/var/machine-learning/puppet/environment/development/modules --confdir=/var/machine-learning/test
+RUN /opt/puppetlabs/bin/puppet apply /var/machine-learning/puppet/environment/development/manifests/compile_asset.pp --modulepath=/var/machine-learning/puppet/environment/development/modules_contrib:/var/machine-learning/puppet/environment/development/modules --confdir=/var/machine-learning/test
+
+## show directory
+RUN ls -l /var/machine-learning/interface/static/js
+RUN ls -l /var/machine-learning/interface/static/css
+RUN ls -l /var/machine-learning/interface/static/img

docker/redis.dockerfile

@@ -0,0 +1,5 @@
+## custom container built locally
+FROM container-default
+
+## provision with puppet
+RUN /opt/puppetlabs/bin/puppet apply /var/machine-learning/puppet/environment/development/manifests/configure_redis.pp --modulepath=/var/machine-learning/puppet/environment/development/modules_contrib:/var/machine-learning/puppet/environment/development/modules --confdir=/var/machine-learning/test

docker/webserver.dockerfile

@@ -0,0 +1,5 @@
+## custom container built locally
+FROM container-default
+
+## provision with puppet
+RUN /opt/puppetlabs/bin/puppet apply /var/machine-learning/puppet/environment/development/manifests/start_webserver.pp --modulepath=/var/machine-learning/puppet/environment/development/modules_contrib:/var/machine-learning/puppet/environment/development/modules --confdir=/var/machine-learning/test

hiera.yaml

@@ -2,8 +2,8 @@
 :backends:
   - yaml
 
-:hierarchy:
-  - settings
-
 :yaml:
-  :datadir: /vagrant/hiera
+  :datadir: /vagrant/hiera
+
+:hierarchy:
+  - settings

hiera/settings.yaml

@@ -10,6 +10,7 @@
 ## variables
 general:
     environment: 'development'
+    vagrant_implement: true
     root: '/vagrant'
     host: 'localhost'
     user: 'vagrant'

interface/__init__.py

@@ -1,14 +1,10 @@
 '''@__init__
-
 This file allows the containing directory to be considered a python package,
 consisting of python module(s). Also, this file initializes flask object, which
 allows each module to import it.
-
 Note: the last 'views' import is required to be after initializing flask.
-
 Note: the use of '# noqa', at the end of specified line(s) of code, causes
       flake8 to ignore the corresponding line, during pep8 linting.
-
 '''
 
 from flask import Flask
@@ -17,4 +13,4 @@
 app = Flask(__name__)
 
 # required circular import
-import interface.views    # noqa
+import interface.views  # noqa

puppet/environment/development/manifests/compile_asset.pp

@@ -18,7 +18,7 @@
     require dependencies
 
     ## configure webcompilers	
-    contain system::webcompiler_directory
+    require system::webcompiler_directory
     contain compiler::webcompilers
 }
 

puppet/environment/development/manifests/install_packages.pp

@@ -24,6 +24,7 @@
     ## set dependency
     require apt
     require install_nodejs
+    require system::webcompiler_directory
 
     ## install packages
     contain package::inotify_tools
@@ -32,6 +33,8 @@
     contain package::xmltodict
     contain package::six
     contain package::fetch
+    contain package::pyyaml
+    contain package::pytest
 }
 
 ## initiate

puppet/environment/development/modules/compiler/manifests/initial_compile.pp

@@ -4,32 +4,50 @@
 ###
 class compiler::initial_compile {
     ## local variables
-    $root_dir = hiera('general')['root']
+    $hiera_general   = hiera('general')
+    $root_dir        = $hiera_general['root']
+    $vagrant_mounted = $hiera_general['vagrant_implement']
+    $dev_env_path    = "${root_dir}/puppet/environment/development"
+
     $sources  = [
         'jsx',
         'img',
         'scss',
         'js'
     ]
 
-    $sources.each |String $source| {
-        ## variables
-        $check_files = "if [ \"$(ls -A ${root_dir}/src/${source}/)\" ];"
-        $touch_files = "then touch ${root_dir}/src/${source}/*; fi"
+    ## vagrant container
+    if $vagrant_mounted {
+        $sources.each |String $source| {
+            ## variables
+            $check_files = "if [ \"$(ls -A ${root_dir}/src/${source}/)\" ];"
+            $touch_files = "then touch ${root_dir}/src/${source}/*; fi"
 
-        ## touch source: ensure initial build compiles every source file.
-        #
-        #  @touch, changes the modification time to the current system time.
-        #
-        #  Note: the current inotifywait implementation watches close_write,
-        #        move, and create. However, the source files will already exist
-        #        before this 'inotifywait', since the '/vagrant' directory will
-        #        already have been mounted on the initial build.
-        #
-        #  Note: every 'command' implementation checks if directory is
-        #        nonempty, then touch all files in the directory, respectively.
-        exec { "touch-${source}-files":
-            command  => "${check_files} ${touch_files}",
+            ## touch source: ensure initial build compiles every source file.
+            #
+            #  @touch, changes the modification time to the current system time.
+            #
+            #  Note: the current inotifywait implementation watches close_write,
+            #        move, and create. However, the source files will already exist
+            #        before this 'inotifywait', since the '/vagrant' directory will
+            #        already have been mounted on the initial build.
+            #
+            #  Note: every 'command' implementation checks if directory is
+            #        nonempty, then touch all files in the directory, respectively.
+            exec { "touch-${source}-files":
+                command  => "${check_files} ${touch_files}",
+                provider => shell,
+            }
+        }
+    }
+    ## docker container
+    else {
+        ## manually compile jsx asset, since first pass through via
+        # 'start_uglifyjs.pp' does not have adequate scope resolution.
+        exec { 'rerun-uglifyjs':
+            command  => "./uglifyjs ${root_dir}",
+            cwd      => "${dev_env_path}/modules/compiler/scripts",
+            path     => '/usr/bin',
             provider => shell,
         }
     }

puppet/environment/development/modules/compiler/manifests/start_browserify.pp

@@ -3,8 +3,27 @@
 ###       https://github.com/jeff1evesque/machine-learning/issues/2349
 ###
 class compiler::start_browserify {
-    service { 'browserify':
-        ensure => 'running',
-        enable => true,
+    # variables
+    $hiera_general   = hiera('general')
+    $root_dir        = $hiera_general['root']
+    $vagrant_mounted = $hiera_general['vagrant_implement']
+    $dev_env_path    = "${root_dir}/puppet/environment/development"
+
+    # run sass
+    if $vagrant_mounted {
+        # ensure service starts at boot
+        service { 'browserify':
+            ensure => 'running',
+            enable => true,
+        }
+    }
+    else {
+        # manually compile
+        exec { 'browserify':
+            command  => "./browserify ${root_dir}",
+            cwd      => "${dev_env_path}/modules/compiler/scripts",
+            path     => '/usr/bin',
+            provider => shell,
+        }
     }
 }

puppet/environment/development/modules/compiler/manifests/start_imagemin.pp

@@ -3,8 +3,26 @@
 ###       https://github.com/jeff1evesque/machine-learning/issues/2349
 ###
 class compiler::start_imagemin {
-    service { 'imagemin':
-        ensure => 'running',
-        enable => true,
+    # variables
+    $hiera_general   = hiera('general')
+    $root_dir        = $hiera_general['root']
+    $vagrant_mounted = $hiera_general['vagrant_implement']
+    $dev_env_path    = "${root_dir}/puppet/environment/development"
+
+    # run imagemin
+    if $vagrant_mounted {
+        # ensure service starts at boot
+        service { 'imagemin':
+            ensure => 'running',
+            enable => true,
+        }
+    }
+    else {
+        # manually compile
+        exec { 'imagemin':
+            command  => "./imagemin ${root_dir}",
+            cwd      => "${dev_env_path}/modules/compiler/scripts",
+            provider => shell,
+        }
     }
 }

puppet/environment/development/modules/compiler/manifests/start_sass.pp

@@ -3,8 +3,26 @@
 ###       https://github.com/jeff1evesque/machine-learning/issues/2349
 ###
 class compiler::start_sass {
-    service { 'sass':
-        ensure => 'running',
-        enable => true,
+    # variables
+    $hiera_general   = hiera('general')
+    $root_dir        = $hiera_general['root']
+    $vagrant_mounted = $hiera_general['vagrant_implement']
+    $dev_env_path    = "${root_dir}/puppet/environment/development"
+
+    # run sass
+    if $vagrant_mounted {
+        # ensure service starts at boot
+        service { 'sass':
+            ensure => 'running',
+            enable => true,
+        }
+    }
+    else {
+        # manually compile
+        exec { 'sass':
+            command  => "./sass ${root_dir}",
+            cwd      => "${dev_env_path}/modules/compiler/scripts",
+            provider => shell,
+        }
     }
 }