Serverspec fixes (#165)

* Fix parallel execution environment
* Add more verbose messages for spec init/run
* Reduce python cpu usage for wait loop
* Set threads to auto for test target

Makefile

@@ -36,7 +36,7 @@ requirements:
 	cd tests/serverspec && bundle install --path=vendor
 
 test:
-	python bin/console test:serverspec --threads=auto/2 -v
+	python bin/console test:serverspec --threads=auto -v
 
 baselayout:
 	python bin/console generate:provision --baselayout

bin/webdevops/Command.py

@@ -18,7 +18,7 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import os, subprocess, tempfile
+import os, subprocess, tempfile, copy, time
 
 def execute(cmd, cwd=False, env=None):
     """
@@ -27,9 +27,20 @@ def execute(cmd, cwd=False, env=None):
 
     print 'Execute: %s' % ' '.join(cmd)
 
-    # remove _ from env (prevent errors)
-    if env is not None and '_' in env:
-        del env['_']
+    if env is not None:
+        env = copy.deepcopy(env)
+        env['PWD'] = os.environ['PWD']
+        env['LC_CTYPE'] = os.environ['LC_CTYPE']
+        env['SHELL'] = os.environ['SHELL']
+
+        if '_' in env:
+            del env['_']
+
+        # add system env vars
+        system_env_vars = ['PWD', 'PATH', 'LC_CTYPE', 'SHELL', 'DOCKER_HOST', 'DOCKER_CERT_PATH', 'DOCKER_MACHINE_NAME', 'DOCKER_TLS_VERIFY']
+        for var_name in system_env_vars:
+            if not var_name in env and var_name in os.environ:
+                env[var_name] = os.environ[var_name]
 
     # set current working directory
     path_current = os.getcwd()
@@ -51,7 +62,7 @@ def execute(cmd, cwd=False, env=None):
 
     # wait for process end
     while proc.poll() is None:
-        pass
+        time.sleep(1)
 
     # output stdout
     with open(file_stdout.name, 'r') as f:

bin/webdevops/command/DoitCommand.py

@@ -28,7 +28,7 @@ def run_doit(self, task_loader, configuration):
         extra_configuration = {}
 
         if 'threads' in configuration and configuration.get('threads') > 1:
-            arguments.extend(['-n', str(configuration.get('threads'))])
+            arguments.extend(['-n', str(configuration.get('threads')), '--parallel-type', 'process'])
 
         if 'doitConfig' in configuration:
             extra_configuration = configuration.get('doitConfig')

bin/webdevops/taskloader/DockerTestServerspecTaskLoader.py

@@ -18,7 +18,7 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import os, re, tempfile, json
+import os, re, tempfile, json, base64
 from webdevops import Command
 from .BaseDockerTaskLoader import BaseDockerTaskLoader
 from .BaseTaskLoader import BaseTaskLoader
@@ -75,18 +75,18 @@ def task_run(dockerfile, configuration, task):
         tmp_suffix = '.%s_%s_%s.tmp' % (dockerfile['image']['repository'], dockerfile['image']['imageName'], dockerfile['image']['tag'])
         test_dockerfile = tempfile.NamedTemporaryFile(prefix='Dockerfile.', suffix=tmp_suffix, dir=configuration.get('serverspecPath'), bufsize=0, delete=False)
 
-        # serverspec options
-        serverspec_opts = []
-        serverspec_opts.extend(['--pattern', spec_path])
-
-        # serverspec env
-        serverspec_env = DockerTestServerspecTaskLoader.generate_serverspec_environment(
+        # serverspec conf
+        serverspec_conf = DockerTestServerspecTaskLoader.generate_serverspec_configuration(
             path=os.path.basename(test_dockerfile.name),
             dockerfile=dockerfile,
             configuration=configuration,
             is_toolimage=is_toolimage
         )
 
+        # serverspec options
+        serverspec_opts = []
+        serverspec_opts.extend([spec_path, dockerfile['image']['fullname'], base64.b64encode(json.dumps(serverspec_conf)), os.path.basename(test_dockerfile.name)])
+
         # dockerfile content
         dockerfile_content = DockerTestServerspecTaskLoader.generate_dockerfile(
             dockerfile=dockerfile,
@@ -103,15 +103,13 @@ def task_run(dockerfile, configuration, task):
             print '          path: %s' % (spec_path)
             print '          args: %s' % (' '.join(serverspec_opts))
             print ''
-            print 'environment:'
-            print '------------'
-            print json.dumps(serverspec_env, indent=4, sort_keys=True)
+            print 'spec configuration:'
+            print '-------------------'
+            print json.dumps(serverspec_conf, indent=4, sort_keys=True)
             print ''
             print 'Dockerfile:'
             print '-----------'
             print dockerfile_content
-
-            os.remove(test_dockerfile.name)
             return True
 
         # check if we have any tests
@@ -123,10 +121,6 @@ def task_run(dockerfile, configuration, task):
         cmd = ['bash', 'serverspec.sh']
         cmd.extend(serverspec_opts)
 
-        # Set environment variables
-        env = os.environ.copy()
-        env.update(serverspec_env)
-
         # create Dockerfile
         with open(test_dockerfile.name, mode='w', buffering=0) as f:
             f.write(dockerfile_content)
@@ -137,7 +131,7 @@ def task_run(dockerfile, configuration, task):
         test_status = False
         for retry_count in range(0, configuration.get('retry')):
             try:
-                test_status = Command.execute(cmd, cwd=configuration.get('serverspecPath'), env=env)
+                test_status = Command.execute(cmd, cwd=configuration.get('serverspecPath'))
             except Exception as e:
                 print e
                 pass
@@ -149,23 +143,22 @@ def task_run(dockerfile, configuration, task):
             else:
                 print '    failed, giving up'
 
-        os.remove(test_dockerfile.name)
         return test_status
 
     @staticmethod
-    def generate_serverspec_environment(path, dockerfile, configuration, is_toolimage=False):
+    def generate_serverspec_configuration(path, dockerfile, configuration, is_toolimage=False):
         """
-        Generate serverspec environment dict
+        Generate serverspec configuration dict
         """
         ret = {}
 
         # add default vars
-        default_env_list = configuration.get('dockerTest.environment.default', False)
+        default_env_list = configuration.get('dockerTest.configuration.default', False)
         if default_env_list:
             ret = default_env_list.to_dict().copy()
 
         # add docker image specific vars
-        image_env_list = configuration.get('dockerTest.environment.image')
+        image_env_list = configuration.get('dockerTest.configuration.image')
         if image_env_list:
             image_env_list = image_env_list.to_dict().copy()
             for term in image_env_list:
@@ -191,6 +184,7 @@ def generate_dockerfile(dockerfile, configuration, is_toolimage=False):
 
         ret.append('FROM %s' % dockerfile['image']['fullname'])
         ret.append('COPY conf/ /')
+        ret.append('RUN echo "%s" > /DOCKER.IMAGENAME' % dockerfile['image']['fullname'])
 
         if is_toolimage:
             ret.append('RUN chmod +x /loop-entrypoint.sh')

conf/console.yml

@@ -7,6 +7,7 @@ imagePath: 'documentation/docs/resources/images/'
 baselayoutPath: 'baselayout'
 testinfraPath: 'tests/testinfra'
 serverspecPath: 'tests/serverspec'
+testDockerfilePath: 'tests/dockerfile'
 blacklistFile: 'BLACKLIST'
 
 docker:
@@ -35,7 +36,7 @@ dockerTest:
     '/varnish':
       - 'ENV VARNISH_BACKEND_HOST webdevops.io'
 
-  environment:
+  configuration:
     default:
       OS_FAMILY: 'ubuntu'
       OS_VERSION: '16.04'

tests/serverspec/Rakefile

@@ -1,13 +1,36 @@
-require 'rake'
-require 'rspec/core/rake_task'
-
-RSpec.configure do |config|
-  # show retry status in spec process
-  config.verbose_retry = true
-  # show exception that triggers a retry if verbose_retry is set to true
-  config.display_try_failure_messages = true
-end
+begin
+    require 'rake'
+    require 'json'
+    require 'base64'
+    require 'docker'
+    require 'rspec'
+    require 'serverspec'
+    require 'rspec/retry'
+    require 'rspec/core/rake_task'
+
+    RSpec::Core::RakeTask.new(:spec, :pattern, :dockerImageName, :configuration) do |t, task_args|
+        t.pattern = task_args[:pattern]
+        t.fail_on_error = true
+
+        # hide command
+        t.verbose = false
+
+        $specConfiguration = JSON.parse(Base64.decode64(task_args[:configuration]))
+
+        $dockerImage = Docker::Image.build_from_dir('.', { 'dockerfile' => $specConfiguration['DOCKERFILE'] })
+        $specConfiguration['DOCKERIMAGE_ID'] = $dockerImage.id
+
+        print "\n"
+        print "Environment configuration\n"
+        print "-------------------------\n"
+        $specConfiguration.each do |key, value|
+               print " " + key + ':' + value + "\n"
+               ENV[key] = value
+        end
+        print "\n"
+        print "\n"
+   end
 
-RSpec::Core::RakeTask.new(:"spec") do |t, args|
-  #t.pattern = 'spec/base/*_spec.rb'
+rescue LoadError
+    # no rspec available
 end

tests/serverspec/serverspec.sh

@@ -1,51 +1,19 @@
 #!/usr/bin/env bash
 
-# Check if DOCKERFILE is set, needed for test
-if [[ -z "$DOCKERFILE" ]]; then
-    echo "Environment variable 'DOCKERFILE' not set"
-    exit 1
-fi
+set -o pipefail  ## trace ERR through pipes
+set -o errtrace  ## trace ERR through 'time command' and other functions
+set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable
+set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
 
-# Check if dockerfile exists
-if [[ ! -f "${DOCKERFILE}" ]]; then
-    # Filesystem is maybe not synced?
-    sync
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-    # recheck if file is now available
-    if [[ ! -f "${DOCKERFILE}" ]]; then
-        echo "Dockerfile $DOCKERFILE' not found"
-        exit 1
-    fi
-fi
+PARAM_SPEC_FILE="$1"
+PARAM_DOCKER_IMAGE="$2"
+PARAM_SPEC_CONF="$3"
 
-# Check if DOCKER_IMAGE is set, needed for test
-if [[ -z "$DOCKER_IMAGE" ]]; then
-    echo "Environment variable 'DOCKER_IMAGE' not set"
-    exit 1
-fi
+# LOGFILE="${PARAM_DOCKER_IMAGE//:/_}"
+# LOGFILE="${PARAM_DOCKER_IMAGE//\//_}"
+# LOGFILE="${SCRIPT_DIR}/logs/${LOGFILE}.log"
 
-# Check if DOCKER_TAG is set, needed for test
-if [[ -z "$DOCKER_TAG" ]]; then
-    echo "Environment variable 'DOCKER_TAG' not set"
-    exit 1
-fi
+exec bundle exec rake spec["$PARAM_SPEC_FILE","$PARAM_DOCKER_IMAGE","$PARAM_SPEC_CONF"]
 
-# Check if OS_FAMILY is set, needed for test
-if [[ -z "$OS_FAMILY" ]]; then
-    echo "Environment variable 'OS_FAMILY' not set"
-    exit 1
-fi
-
-# Check if OS_FAMILY is set, needed for test
-if [[ -z "$OS_VERSION" ]]; then
-    echo "Environment variable 'OS_FAMILY' not set"
-    exit 1
-fi
-
-echo "Starting serverspec"
-echo "            OS: ${OS_FAMILY} Version ${OS_VERSION}"
-echo "  Docker image: ${DOCKER_IMAGE}:${DOCKER_TAG}"
-echo "    Dockerfile: ${DOCKERFILE}"
-echo ""
-
-exec bundle exec rspec "$@"

tests/serverspec/spec/docker/ansible_spec.rb

@@ -1,11 +1,10 @@
 require 'serverspec'
 require 'docker'
-require 'spec_helper'
+require 'spec_init'
 
 describe "Dockerfile" do
     before(:all) do
-        @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] })
-        set :docker_image, @image.id
+        set :docker_image, ENV['DOCKERIMAGE_ID']
     end
 
     include_examples 'collection::bootstrap'

tests/serverspec/spec/docker/apache-dev_spec.rb

@@ -1,11 +1,10 @@
 require 'serverspec'
 require 'docker'
-require 'spec_helper'
+require 'spec_init'
 
 describe "Dockerfile" do
     before(:all) do
-        @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] })
-        set :docker_image, @image.id
+        set :docker_image, ENV['DOCKERIMAGE_ID']
     end
 
     include_examples 'collection::bootstrap'

tests/serverspec/spec/docker/apache_spec.rb

@@ -1,11 +1,10 @@
 require 'serverspec'
 require 'docker'
-require 'spec_helper'
+require 'spec_init'
 
 describe "Dockerfile" do
     before(:all) do
-        @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] })
-        set :docker_image, @image.id
+        set :docker_image, ENV['DOCKERIMAGE_ID']
     end
 
     include_examples 'collection::bootstrap'

tests/serverspec/spec/docker/base-app_spec.rb

@@ -1,11 +1,10 @@
 require 'serverspec'
 require 'docker'
-require 'spec_helper'
+require 'spec_init'
 
 describe "Dockerfile" do
     before(:all) do
-        @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] })
-        set :docker_image, @image.id
+        set :docker_image, ENV['DOCKERIMAGE_ID']
     end
 
     include_examples 'collection::bootstrap'

tests/serverspec/spec/docker/base_spec.rb

@@ -1,11 +1,10 @@
 require 'serverspec'
 require 'docker'
-require 'spec_helper'
+require 'spec_init'
 
 describe "Dockerfile" do
     before(:all) do
-        @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] })
-        set :docker_image, @image.id
+        set :docker_image, ENV['DOCKERIMAGE_ID']
     end
 
     include_examples 'collection::bootstrap'

tests/serverspec/spec/docker/bootstrap_spec.rb

@@ -1,11 +1,10 @@
 require 'serverspec'
 require 'docker'
-require 'spec_helper'
+require 'spec_init'
 
 describe "Dockerfile" do
     before(:all) do
-        @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] })
-        set :docker_image, @image.id
+        set :docker_image, ENV['DOCKERIMAGE_ID']
     end
 
     include_examples 'collection::bootstrap'

tests/serverspec/spec/docker/certbot_spec.rb

@@ -1,11 +1,10 @@
 require 'serverspec'
 require 'docker'
-require 'spec_helper'
+require 'spec_init'
 
 describe "Dockerfile" do
     before(:all) do
-        @image = Docker::Image.build_from_dir('.', { 'dockerfile' => ENV['DOCKERFILE'] })
-        set :docker_image, @image.id
+        set :docker_image, ENV['DOCKERIMAGE_ID']
     end
 
     include_examples 'collection::bootstrap'