updates to ci process including docker tpl builds

.circleci/config.yml

@@ -1,14 +1,14 @@
-version: 2
-jobs:
-  build:
-    docker:
-      - image: visitdav/visit-ci-develop:current
-    steps:
-      - checkout
-      - run:
-          name: Submodule Init
-          command: git submodule update --init --recursive
-      - run:
-          name: Test Build
-          command: scripts/ci/circle/circle-test-visit-ci-develop.sh 
+version: 2	
+jobs:	
+  build:	
+    docker:	
+      - image: visitdav/visit-ci-develop:current	
+    steps:	
+      - checkout	
+      - run:	
+          name: Submodule Init	
+          command: git submodule update --init --recursive	
+      - run:	
+          name: Test Build	
+          command: scripts/ci/circle/circle-test-visit-ci-develop.sh 	
           no_output_timeout: 30m

.gitignore

@@ -10,5 +10,6 @@ build_branch_*/
 /src/test/results.json
 /src/test/tests.json
 visit.masonry.docker.src.tar
+visit.build_visit.docker.src.tar
 build-mb-*
 _logs

azure-pipelines.yml

@@ -0,0 +1,90 @@
+# Copyright (c) Lawrence Livermore National Security, LLC and other VisIt
+# Project developers.  See the top-level LICENSE file for dates and other
+# details.  No copyright assignment is required to contribute to VisIt.
+
+###############################################################################
+# VisIt Azure CI Checks
+###############################################################################
+# Azure Commands Ref:
+# https://aka.ms/yaml
+
+#####
+# TO USE A NEW CONTAINER, UPDATE TAG NAME HERE AS PART OF YOUR PR!
+#####
+variables:
+  container_tag: visitdav/visit-ci-develop:2020-08-24-shac49c21
+
+# only build merge target pr to develop
+trigger: none
+pr:
+  branches:
+    include:
+    - develop
+
+# add fast fail checks here
+stages:
+- stage: Quick
+  jobs:
+  - job: Fast_Check
+    pool:
+      vmImage: 'ubuntu-16.04'
+    steps:
+      - checkout: self
+        clean: boolean
+        submodules: recursive
+
+      - script: |
+          #######################################
+          # run our quick check
+          #######################################
+          echo "OK"
+          # NOTE FOR AZURE: 
+          # only return code of the command is checked!
+        displayName: 'Example Fast Check'
+
+# main checks build using docker containers that
+# include tpls built with build_visit
+- stage: Main
+  jobs:
+  - job: Main_Build
+    pool:
+      vmImage: 'ubuntu-16.04'
+    timeoutInMinutes: 0
+    container: ${{ variables.container_tag }}
+    variables:
+      TPLS_PATH: /masonry/build-mb-develop-ci-smoke/thirdparty_shared/third_party/
+      CMAKE_VERSION: 3.9.3
+    steps:
+      - checkout: self
+        clean: boolean
+        submodules: recursive
+
+      - script: |
+           #################################
+           # configure
+           #################################
+           # setup path to cmake
+           export PATH=$PATH:${TPLS_PATH}/cmake/${CMAKE_VERSION}/ci/bin/
+           # make dir for build
+           mkdir build
+           cd build
+           # configure with cmake
+           cmake -DVISIT_CONFIG_SITE:PATH="/visit-ci-cfg.cmake" ../src
+        displayName: 'Configure with CMake'
+
+      - script: |
+           #################################
+           # build
+           #################################
+           # build
+           cd build
+           make -j 4
+        displayName: 'Build'
+
+      # - script: |
+      #      #################################
+      #      # install
+      #      #################################
+      #      cd build
+      #      make install
+      #   displayName: 'Install'

scripts/ci/docker/Dockerfile

@@ -1,3 +1,6 @@
+# Copyright (c) Lawrence Livermore National Security, LLC and other VisIt
+# Project developers.  See the top-level LICENSE file for dates and other
+# details.  No copyright assignment is required to contribute to VisIt.
 FROM ubuntu:xenial
 MAINTAINER Cyrus Harrison <cyrush@llnl.gov>
 
@@ -21,6 +24,7 @@ RUN apt-get update && apt-get install -y \
     libx11-xcb-dev \
     libxcb-dri2-0-dev \
     libxcb-xfixes0-dev \
+    libffi-dev \
     xutils-dev \
     xorg-dev \
     libfreetype6-dev \
@@ -35,16 +39,26 @@ RUN apt-get update && apt-get install -y \
 
 RUN cd /usr/include && ln -s freetype2 freetype
 
-RUN groupadd -r ci && useradd -ms /bin/bash --no-log-init -r -g ci ci
-USER ci
-WORKDIR /home/ci
+#  -- Azure will add its own user, so skip this -- 
+# RUN groupadd -r ci && useradd -ms /bin/bash --no-log-init -r -g ci ci
+# USER ci
+# WORKDIR /home/ci
 
-# untar the current masonry source  (created as part of build_docker_visit_ci.sh)
-COPY visit.masonry.docker.src.tar /home/ci
+
+# untar the current masonry source  (created as part of build_docker_visit_ci.py)
+COPY visit.masonry.docker.src.tar /
 RUN tar -xzf visit.masonry.docker.src.tar
-RUN rm -rf visit.masonry.docker.src.tar
+# untar the current build_visit source (created as part of build_docker_visit_ci.py)
+RUN mkdir -p /masonry/build-mb-develop-ci-smoke/visit/src/tools/dev/scripts/
+COPY visit.build_visit.docker.src.tar /masonry
+COPY masonry_docker_ci_cleanup.py /
+RUN cd /masonry/build-mb-develop-ci-smoke/visit/src/tools/dev/scripts/ && tar -xzf /masonry/visit.build_visit.docker.src.tar
+
 # call masonry to build tpls
 RUN cd masonry && python bootstrap_visit.py opts/mb-develop-ci-smoke.json
+
 # cleanup extract build dirs and obtain the config site file
-COPY masonry_docker_ci_cleanup.py /home/ci
 RUN python masonry_docker_ci_cleanup.py
+# change perms
+RUN chmod -R a+rX /masonry
+

scripts/ci/docker/build_docker_visit_ci.py

@@ -0,0 +1,100 @@
+# Copyright (c) Lawrence Livermore National Security, LLC and other VisIt
+# Project developers.  See the top-level LICENSE file for dates and other
+# details.  No copyright assignment is required to contribute to VisIt.
+
+#
+# Helper script that drives our docker build and tag for ci
+#
+
+import os
+import sys
+import subprocess
+import shutil
+import datetime
+
+from os.path import join as pjoin
+
+def remove_if_exists(path):
+    """
+    Removes a file system path if it exists.
+    """
+    if os.path.isfile(path):
+        os.remove(path) 
+    if os.path.isdir(path):
+        shutil.rmtree(path)
+
+def timestamp(t=None,sep="-"):
+    """ Creates a timestamp that can easily be included in a filename. """
+    if t is None:
+        t = datetime.datetime.now()
+    sargs = (t.year,t.month,t.day)
+    sbase = "".join(["%04d",sep,"%02d",sep,"%02d"])
+    return  sbase % sargs
+
+def sexe(cmd,ret_output=False,echo = True):
+    """ Helper for executing shell commands. """
+    if echo:
+        print("[exe: {}]".format(cmd))
+    if ret_output:
+        p = subprocess.Popen(cmd,
+                             shell=True,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.STDOUT)
+        res = p.communicate()[0]
+        res = res.decode('utf8')
+        return p.returncode,res
+    else:
+        return subprocess.call(cmd,shell=True)
+
+def git_hash():
+    """
+    Returns the current git repo hash, or UNKNOWN
+    """
+    res = "UNKNOWN"
+    rcode,rout = sexe("git rev-parse HEAD",ret_output=True)
+    if rcode == 0:
+        res = rout
+    return res;
+
+def gen_docker_tag():
+    """
+    Creates a useful docker tag for the current build.
+    """
+    ghash = git_hash()
+    if ghash != "UNKNOWN":
+        ghash = ghash[:6]
+    return timestamp() + "-sha" + ghash
+
+def main():
+    # clean up tarballs we push into the container
+    remove_if_exists("visit.masonry.docker.src.tar")
+    remove_if_exists("visit.build_visit.docker.src.tar")
+
+    # save current working dir so we can get back here
+    orig_dir = os.path.abspath(os.getcwd())
+
+    # move to dir with masonry and build_viist
+    os.chdir("../../../src/tools/dev")
+
+    # get current copy of masonry
+    cmd ='tar -czvf {0} --exclude "build-*" --exclude "*.pyc" masonry'
+    sexe(cmd.format(pjoin(orig_dir, "visit.masonry.docker.src.tar")))
+
+    # get current copy of build_visit from this branch
+    os.chdir("scripts")
+    cmd = 'tar -czvf {0} --exclude "build-*" --exclude "*.pyc" build_visit bv_support' 
+    sexe(cmd.format(pjoin(orig_dir, "visit.build_visit.docker.src.tar")))
+
+    # change back to orig working dir
+    os.chdir(orig_dir)
+
+    # exec docker build to create image
+    # note: --squash requires docker runtime with experimental 
+    # docker features enabled. It combines all the layers into
+    # a more compact final image to save disk space.
+    # tag with date + git hash
+    sexe('docker build -t visitdav/visit-ci-develop:{0} . --squash'.format(gen_docker_tag()))
+
+if __name__ == "__main__":
+    main()
+

scripts/ci/docker/masonry_docker_ci_cleanup.py

@@ -1,3 +1,7 @@
+# Copyright (c) Lawrence Livermore National Security, LLC and other VisIt
+# Project developers.  See the top-level LICENSE file for dates and other
+# details.  No copyright assignment is required to contribute to VisIt.
+
 #
 # Helper script to cleanup tpl build dirs to reduce
 # our docker image size
@@ -9,18 +13,13 @@
 
 from os.path import join as pjoin
 
-def home_dir():
-    return "/home/ci/"
-
 def tpl_build_dir():
-    return pjoin(home_dir(),"masonry/build-mb-develop-ci-smoke/thirdparty_shared")
+    return "/masonry/build-mb-develop-ci-smoke/thirdparty_shared"
 
 def check_to_keep(path):
     res = False
     if path.count("third_party") > 0:
         res = True
-    elif path.endswith(".cmake"):
-        res = True
     return res
 
 def cleanup_tpl_build_dirs():
@@ -35,13 +34,14 @@ def cleanup_tpl_build_dirs():
 
 def copy_config_site():
      cfg_site = glob.glob( pjoin(tpl_build_dir(),"*.cmake") )[0]
-     dest = pjoin(home_dir(),"visit-ci-cfg.cmake")
+     dest = "/visit-ci-cfg.cmake"
      print "[copying %s to %s]" % (cfg_site,dest)
      shutil.copyfile(cfg_site,dest)
 
 def main():
-    cleanup_tpl_build_dirs()
     copy_config_site()
+    cleanup_tpl_build_dirs()
+
 
 if __name__ == "__main__":
     main()

src/doc/dev_manual/CI.rst

@@ -0,0 +1,47 @@
+Docker Containers For CI Testing
+=================================
+
+We use Azure Pipelines for CI testing Visit's Pull Requests.
+
+* `Ascent Azure DevOps Space <https://dev.azure.com/visit-dav/VisIt/>`_
+
+
+To speed up our CI testing we use Docker containers with pre-built third party
+libraries (TPLs). These containers leverage our ``build_visit`` third party
+build process. The Docker files and build scripts used to create 
+these containers are in ``scripts/ci/docker``. To update the containers
+(assuming you have Docker installed):
+
+ * Run ``python build_docker_visit_ci.py`` to build and tag a new version
+   of our TPL container.
+
+  This tag will include today's day and a short substring of the
+  current git hash. 
+  Example Tag: ``visitdav/visit-ci-develop:2020-08-24-shac49c21``
+
+ * Run ``docker push <container-name>`` to push the container image
+   to `VisIt's DockerHub Registry <https://hub.docker.com/orgs/visitdav>`_.
+
+  If you do not already have a DockerHub account, go
+  `here <https://hub.docker.com/signup >`_  and sign up for one. Then
+  contact another member of `visitdav <https://hub.docker.com/orgs/visitdav>`_
+  and ask to be added to the organization.
+
+  You will need to be logged into DockerHub to successfully push. If you
+  are not logged in the process will ask for your DockerHub username
+  and password on the command line.
+  Example Push Command:
+   ``docker push visitdav/visit-ci-develop:2020-08-24-shac49c21``
+
+ * To change which Docker Image is used by Azure, edit ``azure-pipelines.yml``
+   and change `container_tag` variable. ::
+
+    #####
+    # TO USE A NEW CONTAINER, UPDATE TAG NAME HERE AS PART OF YOUR PR!
+    #####
+    variables:
+      container_tag: visitdav/visit-ci-develop:2020-08-24-shac49c21
+
+When the PR is merged, the azure changes will be merged and PRs to develop 
+will now use the new container.
+