updates ci subcommand to install singularity (#708)

fixes #702

ci/test/ci

@@ -6,38 +6,47 @@ init_test_repo
 
 cd "$test_repo_path"
 
-popper ci --service travis
+popper ci travis
 test -f .travis.yml
 cat .travis.yml
 
-popper ci --service circle
+popper ci circle
 test -f .circleci/config.yml
 cat .circleci/config.yml
 
-popper ci --service jenkins
+popper ci jenkins
 test -f Jenkinsfile
 cat Jenkinsfile
 
-popper ci --service gitlab
+popper ci gitlab
 test -f .gitlab-ci.yml
 cat .gitlab-ci.yml
 
-popper ci --service brigade
+popper ci brigade
 test -f brigade.js
 cat brigade.js
 
-popper ci --service travis --with-singularity
+popper ci travis --install singularity
+test -f .travis.yml
+test -f scripts/install_scripts.sh
 cat .travis.yml
+cat scripts/install_scripts.sh
 
-popper ci --service circle --with-singularity
+popper ci circle --install singularity
+test -f .circleci/config.yml
+test -f scripts/install_scripts.sh
 cat .circleci/config.yml
+cat scripts/install_scripts.sh
 
-popper ci --service brigade --with-singularity
+popper ci brigade --install singularity
+test -f brigade.js
+test -f scripts/install_scripts.sh
 cat brigade.js
+cat scripts/install_scripts.sh
 
-! popper ci --service jenkins --with-singularity
+! popper ci jenkins --install singularity
 
-! popper ci --service gitlab --with-singularity
+! popper ci gitlab --install singularity
 
 
 # Test blacklisting/skip in CI environment

cli/popper/commands/cmd_ci.py

@@ -1,16 +1,40 @@
 import os
+import stat
 from string import Template
 
 import click
 
 from popper import scm, utils as pu
 from popper.cli import pass_context, log
 
-install_scripts = {
-    'singularity': """wget http://neurodeb.pirsquared.org/\
-pool/main/s/singularity-container/\
-singularity-container_2.6.1-2~nd16.04+1_amd64.deb && \
-sudo apt-get -f install ./singularity-container_2.6.1-2~nd16.04+1_amd64.deb"""}
+base_script_content = """
+#!/bin/bash
+set -ex
+"""
+
+install_scripts_content = {
+    'singularity': """
+sudo apt-get update
+sudo apt-get install -y build-essential \
+                        libssl-dev \
+                        uuid-dev \
+                        libgpgme11-dev \
+                        libseccomp-dev \
+                        pkg-config \
+                        squashfs-tools
+mkdir -p ${GOPATH}/src/github.com/sylabs
+cd ${GOPATH}/src/github.com/sylabs
+git clone https://github.com/sylabs/singularity.git
+cd singularity
+git checkout v3.2.0
+cd ${GOPATH}/src/github.com/sylabs/singularity
+./mconfig
+cd ./builddir
+make
+sudo make install
+singularity version
+cd $TRAVIS_BUILD_DIR
+"""}
 
 ci_files = {
     'travis': {
@@ -20,11 +44,11 @@
 language: python
 python: 3.7
 services: docker
+$install_scripts
 install:
 - git clone https://github.com/systemslab/popper /tmp/popper
 - export PYTHONUNBUFFERED=1
 - pip install /tmp/popper/cli
-$install_scripts
 script: popper run
 """)
     },
@@ -39,10 +63,10 @@
     - checkout
     - run:
         command: |
+        $install_scripts
         git clone https://github.com/systemslab/popper /tmp/popper
         export PYTHONUNBUFFERED=1
         pip install /tmp/popper/cli
-        $install_scripts
         popper run
 """)
     },
@@ -88,10 +112,10 @@
     popper.tasks = [
         "apt-get update",
         "apt-get install -y git",
+        $install_scripts
         "git clone https://github.com/systemslab/popper /tmp/popper",
         "export PYTHONUNBUFFERED=1",
         "pip install /tmp/popper/cli",
-        $install_scripts
         "popper run"
     ]
     popper.run()
@@ -102,26 +126,24 @@
 
 
 @click.command('ci', short_help='Generate CI service configuration files.')
-@click.option(
-    '--service',
-    help='Name of CI service for which config files get generated.',
+@click.argument(
+    'service',
     type=click.Choice(['travis', 'circle', 'jenkins', 'gitlab', 'brigade']),
     required=True
 )
 @click.option(
-    '--with-singularity',
-    help='Add singularity install scripts in generated config files.',
+    '--install',
+    help='Specify the runtime dependencies to install.',
     required=False,
-    is_flag=True
+    type=click.Choice(['singularity']),
+    default=list(),
+    multiple=True
 )
 @pass_context
-def cli(ctx, service, with_singularity):
+def cli(ctx, service, install):
     """Generates configuration files for distinct CI services. This command
     needs to be executed on the root of your Git repository folder.
     """
-    if service not in ci_files:
-        log.fail("Unrecognized service " + service)
-
     project_root = scm.get_git_root_folder()
 
     if project_root != os.getcwd():
@@ -130,35 +152,51 @@ def cli(ctx, service, with_singularity):
             'Git project folder (where the .git/ folder is located).')
 
     for ci_file, ci_file_content in pu.get_items(ci_files[service]):
-        ci_file_content = ci_file_content
+
+        # Prepare and write the CI config file.
         ci_file = os.path.join(project_root, ci_file)
-        # Create parent folder
         if not os.path.isdir(os.path.dirname(ci_file)):
             os.makedirs(os.path.dirname(ci_file))
 
-        # Customize content
-        scripts = []
-        if with_singularity:
-            if service in ['jenkins', 'gitlab']:
+        install_script_cmd = ''
+        if install:
+            if service == 'jenkins' or service == 'gitlab':
                 log.fail(
-                    'Scaffolding of Singularity install script is not '
+                    'Scaffolding of custom install scripts is not '
                     'supported for Jenkins and Gitlab CI. Include it '
                     'manually depending upon the CI\'s OS.')
-            scripts.append(install_scripts['singularity'])
-
-        if scripts:
-            scripts = ' && '.join(scripts)
-            if service == 'travis':
-                scripts = '- ' + scripts
-            if service == 'brigade':
-                scripts = '"' + scripts + '",'
-        else:
-            scripts = ''
-
-        # Write content
+
+            elif service == 'travis':
+                install_script_cmd = ('before_install: scripts/'
+                                      'install_scripts.sh')
+
+            elif service == 'circle':
+                install_script_cmd = 'bash scripts/install_scripts.sh'
+
+            elif service == 'brigade':
+                install_script_cmd = '"bash scripts/install_scripts.sh",'
+
         with open(ci_file, 'w') as f:
             f.write(reformat(ci_file_content.safe_substitute(
-                {'install_scripts': scripts})))
+                {'install_scripts': install_script_cmd})))
+
+        # Prepare and Write the install scripts.
+        if install:
+            install = set(install)
+            install_script_file = os.path.join(
+                project_root, 'scripts', 'install_scripts.sh')
+            script_content = base_script_content
+            for runtime in install:
+                script_content += install_scripts_content[runtime]
+
+            if not os.path.isdir(os.path.dirname(install_script_file)):
+                os.makedirs(os.path.dirname(install_script_file))
+
+            with open(install_script_file, 'w') as f:
+                f.write(script_content)
+
+            st = os.stat(install_script_file)
+            os.chmod(install_script_file, st.st_mode | stat.S_IEXEC)
 
     log.info('Wrote {} configuration successfully.'.format(service))