Merge pull request #2 from singularityware/development

Development

.github/SUPPORT.md

@@ -14,6 +14,27 @@ For issues with code (and especially if you need to share debug output) we recom
  - [Singularity Registry Issues](https://github.com/singularityhub/sregistry/issues): is the board for issues relevant to Singularity Registry
  - [Documentation Issues](https://github.com/singularityware/singularityware.github.io/issues): documentation questions, feedback, and suggestions should go here. Feel free to create an issue on a board and additionally request updated content here.
 
+### How do I ask for help?
+After you identify a bug, you should search the respective issue board for similar problems reported by other users. Another user may be facing the same issue, and you can add a +1 (in message or icon) to indicate to the maintainers that the issue is pressing for you as well. The squeaky wheel gets the grease!
+
+### How is time allocated to addressing issues?
+While we wish we could address every issue, there are only so many hours in the day. We rank issues based on the following questions:
+
+ 1. How many users are affected?
+ 2. Is there a proposed work-around?
+ 2. In how many instances does the proposed work-around fail?
+
+With these simple questions, we can ensure that work is directed and has the maximum impact! However, if your issue doesn't seem to be getting attention you can still move it along using some of the strategies discussed below.
+
+### What if my issue goes stale?
+Issues can go stale for a number of reasons. In the bullets below, we will review some of these reasons, along with strategies for managing them:
+
+ 1. *The issue needs a gentle reminder*. Try targeting a few people with a "`ping @username any thoughts about this?`" in the case that it was forgotten.
+ 2. *Was your issue properly explained*? You are much more likely to get help when you give clear instructions for reproducing the issue, and show effort on your part to think about what the problem might be. If possible, try to come up with a way to reproduce the issue that does not involve a special environment or exotic hardware. 
+ 3. *Is there broad need*? It could be that your issue isn't having a big enough impact for other users to warrant the time for the small development team. In this case, you might try implementing a suggested fix, and then asking for help with the details.
+ 4. *Is your issue scattered?* When many issues pile up on boards, it sometimes is the case that issues are duplicated. It's important to find these duplicates and merge them into one, because in finding the duplicate you find another user to talk to about the issue.
+ 5. *Does your issue need to have scope?* The idea of scoping an issue means framing it with respect to other components of the software. For example, if you have a feature request to see metadata about an object, you might frame that in the context of container introspection, and suggest an addition to the software that fits with the "inspect" command. A very powerful thing to do would be to open up an issue that (not only discusses your specific addition) but also opens up discussion to the general community for "How we can do introspection" better. Then create a set of issues and add them to a [Github milestone](https://help.github.com/articles/about-milestones/).  This kind of contribution is much more powerful than simply asking for something.
+
 ## Google Group
 You can reach the community quickly by way of joining our [Google Group](https://groups.google.com/a/lbl.gov/forum/#!forum/singularity).
 

.gitignore

@@ -33,6 +33,7 @@ missing
 singularity.spec
 src/util/config_defaults.h
 test.sh
+secbuildimg.sh
 
 src/action
 src/action-suid
@@ -51,9 +52,8 @@ src/verify
 src/stamp-h1
 src/include/stamp-h1
 src/.dirstamp
-src/sinit
-src/start
-src/start-suid
+src/wrapper
+src/wrapper-suid
 
 src/.deps/
 src/.libs/

.travis.yml

@@ -8,6 +8,7 @@ addons:
       - squashfs-tools
       - uuid-dev
       - libssl-dev
+      - libarchive-dev
 
 env:
   global:
@@ -24,9 +25,6 @@ matrix:
     - os: linux
       env: OS_TYPE=centos OS_VERSION=7
 
-services:
-  - docker
-
 before_install:
   - .travis/before_install
 

.travis/before_install

@@ -13,12 +13,8 @@ if [ -z "$OS_TYPE" ]; then
     exit
 fi
 
-# run docker as shown at
-#  https://djw8605.github.io/2016/05/03/building-centos-packages-on-travisci/
-
 sudo apt-get update
-echo 'DOCKER_OPTS="-H tcp://127.0.0.1:2375 -H unix:///var/run/docker.sock -s devicemapper"' | \
-        sudo tee /etc/default/docker > /dev/null
-sudo service docker restart
-sleep 5
-sudo docker pull ${OS_TYPE}:${OS_VERSION}
+
+./autogen.sh
+./configure --prefix=/usr/local
+sudo make install

.travis/centos_run

@@ -1,25 +1,21 @@
 #!/bin/bash -ex
 
-# This script starts docker and (on el7) systemd and runs a test
+DIST="${OS_TYPE}${OS_VERSION}"
 
-if [ "$OS_VERSION" = "6" ]; then
-    sudo docker run --privileged --rm=true -v `pwd`:/build:rw centos:${OS_VERSION} /bin/bash -exc "cd /build;.travis/rpmbuild_test $OS_VERSION"
-    exit
-fi
+mkdir /tmp/$DIST
 
-# Mount /var/run/docker.sock and set --network=host so we can call docker from inside
-# cause some tests need it. Cannot mount to /var/run/docker.sock inside cause CentOS
-# /usr/sbin/init mounts another overlayfs on top of it
-docker run --privileged -d -ti -e "container=docker" -v /var/run/docker.sock:/docker.sock --network=host \
-       -v /sys/fs/cgroup:/sys/fs/cgroup -v `pwd`:/build:rw  centos:${OS_VERSION}   /usr/sbin/init
-DOCKER_CONTAINER_ID=$(docker ps | grep centos | awk '{print $1}')
-docker logs $DOCKER_CONTAINER_ID
-docker exec -ti $DOCKER_CONTAINER_ID /bin/bash -exc "
-    export DOCKER_HOST=unix:///docker.sock
-    chmod o+rw /docker.sock &&
-    cd /build &&
-    .travis/rpmbuild_test $OS_VERSION
-"
-docker ps -a
-docker stop $DOCKER_CONTAINER_ID
-docker rm -v $DOCKER_CONTAINER_ID
+cp -a . /tmp/$DIST/singularity
+cat > "/tmp/$DIST/script" << SCRIPT
+#!/bin/bash
+
+echo "%_var /var" > /root/.rpmmacros
+echo "%_dbpath %{_var}/lib/rpm" >> /root/.rpmmacros
+
+.travis/rpmbuild_test $OS_VERSION
+SCRIPT
+
+cd /tmp/$DIST/singularity
+
+# mounting /run for /dev/shm (python multiprocessing)
+sudo singularity exec -w -B /run --keep-privs --allow-setuid docker://${OS_TYPE}:${OS_VERSION} /bin/bash -ex /tmp/$DIST/script
+sudo rm -rf /tmp/$DIST

.travis/install_test

@@ -3,4 +3,5 @@
 ./autogen.sh
 ./configure --prefix=/usr/local
 sudo make install
+sudo make secbuildimg
 make test

.travis/rpmbuild_test

@@ -1,12 +1,13 @@
 #!/bin/bash -ex
 
-# this script runs as root under docker 
+# this script runs as root under docker
 
 OS_VERSION="$1"
 
 # build and install
-yum install -y libtool rpm-build make squashfs-tools openssl-devel libuuid-devel
+yum install -y libtool rpm-build make yum-utils squashfs-tools
 ./autogen.sh
+yum-builddep -y singularity.spec
 ./configure
 make dist
 rpmbuild -ta *.tar.gz
@@ -30,4 +31,11 @@ sed -i 's,^localstatedir=.*,localstatedir=/var,' test.sh
 useradd testuser
 echo "Defaults:testuser env_keep=DOCKER_HOST" >>/etc/sudoers
 echo "testuser ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers
+
+# build image for isolated builds
+sed -i 's,^prefix=.*,prefix=/usr,' secbuildimg.sh
+sed -i 's,^sysconfdir=.*,sysconfdir=/etc,' secbuildimg.sh
+sed -i 's,^localstatedir=.*,localstatedir=/var,' secbuildimg.sh
+make secbuildimg
+
 su testuser -c "make test"

CHANGELOG.md

@@ -13,7 +13,37 @@ and changes prior to that are (unfortunately) done retrospectively. Critical ite
  - changed behaviour (recipe sections work differently)
 
 
-## [v2.4.1](https://github.com/singularityware/singularity/tree/development) (development)
+## [v3.0](https://github.com/singularityware/singularity/tree/development)
+
+### Implemented enhancements
+ - Feature sif sign #1143
+ - Boot/start instance #1032
+ - --nv option will use [nvidia-container-cli](https://github.com/NVIDIA/libnvidia-container) if installed
+ - [nvliblist.conf](https://github.com/singularityware/singularity/blob/master/etc/nvliblist.conf) now has a section for binaries
+ - --nv can be made default with all action commands in singularity.conf
+ - --nv can be controlled by env vars `$SINGULARITY_NV` and `$SINGULARITY_NV_OFF`
+ - Adjustments to SCIF (Scientific Filesystem) integration for broader use
+
+### Security related fixes
+ - Add capability support and secure build #934
+ - Put /usr/local/{bin,sbin} in front of the default PATH
+ - Add capability to support all tar compression formats #1155
+ - Handle docker layer aufs whiteout files correctly (requires libarchive).
+
+### Bug Fixes
+ - Put /usr/local/{bin,sbin} in front of the default PATH
+ - Fixed bug that did not export environment variables for apps with "-" in name
+
+## [v2.4.2](https://github.com/singularityware/singularity/tree/release-2.4)
+
+ - This fixed an issue for support of older distributions and kernels with regards to `setns()`
+   functionality.
+ - Fixed autofs bug path (lost during merge)
+
+## [v2.4.1](https://github.com/singularityware/singularity/tree/release-2.4) (2017-11-22)
+
+### apprun script backslash removal fix
+ - Fixed the unwanted removal of backslashes in apprun scripts
 
 ### Security related fixes
  - Fixed container path and owner limitations (original merge was lost)
@@ -25,7 +55,6 @@ and changes prior to that are (unfortunately) done retrospectively. Critical ite
  - Exposing labels for SCI-F in environment
 
 ### Bug Fixes
- - Tar creation uses GNU tar-1.28 to not add a timestamp to the header for more reproducible builds
  - Adjusting environment parsing regular expression for Docker to allow for "=" sign in variable
  - Try overlayFS now default option
  - Confirm that localstate directories were properly packaged
@@ -36,9 +65,13 @@ and changes prior to that are (unfortunately) done retrospectively. Critical ite
  - Fix for empty docker namespaces in private repositories
  - Fix Docker environment parsing
  - Revert lolcow easter egg
-
-
-
+ - Fix "Duplicate bootstrap definition key" triggered by comments and blank spaces
+ - Fix for docker permission error when downloading multiple layers
+ - Fix parsing of registry (including port), namespace, tags, and version
+ - Add "$@" to any CMD/ENTRYPOINT found when building from Docker
+ - Added sqaushfs-tools as a dependency for building deb files
+ - Fix terminal echo problem when using PID namespace and killing shell
+ - Fix SuSE squashFS package name in RPM spec
 
 ## [v2.4](https://github.com/singularityware/singularity/tree/v2.4) (2017-10-02)
 [Full Changelog](https://github.com/singularityware/singularity/compare/2.3.2...2.4)

CONTRIBUTORS.md

@@ -7,13 +7,15 @@
     - Cedric Clerget <cedric.clerget@univ-fcomte.fr>
     - Dave Dykstra <dwd@fnal.gov>
     - Dave Godlove <davidgodlove@gmail.com>
+    - Divya Cote <divya.cote@gmail.com>
     - Krishna Muriki <kmuriki@lbl.gov>
     - Michael Bauer <bauerm@umich.edu>
     - Vanessa Sochat <vsochat@stanford.edu>
     - Yannick Cote <yhcote@gmail.com>
 
 #Contributors:
 
+    - Afif Elghraoui <afif.elghraoui@nih.gov>
     - Amanda Duffy <aduffy@lenovo.com>
     - Ángel Bejarano <abejarano@ontropos.com>
     - Bernard Li <bernardli@lbl.gov>
@@ -25,6 +27,7 @@
     - George Hartzell <hartzell@alerce.com>
     - Jarrod Johnson <jjohnson2@lenovo.com>
     - Jason Stover <jason.stover@gmail.com>
+    - Jeff Kriske <jekriske@gmail.com>
     - Maciej Sieczka <msieczka@sieczka.org>
     - Mark Egan-Fuller <markeganfuller@googlemail.com>
     - Nathan Lin <nathan.lin@yale.edu>
@@ -36,4 +39,7 @@
     - Yaroslav Halchenko <debian@onerussian.com>
     - Josef Hrabal <josef.hrabal@vsb.cz>
     - Daniele Tamino <daniele.tamino@gmail.com>
-
+    - Thomas Hamel <hmlth@t-hamel.fr>
+    - David Trudgian <david.trudgian@utsouthwestern.edu>
+    - Tarcisio Fedrizzi <tarcisio.fedrizzi@gmail.com>
+

Makefile.am

@@ -1,7 +1,7 @@
 SUBDIRS = bin etc libexec man src
 
-MAINTAINERCLEANFILES = Makefile.in aclocal.m4 configure ltmain.sh depcomp install-sh missing config.* *.m4 singularity-*.tar.gz singularity-*.rpm m4/* test.sh
-DISTCLEANFILES = Makefile test.sh
+MAINTAINERCLEANFILES = Makefile.in aclocal.m4 configure ltmain.sh compile depcomp install-sh missing config.* *.m4 singularity-*.tar.gz singularity-*.rpm m4/* test.sh secbuildimg.sh
+DISTCLEANFILES = Makefile test.sh secbuildimg.sh
 CLEANFILES = 
 EXTRA_DIST = singularity.spec autogen.sh examples debian CONTRIBUTORS.md CONTRIBUTING.md COPYRIGHT.md INSTALL.md LICENSE-LBNL.md LICENSE.md README.md tests
 
@@ -13,18 +13,32 @@ distclean-local: maintainer-clean-local
 test: test.sh
 	sh ./test.sh
 
+secbuildimg: secbuildimg.sh
+	sh ./secbuildimg.sh
+
 install-perms:
 	@echo
 	@echo "install-perms is no longer required"
 	@echo
 
+all-local:
+	@echo
+	@echo "******"
+	@echo "  mksquashfs from squash-tools is required for full functionality"
+	@echo "******"
+	@echo
+
 install-data-hook:
 	install -d -m 0755 $(DESTDIR)$(CONTAINER_MOUNTDIR)
 	install -d -m 0755 $(DESTDIR)$(CONTAINER_FINALDIR)
 	install -d -m 0755 $(DESTDIR)$(CONTAINER_OVERLAY)
 	install -d -m 0755 $(DESTDIR)$(SESSIONDIR)
 	test -f $(DESTDIR)$(libexecdir)/singularity/sexec-suid && rm -f $(DESTDIR)$(libexecdir)/singularity/sexec-suid || :
 	test -f $(DESTDIR)$(libexecdir)/singularity/bin/copy-suid && rm -f $(DESTDIR)$(libexecdir)/singularity/bin/copy-suid || :
-
+	test -f $(DESTDIR)$(libexecdir)/singularity/bin/action-suid && rm -f $(DESTDIR)$(libexecdir)/singularity/bin/action-suid || :
+	test -f $(DESTDIR)$(libexecdir)/singularity/bin/start-suid && rm -f $(DESTDIR)$(libexecdir)/singularity/bin/start-suid || :
+	test -f $(DESTDIR)$(libexecdir)/singularity/bin/mount-suid && rm -f $(DESTDIR)$(libexecdir)/singularity/bin/mount-suid || :
+	test -f $(DESTDIR)$(libexecdir)/singularity/bin/import-suid && rm -f $(DESTDIR)$(libexecdir)/singularity/bin/import-suid || :
+	test -f $(DESTDIR)$(libexecdir)/singularity/bin/export-suid && rm -f $(DESTDIR)$(libexecdir)/singularity/bin/export-suid || :
 
 ACLOCAL_AMFLAGS = -I m4

bin/singularity.in

@@ -83,6 +83,16 @@ while true; do
             message 4 "Enabling shell debugging\n"
             shift
         ;;
+        -c|--conf)
+            shift
+            if [ $(id -ru) != 0 ]; then
+                message ERROR "Only root can set a custom configuration file\n"
+                exit 1
+            else
+                export SINGULARITY_CONFIG_FILE="${1:-}"
+            fi
+            shift
+        ;;
         -v|--verbose)
             SINGULARITY_MESSAGELEVEL=`expr $SINGULARITY_MESSAGELEVEL + 1`
             message 2 "Increasing verbosity level ($SINGULARITY_MESSAGELEVEL)\n"
@@ -137,3 +147,4 @@ fi
 
 # We should never get here...
 exit 255
+