Migration to Singularity Hub 2.0 is done

.gitignore

@@ -1,2 +1,6 @@
+singularity.egg-info
+__pycache__
+pypi.sh
+dist
 *.pyc
 OLD

.travis.yml

@@ -5,20 +5,17 @@ sudo: true
 os: linux
 
 language: python
-python:
-  - "2.7"
-  - "3.5"
 
-install:
-  - pip install pylint
-  - cd $TRAVIS_BUILD_DIR/
-  - cd $TRAVIS_BUILD_DIR && pip install -r requirements.txt
-  - cd $TRAVIS_BUILD_DIR && pip3 install -r requirements.txt
-  - python setup.py sdist
-  - python setup.py install
-  - pylint --version
-  - cd /tmp && git clone -b development https://github.com/singularityware/singularity.git && cd singularity && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install
+matrix:
+  include:
+    - os: linux
+      env: PYTHON_VERSION=2
+    - os: linux
+      env: PYTHON_VERSION=3
+
+before_install:
+  - sudo chmod u+x .travis/*
+  - .travis/before_install
 
 script:
-  - python -m unittest discover -s $TRAVIS_BUILD_DIR/singularity/tests/ -p '[t|T]est*.py'
-  #bash $TRAVIS_BUILD_DIR/singularity/tests/run_tests.sh $TRAVIS_BUILD_DIR/singularity/tests /tmp
+  - .travis/script

.travis/before_install

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+apt-get update && apt-get install -y wget git build-essential squashfs-tools \
+                                          libtool \
+                                          autotools-dev \
+                                          automake \
+                                          autoconf
+
+sudo sed -i -e 's/^Defaults\tsecure_path.*$//' /etc/sudoers
+
+if [ "${PYTHON_VERSION}" == "2" ];
+then
+    wget https://repo.continuum.io/archive/Anaconda2-5.0.0.1-Linux-x86_64.sh
+    sudo bash Anaconda2-5.0.0.1-Linux-x86_64.sh -b -p /opt/anaconda2 && sudo chown -R 0755 /opt/anaconda2
+    PATH=/opt/anaconda3/bin:$PATH
+    rm Anaconda2-5.0.0.1-Linux-x86_64.sh 
+    export PATH
+else
+    wget https://repo.continuum.io/archive/Anaconda3-5.0.0.1-Linux-x86_64.sh
+    sudo bash Anaconda3-5.0.0.1-Linux-x86_64.sh -b -p /opt/anaconda3 && sudo chown -R 0755 /opt/anaconda3
+    PATH=/opt/anaconda3/bin:$PATH
+    rm Anaconda3-5.0.0.1-Linux-x86_64.sh
+    export PATH
+fi
+
+# Install python dependencies
+which python
+
+conda install -y pandas scikit-learn pygments
+cd $TRAVIS_BUILD_DIR && sudo python -m pip install -r requirements.txt
+python setup.py sdist && python setup.py install
+pylint --version
+
+# Install Singularity (development)
+cd /tmp && git clone -b development https://github.com/singularityware/singularity.git && cd singularity && ./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install
+

.travis/script

@@ -0,0 +1,3 @@
+#!/bin/bash
+sudo apt-get install -y squashfs-tools
+python -m unittest discover -s $TRAVIS_BUILD_DIR/singularity/tests/ -p '[t|T]est*.py'

MANIFEST.in

@@ -1,10 +1,5 @@
-include README.md
-include LICENSE
-recursive-include singularity/tests *
-recursive-include singularity/hub *
-recursive-include singularity/templates *
-recursive-include singularity/static *
-recursive-include singularity/build *
-recursive-include singularity/analysis *
-recursive-include singularity/views *
-recursive-include singularity/testing *
+include README.md LICENSE
+recursive-include singularity *
+recursive-exclude * __pycache__
+recursive-exclude * *.pyc
+recursive-exclude * *.pyo

README.md

@@ -1,5 +1,3 @@
-
-
 # Singularity Python
 
 [![Build Status](https://travis-ci.org/singularityware/singularity-python.svg?branch=master)](https://travis-ci.org/singularityware/singularity-python)
@@ -12,4 +10,3 @@ Please see our [complete docs](https://github.com/singularityware/singularity-py
 
 ## Help and Contribution
 Please contribute to the package, or post feedback and questions as <a href="https://github.com/singularityware/singularity-python" target="_blank">issues</a>. For points that require discussion of the larger group, please use the <a href="https://groups.google.com/a/lbl.gov/forum/#!forum/singularity" target="_blank">Singularity List</a>
-

examples/cli/run_singularity/singularity_client.py

@@ -11,36 +11,46 @@
 S.help()
 
 # These are the defaults, which can be specified
-S = Singularity(sudo=False,sudopw=None,debug=False)
+S = Singularity(sudo=False,debug=False)
 
-# Create an image
-image = S.create('myimage.img')
+# Note that the "create" and "import" workflow is deprecated in favor of build
+# https://singularityware.github.io/docs-build
 
-# Import into it
+image = S.build('myimage.simg', 'docker://ubuntu:latest') # requires sudo
+
+# (Deprecated) create an image and import into it
+image = S.create('myimage.simg')
 S.importcmd(image,'docker://ubuntu:latest')
 
 # Execute command to container
 result = S.execute(image,command='cat /singularity')
 print(result)
 '''
-#!/bin/sh
-
-if test -x /bin/bash; then
-    exec /bin/bash "$@"
-elif test -x /bin/sh; then
-    exec /bin/sh "$@"
-else
-    echo "ERROR: No valid shell within container"
-    exit 255
-fi
+'#!/bin/sh\n\nexec "/bin/bash"\n'
 '''
 
 # For any function you can get the docs:
 S.help(command="exec")
 
-# export an image as a byte array
-byte_array = S.export(image)
+# export an image to tar
+tar = S.export(image)
+
+# Show apps and inspect
+S.apps(image)
+S.inspect(image)
 
-# Get an in memory tar
-from singularity.reproduce import get_memory_tar
-tar = get_memory_tar(image)
+'''
+{
+    "data": {
+        "attributes": {
+            "deffile": null,
+            "help": null,
+            "labels": null,
+            "environment": "# Custom environment shell code should follow\n\n",
+            "runscript": "#!/bin/sh\n\nexec \"/bin/bash\"\n",
+            "test": null
+        },
+        "type": "container"
+    }
+}
+'''

examples/create/make_runscript/make_runscript.sh

@@ -1,7 +1,40 @@
 #!/bin/sh
 
-# Generate a python run script in the present working directory
-shub --runscript py
+# See help
+shub create --help
 
-# Generate a python run script somewhere else
-shub --runscript py --outfolder /home/vanessa/Desktop
+'''
+Usage: shub create [-h] [--recipe] [--app APP] [--from BOOTSTRAP_FROM]
+                   [--bootstrap BOOTSTRAP] [--outfolder OUTFOLDER]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --recipe              create template recipe
+  --app APP             the name of an app to include in the recipe
+  --from BOOTSTRAP_FROM
+                        the bootstrap "from", should coincide with "bootstrap"
+                        type
+  --bootstrap BOOTSTRAP
+                        the bootstrap type, default is docker
+  --outfolder OUTFOLDER
+                        full path to folder for output, stays in tmp (or pwd)
+                        if not specified
+
+'''
+
+# Generate a relatively blank template, default bootstrap is docker
+shub create
+'''
+DEBUG bootstrap: docker
+Output file written to /home/vanessa/Documents/Dropbox/Code/singularity/singularity-python/examples/create/make_runscript/Singularity
+'''
+
+# Create a recipe template for an app
+shub create --app foo
+'''
+DEBUG bootstrap: docker
+Output file written to /home/vanessa/Documents/Dropbox/Code/singularity/singularity-python/examples/create/make_runscript/Singularity.foo
+'''
+
+# Specify a from, bootstrap
+shub create --app foo --from ubuntu --bootstrap docker

examples/create/package_image/package_images.py

@@ -79,9 +79,4 @@
     if not os.path.exists(package_name):
         image_package = package(image_path=image,
                                 output_folder=output_folder,
-                                remove_image=True,
-                                runscript=True,
-                                software=True)
-    tmpfolder = os.path.dirname(image)
-    shutil.rmtree(tmpfolder)
-
+                                old_version=True) # makes file tar instead of memory

examples/create/package_node/README.md

@@ -0,0 +1,76 @@
+# Package a Node
+
+This is a test for functions that will package a node, meaning sitting on it, and then having it packaged and built into a singularity image (on a host with Singularity). This is only relevant for `vsoch/singularity-python` development branch.
+
+## Install Development Singularity Python
+```
+git clone -b development https://www.github.com/vsoch/singularity-python
+cd singularity-python
+python3 setup.py sdist
+python3 setup.py install
+```
+
+## Package Node
+Then in python3 (this is what the builder on Singularity Hub uses, so I am using that image to test)
+
+```
+from singularity.package import package_node
+
+package=package_node()
+
+package
+'/tmp/tmp06ww_8_n/vanessa-testing-self-cloning.tgz'
+```
+
+## Unpack Node into Container
+Then this file can be turned into a Singularity image:
+
+```
+from singularity.package import unpack_node
+image = unpack_node(image_path=package,size=9000)
+```
+
+We see unpack, image creation, and a break at the end when it's importing:
+
+```
+DEBUG Preparing to unpack vanessa-testing-self-cloning.tgz to vanessa-testing-self-cloning.img.
+Initializing Singularity image subsystem
+Opening image file: vanessa-testing-self-cloning.img
+Creating 2000MiB image
+Binding image to loop
+Creating file system within image
+Image is done: /vanessa-testing-self-cloning.img
+```
+
+The image is named the same as the original `.tgz` but with an `img` extension if `name` is not specified, and in the same folder as the host if `output_folder` is not specified.
+
+```
+image
+$ '/home/vanessa/Desktop/vanessa-testing-self-cloning.img'
+
+```
+
+## Testing Image
+Then on the host, there isn't a run driver so we can't run:
+
+```
+./vanessa-testing-self-cloning.img 
+ERROR  : No run driver found inside container
+ABORT  : Retval = 255
+```
+
+But we can shell!
+
+```
+$ sudo singularity shell vanessa-testing-self-cloning.
+Singularity: Invoking an interactive shell within container..
+#
+ls /
+# ls /
+bin   home	      lib64	  opt	sbin  tmp      vmlinuz.old
+boot  initrd.img      lost+found  proc	snap  usr
+dev   initrd.img.old  media	  root	srv   var
+etc   lib	      mnt	  run	sys   vmlinuz
+```
+
+Proposal: we would want to have security checks, and then generation of the needed commands (possibly via bootstrap functions) that would be needed to drive the image.