automate packaging section (#52)

This adds a script for the packaging section of the tutorial (save one
bit of output) and reworks tutorial_packaging to include from generated
files.

- [x] automate all but one example from packaging
- [x] add output files for packaging
- [x] rework tutorial_packaging.rst too use new files

outputs/Makefile

@@ -1,7 +1,7 @@
 .PHONY: all replay
 
 raw_output = raw/*/*.out
-subdirs = basics
+subdirs = basics packaging
 
 
 all:

outputs/basics.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+pip3 install boto3
+
 # Source definitions
 project=$(dirname "$0")
 . $project/defs.sh

outputs/package-py-files/0.package.py

@@ -0,0 +1,46 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+# ----------------------------------------------------------------------------
+# If you submit this package back to Spack as a pull request,
+# please first remove this boilerplate and all FIXME comments.
+#
+# This is a template package file for Spack.  We've put "FIXME"
+# next to all the things you'll want to change. Once you've handled
+# them, you can save this file and test your package like this:
+#
+#     spack install mpileaks
+#
+# You can edit this file again by typing:
+#
+#     spack edit mpileaks
+#
+# See the Spack documentation for more information on packaging.
+# ----------------------------------------------------------------------------
+
+from spack import *
+
+
+class Mpileaks(AutotoolsPackage):
+    """FIXME: Put a proper description of your package here."""
+
+    # FIXME: Add a proper url for your package's homepage here.
+    homepage = "https://www.example.com"
+    url      = "https://github.com/LLNL/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz"
+
+    # FIXME: Add a list of GitHub accounts to
+    # notify when the package is updated.
+    # maintainers = ['github_user1', 'github_user2']
+
+    version('1.0', sha256='2e34cc4505556d1c1f085758e26f2f8eea0972db9382f051b2dcfb1d7d9e1825')
+
+    # FIXME: Add dependencies if required.
+    # depends_on('foo')
+
+    def configure_args(self):
+        # FIXME: Add arguments other than --prefix
+        # FIXME: If not needed delete this function
+        args = []
+        return args

outputs/package-py-files/1.package.py

@@ -0,0 +1,27 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Mpileaks(AutotoolsPackage):
+    """Tool to detect and report MPI objects like MPI_Requests and
+    MPI_Datatypes."""
+
+    homepage = "https://github.com/LLNL/mpileaks"
+    url      = "https://github.com/LLNL/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz"
+
+    maintainers = ['adamjstewart']
+
+    version('1.0', sha256='2e34cc4505556d1c1f085758e26f2f8eea0972db9382f051b2dcfb1d7d9e1825')
+
+    # FIXME: Add dependencies if required.
+    # depends_on('foo')
+
+    def configure_args(self):
+        # FIXME: Add arguments other than --prefix
+        # FIXME: If not needed delete this function
+        args = []
+        return args

outputs/package-py-files/2.package.py

@@ -0,0 +1,28 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Mpileaks(AutotoolsPackage):
+    """Tool to detect and report MPI objects like MPI_Requests and
+    MPI_Datatypes."""
+
+    homepage = "https://github.com/LLNL/mpileaks"
+    url      = "https://github.com/LLNL/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz"
+
+    maintainers = ['adamjstewart']
+
+    version('1.0', sha256='2e34cc4505556d1c1f085758e26f2f8eea0972db9382f051b2dcfb1d7d9e1825')
+
+    depends_on('mpi')
+    depends_on('adept-utils')
+    depends_on('callpath')
+
+    def configure_args(self):
+        # FIXME: Add arguments other than --prefix
+        # FIXME: If not needed delete this function
+        args = []
+        return args

outputs/package-py-files/3.package.py

@@ -0,0 +1,30 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Mpileaks(AutotoolsPackage):
+    """Tool to detect and report MPI objects like MPI_Requests and
+    MPI_Datatypes."""
+
+    homepage = "https://github.com/LLNL/mpileaks"
+    url      = "https://github.com/LLNL/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz"
+
+    maintainers = ['adamjstewart']
+
+    version('1.0', sha256='2e34cc4505556d1c1f085758e26f2f8eea0972db9382f051b2dcfb1d7d9e1825')
+
+    depends_on('mpi')
+    depends_on('adept-utils')
+    depends_on('callpath')
+
+    def configure_args(self):
+        args = [
+            '--with-adept-utils={0}'.format(self.spec['adept-utils'].prefix),
+            '--with-callpath={0}'.format(self.spec['callpath'].prefix)
+        ]
+
+        return args

outputs/package-py-files/4.package.py

@@ -0,0 +1,40 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Mpileaks(AutotoolsPackage):
+    """Tool to detect and report MPI objects like MPI_Requests and
+    MPI_Datatypes."""
+
+    homepage = "https://github.com/LLNL/mpileaks"
+    url      = "https://github.com/LLNL/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz"
+
+    maintainers = ['adamjstewart']
+
+    version('1.0', sha256='2e34cc4505556d1c1f085758e26f2f8eea0972db9382f051b2dcfb1d7d9e1825')
+
+    variant('stackstart', values=int, default=0,
+            description='Specify the number of stack frames to truncate')
+
+    depends_on('mpi')
+    depends_on('adept-utils')
+    depends_on('callpath')
+
+    def configure_args(self):
+        args = [
+            '--with-adept-utils={0}'.format(self.spec['adept-utils'].prefix),
+            '--with-callpath={0}'.format(self.spec['callpath'].prefix),
+        ]
+
+        stackstart = int(self.spec.variants['stackstart'].value)
+        if stackstart:
+            args.extend([
+                '--with-stack-start-c={0}'.format(stackstart),
+                '--with-stack-start-fortran={0}'.format(stackstart)
+            ])
+
+        return args

outputs/packaging.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Source definitions
+project=$(dirname "$0")
+. $project/defs.sh
+
+rm -rf $raw_outputs/packaging
+. $project/init_spack.sh
+mpileaks_package_py=$SPACK_ROOT/var/spack/repos/tutorial/packages/mpileaks/package.py
+
+example packaging/repo-add   "spack repo add \$SPACK_ROOT/var/spack/repos/tutorial/"
+
+# make the editor automatically exit
+export EDITOR="bash -c exit 0"
+example packaging/create     "spack create https://github.com/LLNL/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz"
+
+example packaging/install-mpileaks-1  "spack install mpileaks"
+
+cp /project/package-py-files/1.package.py $mpileaks_package_py
+example packaging/info-mpileaks       "spack info mpileaks"
+
+spack uninstall -ay mpileaks
+cp /project/package-py-files/2.package.py $mpileaks_package_py
+example packaging/install-mpileaks-2  "spack install --keep-stage mpileaks"
+
+stage_dir=$(spack location -s mpileaks)
+example packaging/build-output        "cat $stage_dir/spack-build-out.txt"
+
+#prefix=$(spack python -c \
+#    'import spack.spec; print(spack.spec.Spec("mpileaks").concretized().prefix)')
+#spack cd mpileaks
+#echo "configure --prefix=$prefix" | example packaging/build-env-configure "spack build-env mpileaks bash"
+
+cp /project/package-py-files/3.package.py $mpileaks_package_py
+example packaging/install-mpileaks-3  "spack install mpileaks"
+
+cp /project/package-py-files/4.package.py $mpileaks_package_py
+example packaging/install-mpileaks-4  "spack install --verbose mpileaks stackstart=4"
+
+example packaging/cleanup  "spack uninstall -ay mpileaks"
+example packaging/cleanup  "spack repo remove tutorial"
+example packaging/cleanup  "rm -rf \$SPACK_ROOT/var/spack/repos/tutorial/packages/mpileaks"

outputs/packaging/build-env-configure.out

@@ -0,0 +1,27 @@
+$ ./configure --prefix=/home/spack/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/mpileaks-1.0-2e2wmy4spaei2ov4g57hqz7fn3jb43hi
+checking metadata... no
+checking installation directory variables... yes
+checking for a BSD-compatible install... /usr/bin/install -c
+checking whether build environment is sane... yes
+checking for a thread-safe mkdir -p... /bin/mkdir -p
+checking for gawk... no
+checking for mawk... mawk
+checking whether make sets $(MAKE)... yes
+checking for gcc... /home/spack/spack/lib/spack/env/gcc/gcc
+checking for C compiler default output file name... a.out
+checking whether the C compiler works... yes
+checking whether we are cross compiling... no
+checking for suffix of executables...
+checking for suffix of object files... o
+checking whether we are using the GNU C compiler... yes
+checking whether /home/spack/spack/lib/spack/env/gcc/gcc accepts -g... yes
+checking for /home/spack/spack/lib/spack/env/gcc/gcc option to accept ISO C89... none needed
+checking for style of include used by make... GNU
+checking dependency style of /home/spack/spack/lib/spack/env/gcc/gcc... gcc3
+checking whether /home/spack/spack/lib/spack/env/gcc/gcc and cc understand -c and -o together... yes
+checking whether we are using the GNU C++ compiler... yes
+checking whether /home/spack/spack/lib/spack/env/gcc/g++ accepts -g... yes
+checking dependency style of /home/spack/spack/lib/spack/env/gcc/g++... gcc3
+checking for /home/spack/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/openmpi-3.1.6-wudo6kygednztgzemlnq4l7rlwrwu3zw/bin/mpicc... /home/spack/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/openmpi-3.1.6-wudo6kygednztgzemlnq4l7rlwrwu3zw/bin/mpicc
+Checking whether /home/spack/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/openmpi-3.1.6-wudo6kygednztgzemlnq4l7rlwrwu3zw/bin/mpicc responds to '-showme:compile'... yes
+configure: error: unable to locate adept-utils installation

outputs/packaging/build-output.out

@@ -0,0 +1,30 @@
+$ cat /tmp/spack/spack-stage/spack-stage-mpileaks-1.0-2e2wmy4spaei2ov4g57hqz7fn3jb43hi/spack-build-out.txt
+==> mpileaks: Executing phase: 'autoreconf'
+==> mpileaks: Executing phase: 'configure'
+==> [2020-07-28-07:13:44.201951] '/tmp/spack/spack-stage/spack-stage-mpileaks-1.0-2e2wmy4spaei2ov4g57hqz7fn3jb43hi/spack-src/configure' '--prefix=/home/spack/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/mpileaks-1.0-2e2wmy4spaei2ov4g57hqz7fn3jb43hi'
+checking metadata... no
+checking installation directory variables... yes
+checking for a BSD-compatible install... /usr/bin/install -c
+checking whether build environment is sane... yes
+checking for a thread-safe mkdir -p... /bin/mkdir -p
+checking for gawk... no
+checking for mawk... mawk
+checking whether make sets $(MAKE)... yes
+checking for gcc... /home/spack/spack/lib/spack/env/gcc/gcc
+checking for C compiler default output file name... a.out
+checking whether the C compiler works... yes
+checking whether we are cross compiling... no
+checking for suffix of executables...
+checking for suffix of object files... o
+checking whether we are using the GNU C compiler... yes
+checking whether /home/spack/spack/lib/spack/env/gcc/gcc accepts -g... yes
+checking for /home/spack/spack/lib/spack/env/gcc/gcc option to accept ISO C89... none needed
+checking for style of include used by make... GNU
+checking dependency style of /home/spack/spack/lib/spack/env/gcc/gcc... gcc3
+checking whether /home/spack/spack/lib/spack/env/gcc/gcc and cc understand -c and -o together... yes
+checking whether we are using the GNU C++ compiler... yes
+checking whether /home/spack/spack/lib/spack/env/gcc/g++ accepts -g... yes
+checking dependency style of /home/spack/spack/lib/spack/env/gcc/g++... gcc3
+checking for /home/spack/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/openmpi-3.1.6-wudo6kygednztgzemlnq4l7rlwrwu3zw/bin/mpicc... /home/spack/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/openmpi-3.1.6-wudo6kygednztgzemlnq4l7rlwrwu3zw/bin/mpicc
+Checking whether /home/spack/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/openmpi-3.1.6-wudo6kygednztgzemlnq4l7rlwrwu3zw/bin/mpicc responds to '-showme:compile'... yes
+configure: error: unable to locate adept-utils installation

outputs/packaging/cleanup.out

@@ -0,0 +1,6 @@
+$ spack uninstall -ay mpileaks
+==> Successfully uninstalled mpileaks@1.0%gcc@7.5.0 arch=linux-ubuntu18.04-x86_64/2e2wmy4
+==> Successfully uninstalled mpileaks@1.0%gcc@7.5.0 stackstart=4 arch=linux-ubuntu18.04-x86_64/6iokq53
+$ spack repo remove tutorial
+==> Removed repository /home/spack/spack/var/spack/repos/tutorial with namespace 'tutorial'.
+$ rm -rf $SPACK_ROOT/var/spack/repos/tutorial/packages/mpileaks

outputs/packaging/create.out

@@ -0,0 +1,6 @@
+$ spack create https://github.com/LLNL/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz
+==> This looks like a URL for mpileaks
+
+==> This package looks like it uses the autotools build system
+==> Created template for mpileaks package
+==> Created package file: /home/spack/spack/var/spack/repos/tutorial/packages/mpileaks/package.py

outputs/packaging/info-mpileaks.out

@@ -0,0 +1,38 @@
+$ spack info mpileaks
+AutotoolsPackage:   mpileaks
+
+Description:
+    Tool to detect and report MPI objects like MPI_Requests and
+    MPI_Datatypes.
+
+Homepage: https://github.com/LLNL/mpileaks
+
+Maintainers: @adamjstewart
+
+Tags:
+    None
+
+Preferred version:
+    1.0    https://github.com/LLNL/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz
+
+Safe versions:
+    1.0    https://github.com/LLNL/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz
+
+Variants:
+    None
+
+Installation Phases:
+    autoreconf	  configure    build	install
+
+Build Dependencies:
+    None
+
+Link Dependencies:
+    None
+
+Run Dependencies:
+    None
+
+Virtual Packages:
+    None
+

outputs/packaging/install-mpileaks-1.out

@@ -0,0 +1,21 @@
+$ spack install mpileaks
+==> Installing mpileaks
+==> No binary for mpileaks found: installing from source
+==> mpileaks: Executing phase: 'autoreconf'
+==> mpileaks: Executing phase: 'configure'
+==> Error: ProcessError: Command exited with status 1:
+    '/tmp/spack/spack-stage/spack-stage-mpileaks-1.0-2zeskz65ktjgqj6u57venocpndampdt3/spack-src/configure' '--prefix=/home/spack/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/mpileaks-1.0-2zeskz65ktjgqj6u57venocpndampdt3'
+
+1 error found in build log:
+     30    checking for mpipgcc... no
+     31    Checking whether not-found responds to '-showme:compile'... no
+     32    Checking whether not-found responds to '-showme'... no
+     33    Checking whether not-found responds to '-compile-info'... no
+     34    Checking whether not-found responds to '-show'... no
+     35    /tmp/spack/spack-stage/spack-stage-mpileaks-1.0-2zeskz65ktjgqj6u57venocpndampdt3/spack-s
+	   rc/configure: line 4931: Echo: command not found
+  >> 36    configure: error: unable to locate adept-utils installation
+
+See build log for details:
+  /tmp/spack/spack-stage/spack-stage-mpileaks-1.0-2zeskz65ktjgqj6u57venocpndampdt3/spack-build-out.txt
+