update environments, stacks, dev workflows, scripting (#60)

* update scripts to automate environments section
* update outputs for env tutorial
* Update and automate dev workflow section (#56)
* update dev section
* update scripting section
* update environment and stack scripts

Co-authored-by: Massimiliano Culpo <massimiliano.culpo@gmail.com>

outputs/Makefile

@@ -1,8 +1,7 @@
 .PHONY: all replay
 
 raw_output = raw/*/*.out
-subdirs = basics packaging
-
+subdirs = basics packaging environments stacks dev scripting
 
 all:
 	echo "Filtering raw outputs though col"

outputs/basics.sh

@@ -9,6 +9,9 @@ project=$(dirname "$0")
 # clean things up before starting this first script
 rm -rf $raw_outputs ~/spack ~/.spack ~/.gnupg
 
+# install boto3
+pip3 install boto3
+
 # basic installation
 example basics/clone         "git clone https://github.com/spack/spack ~/spack"
 

outputs/basics/find-d-tcl.out

@@ -7,8 +7,8 @@ tcl@8.6.8
 
 -- linux-ubuntu18.04-x86_64 / gcc@7.5.0 -------------------------
 tcl@8.6.8
-    zlib@1.2.11
+    zlib@1.2.8
 
 tcl@8.6.8
-    zlib@1.2.8
+    zlib@1.2.11
 

outputs/basics/find-ldf-2.out

@@ -120,12 +120,12 @@ crhlefo 	    ncurses@6.2%gcc
 t54jzdy readline@8.0%gcc
 crhlefo     ncurses@6.2%gcc
 
-aszidzn tcl@8.6.8%gcc
-smoyzzo     zlib@1.2.11%gcc
-
 xthyjrq tcl@8.6.8%gcc
 h6i53if     zlib@1.2.8%gcc  cppflags="-O3"
 
+aszidzn tcl@8.6.8%gcc
+smoyzzo     zlib@1.2.11%gcc
+
 cr3i7es texinfo@6.5%gcc
 hyrsxn4     perl@5.30.3%gcc
 4av4gyw 	gdbm@1.18.1%gcc

outputs/defs.sh

@@ -3,6 +3,15 @@
 raw_outputs=/project/raw
 
 example() {
+    if [[ "$1" == "-tee" ]]; then
+        # tee option allows us to use bash function; only use when necessary
+        tee=1
+        shift
+    else
+        # non-tee option uses a pty, so we get full outputs
+        tee=0
+    fi
+
     filename="$raw_outputs/$1.out"
     shift
 
@@ -12,11 +21,52 @@ example() {
     # print the command to the file
     echo "$ $@" &>> "$filename"
 
-    # get the command's output
-    script -q -a "$filename" -c "$@"
+    if [[ "$tee" == "1" ]]; then
+        cmd="$@"
+
+        # Don't use despacktivate alias (and strip comment)
+        if [[ "$@" == despacktivate* ]]; then
+            cmd="spack env deactivate"
+        fi
+
+        echo $cmd
+        # get the command's output
+        $cmd 2>&1 | tee -a "$filename"
+    else
+        script -q -a "$filename" -c "$@"
+
+        # strip "script started/done" output from the file
+        grep -v '^Script started\|^Script done' "$filename" > "${filename}.tmp"
+        sed -i~ '$d' "${filename}.tmp"
+        mv "${filename}.tmp" "$filename"
+    fi
+}
+
+# This allows us to echo a different command than we run
+# Used to stay out of the build environment subshell
+fake_example(){
+    filename="$raw_outputs/$1.out"
+    shift
+
+    fake_cmd="$1"
+    shift
+
+    parent=$(dirname $filename)
+    mkdir -p $parent
+
+    # print the command to the file
+    echo "$ $fake_cmd" &>> "$filename"
+
+    cmd="$@"
+
+    # Don't use despacktivate alias (and strip comment)
+    if [[ "$cmd" == despacktivate* ]]; then
+        cmd="spack env deactivate"
+    fi
 
-    # strip "script started/done" output from the file
-    grep -v '^Script started\|^Script done' "$filename" > "${filename}.tmp"
-    sed -i~ '$d' "${filename}.tmp"
-    mv "${filename}.tmp" "$filename"
+    # print command to stdout to help debugging
+    echo "$cmd"
+
+    # get the command's output
+    $cmd | tee -a "$filename"
 }

outputs/dev.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Source definitions
+project=$(dirname "$0")
+. $project/defs.sh
+
+rm -rf $raw_outputs/dev
+pip install boto3
+
+example dev/up-to-date "git clone https://github.com/spack/spack ~/spack"
+example dev/up-to-date "cd ~/spack"
+cd ~/spack
+example dev/up-to-date "git checkout releases/v0.15"
+example dev/up-to-date ". share/spack/setup-env.sh"
+. share/spack/setup-env.sh
+spack config add "config:suppress_gpg_warnings:true"
+spack config add "packages:all:target:[x86_64]"
+
+example dev/up-to-date "spack mirror add tutorial s3://spack-tutorial-container/mirror/"
+example dev/up-to-date "spack gpg trust share/spack/keys/tutorial.pub"
+
+example dev/setup-hwloc "cd ~"
+cd ~
+example dev/setup-hwloc "git clone https://github.com/open-mpi/hwloc.git"
+example dev/setup-hwloc "cd hwloc"
+cd hwloc
+
+example dev/dev-build-1 "spack dev-build hwloc@master"
+
+example dev/info "spack info hwloc"
+
+echo y | example dev/dev-build-2 "spack uninstall hwloc"
+fake_example dev/dev-build-2 "spack dev-build --until configure --drop-in bash hwloc@master" "spack dev-build --until configure hwloc@master"
+
+export EDITOR="bash -c exit 0"
+fake_example dev/edit-1 '$EDITOR hwloc/base64.c' "/bin/true"
+sed -i~ s'|\("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"\);|\1|' hwloc/base64.c | head -n 70
+
+fake_example dev/hand-build-1 "make" "spack build-env hwloc@master -- make"
+
+fake_example dev/hand-build-2 '$EDITOR hwloc/base64.c' "/bin/true"
+sed -i~ s'|\("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"\)|\1;|' hwloc/base64.c | head -n 70
+fake_example dev/hand-build-2 "make" "spack build-env hwloc@master -- make"