Updating spliced to use library (#4)

* prepare for unconventional splicing
Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.github/workflows/splice.yaml

@@ -3,7 +3,7 @@ on:
   workflow_dispatch: 
     inputs:
       splice:
-        description: 'Name of YAML splice config in splices (e.g., curl.yaml)'
+        description: 'Name of YAML splice config in splices (e.g., splices/curl.yaml)'
         required: true
 
 jobs:
@@ -12,30 +12,22 @@ jobs:
     permissions:
       packages: read
     outputs:
-      containers: ${{ steps.generate.outputs.containers }}
-      package: ${{ steps.setup.outputs.package }}
-      splice: ${{ steps.setup.outputs.splice }}
-      command: ${{ steps.setup.outputs.command }}
-      experiment: ${{ steps.setup.outputs.experiment }}
+      matrix: ${{ steps.generate.outputs.matrix }}
 
-    name: Build Matrix
+    name: Generate Spliced Matrix
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - name: Setup splice
-        id: setup
-        env:
-          package: ${{ github.event.inputs.splice }}
-        run: |
-            pip install pyaml
-            python scripts/prepare-splice.py ${package}
-      - name: Generate matrix
+      - name: Install Spliced
+        uses: buildsi/spliced/action/install@main
+      - name: Generate Matrix
+        uses: buildsi/spliced/action/matrix@main
         id: generate
-        run: |
-            pip install requests
-            python scripts/generate-matrix.py ${{ steps.setup.outputs.package }} ${{ steps.setup.outputs.splice }} ${{ steps.setup.output.experiment }} ${{ steps.setup.outputs.command }}
-            
-  build:
+        with:
+          yaml: ${{ github.event.inputs.splice }}
+          container: ghcr.io/buildsi/spliced-ubuntu-20.04:latest            
+
+  run-splices:
     runs-on: ubuntu-latest
     needs:
       - prepare
@@ -44,32 +36,45 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Each entry is a container, and then compiler within it, and a tag for the version, and the version
-        # [container, label, container_name, pkg, version, splice, command, experiment]
-        container: ${{ fromJson(needs.prepare.outputs.containers) }}
-        arch: ['linux/amd64'] # linux/ppc64le,linux/arm64 these take forever
+        # Each entry has a command, and all the variables provided individually
+        # {
+        #   "command": "spliced splice --package curl@7.52.1 --splice zlib --runner spack --replace zlib --experiment curl curl --head https://linuxize.com/",
+        #   "package": "curl@7.52.1",
+        #   "runner": "spack",
+        #   "splice": "zlib",
+        #   "replace": "zlib",
+        #   "experiment": "curl",
+        #   "container": "ghcr.io..."
+        # },
+        entry: ${{ fromJson(needs.prepare.outputs.matrix) }}
+        arch: ['linux/amd64']
 
     # Run tests inside the container above - we could also build with Dockerfile if desired
     container:
-      image: ${{ matrix.container[0] }}
+      image: ${{ matrix.entry.container }}
       options: "--platform=${{ matrix.arch }}"
 
-    name: ${{ matrix.package }} ${{ matrix.container[0] }} ${{ matrix.container[3]}}@${{ matrix.container[4] }}
+    name: ${{ matrix.entry.package }} ${{ matrix.entry.replace }} -> ${{ matrix.entry.splice }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+      - name: Set Result Name
+        env:
+          splice_id: pkg-${{ matrix.entry.package }}-splice-${{ matrix.entry.splice }}-with-${{ matrix.entry.splice }}-experiment-${{ matrix.entry.experiment }}
+        run: echo "splice_id=${splice_id}" >> $GITHUB_ENV
 
       - name: Run analysis
         env:
           SPACK_ADD_DEBUG_FLAGS: true
+          command: ${{ matrix.entry.command }}
         run: |
           apt-get update && apt-get install -y vim libtool
           cd /opt/spack && git fetch && git checkout vsoch/db-17-splice && cd -
-          printf "spack python splice.py splice ${{ matrix.container[3] }}@${{ matrix.container[4] }} ${{ matrix.container[5] }} --outfile ${{ matrix.container[3] }}-${{ matrix.container[4] }}-splices.json --experiment ${{ matrix.container[7]}} ${{ matrix.container[6] }}\n"
-          spack python splice.py splice ${{ matrix.container[3] }}@${{ matrix.container[4] }} ${{ matrix.container[5] }} --outfile ${{ matrix.container[3] }}-${{ matrix.container[4] }}-${{ matrix.container[7] }}-${{ matrix.container[5] }}-splices.json --experiment ${{ matrix.container[7]}} ${{ matrix.container[6] }}
+          printf "${command} --outfile ${{ env.splice_id }}-splices.json\n"
+          ${command} --outfile ${{ env.splice_id }}-splices.json
 
       - name: Save Result Artifact
         uses: actions/upload-artifact@v2
         with:
-          name: "test-results-${{ matrix.container[3] }}-${{ matrix.container[4] }}-${{ matrix.container[7]}}"
-          path: ./${{ matrix.container[3] }}-${{ matrix.container[4] }}-${{ matrix.container[7] }}-splices.json
+          name: "test-results-${{ env.splice_id }}"
+          path: ./${{ env.splice_id }}-splices.json

README.md

@@ -1,8 +1,8 @@
 # Testing Splicing
 
 This is a testing ground for splicing! You can use the workflow dispatch to run
-a job (and upload artifacts, TBA for saving somewhere). Running locally, instructions are below.
-For running on GitHub, you can add a file to [splices](splices) and then trigger the dispatch
+a job (and upload artifacts, TBA for saving somewhere). Running locally, you should use the [spliced](https://github.com/buildsi/spliced)
+library. For either case, you can add a file to [splices](splices) and then trigger the dispatch
 event for the workflow. Each YAML file in splices should have the following:
 
 ```yaml
@@ -14,6 +14,17 @@ command: curl --head https://linuxize.com/
 It's currently a flat list because we have one of each, and this can be adjusted as needed.
 Each of these is considered one experiment. You should not include versions with the package
 to be spliced, or the library to splice in, as they will be discovered programatically.
+The above says "Take the binary 'curl' for the package curl, and replace the chosen version of
+zlib with all other versions of zlib." You can also ask to splice in a totally different dependency,
+for example "Take the hdf5 package, and replace openmpi with mpich." 
+
+```yaml
+package: hdf4
+splice: openmpi
+replace: mpich
+...
+```
+When you don't include a "replace" field, the replacement library is implied to be the same as the spliced one.
 To then run the workflow, simply input "curl.yaml" as the splice variable in the GitHub
 workflow interface.
 
@@ -25,36 +36,46 @@ Name it whatever you like.
 $ docker build -t splice-test .
 ```
 
-Or change the base image (but be aware the Dockerfile here is for Debian, see the [Dockerfile.centos](Dockerfile.centos)
-for centos.
+Now shell into the container (the entrypoint is bash)
 
 ```bash
-$ docker build --build-arg base=ghcr.io/buildsi/spack-ubuntu-18.04:latest splice-test .
+$ docker run -it splice-test
 ```
 
-Now shell into the container (the entrypoint is bash)
+The command line client "spliced" should already be installed, and you can always re-pull to update it.
+
+## Generate splice runs
+
+In GitHub workflows, we would generate a matrix of runs. However since we are manually testing, let's just sploot out a list and we can
+choose one that we like. As stated above, a splicing experiment is determined by a YAML configuration file. So if we have a set in [splices](splices)
+we can shell into the container and bind and they will be there:
 
 ```bash
-$ docker run -it splice-test
+$ docker run -it -v $PWD:/code splice-test
 ```
 
-If you want to bind the script to change/update and run again:
+Here they are:
 
 ```bash
-$ docker run -it -v $PWD:/code splice-test
+$ ls /code/splices/
 ```
 
+Note that spack and spliced are on the path
+
 ```bash
 $ which spack
 /opt/spack/bin/spack
+
+$ which spliced
+/usr/local/bin/spliced
 ```
 
-This is where the spack install is located if you want to try tweaking things and then re-running.
-Let's rum the splice script for curl and zlib (across all versions):
+Now let's generate a set of commands to play with!
 
 ```bash
-                         # binary  # splice                                  # command
-$ spack python splice.py curl@7.78.0 zlib   --outfile spliced-curl-7.78.json curl --head https://linuxize.com/
+$ spliced command /code/splices/curl.yaml
+...
+spliced splice --package curl@7.74.0 --splice zlib --runner spack --replace zlib --experiment curl curl --head https://linuxize.com/
 ```
 
 This is going to concretize this version of curl, and then perform the splices (see [splice.py](splice.py) for how that works