Add support for fs extraction (#5)

* preparing to test extracting fs
Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.github/workflows/generate.yaml

@@ -1,18 +1,10 @@
 name: Generate Container Guts
 on:
   pull_request: []
-  workflow_dispatch: 
-    inputs:
-      docker_uri:
-        description: 'Docker identifier to generate recipe for'
-        required: true
-        default: "quay.io/autamus/clingo:5.5.1"
+
 jobs:
   generate-recipe:
-    if: (inputs.docker_uri != '')
     runs-on: ubuntu-latest
-    permissions:
-      packages: read
     name: ${{ inputs.docker_uri }}
     steps:
       - name: Checkout Repository
@@ -27,11 +19,18 @@ jobs:
           export PATH="/usr/share/miniconda/bin:$PATH"
           source activate guts
           pip install -e .
+          echo "/usr/share/miniconda/bin" >> ${GITHUB_PATH}
+
+      - name: Test Basic Guts
+        uses: ./action/manifest
+        with:
+          image: centos
 
-      - name: Guts for ${{ inputs.docker_uri }}
-        uses: ./manifest
+      - name: Filesystem Include
+        uses: ./action/manifest
         with:
-          image: ${{ inputs.docker_uri }}
+          image: centos
+          include: fs
 
   generate-recipes:
     runs-on: ubuntu-latest
@@ -56,9 +55,10 @@ jobs:
           export PATH="/usr/share/miniconda/bin:$PATH"
           source activate guts
           pip install -e .
+          echo "/usr/share/miniconda/bin" >> ${GITHUB_PATH}
 
       - name: Guts for ${{ matrix.image }}
-        uses: ./manifest
+        uses: ./action/manifest
         with:
           image: ${{ matrix.image }}
           outfile: ${{ matrix.image }}.json

.github/workflows/main.yml

@@ -33,6 +33,7 @@ jobs:
           export PATH="/usr/share/miniconda/bin:$PATH"
           source activate black
           pyflakes container_guts/*.py
+          pyflakes container_guts/client
           pyflakes container_guts/main/client.py
           pyflakes container_guts/main/templates.py
           pyflakes container_guts/main/container/docker.py

CHANGELOG.md

@@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pip. Only major versions will be released as tags on Github.
 
 ## [0.0.x](https://github.com/singularityhub/guts/tree/main) (0.0.x)
+ - adding fs (filesystem) extraction support (0.0.13)
  - tag should be own file (0.0.12)
  - Support for output directory (so path prepared by guts) (0.0.11)
  - Initial creation of project (0.0.1)

README.md

@@ -3,7 +3,7 @@
 [![PyPI version](https://badge.fury.io/py/container-guts.svg)](https://badge.fury.io/py/container-guts)
 
 Want to know the "guts" inside your container, or more specifically,
-the executables on the PATH inside? 🤓
+the executables on the PATH inside, and filesystem paths? 🤓
 
 ![docs/assets/img/guts-colors-small.png](docs/assets/img/guts-colors-small.png)
 
@@ -12,6 +12,9 @@ This is a library and set of GitHub actions to derive that!
 ## Usage
 
 For client and GitHub Actions usage, see our ⭐️ [documentation](https://singularityhub.github.io/guts/) ⭐️
+There are (or will be) scattered examples under [examples](examples), e.g.,
+
+![examples/os_diffs/os-diffs.png](examples/os_diffs/os-diffs.png)
 
 ## TODO
 

manifest/action.yaml → action/manifest/action.yaml

@@ -10,6 +10,10 @@ inputs:
   outdir:
     description: Root output directory
     required: false
+  include:
+    description: List of (space separated) includes to extract
+    required: false
+    default: paths
 
 outputs:
   outfile:
@@ -29,6 +33,7 @@ runs:
        image: ${{ inputs.image }} 
        outfile: ${{ inputs.outfile }} 
        outdir: ${{ inputs.outdir }} 
+       includes: ${{ inputs.include }}
      run: | 
         cmd="guts manifest"
         if [ "${outfile}" != "" ]; then
@@ -37,6 +42,9 @@ runs:
         if [ "${outdir}" != "" ]; then
             cmd="${cmd} --outdir ${outdir}"
         fi
+        for include in ${includes}; do
+            cmd="${cmd} --include ${include}"
+        done
         cmd="${cmd} ${image}" 
         printf "${cmd}\n"
         ${cmd}

container_guts/client/__init__.py

@@ -55,31 +55,53 @@ def get_parser():
 
     manifest = subparsers.add_parser(
         "manifest",
-        description="export manifest of executables on the PATH, guts!",
+        description="export manifest of guts!",
         formatter_class=argparse.RawTextHelpFormatter,
     )
-    manifest.add_argument(
-        "-c",
-        "--container-tech",
-        dest="container_tech",
-        help="container technology to use for exporting",
-        choices=["docker"],
-        default="docker",
-    )
-    manifest.add_argument(
-        "image",
-        help="Container URI to parse",
-    )
-    manifest.add_argument(
-        "-o",
-        "--outfile",
-        help="Output manifest file, over-rides outdir",
-        dest="outfile",
+    diff = subparsers.add_parser(
+        "diff",
+        description="take a diff of your container against a guts database.",
+        formatter_class=argparse.RawTextHelpFormatter,
     )
-    manifest.add_argument(
-        "--outdir",
-        help="Root to write output structure, not used if not set.",
+    diff.add_argument(
+        "--db",
+        "--database",
+        help="Database root (of json files) to use, either filesystem or git URL to clone",
+        dest="database",
     )
+
+    for command in manifest, diff:
+        command.add_argument(
+            "-i",
+            "--include",
+            help="Type of guts to include in extraction (defaults to paths)",
+            dest="guts",
+            choices=["fs", "paths"],
+            action="append",
+            default=[],
+        )
+        command.add_argument(
+            "-c",
+            "--container-tech",
+            dest="container_tech",
+            help="container technology to use for exporting",
+            choices=["docker"],
+            default="docker",
+        )
+        command.add_argument(
+            "image",
+            help="Container URI to parse",
+        )
+        command.add_argument(
+            "-o",
+            "--outfile",
+            help="Output manifest file, over-rides outdir",
+            dest="outfile",
+        )
+        command.add_argument(
+            "--outdir",
+            help="Root to write output structure, not used if not set.",
+        )
     return parser
 
 
@@ -135,6 +157,8 @@ def help(return_code=0):
     # Does the user want a shell?
     if args.command == "manifest":
         from .manifest import main
+    elif args.command == "diff":
+        from .diff import main
 
     # Pass on to the correct parser
     return_code = 0

container_guts/client/diff.py

@@ -0,0 +1,45 @@
+__author__ = "Vanessa Sochat"
+__copyright__ = "Copyright 2022, Vanessa Sochat"
+__license__ = "MPL 2.0"
+
+import sys
+import json
+import os
+
+import container_guts.utils as utils
+from ..main import ManifestGenerator
+
+
+def main(args, parser, extra, subparser):
+
+    # Show args to the user
+    print("         image: %s" % args.image)
+    print("       outfile: %s" % args.outfile)
+    print("        outdir: %s" % args.outdir)
+    print("container tech: %s" % args.container_tech)
+    print("      database: %s" % args.database)
+
+    print("This command is not fully implemented yet, come back soon!")
+    sys.exit()
+
+    # Derive an initial manifest
+    cli = ManifestGenerator(tech=args.container_tech)
+    manifests = cli.diff(args.image, database=args.database)
+    outfile = None
+
+    # Default to using outfile first, then outdir if defined
+    if args.outfile:
+        outfile = args.outfile
+    elif args.outdir:
+        outfile = os.path.join(args.outdir, "%s.json" % cli.save_path(args.image))
+        dirname = os.path.dirname(outfile)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+
+    # If we have an output file, make sure to set step output
+    if outfile:
+        print(f"Saving to {outfile}...")
+        print(f"::set-output name=outfile::{outfile}")
+        utils.write_json(manifests, outfile)
+    else:
+        print(json.dumps(manifests, indent=4))

container_guts/client/manifest.py

@@ -16,9 +16,10 @@ def main(args, parser, extra, subparser):
     print("       outfile: %s" % args.outfile)
     print("        outdir: %s" % args.outdir)
     print("container tech: %s" % args.container_tech)
+    print("       include: %s" % args.guts)
 
     cli = ManifestGenerator(tech=args.container_tech)
-    manifests = cli.run(args.image)
+    manifests = cli.run(args.image, includes=args.guts)
     outfile = None
 
     # Default to using outfile first, then outdir if defined

container_guts/defaults.py

@@ -0,0 +1,6 @@
+__author__ = "Vanessa Sochat"
+__copyright__ = "Copyright 2021-2022, Vanessa Sochat"
+__license__ = "MPL 2.0"
+
+# Default database for base image
+database = "https://github.com/singularityhub/shpc-guts"