Find packages containing any missing files

I also rewrote the outer driver script in Python instead of shell
because this logic was too painful to implement otherwise.

autobake-nix

@@ -0,0 +1,158 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p python3
+
+from contextlib import closing
+import subprocess
+
+
+def instantiate(*args):
+    """
+    Evaluate a Nix expression which produces a derivation, and return the
+    store-path of that derivation. All arguments are passed directly to
+    nix-instantiate.
+
+    Raises subprocess.CalledProcessError on failure.
+    """
+
+    result = subprocess.run(
+        ('nix-instantiate',) + args,
+        check=True,
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.PIPE,
+        universal_newlines=True,
+    )
+
+    lines = result.stdout.splitlines()
+    assert len(lines) == 1, result.stdout
+    return lines[0]
+
+
+def realise(drv, *args, **kwargs):
+    """
+    Given the store-path of a Nix derivation, return the exit status from
+    building that derivation, which is 0 on success.
+
+    By default, stdout and stderr are inherited from the parent process, but
+    you can pass any keyword arguments for subprocess.run to change that and
+    other behavior.
+
+    Extra positional arguments are passed to nix-store, which might be useful
+    for options like --quiet or --no-build-output.
+    """
+
+    return subprocess.run(
+        ('nix-store', '--realise') + args + (drv,),
+        stdin=subprocess.DEVNULL,
+        universal_newlines=True,
+        **kwargs
+    ).returncode
+
+
+def read_log(drv):
+    """
+    Search the build log of a Nix derivation for agent output and return it.
+    """
+    log_process = subprocess.Popen(
+        ['nix-store', '--read-log', drv],
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.PIPE,
+        universal_newlines=True,
+    )
+
+    in_summary = False
+    with closing(log_process.stdout) as lines:
+        for line in lines:
+            line = line.rstrip('\r\n')
+            if line == '===== begin autobake-agent summary =====':
+                in_summary = True
+            elif line == '===== end autobake-agent summary =====':
+                return
+            elif in_summary:
+                yield line
+
+
+def find_packages_for_file(path):
+    if not path.startswith('/'):
+        path = '/' + path
+
+    log_process = subprocess.Popen(
+	[
+            'nix-locate',
+            '--minimal',
+            '--top-level', '--whole-name',
+            '--type', 'x', '--type', 'r', '--type', 's',
+            path,
+        ],
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.PIPE,
+        universal_newlines=True,
+    )
+
+    with closing(log_process.stdout) as lines:
+        for line in lines:
+            yield line.rstrip('\r\n')
+
+
+if __name__ == '__main__':
+    from collections import defaultdict
+    from pathlib import Path
+    import sys
+
+    drv = instantiate(
+        '--show-trace',
+        '--arg', 'pkg', sys.argv[1],
+        Path(__file__).parent / 'build-agent',
+    )
+
+    exitcode = realise(drv)
+    print()
+    print(' *** trial build', 'SUCCESS' if exitcode == 0 else 'FAILED (status {})'.format(exitcode))
+    print()
+
+    all_packages = defaultdict(list)
+    multiple = defaultdict(list)
+    missing = []
+    for original_path in read_log(drv):
+        path = original_path
+        while True:
+            found = list(find_packages_for_file(path))
+
+            if len(found) == 1:
+                all_packages[found[0]].append(path)
+                break
+            elif found:
+                multiple[path] = found
+                break
+
+            # If there's another '/' left, remove the first component and
+            # check again.
+            try:
+                path = path[path.index('/', 1) + 1:]
+            except ValueError:
+                missing.append(original_path)
+                break
+
+    if all_packages:
+        print(' *** suggested dependencies:')
+        for package, paths in sorted(all_packages.items()):
+            print('-', package, 'for:')
+            for path in paths:
+                print('  -', path)
+        print()
+    else:
+        print(' *** no additional dependencies found')
+
+    if multiple:
+        print(' *** multiple possibly-satisfying choices:')
+        for path, packages in sorted(multiple.items()):
+            print('-', path, 'is available from:')
+            for package in packages:
+                print('  -', package)
+        print()
+
+    if missing:
+        print(' *** unsatisfiable paths:')
+        missing.sort()
+        for path in missing:
+            print('-', path)
+        print()

build-agent/postbuild

@@ -187,42 +187,41 @@ if __name__ == "__main__":
     os.chdir(d.parent)
     base = d.name + '.'
 
-    traces = {}
+    trace = []
     with os.scandir() as it:
         for entry in it:
-            assert entry.name.startswith(base)
-            pid = entry.name[len(base):]
             with open(entry.name) as log:
-                traces[pid] = list(parse_one_trace(log))
-
-    trace = itertools.chain.from_iterable(traces.values())
+                trace.extend(parse_one_trace(log))
 
     parents = set()
     for event in trace:
         if event[1] != 'fork':
             parents.update(event[2].parents)
 
-    trace = itertools.chain.from_iterable(traces.values())
-
     found = [defaultdict(set), defaultdict(set)]
     for event in trace:
-        p = event[2]
         if event[1] == 'fork':
-            pass
-        elif ignore_filenames.fullmatch(p.name) or ignore_pathnames.search(str(p)):
-            pass
-        elif p not in parents:
+            continue
+        p = event[2]
+        if p not in parents and not ignore_filenames.fullmatch(p.name):
             found[event[0]][p.name].add(p.parent)
 
-    singletons = []
+    missing = []
     searches = defaultdict(list)
     for filename, searched in sorted(found[False].items()):
         if filename in found[True]:
             continue
 
-        searched = sorted(searched)
+        searched = sorted(
+            p for p in searched
+            if p.is_absolute()
+            if not ignore_pathnames.search(str(p / filename))
+        )
+        if not searched:
+            continue
+
         if len(searched) == 1:
-            singletons.append(str(searched.pop() / filename))
+            missing.append(str(searched.pop() / filename))
             continue
 
         while True:
@@ -233,23 +232,12 @@ if __name__ == "__main__":
             filename = Path(common_parent, filename)
             searched = [parent.parent for parent in searched]
 
-        searched = tuple(map(str, searched))
-        searches[searched].append(str(filename))
+        missing.append(str(filename))
 
     print()
     print('===== begin autobake-agent summary =====')
-
-    print('not found:')
-    for path in sorted(singletons):
-        print(' -', shlex.quote(path))
-
-    for searched, filenames in sorted(searches.items()):
-        print()
-        print('looked in:')
-        for parent in searched:
-            print(' -', shlex.quote(parent))
-        print('but did not find:')
-        for filename in filenames:
-            print(' -', shlex.quote(filename))
-
+    missing.sort()
+    for path in missing:
+        print(path)
     print('===== end autobake-agent summary =====')
+    print()