initial commit of nested-partition-loader

This include a workaround for cargo-vendor being unable to
vendor compiler's dependencies.

It will load rust's Cargo.lock and manually version every component in a
nix derivation.

Both project and rust cargo vendor are then merged together (with
symlinks) and provided for cargo to use.

Signed-off-by: Arthur Gautier <baloo@superbaloo.net>

packages/cargo-lock-utils/compute-checksum

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+
+import json
+import sys
+import os
+import hashlib
+
+source, checksum = sys.argv[1:3]
+
+BLOCKSIZE=65536
+crate_files = {}
+
+for root, dirs, files in os.walk(source):
+    for f in files:
+        if f in ['.gitattributes', '.gitignore', '.cargo-ok', '.cargo-checksum.json', '.cargo_vcs_info.json']:
+            continue
+        if f.endswith('.rej') or f.endswith('.orig'):
+            continue
+
+        fpath = os.path.join(root, f)
+        rpath = os.path.relpath(fpath, source)
+
+        hasher = hashlib.sha256()
+        with open(fpath, 'rb') as fp:
+            buf = fp.read(BLOCKSIZE)
+            while len(buf) > 0:
+                hasher.update(buf)
+                buf = fp.read(BLOCKSIZE)
+
+        crate_files[rpath] = hasher.hexdigest()
+
+output = {
+  "files": crate_files,
+  "package": checksum
+}
+
+out = json.dumps(output, separators=(',', ':'), sort_keys=True)
+
+sys.stdout.write(out)
+

packages/cargo-lock-utils/default.nix

@@ -0,0 +1,15 @@
+{ pkgs ? import <nixpkgs> {}
+}:
+
+with pkgs.python3Packages;
+
+buildPythonApplication {
+  pname = "merge-cargo-lock";
+  version = "0.0.1";
+
+  propagatedBuildInputs = [
+    toml
+  ];
+
+  src = ./.;
+}

packages/cargo-lock-utils/list-cargo-lock

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+
+import toml
+import sys
+
+source = sys.argv[1]
+
+with open(source) as f:
+    content = toml.load(f)
+
+# first rewrite rust dependencies to make sure they are version pinned
+for p in content['package']:
+    if p.get('source') == 'registry+https://github.com/rust-lang/crates.io-index':
+        print("https://crates.io/api/v1/crates/%s/%s/download %s-%s %s" % (p['name'], p['version'], p['name'], p['version'], p['checksum']))

packages/cargo-lock-utils/merge-cargo-lock

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+
+import toml
+import sys
+
+source, rust = sys.argv[1:3]
+
+with open(source) as f:
+    origin_content = toml.load(f)
+
+with open(rust) as f:
+    rust_content = toml.load(f)
+
+# first rewrite rust dependencies to make sure they are version pinned
+package_versions = {}
+for p in rust_content['package']:
+    if p['name'] in package_versions:
+        package_versions[p['name']].append(p['version'])
+    else:
+        package_versions[p['name']] = [p['version']]
+
+for p in rust_content['package']:
+    if 'dependencies' in p:
+        deps = []
+        for dep in p['dependencies']:
+            if ' ' in dep:
+                deps.append(dep)
+            else:
+                deps.append(dep + " " + package_versions[dep][0])
+        p['dependencies'] = deps
+
+# do the same with the origin ones
+origin_packages = set()
+package_versions = {}
+for p in origin_content['package']:
+    origin_packages.add((p['name'], p['version']))
+
+    if p['name'] in package_versions:
+        package_versions[p['name']].append(p['version'])
+    else:
+        package_versions[p['name']] = [p['version']]
+
+for p in origin_content['package']:
+    if 'dependencies' in p:
+        deps = []
+        for dep in p['dependencies']:
+            if ' ' in dep:
+                deps.append(dep)
+            else:
+                deps.append(dep + " " + package_versions[dep][0])
+        p['dependencies'] = deps
+
+
+# then merge the rust in the origin ones
+# to do that, we can't add the same dependency twice.
+# TODO: do we need to care about the dependencies? (make it a super set?)
+for p in rust_content['package']:
+    tup = (p['name'], p['version'])
+    if tup not in origin_packages:
+        origin_content['package'].append(p)
+
+output = toml.dumps(origin_content)
+print(output)

packages/cargo-lock-utils/setup.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+from setuptools import setup, find_packages
+
+setup(name='cargo-lock-utils',
+      version='0.0.1',
+      packages=find_packages(),
+      scripts=[
+        'compute-checksum',
+        'list-cargo-lock',
+        'merge-cargo-lock',
+      ],
+     )

packages/nested-partition-loader/.cargo/config

@@ -0,0 +1,3 @@
+[unstable]
+build-std = ["core", "compiler_builtins", "alloc"]
+build-std-features = [ "compiler-builtins-mem" ]

packages/nested-partition-loader/.gitignore

@@ -0,0 +1 @@
+target

packages/nested-partition-loader/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "nested-partition-loader"
+version = "0.1.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+uefi = { path = "uefi-rs", features = [ "exts", "logger" ] }
+uefi-services = { path = "uefi-rs/uefi-services" }
+
+log = { version = "0.4.11", default-features = false }
+
+[patch.crates-io]
+uefi-macros = { path = "uefi-rs/uefi-macros" }
+uefi = { path = "uefi-rs" }

packages/nested-partition-loader/cargo-debug-hook.sh

@@ -0,0 +1,17 @@
+cargoRewriteConfigHook() {
+    echo "LASJKDLKAJSD"
+    cat .cargo/config
+
+    set -x
+    echo @mergeCargoLock@
+
+#    cat >> .cargo/config <<EOF
+#[source."library"]
+#"directory" = "/nix/store/iz8h5l1q4r8dqg88rwqapyi1p4ygcfv4-rust-src/lib/rustlib/src/rust/library/"
+#EOF
+    cat .cargo/config
+}
+
+if [ -z "${dontCargoRewriteConfig-}" ]; then
+  postUnpackHooks+=(cargoRewriteConfigHook)
+fi