[python] Rework graminelibos python library

This commit introduces new python APIs for managing manifests and
SGX sigstructs, including MRENCLAVE generation, signing and token
generation.
This required a couple of rewrites, which hopefuly resulted in cleaner
code.

Signed-off-by: Borys Popławski <borysp@invisiblethingslab.com>

.ci/run-pylint

@@ -19,5 +19,7 @@ find . -name \*.py \
     -and -not -path ./LibOS/shim/test/ltp/install/\* \
 | sed 's/./\\&/g' \
 | xargs "${PYLINT}" "$@" \
+    python/gramine-gen-depend \
+    python/gramine-manifest \
     python/gramine-sgx-get-token \
     python/gramine-sgx-sign

Documentation/conf.py

@@ -148,6 +148,8 @@ def setup(app):
 # (source start file, name, description, authors, manual section).
 man_pages = [
     ('manpages/gramine-manifest', 'gramine-manifest', 'Gramine manifest preprocessor', [author], 1),
+    ('manpages/gramine-sgx-sign', 'gramine-sgx-sign', 'Gramine SIGSTRUCT generator', [author], 1),
+    ('manpages/gramine-sgx-get-token', 'gramine-sgx-get-token', 'Gramine SGX Token generator', [author], 1),
     ('manpages/pal_loader', 'pal_loader', 'FIXME Loader', [author], 1),
     ('manpages/is_sgx_available', 'is_sgx_available', 'Check SGX compatibility', [author], 1),
     ('manpages/quote_dump', 'quote_dump', 'Display SGX quote', [author], 1),

Documentation/manpages/gramine-sgx-get-token.rst

@@ -0,0 +1,29 @@
+.. program:: gramine-sgx-get-token
+.. _gramine-sgx-get-token:
+
+===============================================================
+:program:`gramine-sgx-get-token` -- Gramine SGX token generator
+===============================================================
+
+Synopsis
+========
+
+:command:`gramine-sgx-get-token` [*OPTION*]... --sig sigstruct_file
+--output token_file
+
+Description
+===========
+
+:program:`gramine-sgx-get-token` is used to generate the SGX token file for
+given SIGSTRUCT (".sig" file).
+
+Command line arguments
+======================
+
+.. option:: --sig sigstruct_file, -s sigstruct_file
+
+    Path to the input file containing SIGSTRUCT.
+
+.. option:: --output token_file, -o token_file
+
+   Path to the output token file.

Documentation/manpages/gramine-sgx-sign.rst

@@ -0,0 +1,43 @@
+.. program:: gramine-sgx-sign
+.. _gramine-sgx-sign:
+
+==========================================================
+:program:`gramine-sgx-sign` -- Gramine SIGSTRUCT generator
+==========================================================
+
+Synopsis
+========
+
+:command:`gramine-sgx-sign` [*OPTION*]... --output output_manifest
+--key key_file --manifest manifest_file
+
+Description
+===========
+
+:program:`gramine-sgx-sign` is used to expand Trusted Files and generate
+signature file for given input manifest and libpal file (main Gramine binary).
+
+Command line arguments
+======================
+
+.. option:: --output output_manifest, -o output_manifest
+
+   Path to the output manifest file (with Trusted Files expanded).
+
+.. option:: --key key_file, -k key_file
+
+    Path to the private key used for signing.
+
+.. option:: --manifest manifest_file, -m manifest_file
+
+    Input manifest file.
+
+.. option:: --libpal libpal_path, -l libpal_path
+
+    Path to libpal file (main Gramine binary).
+
+.. option:: --sigfile sigfile, -s sigfile
+
+    Path to the output file containing SIGSTRUCT. If not provided,
+    `manifest_file` will be used with ".manifest" (if present) removed from
+    the end and with ".sig" appended.

LibOS/shim/test/ltp/Makefile.Test

@@ -91,10 +91,8 @@ expand_target_to_sgx = $(addsuffix .manifest.sgx,$(call drop_manifest_suffix,$(1
 	@$(SGX_SIGN) --output $*.manifest.sgx --manifest $< > .output.sgx_sign.$*
 
 .PRECIOUS: %.manifest.sgx.d
-
 %.manifest.sgx.d: %.manifest
-	@echo [ $@ ]
-	@$(SGX_SIGN) --depend --output $@ --manifest $<
+	$(call cmd,manifest_gen_depend)
 
 ifeq ($(filter clean,$(MAKECMDGOALS)),)
 include $(addsuffix .manifest.sgx.d,$(call drop_manifest_suffix,$(manifests)))

Pal/regression/Bootstrap3.manifest

@@ -3,6 +3,6 @@ loader.log_level = "debug"
 loader.preload = "file:Preload1.so,file:Preload2.so"
 loader.argv0_override = "Bootstrap3"
 
-sgx.trusted_files.entrypoint = "file:Bootstrap3"
+sgx.trusted_files = [ "file:Bootstrap3" ]
 sgx.nonpie_binary = true
 sgx.debug = true

Pal/regression/Bootstrap6.manifest

@@ -9,4 +9,4 @@ sgx.enclave_size = "8192M"
 sgx.nonpie_binary = true
 sgx.debug = true
 
-sgx.trusted_files.entrypoint = "file:Bootstrap"
+sgx.trusted_files = [ "file:Bootstrap" ]

Pal/regression/Bootstrap7.manifest

@@ -1,7 +1,7 @@
 pal.entrypoint = "file:Bootstrap7"
 loader.argv0_override = "Bootstrap7"
 
-sgx.trusted_files.entrypoint = "file:Bootstrap7"
+sgx.trusted_files = [ "file:Bootstrap7" ]
 sgx.nonpie_binary = true
 sgx.debug = true
 

Pal/regression/Makefile

@@ -77,7 +77,7 @@ gramine = sgx
 else
 gramine = direct
 endif
-pal_lib ?= $(shell gramine-manifest -c '{{ gramine.pkglibdir }}')/$(gramine)/libpal.so
+pal_lib ?= $(shell gramine-manifest -c 'EXTRACT = "{{ gramine.pkglibdir }}"' | grep -F EXTRACT | sed -e "s/EXTRACT = \"\(.*\)\"/\1/")/$(gramine)/libpal.so
 
 .PHONY: all
 all: $(target) $(call expand_target_to_sig,$(target)) \

Pal/regression/Process3.manifest

@@ -6,4 +6,4 @@ loader.insecure__use_cmdline_argv = true
 sgx.nonpie_binary = true
 sgx.debug = true
 
-sgx.trusted_files.entrypoint = "file:Process3"
+sgx.trusted_files = [ "file:Process3" ]

Pal/regression/Thread2.manifest

@@ -6,4 +6,4 @@ sgx.enable_stats = true
 sgx.nonpie_binary = true
 sgx.debug = true
 
-sgx.trusted_files.entrypoint = "file:Thread2"
+sgx.trusted_files = [ "file:Thread2" ]

Pal/regression/Thread2_exitless.manifest

@@ -7,4 +7,4 @@ sgx.enable_stats = true
 sgx.nonpie_binary = true
 sgx.debug = true
 
-sgx.trusted_files.entrypoint = "file:Thread2"
+sgx.trusted_files = [ "file:Thread2" ]

Pal/src/host/Linux-SGX/manifest.mk

@@ -18,7 +18,7 @@ $(SGX_SIGNER_KEY):
 
 .PRECIOUS: %.manifest.sgx.d
 %.manifest.sgx.d: %.manifest
-	$(call cmd,sgx_sign_depend)
+	$(call cmd,manifest_gen_depend)
 
 ifeq ($(filter %clean,$(MAKECMDGOALS)),)
 ifeq ($(target),)

Scripts/Makefile.rules

@@ -128,9 +128,9 @@ quiet_cmd_sgx_sign = [ $*.{sig, manifest.sgx} ]
 quiet_cmd_sgx_get_token = [ Token: $(basename $*) ]
       cmd_sgx_get_token = gramine-sgx-get-token --output $@ --sig $^ > .output.sgx_get_token.$(basename $*)
 
-# sgx manifest dependency
-quiet_cmd_sgx_sign_depend = [ $@ ]
-      cmd_sgx_sign_depend = gramine-sgx-sign --key $(SGX_SIGNER_KEY) --depend --output $@ --manifest $<
+# manifest dependency
+quiet_cmd_manifest_gen_depend = [ $@ ]
+      cmd_manifest_gen_depend = gramine-gen-depend --output $@ --manifest $<
 
 # manifest ($(2) - entrypoint, $(3) - replace rules)
 quiet_cmd_manifest = [ $@ ]

python/gramine-gen-depend

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: LGPL-3.0-or-later
+# Copyright (c) 2021 Intel Corporation
+#                    Borys Popławski <borysp@invisiblethingslab.com>
+
+import os
+
+import click
+
+from graminelibos import Manifest, _CONFIG_PKGLIBDIR
+
+@click.command()
+@click.option('--manifest', '-m', 'manifest_file', type=click.File('r', encoding='utf-8'),
+              required=True, help='Input .manifest file')
+@click.option('--libpal', '-l', type=click.Path(exists=True, dir_okay=False),
+              default=os.path.join(_CONFIG_PKGLIBDIR, 'sgx/libpal.so'), help='Input libpal file',
+              show_default=True)
+@click.option('--output', '-o', type=click.File('w', encoding='utf-8'), required=True,
+              help='Output .manifest.d file')
+def main(manifest_file, libpal, output):
+    manifest = Manifest.load(manifest_file)
+
+    dependencies = manifest.get_dependencies()
+    dependencies.add(libpal)
+
+    output.write(f'{manifest_file.name}:')
+    for filename in dependencies:
+        output.write(f' \\\n\t{filename}')
+    output.write('\n')
+
+if __name__ == '__main__':
+    main() # pylint: disable=no-value-for-parameter

python/gramine-manifest

@@ -1,5 +1,33 @@
 #!/usr/bin/env python3
+# SPDX-License-Identifier: LGPL-3.0-or-later
+# Copyright (c) 2021 Intel Corporation
+#                    Borys Popławski <borysp@invisiblethingslab.com>
 
-import sys
-from graminelibos.manifest import main
-sys.exit(main())
+import click
+
+from graminelibos import Manifest
+
+def validate_define(_ctx, _param, values):
+    ret = {}
+    for value in values:
+        try:
+            k, v = value.split('=', 1)
+        except ValueError:
+            k, v = value, True
+        ret[k] = v
+    return ret
+
+@click.command()
+@click.option('--string', '-c')
+@click.option('--define', '-D', multiple=True, callback=validate_define)
+@click.argument('infile', type=click.File('r'), required=False)
+@click.argument('outfile', type=click.File('w'), default='-')
+def main(string, define, infile, outfile):
+    if not bool(string) ^ bool(infile):
+        click.get_current_context().fail('specify exactly one of (infile, -c)')
+    template = infile.read() if infile else string
+    manifest = Manifest.from_template(template, define)
+    manifest.dump(outfile)
+
+if __name__ == '__main__':
+    main() # pylint: disable=no-value-for-parameter

python/gramine-sgx-get-token

@@ -1,5 +1,19 @@
 #!/usr/bin/env python3
+# SPDX-License-Identifier: LGPL-3.0-or-later
+# Copyright (c) 2021 Intel Corporation
+#                    Borys Popławski <borysp@invisiblethingslab.com>
 
-import sys
-from graminelibos.sgx_get_token import main
-sys.exit(main())
+import click
+
+from graminelibos import Sigstruct, get_token
+
+@click.command()
+@click.option('--sig', '-s', type=click.File('rb'), required=True, help='sigstruct file')
+@click.option('--output', '-o', type=click.File('wb'), required=True, help='Output token file')
+def main(sig, output):
+    sig = Sigstruct.from_bytes(sig.read())
+    token = get_token(sig)
+    output.write(token)
+
+if __name__ == '__main__':
+    main() # pylint: disable=no-value-for-parameter

python/gramine-sgx-sign

@@ -1,5 +1,45 @@
 #!/usr/bin/env python3
+# SPDX-License-Identifier: LGPL-3.0-or-later
+# Copyright (c) 2021 Intel Corporation
+#                    Borys Popławski <borysp@invisiblethingslab.com>
 
-import sys
-from graminelibos.sgx_sign import main
-sys.exit(main())
+import datetime
+
+import click
+
+from graminelibos import Manifest, get_tbssigstruct, sign_with_local_key
+
+@click.command()
+@click.option('--output', '-o', type=click.Path(), required=True,
+              help='Output .manifest.sgx file (manifest augmented with autogenerated fields)')
+@click.option('--libpal', '-l', type=click.Path(exists=True, dir_okay=False),
+              help='Input libpal file')
+@click.option('--key', '-k', type=click.Path(exists=True, dir_okay=False), required=True,
+              help='specify signing key (.pem) file')
+@click.option('--manifest', '-m', 'manifest_file', type=click.File('r', encoding='utf-8'),
+              required=True, help='Input .manifest file')
+@click.option('--sigfile', '-s', help='Output .sig file')
+def main(output, libpal, key, manifest_file, sigfile):
+    manifest = Manifest.load(manifest_file)
+
+    manifest.expand_all_trusted_files()
+
+    with open(output, 'w', encoding='utf-8') as f:
+        manifest.dump(f)
+
+    if not sigfile:
+        if manifest_file.name.endswith('.manifest'):
+            sigfile = manifest_file.name[:-len('.manifest')]
+        else:
+            sigfile = manifest_file.name
+        sigfile += '.sig'
+
+    today = datetime.date.today()
+    sigstruct = get_tbssigstruct(output, today, libpal)
+    sigstruct.sign(sign_with_local_key, key)
+
+    with open(sigfile, 'wb') as f:
+        f.write(sigstruct.to_bytes())
+
+if __name__ == '__main__':
+    main() # pylint: disable=no-value-for-parameter

python/graminelibos/__init__.py

@@ -10,3 +10,14 @@
         'You are attempting to run the tools from repo, without installing. '
         'Please install Gramine before running Python tools. See '
         'https://gramine.readthedocs.io/en/latest/building.html.')
+
+# pylint: disable=wrong-import-position
+from .gen_jinja_env import make_env
+
+_env = make_env()
+
+from .manifest import Manifest
+if '@SGX_ENABLED@' == '1':
+    from .sgx_get_token import get_token
+    from .sgx_sign import get_tbssigstruct, sign_with_local_key
+    from .sigstruct import Sigstruct