Mdsplit to support ADFS (and potentially static MDQ)

.gitignore

@@ -21,3 +21,6 @@ sign.key
 _site
 profile.out*
 localconfig.py
+
+**/.DS_Store
+work

docs/using.rst

@@ -1,7 +1,7 @@
 Using pyFF
 =============
 
-pyFF has two command-line tools: pyff and pyffd.
+pyFF has three command-line tools: pyff, pyffd and pyff_mdsplit.
 
 .. code-block:: bash
 
@@ -45,4 +45,6 @@ Processing steps are called pipes. A pipe can have arguments and options:
 Typically options are used to modify the behaviour of the pipe itself (think macros), while arguments provide
 runtime data to operate on.
 
-Documentation for each pipe is in the :py:mod:`pyff.pipes.builtins` Module. Also take a look at the :doc:`examples`.
+Documentation for each pipe is in the :py:mod:`pyff.pipes.builtins` Module. Also take a look at the :doc:`examples`.
+
+For the documentation of pyff_mdsplit run pyff_mdsplit.py --help

scripts/pyff_mdsplit.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+"""
+usage: pyff_mdsplit.py [-h] [-c CERTFILE] [-k KEYFILE] [-n] [-S] [-v]
+                       [-l LOGFILE] [-L {INFO,DEBUG,CRITICAL,WARNING,ERROR}]
+                       [-o OUTDIR_SIGNED] [-C CACHEDURATION] [-u VALIDUNTIL]
+                       input outdir_unsigned
+
+Metadata Splitter
+
+positional arguments:
+  input                 Metadata aggregate (input)
+  outdir_unsigned       Directory for files containing one unsigned
+                        EntityDescriptor each.
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -c CERTFILE, --certfile CERTFILE
+  -k KEYFILE, --keyfile KEYFILE
+  -n, --nocleanup       do not delete temporary files after signing
+  -S, --nosign          do not sign output
+  -v, --verbose         output to console with DEBUG level
+  -l LOGFILE, --logfile LOGFILE
+  -L {INFO,DEBUG,CRITICAL,WARNING,ERROR}, --loglevel {INFO,DEBUG,CRITICAL,WARNING,ERROR}
+                        default is INFO if env[LOGLEVEL] is not set
+  -o OUTDIR_SIGNED, --outdir_signed OUTDIR_SIGNED
+                        Directory for files containing one signed
+                        EntityDescriptor each.
+  -C CACHEDURATION, --cacheduration CACHEDURATION
+                        override value from input EntitiesDescriptor, if any
+  -u VALIDUNTIL, --validuntil VALIDUNTIL
+                        override iso date value from input EntitiesDescriptor,
+                        if any
+"""
+
+import argparse
+import logging
+import os
+import re
+import sys
+
+import pyff.mdsplit
+
+LOGLEVELS = {'CRITICAL': 50, 'ERROR': 40, 'WARNING': 30, 'INFO': 20, 'DEBUG': 10}
+
+
+class Invocation:
+    """ Get arguments from command line and enviroment """
+    def __init__(self, testargs=None):
+        self.parser = argparse.ArgumentParser(description='Metadata Splitter')
+        self.parser.add_argument('-c', '--certfile', dest='certfile')
+        self.parser.add_argument('-k', '--keyfile', dest='keyfile')
+        self.parser.add_argument('-n', '--nocleanup', action='store_true',
+            help='do not delete temporary files after signing')
+        self.parser.add_argument('-S', '--nosign', action='store_true', help='do not sign output')
+        self.parser.add_argument('-v', '--verbose', action='store_true', help='output to console with DEBUG level')
+        logbasename = re.sub(r'\.py$', '.log', os.path.basename(__file__))
+        self.parser.add_argument('-l', '--logfile', dest='logfile', default=logbasename)
+        loglevel_env = os.environ['LOGLEVEL'] if 'LOGLEVEL' in os.environ else 'INFO'
+        self.parser.add_argument('-L', '--loglevel', dest='loglevel', default=loglevel_env,
+            choices=LOGLEVELS.keys(), help='default is INFO if env[LOGLEVEL] is not set')
+        self.parser.add_argument('-o', '--outdir_signed', default=None,
+            help='Directory for files containing one signed EntityDescriptor each.')
+        self.parser.add_argument('-C', '--cacheduration', dest='cacheDuration', default='PT5H',
+            help='override value from input EntitiesDescriptor, if any')
+        self.parser.add_argument('-u', '--validuntil', dest='validUntil',
+            help='override iso date value from input EntitiesDescriptor, if any')
+        self.parser.add_argument('input', type=argparse.FileType('r'), default=None,
+            help='Metadata aggregate (input)')
+        self.parser.add_argument('outdir_unsigned', default=None,
+            help='Directory for files containing one unsigned EntityDescriptor each.')
+        self.args = self.parser.parse_args()
+        # merge argv with env
+        if not self.args.nosign:
+            self.args.certfile = self._merge_arg('MDSIGN_CERT', self.args.certfile, 'certfile')
+            self.args.keyfile = self._merge_arg('MDSIGN_KEY', self.args.keyfile, 'keyfile')
+            self.args.outdir_signed = self._merge_arg('MDSPLIT_SIGNED', self.args.outdir_signed, 'outdir_signed')
+        self.args.input = self._merge_arg('MD_AGGREGATE', self.args.input, 'input')
+        self.args.outdir_unsigned = self._merge_arg('MDSPLIT_UNSIGNED', self.args.outdir_unsigned, 'outdir_unsigned')
+
+    def _merge_arg(self, env, arg, argname):
+        """ merge argv with env """
+        if env not in os.environ and arg is None:
+            print("Either %s or --%s must be set and point to an existing file" % (env, argname))
+            exit(1)
+        if arg is None:
+            return env
+        else:
+            return arg
+
+
+def main():
+    invocation = Invocation()
+
+    log_args = {'level': LOGLEVELS[invocation.args.loglevel],
+                'format': '%(asctime)s - %(levelname)s  [%(filename)s:%(lineno)s] %(message)s',
+                'filename': invocation.args.logfile,
+    }
+    logging.basicConfig(**log_args)
+    if invocation.args.verbose:
+        console_handler = logging.StreamHandler()
+        console_handler.setLevel(logging.DEBUG)
+        logging.getLogger('').addHandler(console_handler)
+
+    logging.debug('')
+    logging.debug('Input file is ' + invocation.args.input.name)
+    logging.debug('Output directory for unsigned files is ' + os.path.abspath(invocation.args.outdir_unsigned))
+    if not invocation.args.nosign:
+        logging.debug('Output directory for signed files is ' + os.path.abspath(invocation.args.outdir_signed))
+
+    pyff.mdsplit.process_md_aggregate(invocation.args)
+
+if __name__ == "__main__":  # pragma: no cover
+    main()

scripts/pyff_split_sign.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+# create a signed XML file per EntityDescriptor for ADFS
+
+# MDSIGN_CERT, MDSIGN_KEY and MDAGGREGATE must be passed via env
+[ $MDSIGN_CERT ] || echo "MDSIGN_CERT must be set and point to an existing file" && exit 1
+[ $MDSIGN_KEY ] || echo "MDSIGN_KEY must be set and point to an existing file" && exit 1
+[ $MD_AGGREGATE ]|| echo "MD_AGGREGATE must be set and point to an existing file" && exit 1
+# Setting defaults
+[ $MDSPLIT_UNSIGNED ] || MDSPLIT_UNSIGNED='/var/md_source/split/'
+[ $MDSPLIT_SIGNED ] || MDSPLIT_SIGNED='/var/md_feed/split/'
+[ $LOGFILE ] || LOGFILE='/var/log/pyffsplit.log'
+
+# Step 1. Split aggregate and create an XML and a pipeline file per EntityDescriptor
+[ "$LOGLEVEL" == "DEBUG" ] && echo "processing "
+/usr/bin/pyff_mdsplit.py \
+    -c $MDSIGN_CERT -k $MDSIGN_KEY \
+    -l $LOGFILE -L DEBUG \
+    $MD_AGGREGATE $MDSPLIT_UNSIGNED $MDSPLIT_SIGNED
+
+# Step 2. Execute pyff to sign each EntityDescriptor
+cd $MDSPLIT_UNSIGNED
+for fn in *.fd; do
+    echo "running pyff for $fn"
+    /usr/bin/pyff --loglevel=$LOGLEVEL $fn
+done
+
+# make metadata files availabe to nginx container:
+chmod 644 /var/md_feed/split/*.xml 2> /dev/null
+

setup.py

@@ -77,6 +77,7 @@
       entry_points={
           'console_scripts': ['pyff=pyff.md:main', 'pyffd=pyff.mdx:main']
       },
+      scripts=['scripts/pyff_mdsplit.py', ],
       message_extractors={'src': [
           ('**.py', 'python', None),
           ('**/templates/**.html', 'mako', None),

src/pyff/__init__.py

@@ -10,4 +10,7 @@
 __license__ = "BSD"
 __maintainer__ = "leifj@sunet.se"
 __status__ = "Production"
+# Issue when invoking pyff_mdsplit.pyfrom Pycharm
+# "pkg_resources.DistributionNotFound: The 'pyFF' distribution was not found"
+# remove following line temporarily to test the script
 __version__ = pkg_resources.require("pyFF")[0].version

src/pyff/mdsplit.py

@@ -0,0 +1,139 @@
+"""
+mdsplit creates a separate signed XML file for each EntitiyDescriptor from the input md aggregate.
+
+Note: The input file is considered to be trusted, the signature is not verified.
+"""
+
+__author__ = 'r2h2'
+
+import logging
+import lxml.etree as etree
+import os
+import re
+import shutil
+import sys
+
+from pyff.mdrepo import MDRepository
+from pyff.pipes import plumbing
+from pyff.store import MemoryStore
+
+XMLDECLARATION = '<?xml version="1.0" ?>'
+
+
+class Pipeline:
+    def __init__(self, keyfile, certfile, cacheDuration, validUntil):
+        self.keyfile = keyfile
+        self.certfile = certfile
+        self.cacheDuration = cacheDuration
+        self.validUntil = validUntil
+
+    def get(self, loaddir, outfile):
+        # sign a single entity descriptor
+        pipeline = '''- load:
+  - {0}
+- select
+- finalize:
+    cacheDuration: {4}
+    validUntil: {5}
+- sign:
+    key:  {2}
+    cert: {3}
+- publish:
+    {1}
+'''.format(loaddir,
+           outfile,
+           self.keyfile,
+           self.certfile,
+           self.cacheDuration,
+           self.validUntil)
+        return pipeline
+
+
+def entityid_to_dirname(entityid):
+    """
+    Derive a directory name from an HTTP-URL-formed entityID, removing dots and slashes
+    :param entityid:
+    :return: dirname derived from entityID
+    """
+    x = re.sub(r'^https?://', '', entityid)
+    r = ''
+    upper = False
+
+    in_path = False
+    for i in range(0, len(x)):
+        if x[i].isalpha() or x[i].isdigit():
+            if upper:
+                r += x[i].upper()
+            else:
+                r += x[i]
+            upper = False
+        elif not in_path and x[i] == '/':
+            r += '_'
+            in_path = True
+        else:
+            upper = True
+    return r
+
+
+def simple_md(pipeline):
+    """ just a copy of md:main """
+    modules = []
+    modules.append('pyff.builtins')
+    store = MemoryStore()
+    md = MDRepository(store=store)
+    plumbing(pipeline).process(md, state={'batch': True, 'stats': {}})
+
+
+def process_entity_descriptor(ed, pipeline, args):
+    """
+    1. create an unsigned EntityDescriptor XML file
+    2. create a pipline file to sign it
+    3. execute pyff to create an aggregate with the EntityDescriptor
+    4. delete temp files
+    Note: for pyff pipeline processing the entityDescriptor xml file must be alone in a directory
+    TODO: remove EntitiesDecriptor and make EntityDescriptor the rool element.
+    """
+    dirname_temp = os.path.abspath(os.path.join(args.outdir_unsigned,
+                                                entityid_to_dirname(ed.attrib['entityID'])))
+    if not os.path.exists(dirname_temp):
+        os.makedirs(dirname_temp)
+    fn_temp = os.path.join(dirname_temp, 'ed.xml')
+    logging.debug('writing unsigned EntitiyDescriptor ' + ed.attrib['entityID'] + ' to ' + fn_temp)
+    if args.cacheDuration is not None:
+        ed.attrib['cacheDuration'] = args.cacheDuration
+    if args.validUntil is not None:
+        ed.attrib['validUntil'] = args.validUntil
+    if not os.path.exists(os.path.dirname(fn_temp)):
+        os.makedirs(os.path.dirname(fn_temp))
+    with open(fn_temp, 'w') as f:
+        f.write(XMLDECLARATION + '\n' + etree.tostring(ed))
+    if not args.nosign:
+        if not os.path.exists(args.outdir_signed):
+            os.makedirs(args.outdir_signed)
+        fn_out = os.path.abspath(os.path.join(args.outdir_signed,
+                                              entityid_to_dirname(ed.attrib['entityID']) + '.xml'))
+        fn_pipeline = os.path.join(dirname_temp, 'ed.fd')
+        with open(fn_pipeline, 'w') as f_pipeline:
+            f_pipeline.write(pipeline.get(dirname_temp, fn_out))
+        simple_md(fn_pipeline)
+        if not args.nocleanup:
+            shutil.rmtree(dirname_temp)
+
+
+
+def process_md_aggregate(args):
+    """ process each ed; take validUntil and cacheDuration from root level """
+    root = etree.parse(args.input).getroot()
+    if root.tag != '{urn:oasis:names:tc:SAML:2.0:metadata}EntitiesDescriptor':
+        raise Exception('Root element must be EntitiesDescriptor')
+    if 'cacheDuration' in root.attrib and args.cacheDuration is None:
+        args.cacheDuration = root.attrib['cacheDuration']
+    if 'validUntil' in root.attrib and args.validUntil is None:
+        args.validUntil = root.attrib['validUntil']
+    alist = ''
+    for a in root.attrib:
+        alist += ' ' + a + '="' + root.attrib[a] + '"'
+    logging.debug('Root element: ' + root.tag + alist)
+    pipeline = Pipeline(args.keyfile, args.certfile, args.cacheDuration, args.validUntil)
+    for ed in root.findall('{urn:oasis:names:tc:SAML:2.0:metadata}EntityDescriptor'):
+        process_entity_descriptor(ed, pipeline, args)