From 2f653c79c81c0a8d2cf0e72c56077432de3770fe Mon Sep 17 00:00:00 2001 From: r2h2 Date: Mon, 11 Jul 2016 11:32:46 +0200 Subject: [PATCH 01/20] added mdsplit functionality --- scripts/mdsplit.py | 129 +++++++++ scripts/pyffsignsingle.sh | 23 ++ src/pyff/test/data/metadata/test_mdsplit.xml | 268 +++++++++++++++++++ src/pyff/test/data/pki/mycert.pem | 18 ++ src/pyff/test/data/pki/mykey.pem | 15 ++ 5 files changed, 453 insertions(+) create mode 100644 scripts/mdsplit.py create mode 100644 scripts/pyffsignsingle.sh create mode 100644 src/pyff/test/data/metadata/test_mdsplit.xml create mode 100644 src/pyff/test/data/pki/mycert.pem create mode 100644 src/pyff/test/data/pki/mykey.pem diff --git a/scripts/mdsplit.py b/scripts/mdsplit.py new file mode 100644 index 00000000..5b755204 --- /dev/null +++ b/scripts/mdsplit.py @@ -0,0 +1,129 @@ +""" +pyffsplit creates pipeline files for each EntityDescriptor in an aggregate, which then used to +create separate signed XML documents with an EntitiyDescriptor each. +TODO: Signing in the same process +""" +import argparse +import future +import logging +import lxml.etree as etree +import os +import re +import sys + +import pyff +from pyff.mdrepo import MDRepository +from pyff.pipes import plumbing +from pyff.store import MemoryStore + + +class Pipeline: + def __init__(self, keyfile, certfile, idprefix, cacheDuration, validUntil): + self.keyfile = keyfile + self.certfile = certfile + self.idprefix = idprefix + self.cacheDuration = cacheDuration + self.validUntil = validUntil + + def get(self, infile, outfile): + # sign a single entity descriptor + pipeline = '''- load: + - {0} +- select +- finalize: + Name: {4} + cacheDuration: {5} + validUntil: {6} +- sign: + key: {2} + cert: {3} +- publish: + {1} +'''.format(infile, + outfile, + self.keyfile, + self.certfile, + self.idprefix, + self.cacheDuration, + self.validUntil) + return pipeline + +def entityid_to_filename(entityid): + """ + Derive a filename from an entityID, removing dots and slashes + :param entityid: + :return: filename derived from entityID + """ + x = re.sub(r'^https?://', '', entityid) + r = '' + upper = False + + in_path = False + for i in range(0, len(x)): + if x[i].isalpha() or x[i].isdigit(): + if upper: + r += x[i].upper() + else: + r += x[i] + upper = False + elif not in_path and x[i] == '/': + r += '_' + in_path = True + else: + upper = True + return r + '.xml' + +# def simple_md(pipeline): +# """ stupid copy of md:main -> replace this """ +# modules = [] +# modules.append('pyff.builtins') +# store = MemoryStore() +# md = MDRepository(store=store) +# plumbing(pipeline).process(md, state={'batch': True, 'stats': {}}) + + +def main(): + LOGLEVELS = {'CRITICAL': 50, 'ERROR': 40, 'WARNING': 30, 'INFO': 20, 'DEBUG': 10} + XMLDECLARATION = '' + parser = argparse.ArgumentParser(description='Metadata Splitter') + parser.add_argument('-c', '--certfile', dest='certfile', default='pyff_sign-cer.pem') + parser.add_argument('-k', '--keyfile', dest='keyfile', default='pyff_sign-key.pem') + parser.add_argument('-i', '--idprefix', dest='idprefix', default='ourfederation.example.com_') + parser.add_argument('-C', '--cacheduration', dest='cacheduration', default='PT5H') + parser.add_argument('-l', '--logfile', dest='logfile', default='pyffsplit.log') + parser.add_argument('-L', '--loglevel', dest='loglevel', default='INFO', choices=LOGLEVELS.keys()) + parser.add_argument('-u', '--validuntil', dest='validuntil', default='P10D') + parser.add_argument('input', type=argparse.FileType('r'), default=None, + help='Metadata aggregate') + parser.add_argument('outdir_unsigned', default=None) + parser.add_argument('outdir_signed', default=None, + help='Directory for files containing one EntityDescriptor each.') + args = parser.parse_args() + + log_args = {'level': LOGLEVELS[args.loglevel]} + log_args['filename'] = args.logfile + logging.basicConfig(**log_args) + logging.debug('Input file is ' + args.input.name) + logging.debug('Input directory is ' + args.outdir_signed) + + root = etree.parse(args.input).getroot() + if root.tag != '{urn:oasis:names:tc:SAML:2.0:metadata}EntitiesDescriptor': + raise Exception('Root element must be EntitiesDescriptor') + logging.debug('Root element is ' + root.tag) + pipeline = Pipeline(args.keyfile, args.certfile, + args.idprefix, args.cacheduration, args.validuntil) + for e in root.findall('{urn:oasis:names:tc:SAML:2.0:metadata}EntityDescriptor'): + fn_temp = os.path.abspath(os.path.join(args.outdir_unsigned, entityid_to_filename(e.attrib['entityID']))) + fn_out = os.path.abspath(os.path.join(args.outdir_signed, entityid_to_filename(e.attrib['entityID']))) + logging.debug('writing unsigned EntitiyDescriptor ' + e.attrib['entityID'] + ' to ' + fn_temp) + with open(fn_temp, 'w') as f: + f.write(XMLDECLARATION + etree.tostring(e)) + fn_pipeline = fn_temp + '.fd' + with open(fn_pipeline, 'w') as f_pipeline: + f_pipeline.write(pipeline.get(fn_temp, fn_out)) + #simple_md(fn_pipeline) + + +if __name__ == "__main__": # pragma: no cover + #print os.getcwd() + main() \ No newline at end of file diff --git a/scripts/pyffsignsingle.sh b/scripts/pyffsignsingle.sh new file mode 100644 index 00000000..d4b8b150 --- /dev/null +++ b/scripts/pyffsignsingle.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# create a signed XML file per EntityDescriptor for ADFS + +# Step 1. Split aggregate and create an XML and a pipeline file per EntityDescriptor +/usr/bin/mdsplit.py \ + -c /etc/pki/pyff/metadata_signing-crt.pem \ + -k /etc/pki/pyff/metadata_signing-key.pem \ + -l /var/log/pyffsplit.log \ + -L DEBUG \ + /var/md_feed/metadata.xml \ + /var/md_feed/split/ \ + /var/md_source/split/ + +# Step 2. Execute pyff to sign each EntityDescriptor +cd /var/md_source/split/ +for fn in *.fd; do + echo "running pyff for $fn" + /usr/bin/pyff --loglevel=$LOGLEVEL $fn +done + +# make metadata files availabe to nginx container: +chmod 644 /var/md_feed/split/*.xml 2> /dev/null + diff --git a/src/pyff/test/data/metadata/test_mdsplit.xml b/src/pyff/test/data/metadata/test_mdsplit.xml new file mode 100644 index 00000000..a3dec4b6 --- /dev/null +++ b/src/pyff/test/data/metadata/test_mdsplit.xml @@ -0,0 +1,268 @@ + + + + + + + + + MIIC4DCCAcigAwIBAgIQZdBdwaCjAI9Km6hQBawbSjANBgkqhkiG9w0BAQsFADAsMSowKAYDVQQDEyFBREZTIEVuY3J5cHRpb24gLSB3a2lzLmRldi53a28uYXQwHhcNMTUwODI1MjMwNzI2WhcNMTYwODI0MjMwNzI2WjAsMSowKAYDVQQDEyFBREZTIEVuY3J5cHRpb24gLSB3a2lzLmRldi53a28uYXQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCimwdmMHIN/SYSbqS2jhwhSGGPoAKzTgfIc7L3RPvb10l4TMUDLn8q0zi3fGMN9KAp7DHcHpFulDpOsLKzALLQsG6gPY/YVCKatYKDOdNnYV1eq5SI/Sf496YBjxeLFWk1tKd/3epGD0fxpdQ+iK/Yla1vR5HLxeorSfO3+tMDTATxkfh7v8fGrZ29l4QEZjRKH7JUgP22BWffDYsI73fGWt+kniw+TekPfc8olnU8GvopfNhZgtejtzLYbHqqcxB3p06JrQVK5oF/CPUFO/0tdjwbObhkMeB2YWE6NpHUQDoC+aYeoJ7/QMXsJqoGB+CcqQk2Q2db7PV28oRaDiC5AgMBAAEwDQYJKoZIhvcNAQELBQADggEBAJSurTF+OxaTEFKV1iCgNv8D6c1n6dmh71o6EkT1YhgRJtWgXWcBzt850vaAwF+NSlJCE6qELa6GDnDyfd3GVUmwhcgjc0Y4jYAjWRAOm1dGPbgjgTg6VazYjq9LS4rNXftlrz1vLvd3yPWKDJLYBP7anNkzov7Aur2e5EY8NdSJkdAnJpUDLbUymDe4k8fdqwBrs0Ix8RvjJTixwhQOnfd9HCAguaYtZsiCYlWgeIl0+0k7M+aCNxiZWXDk8eFn4f4vmg1XtmPcaSjCsmLv+dFVSF07pX6SidamZscyS0jjqJQlPKXqN0ABZmltNySaI4+mit09JZhG5N03DpGvQHM= + + + + + + + + + MIIC2jCCAcKgAwIBAgIQX1xzEAL1watDffoOg6gqhjANBgkqhkiG9w0BAQsFADApMScwJQYDVQQDEx5BREZTIFNpZ25pbmcgLSB3a2lzLmRldi53a28uYXQwHhcNMTUwODI1MjMwNzI5WhcNMTYwODI0MjMwNzI5WjApMScwJQYDVQQDEx5BREZTIFNpZ25pbmcgLSB3a2lzLmRldi53a28uYXQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCfoWozMPdVIZ2ua7L8DyKROjROnLxw/D/SqHOdV1fmslod3GSKfUZtZhBj60U6lsMuj/G0HMh1I0V06k1HJlnKTPt9tNxkobS3dfXpeKU2WWMCYwHBvwPLiVH2rHfpWFddQl1h5am5Jwc/OFsmIbjzfOTayl375gXlLqPcue66BJaOojJk0r1TdfhidYaOxV8PYZqQQZ7zoiCRkXXgnf0NBOat0sRohyI3rVMcIF9WcGkNP6+gx71bY62y6bLHDCa3CJ8/r5tCb6aGZ7V5dIDEqKjgKOaF+/HfQ10jBTBVx1g8my8DJjD51kDPdERVZYOnlPtaKXPwlsSLffAURy65AgMBAAEwDQYJKoZIhvcNAQELBQADggEBAEFczqnyemiRhArnQpf4s7QPTtqvKZaKDJN2AwFsX2T8OFu6gTOLkwcrKyR7C3r8QTLH7PA9Lk/ckKKwcsoHokIfHpWCW76zw3jmb0NwC1+cHsytDFVEr/TP8f5X/1Hy0sozbWjnhcpqDWU4gUSh6WRt6hjNOj2xRjKQAT6QhTBlkh5Ds9w3ya0+EivvHnePaO8g5rdhkcKfpennWs3SiuNCLMN2z36b5xHHWwYrkQP16x2uFNTCjA6WeQf9o7MKhOckbxeMbH0oRl/h+bpgnAGRYpMm3vV++Dp1aXMCvlX25XEUG/Sy7jbfOIfYsMVBhCV1sjRoP9Ic1jTnwyth4U4= + + + + + + + + urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress + + urn:oasis:names:tc:SAML:2.0:nameid-format:persistent + urn:oasis:names:tc:SAML:2.0:nameid-format:transient + + + + + + + + + + + + + http://wpv.portalverbund.at/ns/names/wpv_standard_attribute_profile + + + + + WPV Web-SSO Testumgebung Testservicew + + WPV Testumgebung Test Service 2 (direct + Shibboleth SP) + + + https://idp1.test.wpv.portalverbund.at/idp/images/wpvlogo.png + + + + + + + + + + MIIC6zCCAdOgAwIBAgIJAIkSELDiPK60MA0GCSqGSIb3DQEBBQUAMBcxFTATBgNV + BAMTDDgyZjdhODMwMmY0ODAeFw0xNjAyMjgxNjE1MjdaFw0yNjAyMjUxNjE1Mjda + MBcxFTATBgNVBAMTDDgyZjdhODMwMmY0ODCCASIwDQYJKoZIhvcNAQEBBQADggEP + ADCCAQoCggEBAMOWMUhYcym8FSGzOedG9suMS+aK/culEXn04B84a/zlm9Dxv8wk + ZXLT071OtQm6Rz9hdkC3e7AqkTeGnLzkpkzvgEjx/ManNoPFBnqfqgL9KE0t+FS2 + gRscy7VKDxdkMvzmXO+3+JjYYqnVVpbFUMFeOMaC74skFhKI8gc/j4a/8DH6TrPd + SsyKi/k7QeRwP8QfnCapLbsGVTW4G+EuMcP6/Wl2zDcelpxh/dHv325yKCBbPvPC + +k/donPsw8OV4ItzehyOrtjWNCsQscgY+wWCidIMCPOkI3WRIGkeClJoz/dg8gSx + HWkYacN6lze/xPYELR6WlzB9r2WOKjuHbRcCAwEAAaM6MDgwFwYDVR0RBBAwDoIM + ODJmN2E4MzAyZjQ4MB0GA1UdDgQWBBQzGCch3EXA37qX6W3AtkE4KBzWSTANBgkq + hkiG9w0BAQUFAAOCAQEAQP46xsyGllK/xk3Y7PCPkj28KrVwMsKeR8geRq8UswB2 + TQ+qhvWPZs5qUlNc1UGZat4nEu7T837ozJ7OeVNtgaLDLElfnerzss/ip/I0at5U + g6g+/memzdipXb/5AF76353nK82dRmhO5M6EmVBdeAF1LGQkonmxBRRdMJ1OtGfq + C9WdsDb3FQWX62QlBFuzf/sq1M/DSn9FHDlLHqtTyFFwGBqX6mlOa1u8e3cpCBpk + o4WHxK0EHAKWMiSisR+VXm7BHqe6Qg6cnu9sipc3rkL3mXNvcWD4ktpkuxL9zEsd + dMe4TwGTdA/GjVIW3ia5ouIVVuEaXrw8MlhsUoiIZQ== + + + + + + + + + urn:oasis:names:tc:SAML:2.0:nameid-format:persistent + + + + + + Austria Pro / AK WPV + Austria Pro / AK WPV + + https://www.wirtschaftsportalverbund.at/ + + + + Rainer + Hörbe + + + + + + + + + + + Test IdP 1 + Test IdP 1 + + + https://idp1.test.wpv.portalverbund.at/ + 81.217.70.0/8 + + + + + + + MIIDZjCCAk6gAwIBAgIVAN63M7mMfFBjceFtOvDl6SQ8xfhxMA0GCSqGSIb3DQEB + CwUAMCkxJzAlBgNVBAMMHmlkcDEudGVzdC53cHYucG9ydGFsdmVyYnVuZC5hdDAe + Fw0xNjAzMDkwOTI1MTFaFw0zNjAzMDkwOTI1MTFaMCkxJzAlBgNVBAMMHmlkcDEu + dGVzdC53cHYucG9ydGFsdmVyYnVuZC5hdDCCASIwDQYJKoZIhvcNAQEBBQADggEP + ADCCAQoCggEBAJtpaS4L0KA8D3r3PDGSgbBtqu4OxRX1hxU+7jVZLEIISsIpfHuW + jOhtY2tuHpegiyiUgQrV+mIDooGuxmr3hxFQJgdhvIjFWkHO2SxqUFPpwoHDs1UV + 1O3WUGEWejypcalMRWhbNSwiG54PwXTyHdcgYVjDqirL8dy6iioV0W/G7xI66acz + yipf8yoiaJINaRiM85PMB98FY+Xtsr+1M5zysWWx0HsC7ruAnwXUS5X3AIhBVu1H + j8CmhCxCdaJHheSr5gmxa96SrLL2FmNU5fhu2Md09R6k3ccbS05AzdzfrTONkDVo + 5H71DeiA83DiykESucqykm88up2Q8sLPe1UCAwEAAaOBhDCBgTAdBgNVHQ4EFgQU + KD55ApKpSoKOF8pqqebtGX7gy2UwYAYDVR0RBFkwV4IeaWRwMS50ZXN0Lndwdi5w + b3J0YWx2ZXJidW5kLmF0hjVodHRwczovL2lkcDEudGVzdC53cHYucG9ydGFsdmVy + YnVuZC5hdC9pZHAvc2hpYmJvbGV0aDANBgkqhkiG9w0BAQsFAAOCAQEAb8zD0oK8 + /vc6JZuvgXDthq474fLaZELopee5wQXQntXkYAkmrpdVvAiOmBKeAJZpBsIbIR6p + WKOavaj1yqBM3xXV7CjXFhihmpEaCvHO1aYA6w1Pn7+XtAHn4p5pd7u9JZYyYBpM + QF9HmEl57tvXdZb0Y7eNYud9KGSuNciUFSOvDN8lKr6EVBP+Ym4NSVbKslHXq4kw + meYD0No3y96EoNIMHX6ZmgfA5h8d5Uyuf3q6MHQE4XgNrLyJvCObrcSIeVTidvoJ + /TnbeBVWyB0F8l9ycTfyQyz08XHWAsrgjgQY/wrOC5GXMWDJjfWdBIhga2qChykE + 9cTCyixjghpGSA== + + + + + + + + + MIIDZTCCAk2gAwIBAgIUTMaFU3su5h9xBQJPalVsbOR7DBMwDQYJKoZIhvcNAQEL + BQAwKTEnMCUGA1UEAwweaWRwMS50ZXN0Lndwdi5wb3J0YWx2ZXJidW5kLmF0MB4X + DTE2MDMwOTA5MjUwOVoXDTM2MDMwOTA5MjUwOVowKTEnMCUGA1UEAwweaWRwMS50 + ZXN0Lndwdi5wb3J0YWx2ZXJidW5kLmF0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A + MIIBCgKCAQEA6PRWhTPw8jDeIs0KNHxFe8vBTKgGJLtTBGz/I2qbHtvzQjDwy7l7 + t3VhC4bvCHU6q6LhprG1yblTs4mZOxNG3FgBSXfnlAz7LYaTksXGHLLrtcrfODJg + gg+SDac6OaOLlN7HuvVIZMeGenQOL82av5FEB1SWl8YEnWx+jWBf4OiHGtUGqJ9F + bFfvmMRUtBEil0xT53gBkNWcZ3T0IvRioVcCrr+mcoxHNH3JHuGmQ8xqZWZ27Iod + dTILEl5MXH9Kc5M3Pc/75lXCDahQsqG1d40gzE3mEG2OmtlTb+FfJB4sXsH3y3pf + O9hTNo/B7PfCv9wNdFk1moa8J2SLX/xCwwIDAQABo4GEMIGBMB0GA1UdDgQWBBQj + GS9lXXW7CryQS/nTjb9mywvR8zBgBgNVHREEWTBXgh5pZHAxLnRlc3Qud3B2LnBv + cnRhbHZlcmJ1bmQuYXSGNWh0dHBzOi8vaWRwMS50ZXN0Lndwdi5wb3J0YWx2ZXJi + dW5kLmF0L2lkcC9zaGliYm9sZXRoMA0GCSqGSIb3DQEBCwUAA4IBAQDI0+RooeNe + Mr5lYOTE55kRFy8iWyx3szlsUw5qm27w8Mmu+lRBMPpwkDnDQ+TyUApOiU41i4EZ + XNwywEnwQGCFp8R42Is2ECYqM6OIpW+Cf5HwsjwkeEj+Vr5o29mEAzzQFBVINF0F + NPEV4xBdyRWI06E4B1FXW5zdn9sB8qIWwO3TpxvMQbQhvgQXva0OryhhmJ8qIaHq + aR3owMZvu6lD7R7ei5z6NEgOlKcbp6K+B79nv8yz8DNvV303f9fn3V29TDVm8O5b + vAFOND6uWGNOG1xQJMeyV8hZMDFWfjp4Q5y/P1JIa7WVwv8dSTuwHbehCp85eKIv + xkaknQcJ1Ksh + + + + + + + + + MIIDZjCCAk6gAwIBAgIVALbYcCosw/d/7xolDLM6db8FUQ/AMA0GCSqGSIb3DQEB + CwUAMCkxJzAlBgNVBAMMHmlkcDEudGVzdC53cHYucG9ydGFsdmVyYnVuZC5hdDAe + Fw0xNjAzMDkwOTI1MTBaFw0zNjAzMDkwOTI1MTBaMCkxJzAlBgNVBAMMHmlkcDEu + dGVzdC53cHYucG9ydGFsdmVyYnVuZC5hdDCCASIwDQYJKoZIhvcNAQEBBQADggEP + ADCCAQoCggEBAMa4SR/cBRG8S8c/BndYvzcP6ArKj0hZ7TIbRFQ7z3blfJY8boAy + sPEcxKfB6NJp1lza3090L9rcKoEhM2lBeuUWjpL0pzcP9kNPU2T9s4ubgz/oXg6h + K+XHI6z/Lt7RuV1gsJ+3OSxbt1kkWKsEl6/bd9qu+GHeXwaZjcrwZCJ5MgeypVuw + fUw1HSCVtYGkkDl/VY91iH3z2QyN7cxGQ3lSghy6/A1HXoAK5If4hsgJL+fuaGWm + k6M/tt7G1SSreIaLB6fflCL8uc2y6/SyN1KKyTwo1lkeWKRpzcQr9lHXeX8U8S4/ + G7xtg3oNZP0d+SJNzPTbuEVr9ZONzZRKhAsCAwEAAaOBhDCBgTAdBgNVHQ4EFgQU + FoOzwiTGMmsqLd64Y8CUaywWuR0wYAYDVR0RBFkwV4IeaWRwMS50ZXN0Lndwdi5w + b3J0YWx2ZXJidW5kLmF0hjVodHRwczovL2lkcDEudGVzdC53cHYucG9ydGFsdmVy + YnVuZC5hdC9pZHAvc2hpYmJvbGV0aDANBgkqhkiG9w0BAQsFAAOCAQEAguamnFjA + fbIGFU6QNl3ZTBW0OakGfdbLEhYQ2xefFBEUf1LFtWua1qL6gzxI4PkPAY9NT+17 + ABN0TY+87MrAzTNHvI/V51W4hCI+0Z7H6zGS/zaJzRsxtL2E8Yi+vue3dkz/o9g8 + Jq01XbTMcSukcmNcPkzEH+65apAwyNP3nv09G7cmBUrgEi0gb3dQ1IvZFQVua4H6 + h0vYbUtWi7r/Epb+6AeCreEF8lW2qxjIEkYocxVt7AoEHLCbUmswBADrYhN+WrNE + WcCNbuQhJZTQZMk2CtUtu7SAEkxTT9jkRq6PVF0jfaGVuN7vErE4zsFyT6FKIHYd + GD7P2VPtNN4/2w== + + + + + + + + + urn:oasis:names:tc:SAML:2.0:nameid-format:transient + urn:oasis:names:tc:SAML:2.0:nameid-format:persistent + + + + + + + Austria Pro / AK WPV + Austria Pro / AK WPV + + https://www.wirtschaftsportalverbund.at/ + + + + Rainer + Hörbe + + + \ No newline at end of file diff --git a/src/pyff/test/data/pki/mycert.pem b/src/pyff/test/data/pki/mycert.pem new file mode 100644 index 00000000..d4a0873c --- /dev/null +++ b/src/pyff/test/data/pki/mycert.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC8jCCAlugAwIBAgIJAJHg2V5J31I8MA0GCSqGSIb3DQEBBQUAMFoxCzAJBgNV +BAYTAlNFMQ0wCwYDVQQHEwRVbWVhMRgwFgYDVQQKEw9VbWVhIFVuaXZlcnNpdHkx +EDAOBgNVBAsTB0lUIFVuaXQxEDAOBgNVBAMTB1Rlc3QgU1AwHhcNMDkxMDI2MTMz +MTE1WhcNMTAxMDI2MTMzMTE1WjBaMQswCQYDVQQGEwJTRTENMAsGA1UEBxMEVW1l +YTEYMBYGA1UEChMPVW1lYSBVbml2ZXJzaXR5MRAwDgYDVQQLEwdJVCBVbml0MRAw +DgYDVQQDEwdUZXN0IFNQMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDkJWP7 +bwOxtH+E15VTaulNzVQ/0cSbM5G7abqeqSNSs0l0veHr6/ROgW96ZeQ57fzVy2MC +FiQRw2fzBs0n7leEmDJyVVtBTavYlhAVXDNa3stgvh43qCfLx+clUlOvtnsoMiiR +mo7qf0BoPKTj7c0uLKpDpEbAHQT4OF1HRYVxMwIDAQABo4G/MIG8MB0GA1UdDgQW +BBQ7RgbMJFDGRBu9o3tDQDuSoBy7JjCBjAYDVR0jBIGEMIGBgBQ7RgbMJFDGRBu9 +o3tDQDuSoBy7JqFepFwwWjELMAkGA1UEBhMCU0UxDTALBgNVBAcTBFVtZWExGDAW +BgNVBAoTD1VtZWEgVW5pdmVyc2l0eTEQMA4GA1UECxMHSVQgVW5pdDEQMA4GA1UE +AxMHVGVzdCBTUIIJAJHg2V5J31I8MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEF +BQADgYEAMuRwwXRnsiyWzmRikpwinnhTmbooKm5TINPE7A7gSQ710RxioQePPhZO +zkM27NnHTrCe2rBVg0EGz7QTd1JIwLPvgoj4VTi/fSha/tXrYUaqc9AqU1kWI4WN ++vffBGQ09mo+6CffuFTZYeOhzP/2stAPwCTU4kxEoiy0KpZMANI= +-----END CERTIFICATE----- diff --git a/src/pyff/test/data/pki/mykey.pem b/src/pyff/test/data/pki/mykey.pem new file mode 100644 index 00000000..d9ec5f8c --- /dev/null +++ b/src/pyff/test/data/pki/mykey.pem @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQDkJWP7bwOxtH+E15VTaulNzVQ/0cSbM5G7abqeqSNSs0l0veHr +6/ROgW96ZeQ57fzVy2MCFiQRw2fzBs0n7leEmDJyVVtBTavYlhAVXDNa3stgvh43 +qCfLx+clUlOvtnsoMiiRmo7qf0BoPKTj7c0uLKpDpEbAHQT4OF1HRYVxMwIDAQAB +AoGAbx9rKH91DCw/ZEPhHsVXJ6cYHxGcMoAWvnMMC9WUN+bNo4gNL205DLfsxXA1 +jqXFXZj3+38vSFumGPA6IvXrN+Wyp3+Lz3QGc4K5OdHeBtYlxa6EsrxPgvuxYDUB +vx3xdWPMjy06G/ML+pR9XHnRaPNubXQX3UxGBuLjwNXVmyECQQD2/D84tYoCGWoq +5FhUBxFUy2nnOLKYC/GGxBTX62iLfMQ3fbQcdg2pJsB5rrniyZf7UL+9FOsAO9k1 +8DO7G12DAkEA7Hkdg1KEw4ZfjnnjEa+KqpyLTLRQ91uTVW6kzR+4zY719iUJ/PXE +PxJqm1ot7mJd1LW+bWtjLpxs7jYH19V+kQJBAIEpn2JnxdmdMuFlcy/WVmDy09pg +0z0imdexeXkFmjHAONkQOv3bWv+HzYaVMo8AgCOksfEPHGqN4eUMTfFeuUMCQF+5 +E1JSd/2yCkJhYqKJHae8oMLXByNqRXTCyiFioutK4JPYIHfugJdLfC4QziD+Xp85 +RrGCU+7NUWcIJhqfiJECQAIgUAzfzhdj5AyICaFPaOQ+N8FVMLcTyqeTXP0sIlFk +JStVibemTRCbxdXXM7OVipz1oW3PBVEO3t/VyjiaGGg= +-----END RSA PRIVATE KEY----- From e089e5c0c9b32df403ccf7ebf910be736a44fd51 Mon Sep 17 00:00:00 2001 From: r2h2 Date: Mon, 11 Jul 2016 11:53:59 +0200 Subject: [PATCH 02/20] added mdsplit functionality --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 42e6f650..4d05f470 100755 --- a/setup.py +++ b/setup.py @@ -77,6 +77,7 @@ entry_points={ 'console_scripts': ['pyff=pyff.md:main', 'pyffd=pyff.mdx:main'] }, + scripts=['scripts/mdsplit.py', ], message_extractors={'src': [ ('**.py', 'python', None), ('**/templates/**.html', 'mako', None), From 2a9b0d436b1229108bc3c696ba1a1fb82df32291 Mon Sep 17 00:00:00 2001 From: r2h2 Date: Mon, 11 Jul 2016 11:54:34 +0200 Subject: [PATCH 03/20] ignore mac .DS_Store --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6eba126c..54740e51 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ sign.key _site profile.out* localconfig.py + +**/.DS_Store \ No newline at end of file From 2ba58b375f230a8bcb3ba85bd3af354ef180341b Mon Sep 17 00:00:00 2001 From: r2h2 Date: Mon, 11 Jul 2016 12:01:46 +0200 Subject: [PATCH 04/20] create out directory for split files if does not exist --- scripts/mdsplit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/mdsplit.py b/scripts/mdsplit.py index 5b755204..ceb39781 100644 --- a/scripts/mdsplit.py +++ b/scripts/mdsplit.py @@ -116,6 +116,7 @@ def main(): fn_temp = os.path.abspath(os.path.join(args.outdir_unsigned, entityid_to_filename(e.attrib['entityID']))) fn_out = os.path.abspath(os.path.join(args.outdir_signed, entityid_to_filename(e.attrib['entityID']))) logging.debug('writing unsigned EntitiyDescriptor ' + e.attrib['entityID'] + ' to ' + fn_temp) + os.makedirs(os.path.dirname(fn_temp), exist_ok=True)¶ with open(fn_temp, 'w') as f: f.write(XMLDECLARATION + etree.tostring(e)) fn_pipeline = fn_temp + '.fd' From e8fdf0f63d213d33661f74b56a60559ef75b674b Mon Sep 17 00:00:00 2001 From: r2h2 Date: Mon, 11 Jul 2016 12:13:01 +0200 Subject: [PATCH 05/20] py27 compatible --- scripts/mdsplit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/mdsplit.py b/scripts/mdsplit.py index ceb39781..840e0656 100644 --- a/scripts/mdsplit.py +++ b/scripts/mdsplit.py @@ -116,7 +116,8 @@ def main(): fn_temp = os.path.abspath(os.path.join(args.outdir_unsigned, entityid_to_filename(e.attrib['entityID']))) fn_out = os.path.abspath(os.path.join(args.outdir_signed, entityid_to_filename(e.attrib['entityID']))) logging.debug('writing unsigned EntitiyDescriptor ' + e.attrib['entityID'] + ' to ' + fn_temp) - os.makedirs(os.path.dirname(fn_temp), exist_ok=True)¶ + if not os.path.exists(os.path.dirname(fn_temp)): + os.makedirs(os.path.dirname(fn_temp)) with open(fn_temp, 'w') as f: f.write(XMLDECLARATION + etree.tostring(e)) fn_pipeline = fn_temp + '.fd' From e2a9e2a231d92f998dd14696a4ead63526fcb072 Mon Sep 17 00:00:00 2001 From: r2h2 Date: Mon, 11 Jul 2016 15:08:00 +0200 Subject: [PATCH 06/20] file location via env --- scripts/pyffsignsingle.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/scripts/pyffsignsingle.sh b/scripts/pyffsignsingle.sh index d4b8b150..af434699 100644 --- a/scripts/pyffsignsingle.sh +++ b/scripts/pyffsignsingle.sh @@ -1,13 +1,23 @@ #!/bin/sh # create a signed XML file per EntityDescriptor for ADFS +if [ ! -e $MDSIGN_CERT ]; then + echo "MDSIGN_CERT must be set and point to an existing file" && exit 1 +fi +if [ ! -e $MDSIGN_KEY ]; then + echo "MDSIGN_KEY must be set and point to an existing file" && exit 1 +fi +if [ ! -e $MD_AGGREGATE ]; then + echo "MD_AGGREGATE must be set and point to an existing file" && exit 1 +fi + # Step 1. Split aggregate and create an XML and a pipeline file per EntityDescriptor /usr/bin/mdsplit.py \ - -c /etc/pki/pyff/metadata_signing-crt.pem \ - -k /etc/pki/pyff/metadata_signing-key.pem \ + -c $MDSIGN_CERT \ + -k $MDSIGN_KEY \ -l /var/log/pyffsplit.log \ -L DEBUG \ - /var/md_feed/metadata.xml \ + $MD_AGGREGATE \ /var/md_feed/split/ \ /var/md_source/split/ From 44435be3772a974f116471eb4bccd5a5b420c6fe Mon Sep 17 00:00:00 2001 From: r2h2 Date: Mon, 11 Jul 2016 15:39:47 +0200 Subject: [PATCH 07/20] refactor --- .../{pyffsignsingle.sh => pyff_split_sign.sh} | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) rename scripts/{pyffsignsingle.sh => pyff_split_sign.sh} (68%) diff --git a/scripts/pyffsignsingle.sh b/scripts/pyff_split_sign.sh similarity index 68% rename from scripts/pyffsignsingle.sh rename to scripts/pyff_split_sign.sh index af434699..86edee55 100644 --- a/scripts/pyffsignsingle.sh +++ b/scripts/pyff_split_sign.sh @@ -1,6 +1,10 @@ #!/bin/sh # create a signed XML file per EntityDescriptor for ADFS +MDSPLITUNSIGNED='/var/md_source/split/' +MDSPLITSIGNED='/var/md_feed/split/' +LOGFILE='/var/log/pyffsplit.log' +# MDSIGN_CERT, MDSIGN_KEY and MDAGGREGATE must be passed via env if [ ! -e $MDSIGN_CERT ]; then echo "MDSIGN_CERT must be set and point to an existing file" && exit 1 fi @@ -12,17 +16,14 @@ if [ ! -e $MD_AGGREGATE ]; then fi # Step 1. Split aggregate and create an XML and a pipeline file per EntityDescriptor +[ "$LOGLEVEL" == "DEBUG" ] && echo "processing " /usr/bin/mdsplit.py \ - -c $MDSIGN_CERT \ - -k $MDSIGN_KEY \ - -l /var/log/pyffsplit.log \ - -L DEBUG \ - $MD_AGGREGATE \ - /var/md_feed/split/ \ - /var/md_source/split/ + -c $CERTFILE -k $KEYFILE \ + -l $LOGFILE -L DEBUG \ + $MD_AGGREGATE $MDSPLITUNSIGNED $MDSPLITSIGNED # Step 2. Execute pyff to sign each EntityDescriptor -cd /var/md_source/split/ +cd $MDSPLITUNSIGNED for fn in *.fd; do echo "running pyff for $fn" /usr/bin/pyff --loglevel=$LOGLEVEL $fn From f241a6b72c98e4d8aa2c850a23322da90bec57b1 Mon Sep 17 00:00:00 2001 From: r2h2 Date: Tue, 12 Jul 2016 11:44:07 +0200 Subject: [PATCH 08/20] refactoring --- .gitignore | 3 +- scripts/pyff_mdsplit.py | 107 +++++++++++++++++++++++++++++++ scripts/pyff_split_sign.sh | 27 ++++---- setup.py | 2 +- src/pyff/__init__.py | 3 +- {scripts => src/pyff}/mdsplit.py | 72 ++++++++------------- 6 files changed, 151 insertions(+), 63 deletions(-) create mode 100644 scripts/pyff_mdsplit.py rename {scripts => src/pyff}/mdsplit.py (53%) diff --git a/.gitignore b/.gitignore index 54740e51..43d2d57a 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,5 @@ _site profile.out* localconfig.py -**/.DS_Store \ No newline at end of file +**/.DS_Store +work \ No newline at end of file diff --git a/scripts/pyff_mdsplit.py b/scripts/pyff_mdsplit.py new file mode 100644 index 00000000..7c100060 --- /dev/null +++ b/scripts/pyff_mdsplit.py @@ -0,0 +1,107 @@ +""" +Invoke mdsplit from the command line. See mdsplit.py for documentation on functionality. +usage: pyff_mdsplit.py [-h] [-c CERTFILE] [-k KEYFILE] [-S] [-l LOGFILE] [-v] + [-L {INFO,DEBUG,CRITICAL,WARNING,ERROR}] + [-o OUTDIR_SIGNED] [-i IDPREFIX] [-C CACHEDURATION] + [-u VALIDUNTIL] + input outdir_unsigned +Metadata Splitter + +positional arguments: + input Metadata aggregate (input) + outdir_unsigned Directory for files containing one unsigned + EntityDescriptor each. + +optional arguments: + -h, --help show this help message and exit + -c CERTFILE, --certfile CERTFILE + -k KEYFILE, --keyfile KEYFILE + -S, --nosign do not sign output + -v, --vebose output to console with DEBUG leve + -l LOGFILE, --logfile LOGFILE + -L {INFO,DEBUG,CRITICAL,WARNING,ERROR}, --loglevel {INFO,DEBUG,CRITICAL,WARNING,ERROR} + default is INFO if env[LOGLEVEL] is not set + -o OUTDIR_SIGNED, --outdir_signed OUTDIR_SIGNED + Directory for files containing one signed + EntityDescriptor each. + -i IDPREFIX, --idprefix IDPREFIX + -C CACHEDURATION, --cacheduration CACHEDURATION + -u VALIDUNTIL, --validuntil VALIDUNTIL +""" + +import argparse +import logging +import os +import re +import sys + +import pyff.mdsplit + +LOGLEVELS = {'CRITICAL': 50, 'ERROR': 40, 'WARNING': 30, 'INFO': 20, 'DEBUG': 10} + + +class Invocation: + """ Get arguments from command line and enviroment """ + def __init__(self, testargs=None): + self.parser = argparse.ArgumentParser(description='Metadata Splitter') + self.parser.add_argument('-c', '--certfile', dest='certfile') + self.parser.add_argument('-k', '--keyfile', dest='keyfile') + self.parser.add_argument('-S', '--nosign', action='store_true', help='do not sign output') + self.parser.add_argument('-v', '--verbose', action='store_true', help='output to console with DEBUG level') + logbasename = re.sub(r'\.py$', '.log', os.path.basename(__file__)) + self.parser.add_argument('-l', '--logfile', dest='logfile', default=logbasename) + loglevel_env = os.environ['LOGLEVEL'] if 'LOGLEVEL' in os.environ else 'INFO' + self.parser.add_argument('-L', '--loglevel', dest='loglevel', default=loglevel_env, + choices=LOGLEVELS.keys(), help='default is INFO if env[LOGLEVEL] is not set') + self.parser.add_argument('-o', '--outdir_signed', default=None, + help='Directory for files containing one signed EntityDescriptor each.') + self.parser.add_argument('-i', '--idprefix', dest='idprefix', default='federation.example.com_') # TODO: take from input + self.parser.add_argument('-C', '--cacheduration', dest='cacheduration', default='PT5H') # TODO: take from input + self.parser.add_argument('-u', '--validuntil', dest='validuntil', default='P10D') # TODO: take from input + self.parser.add_argument('input', type=argparse.FileType('r'), default=None, + help='Metadata aggregate (input)') + self.parser.add_argument('outdir_unsigned', default=None, + help='Directory for files containing one unsigned EntityDescriptor each.') + self.args = self.parser.parse_args() + # merge argv with env + if not self.args.nosign: + self.args.certfile = self._merge_arg('MDSIGN_CERT', self.args.certfile, 'certfile') + self.args.keyfile = self._merge_arg('MDSIGN_KEY', self.args.keyfile, 'keyfile') + self.args.outdir_signed = self._merge_arg('MDSPLIT_SIGNED', self.args.outdir_signed, 'outdir_signed') + self.args.input = self._merge_arg('MD_AGGREGATE', self.args.input, 'input') + self.args.outdir_unsigned = self._merge_arg('MDSPLIT_UNSIGNED', self.args.outdir_unsigned, 'outdir_unsigned') + + def _merge_arg(self, env, arg, argname): + """ merge argv with env """ + if env not in os.environ and arg is None: + print("Either %s or --%s must be set and point to an existing file" % (env, argname)) + exit(1) + if arg is None: + return env + else: + return arg + + +def main(): + invocation = Invocation() + + log_args = {'level': LOGLEVELS[invocation.args.loglevel], + 'format': '%(asctime)s - %(levelname)s [%(filename)s:%(lineno)s] %(message)s', + 'filename': invocation.args.logfile, + } + logging.basicConfig(**log_args) + if invocation.args.verbose: + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.DEBUG) + logging.getLogger('').addHandler(console_handler) + + logging.debug('') + logging.debug('Input file is ' + invocation.args.input.name) + logging.debug('Output directory for unsigned files is ' + os.path.abspath(invocation.args.outdir_unsigned)) + if not invocation.args.nosign: + logging.debug('Output directory for signed files is ' + os.path.abspath(invocation.args.outdir_signed)) + + pyff.mdsplit.process_md_aggregate(invocation.args) + +if __name__ == "__main__": # pragma: no cover + main() \ No newline at end of file diff --git a/scripts/pyff_split_sign.sh b/scripts/pyff_split_sign.sh index 86edee55..55fadcf1 100644 --- a/scripts/pyff_split_sign.sh +++ b/scripts/pyff_split_sign.sh @@ -1,29 +1,24 @@ #!/bin/sh # create a signed XML file per EntityDescriptor for ADFS -MDSPLITUNSIGNED='/var/md_source/split/' -MDSPLITSIGNED='/var/md_feed/split/' -LOGFILE='/var/log/pyffsplit.log' # MDSIGN_CERT, MDSIGN_KEY and MDAGGREGATE must be passed via env -if [ ! -e $MDSIGN_CERT ]; then - echo "MDSIGN_CERT must be set and point to an existing file" && exit 1 -fi -if [ ! -e $MDSIGN_KEY ]; then - echo "MDSIGN_KEY must be set and point to an existing file" && exit 1 -fi -if [ ! -e $MD_AGGREGATE ]; then - echo "MD_AGGREGATE must be set and point to an existing file" && exit 1 -fi +[ $MDSIGN_CERT ] || echo "MDSIGN_CERT must be set and point to an existing file" && exit 1 +[ $MDSIGN_KEY ] || echo "MDSIGN_KEY must be set and point to an existing file" && exit 1 +[ $MD_AGGREGATE ]|| echo "MD_AGGREGATE must be set and point to an existing file" && exit 1 +# Setting defaults +[ $MDSPLIT_UNSIGNED ] || MDSPLIT_UNSIGNED='/var/md_source/split/' +[ $MDSPLIT_SIGNED ] || MDSPLIT_SIGNED='/var/md_feed/split/' +[ $LOGFILE ] || LOGFILE='/var/log/pyffsplit.log' # Step 1. Split aggregate and create an XML and a pipeline file per EntityDescriptor [ "$LOGLEVEL" == "DEBUG" ] && echo "processing " -/usr/bin/mdsplit.py \ - -c $CERTFILE -k $KEYFILE \ +/usr/bin/pyff_mdsplit.py \ + -c $MDSIGN_CERT -k $MDSIGN_KEY \ -l $LOGFILE -L DEBUG \ - $MD_AGGREGATE $MDSPLITUNSIGNED $MDSPLITSIGNED + $MD_AGGREGATE $MDSPLIT_UNSIGNED $MDSPLIT_SIGNED # Step 2. Execute pyff to sign each EntityDescriptor -cd $MDSPLITUNSIGNED +cd $MDSPLIT_UNSIGNED for fn in *.fd; do echo "running pyff for $fn" /usr/bin/pyff --loglevel=$LOGLEVEL $fn diff --git a/setup.py b/setup.py index 4d05f470..ea2be9f7 100755 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ entry_points={ 'console_scripts': ['pyff=pyff.md:main', 'pyffd=pyff.mdx:main'] }, - scripts=['scripts/mdsplit.py', ], + scripts=['scripts/pyff_mdsplit.py', 'pyff_split_sign.sh'], message_extractors={'src': [ ('**.py', 'python', None), ('**/templates/**.html', 'mako', None), diff --git a/src/pyff/__init__.py b/src/pyff/__init__.py index 5ddb1651..acd6065b 100644 --- a/src/pyff/__init__.py +++ b/src/pyff/__init__.py @@ -10,4 +10,5 @@ __license__ = "BSD" __maintainer__ = "leifj@sunet.se" __status__ = "Production" -__version__ = pkg_resources.require("pyFF")[0].version +# removing following line was a quick fix to thes message "pkg_resources.DistributionNotFound: The 'pyFF' distribution was not found" +#__version__ = pkg_resources.require("pyFF")[0].version diff --git a/scripts/mdsplit.py b/src/pyff/mdsplit.py similarity index 53% rename from scripts/mdsplit.py rename to src/pyff/mdsplit.py index 840e0656..955bbf37 100644 --- a/scripts/mdsplit.py +++ b/src/pyff/mdsplit.py @@ -1,21 +1,23 @@ """ pyffsplit creates pipeline files for each EntityDescriptor in an aggregate, which then used to create separate signed XML documents with an EntitiyDescriptor each. +The input file is considered to be trusted, the signature is not verified. TODO: Signing in the same process """ -import argparse -import future + import logging import lxml.etree as etree import os import re import sys -import pyff +#import pyff from pyff.mdrepo import MDRepository from pyff.pipes import plumbing from pyff.store import MemoryStore +XMLDECLARATION = '' + class Pipeline: def __init__(self, keyfile, certfile, idprefix, cacheDuration, validUntil): @@ -48,6 +50,7 @@ def get(self, infile, outfile): self.validUntil) return pipeline + def entityid_to_filename(entityid): """ Derive a filename from an entityID, removing dots and slashes @@ -73,6 +76,7 @@ def entityid_to_filename(entityid): upper = True return r + '.xml' + # def simple_md(pipeline): # """ stupid copy of md:main -> replace this """ # modules = [] @@ -82,50 +86,30 @@ def entityid_to_filename(entityid): # plumbing(pipeline).process(md, state={'batch': True, 'stats': {}}) -def main(): - LOGLEVELS = {'CRITICAL': 50, 'ERROR': 40, 'WARNING': 30, 'INFO': 20, 'DEBUG': 10} - XMLDECLARATION = '' - parser = argparse.ArgumentParser(description='Metadata Splitter') - parser.add_argument('-c', '--certfile', dest='certfile', default='pyff_sign-cer.pem') - parser.add_argument('-k', '--keyfile', dest='keyfile', default='pyff_sign-key.pem') - parser.add_argument('-i', '--idprefix', dest='idprefix', default='ourfederation.example.com_') - parser.add_argument('-C', '--cacheduration', dest='cacheduration', default='PT5H') - parser.add_argument('-l', '--logfile', dest='logfile', default='pyffsplit.log') - parser.add_argument('-L', '--loglevel', dest='loglevel', default='INFO', choices=LOGLEVELS.keys()) - parser.add_argument('-u', '--validuntil', dest='validuntil', default='P10D') - parser.add_argument('input', type=argparse.FileType('r'), default=None, - help='Metadata aggregate') - parser.add_argument('outdir_unsigned', default=None) - parser.add_argument('outdir_signed', default=None, - help='Directory for files containing one EntityDescriptor each.') - args = parser.parse_args() - - log_args = {'level': LOGLEVELS[args.loglevel]} - log_args['filename'] = args.logfile - logging.basicConfig(**log_args) - logging.debug('Input file is ' + args.input.name) - logging.debug('Input directory is ' + args.outdir_signed) - - root = etree.parse(args.input).getroot() - if root.tag != '{urn:oasis:names:tc:SAML:2.0:metadata}EntitiesDescriptor': - raise Exception('Root element must be EntitiesDescriptor') - logging.debug('Root element is ' + root.tag) - pipeline = Pipeline(args.keyfile, args.certfile, - args.idprefix, args.cacheduration, args.validuntil) - for e in root.findall('{urn:oasis:names:tc:SAML:2.0:metadata}EntityDescriptor'): - fn_temp = os.path.abspath(os.path.join(args.outdir_unsigned, entityid_to_filename(e.attrib['entityID']))) - fn_out = os.path.abspath(os.path.join(args.outdir_signed, entityid_to_filename(e.attrib['entityID']))) - logging.debug('writing unsigned EntitiyDescriptor ' + e.attrib['entityID'] + ' to ' + fn_temp) - if not os.path.exists(os.path.dirname(fn_temp)): - os.makedirs(os.path.dirname(fn_temp)) - with open(fn_temp, 'w') as f: - f.write(XMLDECLARATION + etree.tostring(e)) +def process_entity_descriptor(ed, pipeline, args): + fn_temp = os.path.abspath(os.path.join(args.outdir_unsigned, + entityid_to_filename(ed.attrib['entityID']))) + logging.debug('writing unsigned EntitiyDescriptor ' + ed.attrib['entityID'] + ' to ' + fn_temp) + if not os.path.exists(os.path.dirname(fn_temp)): + os.makedirs(os.path.dirname(fn_temp)) + with open(fn_temp, 'w') as f: + f.write(XMLDECLARATION + etree.tostring(ed)) + if not args.nosign: + fn_out = os.path.abspath(os.path.join(args.outdir_signed, + entityid_to_filename(e.attrib['entityID']))) fn_pipeline = fn_temp + '.fd' with open(fn_pipeline, 'w') as f_pipeline: f_pipeline.write(pipeline.get(fn_temp, fn_out)) #simple_md(fn_pipeline) -if __name__ == "__main__": # pragma: no cover - #print os.getcwd() - main() \ No newline at end of file +def process_md_aggregate(args): + root = etree.parse(args.input).getroot() + if root.tag != '{urn:oasis:names:tc:SAML:2.0:metadata}EntitiesDescriptor': + raise Exception('Root element must be EntitiesDescriptor') + logging.debug('Root element: ' + root.tag + ' ' + ' @'.join(root.attrib)) + pipeline = Pipeline(args.keyfile, args.certfile, + args.idprefix, args.cacheduration, + args.validuntil) + for ed in root.findall('{urn:oasis:names:tc:SAML:2.0:metadata}EntityDescriptor'): + process_entity_descriptor(ed, pipeline, args) From b372be58e63361f813920d7e0957d3aac7303115 Mon Sep 17 00:00:00 2001 From: r2h2 Date: Tue, 12 Jul 2016 14:43:03 +0200 Subject: [PATCH 09/20] inherit values for validUntil and cacheDuration from EntitiesDesc --- scripts/pyff_mdsplit.py | 10 ++++++---- src/pyff/mdsplit.py | 17 ++++++++++++----- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/pyff_mdsplit.py b/scripts/pyff_mdsplit.py index 7c100060..0307f2ea 100644 --- a/scripts/pyff_mdsplit.py +++ b/scripts/pyff_mdsplit.py @@ -24,9 +24,10 @@ -o OUTDIR_SIGNED, --outdir_signed OUTDIR_SIGNED Directory for files containing one signed EntityDescriptor each. - -i IDPREFIX, --idprefix IDPREFIX -C CACHEDURATION, --cacheduration CACHEDURATION + override value from input EntitiesDescriptor, if any -u VALIDUNTIL, --validuntil VALIDUNTIL + override value from input EntitiesDescriptor, if any """ import argparse @@ -55,9 +56,10 @@ def __init__(self, testargs=None): choices=LOGLEVELS.keys(), help='default is INFO if env[LOGLEVEL] is not set') self.parser.add_argument('-o', '--outdir_signed', default=None, help='Directory for files containing one signed EntityDescriptor each.') - self.parser.add_argument('-i', '--idprefix', dest='idprefix', default='federation.example.com_') # TODO: take from input - self.parser.add_argument('-C', '--cacheduration', dest='cacheduration', default='PT5H') # TODO: take from input - self.parser.add_argument('-u', '--validuntil', dest='validuntil', default='P10D') # TODO: take from input + self.parser.add_argument('-C', '--cacheduration', dest='cacheDuration', default='PT5H', + help='override value from input EntitiesDescriptor, if any') + self.parser.add_argument('-u', '--validuntil', dest='validUntil', default='P10D', + help='override value from input EntitiesDescriptor, if any') self.parser.add_argument('input', type=argparse.FileType('r'), default=None, help='Metadata aggregate (input)') self.parser.add_argument('outdir_unsigned', default=None, diff --git a/src/pyff/mdsplit.py b/src/pyff/mdsplit.py index 955bbf37..3ea51c12 100644 --- a/src/pyff/mdsplit.py +++ b/src/pyff/mdsplit.py @@ -20,10 +20,9 @@ class Pipeline: - def __init__(self, keyfile, certfile, idprefix, cacheDuration, validUntil): + def __init__(self, keyfile, certfile, cacheDuration, validUntil): self.keyfile = keyfile self.certfile = certfile - self.idprefix = idprefix self.cacheDuration = cacheDuration self.validUntil = validUntil @@ -90,10 +89,14 @@ def process_entity_descriptor(ed, pipeline, args): fn_temp = os.path.abspath(os.path.join(args.outdir_unsigned, entityid_to_filename(ed.attrib['entityID']))) logging.debug('writing unsigned EntitiyDescriptor ' + ed.attrib['entityID'] + ' to ' + fn_temp) + if args.cacheDuration is not None: + ed.attrib['cacheDuration'] = args.cacheDuration + if args.validUntil is not None: + ed.attrib['validUntil'] = args.validUntil if not os.path.exists(os.path.dirname(fn_temp)): os.makedirs(os.path.dirname(fn_temp)) with open(fn_temp, 'w') as f: - f.write(XMLDECLARATION + etree.tostring(ed)) + f.write(XMLDECLARATION + '\n' + etree.tostring(ed)) if not args.nosign: fn_out = os.path.abspath(os.path.join(args.outdir_signed, entityid_to_filename(e.attrib['entityID']))) @@ -104,12 +107,16 @@ def process_entity_descriptor(ed, pipeline, args): def process_md_aggregate(args): + """ process each ed; take validUntil and cacheDuration from root level """ root = etree.parse(args.input).getroot() if root.tag != '{urn:oasis:names:tc:SAML:2.0:metadata}EntitiesDescriptor': raise Exception('Root element must be EntitiesDescriptor') + if 'cacheDuration' in root.attrib and args.cacheDuration is None: + args.cacheDuration = root.attrib['cacheDuration'] + if 'validUntil' in root.attrib and args.validUntil is None: + args.validUntil = root.attrib['validUntil'] logging.debug('Root element: ' + root.tag + ' ' + ' @'.join(root.attrib)) pipeline = Pipeline(args.keyfile, args.certfile, - args.idprefix, args.cacheduration, - args.validuntil) + args.cacheDuration, args.validUntil) for ed in root.findall('{urn:oasis:names:tc:SAML:2.0:metadata}EntityDescriptor'): process_entity_descriptor(ed, pipeline, args) From 200322941abaeb199541d382bcf47a8ebc840765 Mon Sep 17 00:00:00 2001 From: r2h2 Date: Tue, 12 Jul 2016 15:05:12 +0200 Subject: [PATCH 10/20] remov redundant script entry --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ea2be9f7..567fd635 100755 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ entry_points={ 'console_scripts': ['pyff=pyff.md:main', 'pyffd=pyff.mdx:main'] }, - scripts=['scripts/pyff_mdsplit.py', 'pyff_split_sign.sh'], + scripts=['scripts/pyff_mdsplit.py', ], message_extractors={'src': [ ('**.py', 'python', None), ('**/templates/**.html', 'mako', None), From 349e48fdfde960dd1336205093f9a66e416f8d34 Mon Sep 17 00:00:00 2001 From: r2h2 Date: Tue, 12 Jul 2016 15:43:06 +0200 Subject: [PATCH 11/20] added shebang --- scripts/pyff_mdsplit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/pyff_mdsplit.py b/scripts/pyff_mdsplit.py index 0307f2ea..6bddb6e5 100644 --- a/scripts/pyff_mdsplit.py +++ b/scripts/pyff_mdsplit.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python """ Invoke mdsplit from the command line. See mdsplit.py for documentation on functionality. usage: pyff_mdsplit.py [-h] [-c CERTFILE] [-k KEYFILE] [-S] [-l LOGFILE] [-v] From 1ad46ace0b9a836dd4597ec96ee877e1f9b2be92 Mon Sep 17 00:00:00 2001 From: r2h2 Date: Tue, 12 Jul 2016 15:43:58 +0200 Subject: [PATCH 12/20] re-activate pkg_resources.require (does not work in dev env) --- src/pyff/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pyff/__init__.py b/src/pyff/__init__.py index acd6065b..5ddb1651 100644 --- a/src/pyff/__init__.py +++ b/src/pyff/__init__.py @@ -10,5 +10,4 @@ __license__ = "BSD" __maintainer__ = "leifj@sunet.se" __status__ = "Production" -# removing following line was a quick fix to thes message "pkg_resources.DistributionNotFound: The 'pyFF' distribution was not found" -#__version__ = pkg_resources.require("pyFF")[0].version +__version__ = pkg_resources.require("pyFF")[0].version From db7b3d0d3559e832f26368286b1f77e4e39cd6ec Mon Sep 17 00:00:00 2001 From: r2h2 Date: Tue, 12 Jul 2016 16:52:57 +0200 Subject: [PATCH 13/20] improve help text --- scripts/pyff_mdsplit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/pyff_mdsplit.py b/scripts/pyff_mdsplit.py index 6bddb6e5..b16900ca 100644 --- a/scripts/pyff_mdsplit.py +++ b/scripts/pyff_mdsplit.py @@ -59,8 +59,8 @@ def __init__(self, testargs=None): help='Directory for files containing one signed EntityDescriptor each.') self.parser.add_argument('-C', '--cacheduration', dest='cacheDuration', default='PT5H', help='override value from input EntitiesDescriptor, if any') - self.parser.add_argument('-u', '--validuntil', dest='validUntil', default='P10D', - help='override value from input EntitiesDescriptor, if any') + self.parser.add_argument('-u', '--validuntil', dest='validUntil', + help='override iso date value from input EntitiesDescriptor, if any') self.parser.add_argument('input', type=argparse.FileType('r'), default=None, help='Metadata aggregate (input)') self.parser.add_argument('outdir_unsigned', default=None, From 9b918402f32fe741ad387623f7b4a940bdd6dc60 Mon Sep 17 00:00:00 2001 From: r2h2 Date: Tue, 12 Jul 2016 16:54:33 +0200 Subject: [PATCH 14/20] ui + doc improvement --- src/pyff/mdsplit.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/pyff/mdsplit.py b/src/pyff/mdsplit.py index 3ea51c12..ec24aa0f 100644 --- a/src/pyff/mdsplit.py +++ b/src/pyff/mdsplit.py @@ -11,7 +11,6 @@ import re import sys -#import pyff from pyff.mdrepo import MDRepository from pyff.pipes import plumbing from pyff.store import MemoryStore @@ -32,9 +31,8 @@ def get(self, infile, outfile): - {0} - select - finalize: - Name: {4} - cacheDuration: {5} - validUntil: {6} + cacheDuration: {4} + validUntil: {5} - sign: key: {2} cert: {3} @@ -44,7 +42,6 @@ def get(self, infile, outfile): outfile, self.keyfile, self.certfile, - self.idprefix, self.cacheDuration, self.validUntil) return pipeline @@ -76,13 +73,13 @@ def entityid_to_filename(entityid): return r + '.xml' -# def simple_md(pipeline): -# """ stupid copy of md:main -> replace this """ -# modules = [] -# modules.append('pyff.builtins') -# store = MemoryStore() -# md = MDRepository(store=store) -# plumbing(pipeline).process(md, state={'batch': True, 'stats': {}}) +def simple_md(pipeline): + """ just a copy of md:main """ + modules = [] + modules.append('pyff.builtins') + store = MemoryStore() + md = MDRepository(store=store) + plumbing(pipeline).process(md, state={'batch': True, 'stats': {}}) def process_entity_descriptor(ed, pipeline, args): @@ -99,11 +96,11 @@ def process_entity_descriptor(ed, pipeline, args): f.write(XMLDECLARATION + '\n' + etree.tostring(ed)) if not args.nosign: fn_out = os.path.abspath(os.path.join(args.outdir_signed, - entityid_to_filename(e.attrib['entityID']))) + entityid_to_filename(ed.attrib['entityID']))) fn_pipeline = fn_temp + '.fd' with open(fn_pipeline, 'w') as f_pipeline: f_pipeline.write(pipeline.get(fn_temp, fn_out)) - #simple_md(fn_pipeline) + simple_md(fn_pipeline) def process_md_aggregate(args): @@ -115,8 +112,10 @@ def process_md_aggregate(args): args.cacheDuration = root.attrib['cacheDuration'] if 'validUntil' in root.attrib and args.validUntil is None: args.validUntil = root.attrib['validUntil'] - logging.debug('Root element: ' + root.tag + ' ' + ' @'.join(root.attrib)) - pipeline = Pipeline(args.keyfile, args.certfile, - args.cacheDuration, args.validUntil) + alist = '' + for a in root.attrib: + alist += ' ' + a + '="' + root.attrib[a] + '"' + logging.debug('Root element: ' + root.tag + alist) + pipeline = Pipeline(args.keyfile, args.certfile, args.cacheDuration, args.validUntil) for ed in root.findall('{urn:oasis:names:tc:SAML:2.0:metadata}EntityDescriptor'): process_entity_descriptor(ed, pipeline, args) From 8bef8a2a54145831d45f64811ce027140e6ef935 Mon Sep 17 00:00:00 2001 From: r2h2 Date: Tue, 12 Jul 2016 16:55:26 +0200 Subject: [PATCH 15/20] add comment --- src/pyff/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pyff/__init__.py b/src/pyff/__init__.py index 5ddb1651..02f71572 100644 --- a/src/pyff/__init__.py +++ b/src/pyff/__init__.py @@ -10,4 +10,7 @@ __license__ = "BSD" __maintainer__ = "leifj@sunet.se" __status__ = "Production" +# Issue when invoking pyff_mdsplit.pyfrom Pycharm +# "pkg_resources.DistributionNotFound: The 'pyFF' distribution was not found" +# remove following line temporarily to test the script __version__ = pkg_resources.require("pyFF")[0].version From 9650c8c67bb1933b66ba7de392e4a246d00f71e4 Mon Sep 17 00:00:00 2001 From: r2h2 Date: Wed, 13 Jul 2016 18:58:52 +0200 Subject: [PATCH 16/20] add --nocleanup option and cleanup if option is not set --- scripts/pyff_mdsplit.py | 16 +++++++++------- src/pyff/mdsplit.py | 3 +++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/scripts/pyff_mdsplit.py b/scripts/pyff_mdsplit.py index b16900ca..7146570a 100644 --- a/scripts/pyff_mdsplit.py +++ b/scripts/pyff_mdsplit.py @@ -18,7 +18,7 @@ -c CERTFILE, --certfile CERTFILE -k KEYFILE, --keyfile KEYFILE -S, --nosign do not sign output - -v, --vebose output to console with DEBUG leve + -v, --verbose output to console with DEBUG leve -l LOGFILE, --logfile LOGFILE -L {INFO,DEBUG,CRITICAL,WARNING,ERROR}, --loglevel {INFO,DEBUG,CRITICAL,WARNING,ERROR} default is INFO if env[LOGLEVEL] is not set @@ -48,23 +48,25 @@ def __init__(self, testargs=None): self.parser = argparse.ArgumentParser(description='Metadata Splitter') self.parser.add_argument('-c', '--certfile', dest='certfile') self.parser.add_argument('-k', '--keyfile', dest='keyfile') + self.parser.add_argument('-n', '--nocleanup', dest='keyfile', + help='do not delete temporary files after signing') self.parser.add_argument('-S', '--nosign', action='store_true', help='do not sign output') self.parser.add_argument('-v', '--verbose', action='store_true', help='output to console with DEBUG level') logbasename = re.sub(r'\.py$', '.log', os.path.basename(__file__)) self.parser.add_argument('-l', '--logfile', dest='logfile', default=logbasename) loglevel_env = os.environ['LOGLEVEL'] if 'LOGLEVEL' in os.environ else 'INFO' self.parser.add_argument('-L', '--loglevel', dest='loglevel', default=loglevel_env, - choices=LOGLEVELS.keys(), help='default is INFO if env[LOGLEVEL] is not set') + choices=LOGLEVELS.keys(), help='default is INFO if env[LOGLEVEL] is not set') self.parser.add_argument('-o', '--outdir_signed', default=None, - help='Directory for files containing one signed EntityDescriptor each.') + help='Directory for files containing one signed EntityDescriptor each.') self.parser.add_argument('-C', '--cacheduration', dest='cacheDuration', default='PT5H', - help='override value from input EntitiesDescriptor, if any') + help='override value from input EntitiesDescriptor, if any') self.parser.add_argument('-u', '--validuntil', dest='validUntil', - help='override iso date value from input EntitiesDescriptor, if any') + help='override iso date value from input EntitiesDescriptor, if any') self.parser.add_argument('input', type=argparse.FileType('r'), default=None, - help='Metadata aggregate (input)') + help='Metadata aggregate (input)') self.parser.add_argument('outdir_unsigned', default=None, - help='Directory for files containing one unsigned EntityDescriptor each.') + help='Directory for files containing one unsigned EntityDescriptor each.') self.args = self.parser.parse_args() # merge argv with env if not self.args.nosign: diff --git a/src/pyff/mdsplit.py b/src/pyff/mdsplit.py index ec24aa0f..da6784ae 100644 --- a/src/pyff/mdsplit.py +++ b/src/pyff/mdsplit.py @@ -101,6 +101,9 @@ def process_entity_descriptor(ed, pipeline, args): with open(fn_pipeline, 'w') as f_pipeline: f_pipeline.write(pipeline.get(fn_temp, fn_out)) simple_md(fn_pipeline) + if not args.nocleanup: + os.remove(fn_pipeline) + os.remove(fn_temp) def process_md_aggregate(args): From 708a734986c0afc7d375655aeafcdc7bf7fa4248 Mon Sep 17 00:00:00 2001 From: r2h2 Date: Wed, 13 Jul 2016 19:59:30 +0200 Subject: [PATCH 17/20] --nocleanup boolean --- scripts/pyff_mdsplit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyff_mdsplit.py b/scripts/pyff_mdsplit.py index 7146570a..6e436d86 100644 --- a/scripts/pyff_mdsplit.py +++ b/scripts/pyff_mdsplit.py @@ -48,7 +48,7 @@ def __init__(self, testargs=None): self.parser = argparse.ArgumentParser(description='Metadata Splitter') self.parser.add_argument('-c', '--certfile', dest='certfile') self.parser.add_argument('-k', '--keyfile', dest='keyfile') - self.parser.add_argument('-n', '--nocleanup', dest='keyfile', + self.parser.add_argument('-n', '--nocleanup', action='store_true', help='do not delete temporary files after signing') self.parser.add_argument('-S', '--nosign', action='store_true', help='do not sign output') self.parser.add_argument('-v', '--verbose', action='store_true', help='output to console with DEBUG level') From abc7bc9871d9fde39ac0fec3cc09ada976e131e4 Mon Sep 17 00:00:00 2001 From: r2h2 Date: Wed, 13 Jul 2016 20:37:51 +0200 Subject: [PATCH 18/20] create unsigned ed in subdir --- src/pyff/mdsplit.py | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/src/pyff/mdsplit.py b/src/pyff/mdsplit.py index da6784ae..ae23fa60 100644 --- a/src/pyff/mdsplit.py +++ b/src/pyff/mdsplit.py @@ -1,14 +1,16 @@ """ -pyffsplit creates pipeline files for each EntityDescriptor in an aggregate, which then used to +pyffsplit creates pipeline files for each EntityDescriptor in an aggregate, which are used to create separate signed XML documents with an EntitiyDescriptor each. -The input file is considered to be trusted, the signature is not verified. -TODO: Signing in the same process +Note: The input file is considered to be trusted, the signature is not verified. """ +__author__ = 'r2h2' + import logging import lxml.etree as etree import os import re +import shutil import sys from pyff.mdrepo import MDRepository @@ -47,11 +49,11 @@ def get(self, infile, outfile): return pipeline -def entityid_to_filename(entityid): +def entityid_to_dirname(entityid): """ - Derive a filename from an entityID, removing dots and slashes + Derive a directory name from an HTTP-URL-formed entityID, removing dots and slashes :param entityid: - :return: filename derived from entityID + :return: dirname derived from entityID """ x = re.sub(r'^https?://', '', entityid) r = '' @@ -70,7 +72,7 @@ def entityid_to_filename(entityid): in_path = True else: upper = True - return r + '.xml' + return r def simple_md(pipeline): @@ -83,8 +85,19 @@ def simple_md(pipeline): def process_entity_descriptor(ed, pipeline, args): - fn_temp = os.path.abspath(os.path.join(args.outdir_unsigned, - entityid_to_filename(ed.attrib['entityID']))) + """ + 1. create an unsigned EntityDescriptor XML file + 2. create a pipline file to sign it + 3. execute pyff to create an aggregate with the EntityDescriptor + 4. delete temp files + Note: for pyff pipeline processing the entityDescriptor xml file must be alone in a directory + TODO: remove EntitiesDecriptor and make EntityDescriptor the rool element. + """ + dirname_temp = os.path.abspath(os.path.join(args.outdir_unsigned, + entityid_to_dirname(ed.attrib['entityID']))) + if not os.path.exists(dirname_temp): + os.makedirs(dirname_temp) + fn_temp = os.path.join(dirname_temp, 'ed.xml') logging.debug('writing unsigned EntitiyDescriptor ' + ed.attrib['entityID'] + ' to ' + fn_temp) if args.cacheDuration is not None: ed.attrib['cacheDuration'] = args.cacheDuration @@ -95,15 +108,17 @@ def process_entity_descriptor(ed, pipeline, args): with open(fn_temp, 'w') as f: f.write(XMLDECLARATION + '\n' + etree.tostring(ed)) if not args.nosign: + if not os.path.exists(args.outdir_signed): + os.makedirs(args.outdir_signed) fn_out = os.path.abspath(os.path.join(args.outdir_signed, - entityid_to_filename(ed.attrib['entityID']))) - fn_pipeline = fn_temp + '.fd' + entityid_to_dirname(ed.attrib['entityID']) + '.xml')) + fn_pipeline = os.path.join(dirname_temp, 'ed.fd') with open(fn_pipeline, 'w') as f_pipeline: f_pipeline.write(pipeline.get(fn_temp, fn_out)) simple_md(fn_pipeline) if not args.nocleanup: - os.remove(fn_pipeline) - os.remove(fn_temp) + shutil.rmtree(dirname_temp) + def process_md_aggregate(args): From fc8534264ac9497d68594c8cb8d819765bfd8b52 Mon Sep 17 00:00:00 2001 From: r2h2 Date: Wed, 13 Jul 2016 20:46:32 +0200 Subject: [PATCH 19/20] create unsigned ed in subdir (fix fd) --- src/pyff/mdsplit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pyff/mdsplit.py b/src/pyff/mdsplit.py index ae23fa60..2c56c761 100644 --- a/src/pyff/mdsplit.py +++ b/src/pyff/mdsplit.py @@ -27,7 +27,7 @@ def __init__(self, keyfile, certfile, cacheDuration, validUntil): self.cacheDuration = cacheDuration self.validUntil = validUntil - def get(self, infile, outfile): + def get(self, loaddir, outfile): # sign a single entity descriptor pipeline = '''- load: - {0} @@ -40,7 +40,7 @@ def get(self, infile, outfile): cert: {3} - publish: {1} -'''.format(infile, +'''.format(loaddir, outfile, self.keyfile, self.certfile, @@ -114,7 +114,7 @@ def process_entity_descriptor(ed, pipeline, args): entityid_to_dirname(ed.attrib['entityID']) + '.xml')) fn_pipeline = os.path.join(dirname_temp, 'ed.fd') with open(fn_pipeline, 'w') as f_pipeline: - f_pipeline.write(pipeline.get(fn_temp, fn_out)) + f_pipeline.write(pipeline.get(dirname_temp, fn_out)) simple_md(fn_pipeline) if not args.nocleanup: shutil.rmtree(dirname_temp) From f0f7b94773e16827bdf380f133624db992005013 Mon Sep 17 00:00:00 2001 From: r2h2 Date: Wed, 13 Jul 2016 21:45:56 +0200 Subject: [PATCH 20/20] doc update --- docs/using.rst | 6 ++++-- scripts/pyff_mdsplit.py | 15 ++++++++------- src/pyff/mdsplit.py | 4 ++-- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 84b3d3be..3f1641ae 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -1,7 +1,7 @@ Using pyFF ============= -pyFF has two command-line tools: pyff and pyffd. +pyFF has three command-line tools: pyff, pyffd and pyff_mdsplit. .. code-block:: bash @@ -45,4 +45,6 @@ Processing steps are called pipes. A pipe can have arguments and options: Typically options are used to modify the behaviour of the pipe itself (think macros), while arguments provide runtime data to operate on. -Documentation for each pipe is in the :py:mod:`pyff.pipes.builtins` Module. Also take a look at the :doc:`examples`. \ No newline at end of file +Documentation for each pipe is in the :py:mod:`pyff.pipes.builtins` Module. Also take a look at the :doc:`examples`. + +For the documentation of pyff_mdsplit run pyff_mdsplit.py --help \ No newline at end of file diff --git a/scripts/pyff_mdsplit.py b/scripts/pyff_mdsplit.py index 6e436d86..a135e103 100644 --- a/scripts/pyff_mdsplit.py +++ b/scripts/pyff_mdsplit.py @@ -1,11 +1,10 @@ #!/usr/bin/env python """ -Invoke mdsplit from the command line. See mdsplit.py for documentation on functionality. -usage: pyff_mdsplit.py [-h] [-c CERTFILE] [-k KEYFILE] [-S] [-l LOGFILE] [-v] - [-L {INFO,DEBUG,CRITICAL,WARNING,ERROR}] - [-o OUTDIR_SIGNED] [-i IDPREFIX] [-C CACHEDURATION] - [-u VALIDUNTIL] +usage: pyff_mdsplit.py [-h] [-c CERTFILE] [-k KEYFILE] [-n] [-S] [-v] + [-l LOGFILE] [-L {INFO,DEBUG,CRITICAL,WARNING,ERROR}] + [-o OUTDIR_SIGNED] [-C CACHEDURATION] [-u VALIDUNTIL] input outdir_unsigned + Metadata Splitter positional arguments: @@ -17,8 +16,9 @@ -h, --help show this help message and exit -c CERTFILE, --certfile CERTFILE -k KEYFILE, --keyfile KEYFILE + -n, --nocleanup do not delete temporary files after signing -S, --nosign do not sign output - -v, --verbose output to console with DEBUG leve + -v, --verbose output to console with DEBUG level -l LOGFILE, --logfile LOGFILE -L {INFO,DEBUG,CRITICAL,WARNING,ERROR}, --loglevel {INFO,DEBUG,CRITICAL,WARNING,ERROR} default is INFO if env[LOGLEVEL] is not set @@ -28,7 +28,8 @@ -C CACHEDURATION, --cacheduration CACHEDURATION override value from input EntitiesDescriptor, if any -u VALIDUNTIL, --validuntil VALIDUNTIL - override value from input EntitiesDescriptor, if any + override iso date value from input EntitiesDescriptor, + if any """ import argparse diff --git a/src/pyff/mdsplit.py b/src/pyff/mdsplit.py index 2c56c761..d3a56c31 100644 --- a/src/pyff/mdsplit.py +++ b/src/pyff/mdsplit.py @@ -1,6 +1,6 @@ """ -pyffsplit creates pipeline files for each EntityDescriptor in an aggregate, which are used to -create separate signed XML documents with an EntitiyDescriptor each. +mdsplit creates a separate signed XML file for each EntitiyDescriptor from the input md aggregate. + Note: The input file is considered to be trusted, the signature is not verified. """