Merge 0194254 into cc5bf0e

docs/conf.py

@@ -50,9 +50,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '0.9'
+version = '0.10'
 # The full version, including alpha/beta/rc tags.
-release = '0.9.4'
+release = '0.10.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/examples.rst

@@ -6,7 +6,7 @@ Examples are king.
 Example 1 - A simple pull
 -------------------------
 
-Fetch SWAMID metadata, split it up into EntityDescriptor elements and store each as a separate file in /tmp/swamid.
+Fetch SWAMID metadata, split it up into EntityDescriptor elements and store each as a separate file in /tmp/swamid-2.0.xml.
 
 .. code-block:: yaml
 
@@ -19,7 +19,7 @@ Fetch SWAMID metadata, split it up into EntityDescriptor elements and store each
 This is a simple example in 3 steps: load, select, store and stats. Each of these commands operate on a metadata
 repository that starts out as empty. The first command (load) causes a URL to be downloaded and the SAML metadata
 found there is stored in the metadata repository. The next command (select) creates an active document (which in
-this case consists of all EntityDescriptors in the metadata repository). Next, publish is called which causes
+this case consists of all EntityDescriptors in the metadata repository). Next, (publish) is called which causes
 the active document to be stored in an XML file. Finally the stats command prints out some information about
 the metadata repository.
 
@@ -36,8 +36,10 @@ stylesheet (cf below) which cleans up some known problems, sign the result and w
 .. code-block:: yaml
 
     - load:
-       - http://mds.edugain.org edugain-signer.crt
-    - select: "http://mds.edugain.org!//md:EntityDescriptor[md:IDPSSODescriptor]"
+       - http://mds.edugain.org 
+       - edugain-signer.crt
+    - select:
+       - "http://mds.edugain.org!//md:EntityDescriptor[md:IDPSSODescriptor]"
     - xslt:
         stylesheet: tidy.xsl
     - finalize:
@@ -62,16 +64,21 @@ For reference the 'tidy' xsl is included with pyFF and looks like this:
     <?xml version="1.0"?>
     <xsl:stylesheet version="1.0"
                     xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-                    xmlns:shibmeta="urn:mace:shibboleth:metadata:1.0"
-                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                     xmlns:ds="http://www.w3.org/2000/09/xmldsig#"
-                    xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata"
-            xmlns:xi="http://www.w3.org/2001/XInclude"
-                    xmlns:shibmd="urn:mace:shibboleth:metadata:1.0">
+                    xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata">
 
       <xsl:template match="@ID"/>
+      <xsl:template match="@Id"/>
+      <xsl:template match="@xml:id"/>
       <xsl:template match="@validUntil"/>
       <xsl:template match="@cacheDuration"/>
+      <xsl:template match="@xml:base"/>
+      <xsl:template match="ds:Signature"/>
+      <xsl:template match="md:OrganizationName|md:OrganizationURL|md:OrganizationDisplayName">
+        <xsl:if test="normalize-space(text(()) != ''">
+                <xsl:copy><xsl:apply-templates select="node()|@*"/></xsl:copy>
+        </xsl:if>
+      </xsl:template>
 
       <xsl:template match="text()|comment()|@*">
         <xsl:copy/>
@@ -278,9 +285,10 @@ Now start pyffd:
 
 .. code-block:: bash
 
-  # pyffd -f --loglevel=DEBUG -p /var/run/pyffd.pid mdx.fd
+  # pyffd -f -C -p /tmp/pyffd.pid --loglevel=DEBUG --host=0.0.0.0 --port=8080 test.yaml
+  
 
-This should start pyffd in the foreground. If you remove the ``-f`` pyFF should daemonize. For running
+This should start pyffd in the foreground. If you remove the ``-f`` pyFF should daemonize. Setting the cache to ``-c`` will turn it off. For running
 pyFF in production I suggest something like this:
 
 .. code-block:: bash

docs/install.rst

@@ -4,8 +4,8 @@ Installation
 Before you install
 ------------------
 
-Make sure you have a reasonably modern python. pyFF is developed using 2.7 but 2.6
-should work just fine. It is recommended that you install pyFF into a virtualenv
+Make sure you have a reasonably modern python. pyFF is developed using 3.6.
+It is recommended that you install pyFF into a virtualenv
 but there are two ways: with or without site packages.
 
 For both methods start by installing a few basic OS packages. Here we illustrate
@@ -26,9 +26,6 @@ and if you're on a centos system (or other yum-based systems):
 If you want to use OS packages instead of python packages from pypi then
 consider also installing the following packages before you begin:
 
-.. code-block:: bash
-
-  # apt-get install python-lxml python-yaml python-eventlet python-setuptools
 
 With Sitepackages
 ~~~~~~~~~~~~~~~~~
@@ -39,8 +36,7 @@ fewer worries keeping your python environment in sync with OS-level libraries.
 .. code-block:: bash
 
   # apt-get install python-virtualenv
-  # mkdir -p /opt/pyff
-  # virtualenv /opt/pyff
+  # virtualenv python-pyff
 
 Choose this method if you want the OS to keep as many of your packages up to
 date for you.
@@ -53,13 +49,31 @@ are developing pyFF or want to run multiple python-based applications in
 parallell without having to worry about conflicts between packages.
 
 .. code-block:: bash
-
+  
+  # cd $HOME
   # apt-get install python-virtualenv
-  # mkdir -p /opt/pyff
-  # virtualenv /opt/pyff --no-site-packages
+  # virtualenv -p python3 python-pyff --no-site-packages
 
 Choose this method for maximum control - ideal for development setups.
 
+
+Verifying
+----------
+
+To verify that python 3.6 is the default python in the pyFF environment run
+
+.. code-block:: bash
+
+  # python --version
+
+The result should be Python 3.6 or later.
+
+To verify that the version of pip you have is the latest run.
+
+.. code-block:: bash
+  
+ # pip install --upgrade pip
+
 Installing 
 ----------
 
@@ -68,26 +82,39 @@ activating your virtualenv:
 
 .. code-block:: bash
 
-  # . /opt/pyff/bin/activate
+  # source python-pyff/bin/activate
 
 Next install pyFF:
 
 .. code-block:: bash
 
-  # pip install pyFF
+  # cd $HOME
+  # cd pyFF
+  # LANG=en_US.UTF-8 pip install -e .
 
 This will install a bunch of dependencies and compile bindings for both lxml, pyyaml
-aswell as pyXMLSecurity. This may take some time to complete. If there are no errors and if
+as well as pyXMLSecurity. This may take some time to complete. If there are no errors and if
 you have the *pyff* binary in your **$PATH** you should be done.
 
+.. code-block:: bash
+
+ # cd $HOME
+ # mkdir pyff-config
+ # cd pyff-config
+
+pyFF requires the following files which you should copy into the pyff-config directory
+
+- A pipeline file in the *yaml* format
+- Generate a signing key pair from the *genkey* script in the scripts directory 
+
 Upgrading
 ---------
 
 Unless you've made modifications, upgrading should be as simple as running 
 
 .. code-block:: bash
 
-  # . /opt/pyff/bin/activate
+  # source python-pyff/bin/activate
   # pip install -U pyff
 
 This should bring your virtualenv up to the latest version of pyff and its

docs/using.rst

@@ -8,14 +8,28 @@ pyFF has two command-line tools: pyff and pyffd.
   # pyff --loglevel=INFO pipeline.fd [pipeline2.fd]
   # pyffd --loglevel=INFO pipeline.fd [pipeline2.fd]
 
-pyff operates by setting up and running "pipelines". Each pipeline starts with an empty "active repository" - an
+
+pyFF operates by setting up and running "pipelines". Each pipeline starts with an empty "active repository" - an
 in-memory representation of a set of SAML metadata documents - and an empty "working document" - a subset of the
 EntityDescriptor elements in the active repository.
 
 The pyffd tool starts a metadata server with an HTTP-based interface for viewing and downloading metadata. The
 HTTP interface can produce *XML*, *HTML* and *JSON* output (as well as other formats with a bit of configuration) and
 implements the MDX specification for online SAML metadata query.
 
+To start the pyFF daemon
+
+.. code-block:: bash
+
+  # CACHE=-C
+  # PYFF_LOGLEVEL=DEBUG
+  # PYFF_UPDATE_FREQUENCY=28800
+  # PYFF_HOST=0.0.0.0
+  # PYFF_PORT=8080
+  # PYFF_PIDFILE=/tmp/pyff.pid
+
+  # pyffd -f ${CACHE} --loglevel=${PYFF_LOGLEVEL} --frequency=${PYFF_UPDATE_FREQUENCY} --host=${PYFF_HOST} --port=${PYFF_PORT} -p ${PYFF_PIDFILE} --proxy test_mdx.yaml
+
 Pipeline files are *yaml* documents representing a list of processing steps:
 
 .. code-block:: yaml

examples/big.fd

@@ -1,8 +1,8 @@
-- load xrd /tmp/big-loaded.xrd:
+- load:
   - examples/big.xrd
 - select
 - store: 
-     directory: /tmp/big
+     - directory /tmp/big
 - publish:
-     output: /tmp/big.xml
+     - output /tmp/big.xml
 - stats

examples/edugain-idps.fd

@@ -1,8 +1,10 @@
 - load:
-    - http://mds.edugain.org examples/edugain.crt
+    - http://mds.edugain.org 
+    - examples/edugain.crt
 - select: "/tmp/edugain!md:EntityDescriptor[md:IDPSSODescriptor]"
 - store:
-    directory: /tmp/edugain
+    - directory /tmp/edugain
 - certreport
 - publish:
-    output: /tmp/edugain-annotated.xml
+    - output /tmp/edugain-annotated.xml
+- stats

examples/expiration.fd

@@ -1,5 +1,5 @@
 - load:
-  - https://md.swamid.se/md/swamid-2.0.xml
+  - https://mds.swamid.se/md/swamid-2.0.xml
 - select
 - stats
 - certreport

examples/kirei.fd

@@ -1,23 +1,23 @@
 - load:
-  - https://idp.kirei.se/saml2/idp/metadata.php
+  - https://www.sso.uni-erlangen.de/simplesaml/saml2/idp/metadata.php
 - select
 - store:
-    directory: pyFFstore
+    - directory pyFFstore
 - fork:
   - select:
     - "!//md:EntityDescriptor[md:SPSSODescriptor]"
   - sign:
-      key:  sign.key
-      cert: sign.crt
+      key:  test-metadata-signer.key
+      cert: test-metadata-signer.crt
   - publish:
-     output: idfed-sp.xml
+     - output idfed-sp.xml
 - fork:
   - select:
     - "!//md:EntityDescriptor[md:IDPSSODescriptor]"
   - sign:
-      key:  sign.key
-      cert: sign.crt
+      key: test-metadata-signer.key
+      cert: test-metadata-signer.crt
   - publish:
-     output: idfed-xml.xml
+     - output idfed-xml.xml
 - certreport
 - stats

examples/openssl.conf

@@ -8,8 +8,8 @@ pkcs11 = pkcs11_section
 
 [pkcs11_section]
 engine_id = pkcs11
-dynamic_path = /usr/lib/engines/engine_pkcs11.so
-MODULE_PATH = /usr/lib/libsofthsm.so
+dynamic_path = /usr/lib64/engines-1.1/pkcs11.so
+MODULE_PATH = /usr/lib64/libsofthsm2.so
 PIN = secret1
 init = 0